From 6212feb93160385dda8fbde0be81253f96313b4a Mon Sep 17 00:00:00 2001 From: Pirate Praveen Date: Sun, 22 Dec 2019 22:52:31 +0530 Subject: [PATCH] New upstream version 12.4.6 --- elasticsearch-model/.gitignore | 3 + elasticsearch-model/CHANGELOG.md | 74 +- elasticsearch-model/Gemfile | 7 +- elasticsearch-model/README.md | 724 ++++++++++++++++-- elasticsearch-model/Rakefile | 16 +- ...ls.gemspec => elasticsearch-model.gemspec} | 23 +- .../examples/activerecord_article.rb | 77 ++ .../examples/activerecord_associations.rb | 177 +++++ .../activerecord_mapping_completion.rb | 69 ++ .../examples/couchbase_article.rb | 66 ++ .../examples/datamapper_article.rb | 71 ++ .../examples/mongoid_article.rb | 68 ++ elasticsearch-model/examples/ohm_article.rb | 70 ++ elasticsearch-model/examples/riak_article.rb | 52 ++ elasticsearch-model/gemfiles/3.0.gemfile | 13 + elasticsearch-model/gemfiles/4.0.gemfile | 12 + .../lib/elasticsearch/model.rb | 188 +++++ .../lib/elasticsearch/model/adapter.rb | 145 ++++ .../model/adapters/active_record.rb | 114 +++ .../elasticsearch/model/adapters/default.rb | 50 ++ .../elasticsearch/model/adapters/mongoid.rb | 82 ++ .../elasticsearch/model/adapters/multiple.rb | 112 +++ .../lib/elasticsearch/model/callbacks.rb | 35 + .../lib/elasticsearch/model/client.rb | 61 ++ .../elasticsearch/model/ext/active_record.rb | 14 + .../lib/elasticsearch/model/importing.rb | 151 ++++ .../lib/elasticsearch/model/indexing.rb | 434 +++++++++++ .../lib/elasticsearch/model/multimodel.rb | 83 ++ .../lib/elasticsearch/model/naming.rb | 122 +++ .../lib/elasticsearch/model/proxy.rb | 137 ++++ .../lib/elasticsearch/model/response.rb | 83 ++ .../lib/elasticsearch/model/response/base.rb | 44 ++ .../model/response/pagination.rb | 192 +++++ .../elasticsearch/model/response/records.rb | 73 ++ .../elasticsearch/model/response/result.rb | 63 ++ .../elasticsearch/model/response/results.rb | 31 + .../model/response/suggestions.rb | 13 + .../lib/elasticsearch/model/searching.rb | 109 +++ .../lib/elasticsearch/model/serializing.rb | 35 + .../elasticsearch/{rails => model}/version.rb | 2 +- .../lib/elasticsearch/rails.rb | 7 - .../elasticsearch/rails/instrumentation.rb | 36 - .../instrumentation/controller_runtime.rb | 41 - .../rails/instrumentation/log_subscriber.rb | 41 - .../rails/instrumentation/publishers.rb | 36 - .../rails/instrumentation/railtie.rb | 31 - .../lib/elasticsearch/rails/lograge.rb | 44 -- .../lib/elasticsearch/rails/tasks/import.rb | 112 --- .../lib/rails/templates/01-basic.rb | 335 -------- .../lib/rails/templates/02-pretty.rb | 311 -------- .../lib/rails/templates/03-expert.rb | 349 --------- .../lib/rails/templates/04-dsl.rb | 131 ---- .../lib/rails/templates/05-settings-files.rb | 77 -- .../lib/rails/templates/articles.yml.gz | Bin 4224991 -> 0 bytes .../rails/templates/articles_settings.json | 1 - .../lib/rails/templates/index.html.dsl.erb | 160 ---- .../lib/rails/templates/index.html.erb | 160 ---- .../lib/rails/templates/indexer.rb | 27 - .../lib/rails/templates/search.css | 72 -- .../templates/search_controller_test.dsl.rb | 130 ---- .../rails/templates/search_controller_test.rb | 131 ---- .../lib/rails/templates/searchable.dsl.rb | 217 ------ .../lib/rails/templates/searchable.rb | 206 ----- .../lib/rails/templates/seeds.rb | 57 -- ...active_record_associations_parent_child.rb | 139 ++++ .../active_record_associations_test.rb | 326 ++++++++ .../integration/active_record_basic_test.rb | 234 ++++++ ...active_record_custom_serialization_test.rb | 62 ++ .../integration/active_record_import_test.rb | 109 +++ .../active_record_namespaced_model_test.rb | 49 ++ .../active_record_pagination_test.rb | 145 ++++ .../integration/dynamic_index_name_test.rb | 47 ++ .../test/integration/mongoid_basic_test.rb | 177 +++++ .../test/integration/multiple_models_test.rb | 172 +++++ elasticsearch-model/test/support/model.json | 1 + elasticsearch-model/test/support/model.yml | 2 + elasticsearch-model/test/test_helper.rb | 37 +- .../test/unit/adapter_active_record_test.rb | 157 ++++ .../test/unit/adapter_default_test.rb | 41 + .../test/unit/adapter_mongoid_test.rb | 104 +++ .../test/unit/adapter_multiple_test.rb | 106 +++ elasticsearch-model/test/unit/adapter_test.rb | 69 ++ .../test/unit/callbacks_test.rb | 31 + elasticsearch-model/test/unit/client_test.rb | 27 + .../test/unit/importing_test.rb | 203 +++++ .../test/unit/indexing_test.rb | 650 ++++++++++++++++ .../instrumentation/instrumentation_test.rb | 61 -- .../test/unit/instrumentation/lograge_test.rb | 21 - elasticsearch-model/test/unit/module_test.rb | 57 ++ .../test/unit/multimodel_test.rb | 38 + elasticsearch-model/test/unit/naming_test.rb | 103 +++ elasticsearch-model/test/unit/proxy_test.rb | 100 +++ .../test/unit/response_base_test.rb | 40 + .../unit/response_pagination_kaminari_test.rb | 433 +++++++++++ .../response_pagination_will_paginate_test.rb | 398 ++++++++++ .../test/unit/response_records_test.rb | 91 +++ .../test/unit/response_result_test.rb | 90 +++ .../test/unit/response_results_test.rb | 31 + .../test/unit/response_test.rb | 104 +++ .../unit/searching_search_request_test.rb | 78 ++ .../test/unit/searching_test.rb | 41 + .../test/unit/serializing_test.rb | 17 + 102 files changed, 8290 insertions(+), 2908 deletions(-) rename elasticsearch-model/{elasticsearch-rails.gemspec => elasticsearch-model.gemspec} (72%) create mode 100644 elasticsearch-model/examples/activerecord_article.rb create mode 100644 elasticsearch-model/examples/activerecord_associations.rb create mode 100644 elasticsearch-model/examples/activerecord_mapping_completion.rb create mode 100644 elasticsearch-model/examples/couchbase_article.rb create mode 100644 elasticsearch-model/examples/datamapper_article.rb create mode 100644 elasticsearch-model/examples/mongoid_article.rb create mode 100644 elasticsearch-model/examples/ohm_article.rb create mode 100644 elasticsearch-model/examples/riak_article.rb create mode 100644 elasticsearch-model/gemfiles/3.0.gemfile create mode 100644 elasticsearch-model/gemfiles/4.0.gemfile create mode 100644 elasticsearch-model/lib/elasticsearch/model.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/adapter.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/adapters/active_record.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/adapters/default.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/adapters/mongoid.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/adapters/multiple.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/callbacks.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/client.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/ext/active_record.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/importing.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/indexing.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/multimodel.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/naming.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/proxy.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/response.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/response/base.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/response/pagination.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/response/records.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/response/result.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/response/results.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/response/suggestions.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/searching.rb create mode 100644 elasticsearch-model/lib/elasticsearch/model/serializing.rb rename elasticsearch-model/lib/elasticsearch/{rails => model}/version.rb (77%) delete mode 100644 elasticsearch-model/lib/elasticsearch/rails.rb delete mode 100644 elasticsearch-model/lib/elasticsearch/rails/instrumentation.rb delete mode 100644 elasticsearch-model/lib/elasticsearch/rails/instrumentation/controller_runtime.rb delete mode 100644 elasticsearch-model/lib/elasticsearch/rails/instrumentation/log_subscriber.rb delete mode 100644 elasticsearch-model/lib/elasticsearch/rails/instrumentation/publishers.rb delete mode 100644 elasticsearch-model/lib/elasticsearch/rails/instrumentation/railtie.rb delete mode 100644 elasticsearch-model/lib/elasticsearch/rails/lograge.rb delete mode 100644 elasticsearch-model/lib/elasticsearch/rails/tasks/import.rb delete mode 100644 elasticsearch-model/lib/rails/templates/01-basic.rb delete mode 100644 elasticsearch-model/lib/rails/templates/02-pretty.rb delete mode 100644 elasticsearch-model/lib/rails/templates/03-expert.rb delete mode 100644 elasticsearch-model/lib/rails/templates/04-dsl.rb delete mode 100644 elasticsearch-model/lib/rails/templates/05-settings-files.rb delete mode 100644 elasticsearch-model/lib/rails/templates/articles.yml.gz delete mode 100644 elasticsearch-model/lib/rails/templates/articles_settings.json delete mode 100644 elasticsearch-model/lib/rails/templates/index.html.dsl.erb delete mode 100644 elasticsearch-model/lib/rails/templates/index.html.erb delete mode 100644 elasticsearch-model/lib/rails/templates/indexer.rb delete mode 100644 elasticsearch-model/lib/rails/templates/search.css delete mode 100644 elasticsearch-model/lib/rails/templates/search_controller_test.dsl.rb delete mode 100644 elasticsearch-model/lib/rails/templates/search_controller_test.rb delete mode 100644 elasticsearch-model/lib/rails/templates/searchable.dsl.rb delete mode 100644 elasticsearch-model/lib/rails/templates/searchable.rb delete mode 100644 elasticsearch-model/lib/rails/templates/seeds.rb create mode 100644 elasticsearch-model/test/integration/active_record_associations_parent_child.rb create mode 100644 elasticsearch-model/test/integration/active_record_associations_test.rb create mode 100644 elasticsearch-model/test/integration/active_record_basic_test.rb create mode 100644 elasticsearch-model/test/integration/active_record_custom_serialization_test.rb create mode 100644 elasticsearch-model/test/integration/active_record_import_test.rb create mode 100644 elasticsearch-model/test/integration/active_record_namespaced_model_test.rb create mode 100644 elasticsearch-model/test/integration/active_record_pagination_test.rb create mode 100755 elasticsearch-model/test/integration/dynamic_index_name_test.rb create mode 100644 elasticsearch-model/test/integration/mongoid_basic_test.rb create mode 100644 elasticsearch-model/test/integration/multiple_models_test.rb create mode 100644 elasticsearch-model/test/support/model.json create mode 100644 elasticsearch-model/test/support/model.yml create mode 100644 elasticsearch-model/test/unit/adapter_active_record_test.rb create mode 100644 elasticsearch-model/test/unit/adapter_default_test.rb create mode 100644 elasticsearch-model/test/unit/adapter_mongoid_test.rb create mode 100644 elasticsearch-model/test/unit/adapter_multiple_test.rb create mode 100644 elasticsearch-model/test/unit/adapter_test.rb create mode 100644 elasticsearch-model/test/unit/callbacks_test.rb create mode 100644 elasticsearch-model/test/unit/client_test.rb create mode 100644 elasticsearch-model/test/unit/importing_test.rb create mode 100644 elasticsearch-model/test/unit/indexing_test.rb delete mode 100644 elasticsearch-model/test/unit/instrumentation/instrumentation_test.rb delete mode 100644 elasticsearch-model/test/unit/instrumentation/lograge_test.rb create mode 100644 elasticsearch-model/test/unit/module_test.rb create mode 100644 elasticsearch-model/test/unit/multimodel_test.rb create mode 100644 elasticsearch-model/test/unit/naming_test.rb create mode 100644 elasticsearch-model/test/unit/proxy_test.rb create mode 100644 elasticsearch-model/test/unit/response_base_test.rb create mode 100644 elasticsearch-model/test/unit/response_pagination_kaminari_test.rb create mode 100644 elasticsearch-model/test/unit/response_pagination_will_paginate_test.rb create mode 100644 elasticsearch-model/test/unit/response_records_test.rb create mode 100644 elasticsearch-model/test/unit/response_result_test.rb create mode 100644 elasticsearch-model/test/unit/response_results_test.rb create mode 100644 elasticsearch-model/test/unit/response_test.rb create mode 100644 elasticsearch-model/test/unit/searching_search_request_test.rb create mode 100644 elasticsearch-model/test/unit/searching_test.rb create mode 100644 elasticsearch-model/test/unit/serializing_test.rb diff --git a/elasticsearch-model/.gitignore b/elasticsearch-model/.gitignore index d87d4be66f..3934d7e554 100644 --- a/elasticsearch-model/.gitignore +++ b/elasticsearch-model/.gitignore @@ -15,3 +15,6 @@ spec/reports test/tmp test/version_tmp tmp + +gemfiles/3.0.gemfile.lock +gemfiles/4.0.gemfile.lock diff --git a/elasticsearch-model/CHANGELOG.md b/elasticsearch-model/CHANGELOG.md index 5bb4e13bd3..1b28033835 100644 --- a/elasticsearch-model/CHANGELOG.md +++ b/elasticsearch-model/CHANGELOG.md @@ -1,44 +1,74 @@ ## 0.1.9 -* Added checks for proper launch order and other updates to the example application templates -* Updated the example application to work with Elasticsearch 2.x -* Used the `suggest` method instead of `response['suggest']` in the application template +* Added a `suggest` method to wrap the suggestions in response +* Added the `:includes` option to Adapter::ActiveRecord::Records for eagerly loading associated models +* Delegated `max_pages` method properly for Kaminari's `next_page` +* Fixed `#dup` behaviour for Elasticsearch::Model +* Fixed typos in the README and examples ## 0.1.8 -* Added an example application template that loads settings from a file -* Added missing require in the seeds.rb file for the expert template -* Fixed double include of the aliased method (execute_without_instrumentation) -* Fixed the error when getting the search_controller_test.rb asset in `03-expert.rb` template -* Updated URLs for getting raw assets from Github in the `03-expert.rb` template +* Added "default per page" methods for pagination with multi model searches +* Added a convenience accessor for the `aggregations` part of response +* Added a full example with mapping for the completion suggester +* Added an integration test for paginating multiple models +* Added proper support for the new "multi_fields" in the mapping DSL +* Added the `no_timeout` option for `__find_in_batches` in the Mongoid adapter +* Added, that index settings can be loaded from any object that responds to `:read` +* Added, that index settings/mappings can be loaded from a YAML or JSON file +* Added, that String pagination parameters are converted to numbers +* Added, that empty block is not required for setting mapping options +* Added, that on MyModel#import, an exception is raised if the index does not exists +* Changed the Elasticsearch port in the Mongoid example to 9200 +* Cleaned up the tests for multiple fields/properties in mapping DSL +* Fixed a bug where continuous `#save` calls emptied the `@__changed_attributes` variable +* Fixed a buggy test introduced in #335 +* Fixed incorrect deserialization of records in the Multiple adapter +* Fixed incorrect examples and documentation +* Fixed unreliable order of returned results/records in the integration test for the multiple adapter +* Fixed, that `param_name` is used when paginating with WillPaginate +* Fixed the problem where `document_type` configuration was not propagated to mapping [6 months ago by Miguel Ferna +* Refactored the code in `__find_in_batches` to use Enumerable#each_slice +* Refactored the string queries in multiple_models_test.rb to avoid quote escaping ## 0.1.7 -* Updated dependencies for the gem and example applications -* Fixed various small errors in the `01-basic.rb` template -* Fixed error when inserting the Kaminari gem into Gemfile in the 02-pretty.rb template -* Fixed incorrect regex for adding Rails instrumentation into the application.rb in the `02-pretty.rb` template -* Fixed other small errors in the `02-pretty.rb` template -* Improved and added tests for the generated application from the `02-pretty.rb` template -* Added the `04-dsl.rb` template which uses the `elasticsearch-dsl` gem to build the search definition +* Improved examples and instructions in README and code annotations +* Prevented index methods to swallow all exceptions +* Added the `:validate` option to the `save` method for models +* Added support for searching across multiple models (elastic/elasticsearch-rails#345), + including documentation, examples and tests ## 0.1.6 -* Fixed errors in templates for the Rails example applications -* Fixed errors in the importing Rake task -* Refactored and updated the instrumentation support to allow integration with `Persistence::Model` +* Improved documentation +* Added dynamic getter/setter (block/proc) for `MyModel.index_name` +* Added the `update_document_attributes` method +* Added, that records to import can be limited by the `query` option ## 0.1.5 -* Fixed an exception when no suggestions were returned in the `03-expert` example application template +* Improved documentation +* Fixes and improvements to the "will_paginate" integration +* Added a `:preprocess` option to the `import` method +* Changed, that attributes are fetched from `as_indexed_json` in the `update_document` method +* Added an option to the import method to return an array of error messages instead of just count +* Fixed many problems with dependency hell +* Fixed tests so they run on Ruby 2.2 ## 0.1.2 -* Allow passing an ActiveRecord scope to the importing Rake task +* Properly delegate existence methods like `result.foo?` to `result._source.foo` +* Exception is raised when `type` is not passed to Mappings#new +* Allow passing an ActiveRecord scope to the `import` method +* Added, that `each_with_hit` and `map_with_hit` in `Elasticsearch::Model::Response::Records` call `to_a` +* Added support for [`will_paginate`](https://github.com/mislav/will_paginate) pagination library +* Added the ability to transform models during indexing +* Added explicit `type` and `id` methods to Response::Result, aliasing `_type` and `_id` ## 0.1.1 -* Improved the Rake tasks -* Improved the example application templates +* Improved documentation and tests +* Fixed Kaminari implementation bugs and inconsistencies ## 0.1.0 (Initial Version) diff --git a/elasticsearch-model/Gemfile b/elasticsearch-model/Gemfile index 1aeec6c9a0..a54f5084ea 100644 --- a/elasticsearch-model/Gemfile +++ b/elasticsearch-model/Gemfile @@ -1,9 +1,4 @@ source 'https://rubygems.org' -# Specify your gem's dependencies in elasticsearch-rails.gemspec +# Specify your gem's dependencies in elasticsearch-model.gemspec gemspec - -# TODO: Figure out how to specify dependency on local elasticsearch-model without endless "Resolving dependencies" -# if File.exists? File.expand_path("../../elasticsearch-model", __FILE__) -# gem 'elasticsearch-model', :path => File.expand_path("../../elasticsearch-model", __FILE__), :require => true -# end diff --git a/elasticsearch-model/README.md b/elasticsearch-model/README.md index 4760549385..c219a46001 100644 --- a/elasticsearch-model/README.md +++ b/elasticsearch-model/README.md @@ -1,8 +1,11 @@ -# Elasticsearch::Rails +# Elasticsearch::Model -The `elasticsearch-rails` library is a companion for the -the [`elasticsearch-model`](https://github.com/elasticsearch/elasticsearch-rails/tree/master/elasticsearch-model) -library, providing features suitable for Ruby on Rails applications. +The `elasticsearch-model` library builds on top of the +the [`elasticsearch`](https://github.com/elasticsearch/elasticsearch-ruby) library. + +It aims to simplify integration of Ruby classes ("models"), commonly found +e.g. in [Ruby on Rails](http://rubyonrails.org) applications, with the +[Elasticsearch](http://www.elasticsearch.org) search and analytics engine. The library is compatible with Ruby 1.9.3 and higher. @@ -10,107 +13,692 @@ The library is compatible with Ruby 1.9.3 and higher. Install the package from [Rubygems](https://rubygems.org): - gem install elasticsearch-rails + gem install elasticsearch-model To use an unreleased version, either add it to your `Gemfile` for [Bundler](http://bundler.io): - gem 'elasticsearch-rails', git: 'git://github.com/elasticsearch/elasticsearch-rails.git' + gem 'elasticsearch-model', git: 'git://github.com/elasticsearch/elasticsearch-rails.git' or install it from a source code checkout: git clone https://github.com/elasticsearch/elasticsearch-rails.git - cd elasticsearch-rails/elasticsearch-rails + cd elasticsearch-rails/elasticsearch-model bundle install rake install -## Features -### Rake Tasks +## Usage -To facilitate importing data from your models into Elasticsearch, require the task definition in your application, -eg. in the `lib/tasks/elasticsearch.rake` file: +Let's suppose you have an `Article` model: ```ruby -require 'elasticsearch/rails/tasks/import' +require 'active_record' +ActiveRecord::Base.establish_connection( adapter: 'sqlite3', database: ":memory:" ) +ActiveRecord::Schema.define(version: 1) { create_table(:articles) { |t| t.string :title } } + +class Article < ActiveRecord::Base; end + +Article.create title: 'Quick brown fox' +Article.create title: 'Fast black dogs' +Article.create title: 'Swift green frogs' ``` -To import the records from your `Article` model, run: +### Setup -```bash -$ bundle exec rake environment elasticsearch:import:model CLASS='Article' -``` - -To limit the imported records to a certain -ActiveRecord [scope](http://guides.rubyonrails.org/active_record_querying.html#scopes), -pass it to the task: - -```bash -$ bundle exec rake environment elasticsearch:import:model CLASS='Article' SCOPE='published' -``` - -Run this command to display usage instructions: - -```bash -$ bundle exec rake -D elasticsearch -``` - -### ActiveSupport Instrumentation - -To display information about the search request (duration, search definition) during development, -and to include the information in the Rails log file, require the component in your `application.rb` file: +To add the Elasticsearch integration for this model, require `elasticsearch/model` +and include the main module in your class: ```ruby -require 'elasticsearch/rails/instrumentation' +require 'elasticsearch/model' + +class Article < ActiveRecord::Base + include Elasticsearch::Model +end ``` -You should see an output like this in your application log in development environment: +This will extend the model with functionality related to Elasticsearch. - Article Search (321.3ms) { index: "articles", type: "article", body: { query: ... } } +#### Feature Extraction Pattern -Also, the total duration of the request to Elasticsearch is displayed in the Rails request breakdown: - - Completed 200 OK in 615ms (Views: 230.9ms | ActiveRecord: 0.0ms | Elasticsearch: 321.3ms) - -There's a special component for the [Lograge](https://github.com/roidrage/lograge) logger. -Require the component in your `application.rb` file (and set `config.lograge.enabled`): +Instead of including the `Elasticsearch::Model` module directly in your model, +you can include it in a "concern" or "trait" module, which is quite common pattern in Rails applications, +using e.g. `ActiveSupport::Concern` as the instrumentation: ```ruby -require 'elasticsearch/rails/lograge' +# In: app/models/concerns/searchable.rb +# +module Searchable + extend ActiveSupport::Concern + + included do + include Elasticsearch::Model + + mapping do + # ... + end + + def self.search(query) + # ... + end + end +end + +# In: app/models/article.rb +# +class Article + include Searchable +end ``` -You should see the duration of the request to Elasticsearch as part of each log event: +#### The `__elasticsearch__` Proxy - method=GET path=/search ... status=200 duration=380.89 view=99.64 db=0.00 es=279.37 +The `Elasticsearch::Model` module contains a big amount of class and instance methods to provide +all its functionality. To prevent polluting your model namespace, this functionality is primarily +available via the `__elasticsearch__` class and instance level proxy methods; +see the `Elasticsearch::Model::Proxy` class documentation for technical information. -### Rails Application Templates +The module will include important methods, such as `search`, into the class or module only +when they haven't been defined already. Following two calls are thus functionally equivalent: -You can generate a fully working example Ruby on Rails application, with an `Article` model and a search form, -to play with (it even downloads _Elasticsearch_ itself, generates the application skeleton and leaves you with -a _Git_ repository to explore the steps and the code) with the -[`01-basic.rb`](https://github.com/elasticsearch/elasticsearch-rails/blob/master/elasticsearch-rails/lib/rails/templates/01-basic.rb) template: - -```bash -rails new searchapp --skip --skip-bundle --template https://raw.github.com/elasticsearch/elasticsearch-rails/master/elasticsearch-rails/lib/rails/templates/01-basic.rb +```ruby +Article.__elasticsearch__.search 'fox' +Article.search 'fox' ``` -Run the same command again, in the same folder, with the -[`02-pretty`](https://github.com/elasticsearch/elasticsearch-rails/blob/master/elasticsearch-rails/lib/rails/templates/02-pretty.rb) -template to add features such as a custom `Article.search` method, result highlighting and -[_Bootstrap_](http://getbootstrap.com) integration: +See the `Elasticsearch::Model` module documentation for technical information. -```bash -rails new searchapp --skip --skip-bundle --template https://raw.github.com/elasticsearch/elasticsearch-rails/master/elasticsearch-rails/lib/rails/templates/02-pretty.rb +### The Elasticsearch client + +The module will set up a [client](https://github.com/elasticsearch/elasticsearch-ruby/tree/master/elasticsearch), +connected to `localhost:9200`, by default. You can access and use it as any other `Elasticsearch::Client`: + +```ruby +Article.__elasticsearch__.client.cluster.health +# => { "cluster_name"=>"elasticsearch", "status"=>"yellow", ... } ``` -Run the same command with the [`03-expert.rb`](https://github.com/elasticsearch/elasticsearch-rails/blob/master/elasticsearch-rails/lib/rails/templates/03-expert.rb) -template to refactor the application into a more complex use case, -with couple of hundreds of The New York Times articles as the example content. -The template will extract the Elasticsearch integration into a `Searchable` "concern" module, -define complex mapping, custom serialization, implement faceted navigation and suggestions as a part of -a complex query, and add a _Sidekiq_-based worker for updating the index in the background. +To use a client with different configuration, just set up a client for the model: + +```ruby +Article.__elasticsearch__.client = Elasticsearch::Client.new host: 'api.server.org' +``` + +Or configure the client for all models: + +```ruby +Elasticsearch::Model.client = Elasticsearch::Client.new log: true +``` + +You might want to do this during your application bootstrap process, e.g. in a Rails initializer. + +Please refer to the +[`elasticsearch-transport`](https://github.com/elasticsearch/elasticsearch-ruby/tree/master/elasticsearch-transport) +library documentation for all the configuration options, and to the +[`elasticsearch-api`](http://rubydoc.info/gems/elasticsearch-api) library documentation +for information about the Ruby client API. + +### Importing the data + +The first thing you'll want to do is importing your data into the index: + +```ruby +Article.import +# => 0 +``` + +It's possible to import only records from a specific `scope` or `query`, transform the batch with the `transform` +and `preprocess` options, or re-create the index by deleting it and creating it with correct mapping with the `force` option -- look for examples in the method documentation. + +No errors were reported during importing, so... let's search the index! + + +### Searching + +For starters, we can try the "simple" type of search: + +```ruby +response = Article.search 'fox dogs' + +response.took +# => 3 + +response.results.total +# => 2 + +response.results.first._score +# => 0.02250402 + +response.results.first._source.title +# => "Quick brown fox" +``` + +#### Search results + +The returned `response` object is a rich wrapper around the JSON returned from Elasticsearch, +providing access to response metadata and the actual results ("hits"). + +Each "hit" is wrapped in the `Result` class, and provides method access +to its properties via [`Hashie::Mash`](http://github.com/intridea/hashie). + +The `results` object supports the `Enumerable` interface: + +```ruby +response.results.map { |r| r._source.title } +# => ["Quick brown fox", "Fast black dogs"] + +response.results.select { |r| r.title =~ /^Q/ } +# => [#{"title"=>"Quick brown fox"}}>] +``` + +In fact, the `response` object will delegate `Enumerable` methods to `results`: + +```ruby +response.any? { |r| r.title =~ /fox|dog/ } +# => true +``` + +To use `Array`'s methods (including any _ActiveSupport_ extensions), just call `to_a` on the object: + +```ruby +response.to_a.last.title +# "Fast black dogs" +``` + +#### Search results as database records + +Instead of returning documents from Elasticsearch, the `records` method will return a collection +of model instances, fetched from the primary database, ordered by score: + +```ruby +response.records.to_a +# Article Load (0.3ms) SELECT "articles".* FROM "articles" WHERE "articles"."id" IN (1, 2) +# => [#
, #
] +``` + +The returned object is the genuine collection of model instances returned by your database, +i.e. `ActiveRecord::Relation` for ActiveRecord, or `Mongoid::Criteria` in case of MongoDB. + +This allows you to chain other methods on top of search results, as you would normally do: + +```ruby +response.records.where(title: 'Quick brown fox').to_a +# Article Load (0.2ms) SELECT "articles".* FROM "articles" WHERE "articles"."id" IN (1, 2) AND "articles"."title" = 'Quick brown fox' +# => [#
] + +response.records.records.class +# => ActiveRecord::Relation::ActiveRecord_Relation_Article +``` + +The ordering of the records by score will be preserved, unless you explicitly specify a different +order in your model query language: + +```ruby +response.records.order(:title).to_a +# Article Load (0.2ms) SELECT "articles".* FROM "articles" WHERE "articles"."id" IN (1, 2) ORDER BY "articles".title ASC +# => [#
, #
] +``` + +The `records` method returns the real instances of your model, which is useful when you want to access your +model methods -- at the expense of slowing down your application, of course. +In most cases, working with `results` coming from Elasticsearch is sufficient, and much faster. See the +[`elasticsearch-rails`](https://github.com/elasticsearch/elasticsearch-rails/tree/master/elasticsearch-rails) +library for more information about compatibility with the Ruby on Rails framework. + +When you want to access both the database `records` and search `results`, use the `each_with_hit` +(or `map_with_hit`) iterator: + +```ruby +response.records.each_with_hit { |record, hit| puts "* #{record.title}: #{hit._score}" } +# * Quick brown fox: 0.02250402 +# * Fast black dogs: 0.02250402 +``` + +#### Searching multiple models + +It is possible to search across multiple models with the module method: + +```ruby +Elasticsearch::Model.search('fox', [Article, Comment]).results.to_a.map(&:to_hash) +# => [ +# {"_index"=>"articles", "_type"=>"article", "_id"=>"1", "_score"=>0.35136628, "_source"=>...}, +# {"_index"=>"comments", "_type"=>"comment", "_id"=>"1", "_score"=>0.35136628, "_source"=>...} +# ] + +Elasticsearch::Model.search('fox', [Article, Comment]).records.to_a +# Article Load (0.3ms) SELECT "articles".* FROM "articles" WHERE "articles"."id" IN (1) +# Comment Load (0.2ms) SELECT "comments".* FROM "comments" WHERE "comments"."id" IN (1,5) +# => [#
, #, ...] +``` + +By default, all models which include the `Elasticsearch::Model` module are searched. + +NOTE: It is _not_ possible to chain other methods on top of the `records` object, since it + is a heterogenous collection, with models potentially backed by different databases. + +#### Pagination + +You can implement pagination with the `from` and `size` search parameters. However, search results +can be automatically paginated with the [`kaminari`](http://rubygems.org/gems/kaminari) or +[`will_paginate`](https://github.com/mislav/will_paginate) gems. +(The pagination gems must be added before the Elasticsearch gems in your Gemfile, +or loaded first in your application.) + +If Kaminari or WillPaginate is loaded, use the familiar paging methods: + +```ruby +response.page(2).results +response.page(2).records +``` + +In a Rails controller, use the the `params[:page]` parameter to paginate through results: + +```ruby +@articles = Article.search(params[:q]).page(params[:page]).records + +@articles.current_page +# => 2 +@articles.next_page +# => 3 +``` +To initialize and include the Kaminari pagination support manually: + +```ruby +Kaminari::Hooks.init +Elasticsearch::Model::Response::Response.__send__ :include, Elasticsearch::Model::Response::Pagination::Kaminari +``` + +#### The Elasticsearch DSL + +In most situation, you'll want to pass the search definition +in the Elasticsearch [domain-specific language](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html) to the client: + +```ruby +response = Article.search query: { match: { title: "Fox Dogs" } }, + highlight: { fields: { title: {} } } + +response.results.first.highlight.title +# ["Quick brown fox"] +``` + +You can pass any object which implements a `to_hash` method, or you can use your favourite JSON builder +to build the search definition as a JSON string: + +```ruby +require 'jbuilder' + +query = Jbuilder.encode do |json| + json.query do + json.match do + json.title do + json.query "fox dogs" + end + end + end +end + +response = Article.search query +response.results.first.title +# => "Quick brown fox" +``` + +### Index Configuration + +For proper search engine function, it's often necessary to configure the index properly. +The `Elasticsearch::Model` integration provides class methods to set up index settings and mappings. + +```ruby +class Article + settings index: { number_of_shards: 1 } do + mappings dynamic: 'false' do + indexes :title, analyzer: 'english', index_options: 'offsets' + end + end +end + +Article.mappings.to_hash +# => { +# :article => { +# :dynamic => "false", +# :properties => { +# :title => { +# :type => "string", +# :analyzer => "english", +# :index_options => "offsets" +# } +# } +# } +# } + +Article.settings.to_hash +# { :index => { :number_of_shards => 1 } } +``` + +You can use the defined settings and mappings to create an index with desired configuration: + +```ruby +Article.__elasticsearch__.client.indices.delete index: Article.index_name rescue nil +Article.__elasticsearch__.client.indices.create \ + index: Article.index_name, + body: { settings: Article.settings.to_hash, mappings: Article.mappings.to_hash } +``` + +There's a shortcut available for this common operation (convenient e.g. in tests): + +```ruby +Article.__elasticsearch__.create_index! force: true +Article.__elasticsearch__.refresh_index! +``` + +By default, index name and document type will be inferred from your class name, +you can set it explicitely, however: + +```ruby +class Article + index_name "articles-#{Rails.env}" + document_type "post" +end +``` + +### Updating the Documents in the Index + +Usually, we need to update the Elasticsearch index when records in the database are created, updated or deleted; +use the `index_document`, `update_document` and `delete_document` methods, respectively: + +```ruby +Article.first.__elasticsearch__.index_document +# => {"ok"=>true, ... "_version"=>2} +``` + +#### Automatic Callbacks + +You can automatically update the index whenever the record changes, by including +the `Elasticsearch::Model::Callbacks` module in your model: + +```ruby +class Article + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks +end + +Article.first.update_attribute :title, 'Updated!' + +Article.search('*').map { |r| r.title } +# => ["Updated!", "Lime green frogs", "Fast black dogs"] +``` + +The automatic callback on record update keeps track of changes in your model +(via [`ActiveModel::Dirty`](http://api.rubyonrails.org/classes/ActiveModel/Dirty.html)-compliant implementation), +and performs a _partial update_ when this support is available. + +The automatic callbacks are implemented in database adapters coming with `Elasticsearch::Model`. You can easily +implement your own adapter: please see the relevant chapter below. + +#### Custom Callbacks + +In case you would need more control of the indexing process, you can implement these callbacks yourself, +by hooking into `after_create`, `after_save`, `after_update` or `after_destroy` operations: + +```ruby +class Article + include Elasticsearch::Model + + after_save { logger.debug ["Updating document... ", index_document ].join } + after_destroy { logger.debug ["Deleting document... ", delete_document].join } +end +``` + +For ActiveRecord-based models, use the `after_commit` callback to protect +your data against inconsistencies caused by transaction rollbacks: + +```ruby +class Article < ActiveRecord::Base + include Elasticsearch::Model + + after_commit on: [:create] do + __elasticsearch__.index_document if self.published? + end + + after_commit on: [:update] do + __elasticsearch__.update_document if self.published? + end + + after_commit on: [:destroy] do + __elasticsearch__.delete_document if self.published? + end +end +``` + +#### Asynchronous Callbacks + +Of course, you're still performing an HTTP request during your database transaction, which is not optimal +for large-scale applications. A better option would be to process the index operations in background, +with a tool like [_Resque_](https://github.com/resque/resque) or [_Sidekiq_](https://github.com/mperham/sidekiq): + +```ruby +class Article + include Elasticsearch::Model + + after_save { Indexer.perform_async(:index, self.id) } + after_destroy { Indexer.perform_async(:delete, self.id) } +end +``` + +An example implementation of the `Indexer` worker class could look like this: + +```ruby +class Indexer + include Sidekiq::Worker + sidekiq_options queue: 'elasticsearch', retry: false + + Logger = Sidekiq.logger.level == Logger::DEBUG ? Sidekiq.logger : nil + Client = Elasticsearch::Client.new host: 'localhost:9200', logger: Logger + + def perform(operation, record_id) + logger.debug [operation, "ID: #{record_id}"] + + case operation.to_s + when /index/ + record = Article.find(record_id) + Client.index index: 'articles', type: 'article', id: record.id, body: record.as_indexed_json + when /delete/ + Client.delete index: 'articles', type: 'article', id: record_id + else raise ArgumentError, "Unknown operation '#{operation}'" + end + end +end +``` + +Start the _Sidekiq_ workers with `bundle exec sidekiq --queue elasticsearch --verbose` and +update a model: + +```ruby +Article.first.update_attribute :title, 'Updated' +``` + +You'll see the job being processed in the console where you started the _Sidekiq_ worker: + +``` +Indexer JID-eb7e2daf389a1e5e83697128 DEBUG: ["index", "ID: 7"] +Indexer JID-eb7e2daf389a1e5e83697128 INFO: PUT http://localhost:9200/articles/article/1 [status:200, request:0.004s, query:n/a] +Indexer JID-eb7e2daf389a1e5e83697128 DEBUG: > {"id":1,"title":"Updated", ...} +Indexer JID-eb7e2daf389a1e5e83697128 DEBUG: < {"ok":true,"_index":"articles","_type":"article","_id":"1","_version":6} +Indexer JID-eb7e2daf389a1e5e83697128 INFO: done: 0.006 sec +``` + +### Model Serialization + +By default, the model instance will be serialized to JSON using the `as_indexed_json` method, +which is defined automatically by the `Elasticsearch::Model::Serializing` module: + +```ruby +Article.first.__elasticsearch__.as_indexed_json +# => {"id"=>1, "title"=>"Quick brown fox"} +``` + +If you want to customize the serialization, just implement the `as_indexed_json` method yourself, +for instance with the [`as_json`](http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html#method-i-as_json) method: + +```ruby +class Article + include Elasticsearch::Model + + def as_indexed_json(options={}) + as_json(only: 'title') + end +end + +Article.first.as_indexed_json +# => {"title"=>"Quick brown fox"} +``` + +The re-defined method will be used in the indexing methods, such as `index_document`. + +Please note that in Rails 3, you need to either set `include_root_in_json: false`, or prevent adding +the "root" in the JSON representation with other means. + +#### Relationships and Associations + +When you have a more complicated structure/schema, you need to customize the `as_indexed_json` method - +or perform the indexing separately, on your own. +For example, let's have an `Article` model, which _has_many_ `Comment`s, +`Author`s and `Categories`. We might want to define the serialization like this: + +```ruby +def as_indexed_json(options={}) + self.as_json( + include: { categories: { only: :title}, + authors: { methods: [:full_name], only: [:full_name] }, + comments: { only: :text } + }) +end + +Article.first.as_indexed_json +# => { "id" => 1, +# "title" => "First Article", +# "created_at" => 2013-12-03 13:39:02 UTC, +# "updated_at" => 2013-12-03 13:39:02 UTC, +# "categories" => [ { "title" => "One" } ], +# "authors" => [ { "full_name" => "John Smith" } ], +# "comments" => [ { "text" => "First comment" } ] } +``` + +Of course, when you want to use the automatic indexing callbacks, you need to hook into the appropriate +_ActiveRecord_ callbacks -- please see the full example in `examples/activerecord_associations.rb`. + +### Other ActiveModel Frameworks + +The `Elasticsearch::Model` module is fully compatible with any ActiveModel-compatible model, such as _Mongoid_: + +```ruby +require 'mongoid' + +Mongoid.connect_to 'articles' + +class Article + include Mongoid::Document + + field :id, type: String + field :title, type: String + + attr_accessible :id, :title, :published_at + + include Elasticsearch::Model + + def as_indexed_json(options={}) + as_json(except: [:id, :_id]) + end +end + +Article.create id: '1', title: 'Quick brown fox' +Article.import + +response = Article.search 'fox'; +response.records.to_a +# MOPED: 127.0.0.1:27017 QUERY database=articles collection=articles selector={"_id"=>{"$in"=>["1"]}} ... +# => [#
] +``` + +Full examples for CouchBase, DataMapper, Mongoid, Ohm and Riak models can be found in the `examples` folder. + +### Adapters + +To support various "OxM" (object-relational- or object-document-mapper) implementations and frameworks, +the `Elasticsearch::Model` integration supports an "adapter" concept. + +An adapter provides implementations for common behaviour, such as fetching records from the database, +hooking into model callbacks for automatic index updates, or efficient bulk loading from the database. +The integration comes with adapters for _ActiveRecord_ and _Mongoid_ out of the box. + +Writing an adapter for your favourite framework is straightforward -- let's see +a simplified adapter for [_DataMapper_](http://datamapper.org): + +```ruby +module DataMapperAdapter + + # Implement the interface for fetching records + # + module Records + def records + klass.all(id: @ids) + end + + # ... + end +end + +# Register the adapter +# +Elasticsearch::Model::Adapter.register( + DataMapperAdapter, + lambda { |klass| defined?(::DataMapper::Resource) and klass.ancestors.include?(::DataMapper::Resource) } +) +``` + +Require the adapter and include `Elasticsearch::Model` in the class: + +```ruby +require 'datamapper_adapter' + +class Article + include DataMapper::Resource + include Elasticsearch::Model + + property :id, Serial + property :title, String +end +``` + +When accessing the `records` method of the response, for example, +the implementation from our adapter will be used now: + +```ruby +response = Article.search 'foo' + +response.records.to_a +# ~ (0.000057) SELECT "id", "title", "published_at" FROM "articles" WHERE "id" IN (3, 1) ORDER BY "id" +# => [#
, #
] + +response.records.records.class +# => DataMapper::Collection +``` + +More examples can be found in the `examples` folder. Please see the `Elasticsearch::Model::Adapter` +module and its submodules for technical information. + +## Development and Community + +For local development, clone the repository and run `bundle install`. See `rake -T` for a list of +available Rake tasks for running tests, generating documentation, starting a testing cluster, etc. + +Bug fixes and features must be covered by unit tests. + +Github's pull requests and issues are used to communicate, send bug reports and code contributions. + +To run all tests against a test Elasticsearch cluster, use a command like this: ```bash -rails new searchapp --skip --skip-bundle --template https://raw.github.com/elasticsearch/elasticsearch-rails/master/elasticsearch-rails/lib/rails/templates/03-expert.rb +curl -# https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.0.0.RC1.tar.gz | tar xz -C tmp/ +SERVER=start TEST_CLUSTER_COMMAND=$PWD/tmp/elasticsearch-1.0.0.RC1/bin/elasticsearch bundle exec rake test:all ``` ## License diff --git a/elasticsearch-model/Rakefile b/elasticsearch-model/Rakefile index 3cf581a919..2825f58b01 100644 --- a/elasticsearch-model/Rakefile +++ b/elasticsearch-model/Rakefile @@ -22,16 +22,24 @@ namespace :test do # test.warning = true end - Rake::TestTask.new(:integration) do |test| + Rake::TestTask.new(:run_integration) do |test| Rake::Task['test:ci_reporter'].invoke if ENV['CI'] test.libs << 'lib' << 'test' test.test_files = FileList["test/integration/**/*_test.rb"] end - Rake::TestTask.new(:all) do |test| + desc "Run integration tests against ActiveModel 3 and 4" + task :integration do + sh "BUNDLE_GEMFILE='#{File.expand_path('../gemfiles/3.0.gemfile', __FILE__)}' bundle exec rake test:run_integration" unless defined?(RUBY_VERSION) && RUBY_VERSION > '2.2' + sh "BUNDLE_GEMFILE='#{File.expand_path('../gemfiles/4.0.gemfile', __FILE__)}' bundle exec rake test:run_integration" + end + + desc "Run unit and integration tests" + task :all do Rake::Task['test:ci_reporter'].invoke if ENV['CI'] - test.libs << 'lib' << 'test' - test.test_files = FileList["test/unit/**/*_test.rb", "test/integration/**/*_test.rb"] + + Rake::Task['test:unit'].invoke + Rake::Task['test:integration'].invoke end end diff --git a/elasticsearch-model/elasticsearch-rails.gemspec b/elasticsearch-model/elasticsearch-model.gemspec similarity index 72% rename from elasticsearch-model/elasticsearch-rails.gemspec rename to elasticsearch-model/elasticsearch-model.gemspec index 14dee1ddf7..df9509f064 100644 --- a/elasticsearch-model/elasticsearch-rails.gemspec +++ b/elasticsearch-model/elasticsearch-model.gemspec @@ -1,15 +1,15 @@ # coding: utf-8 lib = File.expand_path('../lib', __FILE__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) -require 'elasticsearch/rails/version' +require 'elasticsearch/model/version' Gem::Specification.new do |s| - s.name = "elasticsearch-rails" - s.version = Elasticsearch::Rails::VERSION + s.name = "elasticsearch-model" + s.version = Elasticsearch::Model::VERSION s.authors = ["Karel Minarik"] s.email = ["karel.minarik@elasticsearch.org"] - s.description = "Ruby on Rails integrations for Elasticsearch." - s.summary = "Ruby on Rails integrations for Elasticsearch." + s.description = "ActiveModel/Record integrations for Elasticsearch." + s.summary = "ActiveModel/Record integrations for Elasticsearch." s.homepage = "https://github.com/elasticsearch/elasticsearch-rails/" s.license = "Apache 2" @@ -23,16 +23,21 @@ Gem::Specification.new do |s| s.required_ruby_version = ">= 1.9.3" + s.add_dependency "elasticsearch", '> 0.4' + s.add_dependency "activesupport", '> 3' + s.add_dependency "hashie" + s.add_development_dependency "bundler", "~> 1.3" s.add_development_dependency "rake", "< 11.0" s.add_development_dependency "elasticsearch-extensions" - s.add_development_dependency "elasticsearch-model" + + s.add_development_dependency "sqlite3" + s.add_development_dependency "activemodel", "> 3.0" s.add_development_dependency "oj" - s.add_development_dependency "rails", ">= 3.1" - - s.add_development_dependency "lograge" + s.add_development_dependency "kaminari" + s.add_development_dependency "will_paginate" s.add_development_dependency "minitest", "~> 4.2" s.add_development_dependency "test-unit" if defined?(RUBY_VERSION) && RUBY_VERSION > '2.2' diff --git a/elasticsearch-model/examples/activerecord_article.rb b/elasticsearch-model/examples/activerecord_article.rb new file mode 100644 index 0000000000..b18ee9c7bd --- /dev/null +++ b/elasticsearch-model/examples/activerecord_article.rb @@ -0,0 +1,77 @@ +# ActiveRecord and Elasticsearch +# ============================== +# +# https://github.com/rails/rails/tree/master/activerecord + +$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) + +require 'pry' +Pry.config.history.file = File.expand_path('../../tmp/elasticsearch_development.pry', __FILE__) + +require 'logger' +require 'ansi/core' +require 'active_record' +require 'kaminari' + +require 'elasticsearch/model' + +ActiveRecord::Base.logger = ActiveSupport::Logger.new(STDOUT) +ActiveRecord::Base.establish_connection( adapter: 'sqlite3', database: ":memory:" ) + +ActiveRecord::Schema.define(version: 1) do + create_table :articles do |t| + t.string :title + t.date :published_at + t.timestamps + end +end + +Kaminari::Hooks.init + +class Article < ActiveRecord::Base +end + +# Store data +# +Article.delete_all +Article.create title: 'Foo' +Article.create title: 'Bar' +Article.create title: 'Foo Foo' + +# Index data +# +client = Elasticsearch::Client.new log:true + +# client.indices.delete index: 'articles' rescue nil +# client.indices.create index: 'articles', body: { mappings: { article: { dynamic: 'strict' }, properties: {} } } + +client.indices.delete index: 'articles' rescue nil +client.bulk index: 'articles', + type: 'article', + body: Article.all.as_json.map { |a| { index: { _id: a.delete('id'), data: a } } }, + refresh: true + +# Extend the model with Elasticsearch support +# +Article.__send__ :include, Elasticsearch::Model +# Article.__send__ :include, Elasticsearch::Model::Callbacks + +# ActiveRecord::Base.logger.silence do +# 10_000.times do |i| +# Article.create title: "Foo #{i}" +# end +# end + +puts '', '-'*Pry::Terminal.width! + +Elasticsearch::Model.client = Elasticsearch::Client.new log: true + +response = Article.search 'foo'; + +p response.size +p response.results.size +p response.records.size + +Pry.start(binding, prompt: lambda { |obj, nest_level, _| '> ' }, + input: StringIO.new('response.records.to_a'), + quiet: true) diff --git a/elasticsearch-model/examples/activerecord_associations.rb b/elasticsearch-model/examples/activerecord_associations.rb new file mode 100644 index 0000000000..6143a03560 --- /dev/null +++ b/elasticsearch-model/examples/activerecord_associations.rb @@ -0,0 +1,177 @@ +# ActiveRecord associations and Elasticsearch +# =========================================== +# +# https://github.com/rails/rails/tree/master/activerecord +# http://guides.rubyonrails.org/association_basics.html +# +# Run me with: +# +# ruby -I lib examples/activerecord_associations.rb +# + +$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) + +require 'pry' +Pry.config.history.file = File.expand_path('../../tmp/elasticsearch_development.pry', __FILE__) + +require 'logger' +require 'ansi/core' +require 'active_record' + +require 'elasticsearch/model' + +ActiveRecord::Base.logger = ActiveSupport::Logger.new(STDOUT) +ActiveRecord::Base.establish_connection( adapter: 'sqlite3', database: ":memory:" ) + +# ----- Schema definition ------------------------------------------------------------------------- + +ActiveRecord::Schema.define(version: 1) do + create_table :categories do |t| + t.string :title + t.timestamps + end + + create_table :authors do |t| + t.string :first_name, :last_name + t.timestamps + end + + create_table :authorships do |t| + t.references :article + t.references :author + t.timestamps + end + + create_table :articles do |t| + t.string :title + t.timestamps + end + + create_table :articles_categories, id: false do |t| + t.references :article, :category + end + + create_table :comments do |t| + t.string :text + t.references :article + t.timestamps + end + add_index(:comments, :article_id) +end + +# ----- Elasticsearch client setup ---------------------------------------------------------------- + +Elasticsearch::Model.client = Elasticsearch::Client.new log: true +Elasticsearch::Model.client.transport.logger.formatter = proc { |s, d, p, m| "\e[32m#{m}\n\e[0m" } + +# ----- Search integration ------------------------------------------------------------------------ + +module Searchable + extend ActiveSupport::Concern + + included do + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks + + include Indexing + after_touch() { __elasticsearch__.index_document } + end + + module Indexing + + # Customize the JSON serialization for Elasticsearch + def as_indexed_json(options={}) + self.as_json( + include: { categories: { only: :title}, + authors: { methods: [:full_name], only: [:full_name] }, + comments: { only: :text } + }) + end + end +end + +# ----- Model definitions ------------------------------------------------------------------------- + +class Category < ActiveRecord::Base + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks + + has_and_belongs_to_many :articles +end + +class Author < ActiveRecord::Base + has_many :authorships + + after_update { self.authorships.each(&:touch) } + + def full_name + [first_name, last_name].compact.join(' ') + end +end + +class Authorship < ActiveRecord::Base + belongs_to :author + belongs_to :article, touch: true +end + +class Article < ActiveRecord::Base + include Searchable + + has_and_belongs_to_many :categories, after_add: [ lambda { |a,c| a.__elasticsearch__.index_document } ], + after_remove: [ lambda { |a,c| a.__elasticsearch__.index_document } ] + has_many :authorships + has_many :authors, through: :authorships + has_many :comments +end + +class Comment < ActiveRecord::Base + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks + + belongs_to :article, touch: true +end + +# ----- Insert data ------------------------------------------------------------------------------- + +# Create category +# +category = Category.create title: 'One' + +# Create author +# +author = Author.create first_name: 'John', last_name: 'Smith' + +# Create article + +article = Article.create title: 'First Article' + +# Assign category +# +article.categories << category + +# Assign author +# +article.authors << author + +# Add comment +# +article.comments.create text: 'First comment for article One' +article.comments.create text: 'Second comment for article One' + +Elasticsearch::Model.client.indices.refresh index: Elasticsearch::Model::Registry.all.map(&:index_name) + +puts "\n\e[1mArticles containing 'one':\e[0m", Article.search('one').records.to_a.map(&:inspect), "" + +puts "\n\e[1mModels containing 'one':\e[0m", Elasticsearch::Model.search('one').records.to_a.map(&:inspect), "" + +# Load model +# +article = Article.all.includes(:categories, :authors, :comments).first + +# ----- Pry --------------------------------------------------------------------------------------- + +puts '', '-'*Pry::Terminal.width! + +Pry.start(binding, prompt: lambda { |obj, nest_level, _| '> ' }, + input: StringIO.new("article.as_indexed_json\n"), + quiet: true) diff --git a/elasticsearch-model/examples/activerecord_mapping_completion.rb b/elasticsearch-model/examples/activerecord_mapping_completion.rb new file mode 100644 index 0000000000..46d986011c --- /dev/null +++ b/elasticsearch-model/examples/activerecord_mapping_completion.rb @@ -0,0 +1,69 @@ +require 'ansi' +require 'active_record' +require 'elasticsearch/model' + +ActiveRecord::Base.logger = ActiveSupport::Logger.new(STDOUT) +ActiveRecord::Base.establish_connection( adapter: 'sqlite3', database: ":memory:" ) + +ActiveRecord::Schema.define(version: 1) do + create_table :articles do |t| + t.string :title + t.date :published_at + t.timestamps + end +end + +class Article < ActiveRecord::Base + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks + + mapping do + indexes :title + indexes :title_suggest, type: 'completion', payloads: true + end + + def as_indexed_json(options={}) + as_json.merge \ + title_suggest: { + input: title, + output: title, + payload: { url: "/articles/#{id}" } + } + end +end + +Article.__elasticsearch__.client = Elasticsearch::Client.new log: true + +# Create index + +Article.__elasticsearch__.create_index! force: true + +# Store data + +Article.delete_all +Article.create title: 'Foo' +Article.create title: 'Bar' +Article.create title: 'Foo Foo' +Article.__elasticsearch__.refresh_index! + +# Search and suggest + +response_1 = Article.search 'foo'; + +puts "Article search:".ansi(:bold), + response_1.to_a.map { |d| "Title: #{d.title}" }.inspect.ansi(:bold, :yellow) + +response_2 = Article.__elasticsearch__.client.suggest \ + index: Article.index_name, + body: { + articles: { + text: 'foo', + completion: { field: 'title_suggest', size: 25 } + } + }; + +puts "Article suggest:".ansi(:bold), + response_2['articles'].first['options'].map { |d| "#{d['text']} -> #{d['payload']['url']}" }. + inspect.ansi(:bold, :green) + +require 'pry'; binding.pry; diff --git a/elasticsearch-model/examples/couchbase_article.rb b/elasticsearch-model/examples/couchbase_article.rb new file mode 100644 index 0000000000..57cc421b01 --- /dev/null +++ b/elasticsearch-model/examples/couchbase_article.rb @@ -0,0 +1,66 @@ +# Couchbase and Elasticsearch +# =========================== +# +# https://github.com/couchbase/couchbase-ruby-model + +$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) + +require 'pry' +Pry.config.history.file = File.expand_path('../../tmp/elasticsearch_development.pry', __FILE__) + +require 'logger' +require 'couchbase/model' + +require 'elasticsearch/model' + +# Documents are stored as JSON objects in Riak but have rich +# semantics, including validations and associations. +class Article < Couchbase::Model + attribute :title + attribute :published_at + + # view :all, :limit => 10, :descending => true + # TODO: Implement view a la + # bucket.save_design_doc <<-JSON + # { + # "_id": "_design/article", + # "language": "javascript", + # "views": { + # "all": { + # "map": "function(doc, meta) { emit(doc.id, doc.title); }" + # } + # } + # } + # JSON + +end + +# Extend the model with Elasticsearch support +# +Article.__send__ :extend, Elasticsearch::Model::Client::ClassMethods +Article.__send__ :extend, Elasticsearch::Model::Searching::ClassMethods +Article.__send__ :extend, Elasticsearch::Model::Naming::ClassMethods + +# Create documents in Riak +# +Article.create id: '1', title: 'Foo' rescue nil +Article.create id: '2', title: 'Bar' rescue nil +Article.create id: '3', title: 'Foo Foo' rescue nil + +# Index data into Elasticsearch +# +client = Elasticsearch::Client.new log:true + +client.indices.delete index: 'articles' rescue nil +client.bulk index: 'articles', + type: 'article', + body: Article.find(['1', '2', '3']).map { |a| + { index: { _id: a.id, data: a.attributes } } + }, + refresh: true + +response = Article.search 'foo', index: 'articles', type: 'article'; + +Pry.start(binding, prompt: lambda { |obj, nest_level, _| '> ' }, + input: StringIO.new('response.records.to_a'), + quiet: true) diff --git a/elasticsearch-model/examples/datamapper_article.rb b/elasticsearch-model/examples/datamapper_article.rb new file mode 100644 index 0000000000..383b3738f0 --- /dev/null +++ b/elasticsearch-model/examples/datamapper_article.rb @@ -0,0 +1,71 @@ +# DataMapper and Elasticsearch +# ============================ +# +# https://github.com/datamapper/dm-core +# https://github.com/datamapper/dm-active_model + + +$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) + +require 'pry' +Pry.config.history.file = File.expand_path('../../tmp/elasticsearch_development.pry', __FILE__) + +require 'logger' +require 'ansi/core' + +require 'data_mapper' +require 'dm-active_model' + +require 'active_support/all' + +require 'elasticsearch/model' + +DataMapper::Logger.new(STDOUT, :debug) +DataMapper.setup(:default, 'sqlite::memory:') + +class Article + include DataMapper::Resource + + property :id, Serial + property :title, String + property :published_at, DateTime +end + +DataMapper.auto_migrate! +DataMapper.finalize + +Article.create title: 'Foo' +Article.create title: 'Bar' +Article.create title: 'Foo Foo' + +# Extend the model with Elasticsearch support +# +Article.__send__ :include, Elasticsearch::Model + +# The DataMapper adapter +# +module DataMapperAdapter + + # Implement the interface for fetching records + # + module Records + def records + klass.all(id: @ids) + end + + # ... + end +end + +# Register the adapter +# +Elasticsearch::Model::Adapter.register( + DataMapperAdapter, + lambda { |klass| defined?(::DataMapper::Resource) and klass.ancestors.include?(::DataMapper::Resource) } +) + +response = Article.search 'foo'; + +Pry.start(binding, prompt: lambda { |obj, nest_level, _| '> ' }, + input: StringIO.new('response.records.to_a'), + quiet: true) diff --git a/elasticsearch-model/examples/mongoid_article.rb b/elasticsearch-model/examples/mongoid_article.rb new file mode 100644 index 0000000000..5cd12ca4fa --- /dev/null +++ b/elasticsearch-model/examples/mongoid_article.rb @@ -0,0 +1,68 @@ +# Mongoid and Elasticsearch +# ========================= +# +# http://mongoid.org/en/mongoid/index.html + +$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) + +require 'pry' +Pry.config.history.file = File.expand_path('../../tmp/elasticsearch_development.pry', __FILE__) + +require 'benchmark' +require 'logger' +require 'ansi/core' +require 'mongoid' + +require 'elasticsearch/model' +require 'elasticsearch/model/callbacks' + +Mongoid.logger.level = Logger::DEBUG +Moped.logger.level = Logger::DEBUG + +Mongoid.connect_to 'articles' + +Elasticsearch::Model.client = Elasticsearch::Client.new host: 'localhost:9200', log: true + +class Article + include Mongoid::Document + field :id, type: String + field :title, type: String + field :published_at, type: DateTime + attr_accessible :id, :title, :published_at if respond_to? :attr_accessible + + def as_indexed_json(options={}) + as_json(except: [:id, :_id]) + end +end + +# Extend the model with Elasticsearch support +# +Article.__send__ :include, Elasticsearch::Model +# Article.__send__ :include, Elasticsearch::Model::Callbacks + +# Store data +# +Article.delete_all +Article.create id: '1', title: 'Foo' +Article.create id: '2', title: 'Bar' +Article.create id: '3', title: 'Foo Foo' + +# Index data +# +client = Elasticsearch::Client.new host:'localhost:9200', log:true + +client.indices.delete index: 'articles' rescue nil +client.bulk index: 'articles', + type: 'article', + body: Article.all.map { |a| { index: { _id: a.id, data: a.attributes } } }, + refresh: true + +# puts Benchmark.realtime { 9_875.times { |i| Article.create title: "Foo #{i}" } } + +puts '', '-'*Pry::Terminal.width! + +response = Article.search 'foo'; + +Pry.start(binding, prompt: lambda { |obj, nest_level, _| '> ' }, + input: StringIO.new('response.records.to_a'), + quiet: true) diff --git a/elasticsearch-model/examples/ohm_article.rb b/elasticsearch-model/examples/ohm_article.rb new file mode 100644 index 0000000000..3145085e79 --- /dev/null +++ b/elasticsearch-model/examples/ohm_article.rb @@ -0,0 +1,70 @@ +# Ohm for Redis and Elasticsearch +# =============================== +# +# https://github.com/soveran/ohm#example + +$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) + +require 'pry' +Pry.config.history.file = File.expand_path('../../tmp/elasticsearch_development.pry', __FILE__) + +require 'logger' +require 'ansi/core' +require 'active_model' +require 'ohm' + +require 'elasticsearch/model' + +class Article < Ohm::Model + # Include JSON serialization from ActiveModel + include ActiveModel::Serializers::JSON + + attribute :title + attribute :published_at +end + +# Extend the model with Elasticsearch support +# +Article.__send__ :include, Elasticsearch::Model + +# Register a custom adapter +# +module Elasticsearch + module Model + module Adapter + module Ohm + Adapter.register self, + lambda { |klass| defined?(::Ohm::Model) and klass.ancestors.include?(::Ohm::Model) } + module Records + def records + klass.fetch(@ids) + end + end + end + end + end +end + +# Configure the Elasticsearch client to log operations +# +Elasticsearch::Model.client = Elasticsearch::Client.new log: true + +puts '', '-'*Pry::Terminal.width! + +Article.all.map { |a| a.delete } +Article.create id: '1', title: 'Foo' +Article.create id: '2', title: 'Bar' +Article.create id: '3', title: 'Foo Foo' + +Article.__elasticsearch__.client.indices.delete index: 'articles' rescue nil +Article.__elasticsearch__.client.bulk index: 'articles', + type: 'article', + body: Article.all.map { |a| { index: { _id: a.id, data: a.attributes } } }, + refresh: true + + +response = Article.search 'foo', index: 'articles', type: 'article'; + +Pry.start(binding, prompt: lambda { |obj, nest_level, _| '> ' }, + input: StringIO.new('response.records.to_a'), + quiet: true) diff --git a/elasticsearch-model/examples/riak_article.rb b/elasticsearch-model/examples/riak_article.rb new file mode 100644 index 0000000000..8013cda7ea --- /dev/null +++ b/elasticsearch-model/examples/riak_article.rb @@ -0,0 +1,52 @@ +# Riak and Elasticsearch +# ====================== +# +# https://github.com/basho-labs/ripple + +$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) + +require 'pry' +Pry.config.history.file = File.expand_path('../../tmp/elasticsearch_development.pry', __FILE__) + +require 'logger' +require 'ripple' + +require 'elasticsearch/model' + +# Documents are stored as JSON objects in Riak but have rich +# semantics, including validations and associations. +class Article + include Ripple::Document + + property :title, String + property :published_at, Time, :default => proc { Time.now } +end + +# Extend the model with Elasticsearch support +# +Article.__send__ :include, Elasticsearch::Model + +# Create documents in Riak +# +Article.destroy_all +Article.create id: '1', title: 'Foo' +Article.create id: '2', title: 'Bar' +Article.create id: '3', title: 'Foo Foo' + +# Index data into Elasticsearch +# +client = Elasticsearch::Client.new log:true + +client.indices.delete index: 'articles' rescue nil +client.bulk index: 'articles', + type: 'article', + body: Article.all.map { |a| + { index: { _id: a.key, data: JSON.parse(a.robject.raw_data) } } + }.as_json, + refresh: true + +response = Article.search 'foo'; + +Pry.start(binding, prompt: lambda { |obj, nest_level, _| '> ' }, + input: StringIO.new('response.records.to_a'), + quiet: true) diff --git a/elasticsearch-model/gemfiles/3.0.gemfile b/elasticsearch-model/gemfiles/3.0.gemfile new file mode 100644 index 0000000000..23cbdf53d5 --- /dev/null +++ b/elasticsearch-model/gemfiles/3.0.gemfile @@ -0,0 +1,13 @@ +# Usage: +# +# $ BUNDLE_GEMFILE=./gemfiles/3.0.gemfile bundle install +# $ BUNDLE_GEMFILE=./gemfiles/3.0.gemfile bundle exec rake test:integration + +source 'https://rubygems.org' + +gemspec path: '../' + +gem 'activemodel', '>= 3.0' +gem 'activerecord', '~> 3.2' +gem 'mongoid', '>= 3.0' +gem 'sqlite3' diff --git a/elasticsearch-model/gemfiles/4.0.gemfile b/elasticsearch-model/gemfiles/4.0.gemfile new file mode 100644 index 0000000000..89044bb19e --- /dev/null +++ b/elasticsearch-model/gemfiles/4.0.gemfile @@ -0,0 +1,12 @@ +# Usage: +# +# $ BUNDLE_GEMFILE=./gemfiles/4.0.gemfile bundle install +# $ BUNDLE_GEMFILE=./gemfiles/4.0.gemfile bundle exec rake test:integration + +source 'https://rubygems.org' + +gemspec path: '../' + +gem 'activemodel', '~> 4' +gem 'activerecord', '~> 4' +gem 'sqlite3' diff --git a/elasticsearch-model/lib/elasticsearch/model.rb b/elasticsearch-model/lib/elasticsearch/model.rb new file mode 100644 index 0000000000..9d2b93da51 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model.rb @@ -0,0 +1,188 @@ +require 'elasticsearch' + +require 'hashie' + +require 'active_support/core_ext/module/delegation' + +require 'elasticsearch/model/version' + +require 'elasticsearch/model/client' + +require 'elasticsearch/model/multimodel' + +require 'elasticsearch/model/adapter' +require 'elasticsearch/model/adapters/default' +require 'elasticsearch/model/adapters/active_record' +require 'elasticsearch/model/adapters/mongoid' +require 'elasticsearch/model/adapters/multiple' + +require 'elasticsearch/model/importing' +require 'elasticsearch/model/indexing' +require 'elasticsearch/model/naming' +require 'elasticsearch/model/serializing' +require 'elasticsearch/model/searching' +require 'elasticsearch/model/callbacks' + +require 'elasticsearch/model/proxy' + +require 'elasticsearch/model/response' +require 'elasticsearch/model/response/base' +require 'elasticsearch/model/response/result' +require 'elasticsearch/model/response/results' +require 'elasticsearch/model/response/records' +require 'elasticsearch/model/response/pagination' +require 'elasticsearch/model/response/suggestions' + +require 'elasticsearch/model/ext/active_record' + +case +when defined?(::Kaminari) + Elasticsearch::Model::Response::Response.__send__ :include, Elasticsearch::Model::Response::Pagination::Kaminari +when defined?(::WillPaginate) + Elasticsearch::Model::Response::Response.__send__ :include, Elasticsearch::Model::Response::Pagination::WillPaginate +end + +module Elasticsearch + + # Elasticsearch integration for Ruby models + # ========================================= + # + # `Elasticsearch::Model` contains modules for integrating the Elasticsearch search and analytical engine + # with ActiveModel-based classes, or models, for the Ruby programming language. + # + # It facilitates importing your data into an index, automatically updating it when a record changes, + # searching the specific index, setting up the index mapping or the model JSON serialization. + # + # When the `Elasticsearch::Model` module is included in your class, it automatically extends it + # with the functionality; see {Elasticsearch::Model.included}. Most methods are available via + # the `__elasticsearch__` class and instance method proxies. + # + # It is possible to include/extend the model with the corresponding + # modules directly, if that is desired: + # + # MyModel.__send__ :extend, Elasticsearch::Model::Client::ClassMethods + # MyModel.__send__ :include, Elasticsearch::Model::Client::InstanceMethods + # MyModel.__send__ :extend, Elasticsearch::Model::Searching::ClassMethods + # # ... + # + module Model + METHODS = [:search, :mapping, :mappings, :settings, :index_name, :document_type, :import] + + # Adds the `Elasticsearch::Model` functionality to the including class. + # + # * Creates the `__elasticsearch__` class and instance methods, pointing to the proxy object + # * Includes the necessary modules in the proxy classes + # * Sets up delegation for crucial methods such as `search`, etc. + # + # @example Include the module in the `Article` model definition + # + # class Article < ActiveRecord::Base + # include Elasticsearch::Model + # end + # + # @example Inject the module into the `Article` model during run time + # + # Article.__send__ :include, Elasticsearch::Model + # + # + def self.included(base) + base.class_eval do + include Elasticsearch::Model::Proxy + + Elasticsearch::Model::Proxy::ClassMethodsProxy.class_eval do + include Elasticsearch::Model::Client::ClassMethods + include Elasticsearch::Model::Naming::ClassMethods + include Elasticsearch::Model::Indexing::ClassMethods + include Elasticsearch::Model::Searching::ClassMethods + end + + Elasticsearch::Model::Proxy::InstanceMethodsProxy.class_eval do + include Elasticsearch::Model::Client::InstanceMethods + include Elasticsearch::Model::Naming::InstanceMethods + include Elasticsearch::Model::Indexing::InstanceMethods + include Elasticsearch::Model::Serializing::InstanceMethods + end + + Elasticsearch::Model::Proxy::InstanceMethodsProxy.class_eval <<-CODE, __FILE__, __LINE__ + 1 + def as_indexed_json(options={}) + target.respond_to?(:as_indexed_json) ? target.__send__(:as_indexed_json, options) : super + end + CODE + + # Delegate important methods to the `__elasticsearch__` proxy, unless they are defined already + # + class << self + METHODS.each do |method| + delegate method, to: :__elasticsearch__ unless self.public_instance_methods.include?(method) + end + end + + # Mix the importing module into the proxy + # + self.__elasticsearch__.class_eval do + include Elasticsearch::Model::Importing::ClassMethods + include Adapter.from_class(base).importing_mixin + end + + # Add to the registry if it's a class (and not in intermediate module) + Registry.add(base) if base.is_a?(Class) + end + end + + module ClassMethods + + # Get the client common for all models + # + # @example Get the client + # + # Elasticsearch::Model.client + # => # + # + def client + @client ||= Elasticsearch::Client.new + end + + # Set the client for all models + # + # @example Configure (set) the client for all models + # + # Elasticsearch::Model.client = Elasticsearch::Client.new host: 'http://localhost:9200', tracer: true + # => # + # + # @note You have to set the client before you call Elasticsearch methods on the model, + # or set it directly on the model; see {Elasticsearch::Model::Client::ClassMethods#client} + # + def client=(client) + @client = client + end + + # Search across multiple models + # + # By default, all models which include the `Elasticsearch::Model` module are searched + # + # @param query_or_payload [String,Hash,Object] The search request definition + # (string, JSON, Hash, or object responding to `to_hash`) + # @param models [Array] The Array of Model objects to search + # @param options [Hash] Optional parameters to be passed to the Elasticsearch client + # + # @return [Elasticsearch::Model::Response::Response] + # + # @example Search across specific models + # + # Elasticsearch::Model.search('foo', [Author, Article]) + # + # @example Search across all models which include the `Elasticsearch::Model` module + # + # Elasticsearch::Model.search('foo') + # + def search(query_or_payload, models=[], options={}) + models = Multimodel.new(models) + request = Searching::SearchRequest.new(models, query_or_payload, options) + Response::Response.new(models, request) + end + end + extend ClassMethods + + class NotImplemented < NoMethodError; end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/adapter.rb b/elasticsearch-model/lib/elasticsearch/model/adapter.rb new file mode 100644 index 0000000000..3a25e5d97b --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/adapter.rb @@ -0,0 +1,145 @@ +module Elasticsearch + module Model + + # Contains an adapter which provides OxM-specific implementations for common behaviour: + # + # * {Adapter::Adapter#records_mixin Fetching records from the database} + # * {Adapter::Adapter#callbacks_mixin Model callbacks for automatic index updates} + # * {Adapter::Adapter#importing_mixin Efficient bulk loading from the database} + # + # @see Elasticsearch::Model::Adapter::Default + # @see Elasticsearch::Model::Adapter::ActiveRecord + # @see Elasticsearch::Model::Adapter::Mongoid + # + module Adapter + + # Returns an adapter based on the Ruby class passed + # + # @example Create an adapter for an ActiveRecord-based model + # + # class Article < ActiveRecord::Base; end + # + # myadapter = Elasticsearch::Model::Adapter.from_class(Article) + # myadapter.adapter + # # => Elasticsearch::Model::Adapter::ActiveRecord + # + # @see Adapter.adapters The list of included adapters + # @see Adapter.register Register a custom adapter + # + def from_class(klass) + Adapter.new(klass) + end; module_function :from_class + + # Returns registered adapters + # + # @see ::Elasticsearch::Model::Adapter::Adapter.adapters + # + def adapters + Adapter.adapters + end; module_function :adapters + + # Registers an adapter + # + # @see ::Elasticsearch::Model::Adapter::Adapter.register + # + def register(name, condition) + Adapter.register(name, condition) + end; module_function :register + + # Contains an adapter for specific OxM or architecture. + # + class Adapter + attr_reader :klass + + def initialize(klass) + @klass = klass + end + + # Registers an adapter for specific condition + # + # @param name [Module] The module containing the implemented interface + # @param condition [Proc] An object with a `call` method which is evaluated in {.adapter} + # + # @example Register an adapter for DataMapper + # + # module DataMapperAdapter + # + # # Implement the interface for fetching records + # # + # module Records + # def records + # klass.all(id: @ids) + # end + # + # # ... + # end + # end + # + # # Register the adapter + # # + # Elasticsearch::Model::Adapter.register( + # DataMapperAdapter, + # lambda { |klass| + # defined?(::DataMapper::Resource) and klass.ancestors.include?(::DataMapper::Resource) + # } + # ) + # + def self.register(name, condition) + self.adapters[name] = condition + end + + # Return the collection of registered adapters + # + # @example Return the currently registered adapters + # + # Elasticsearch::Model::Adapter.adapters + # # => { + # # Elasticsearch::Model::Adapter::ActiveRecord => #, + # # Elasticsearch::Model::Adapter::Mongoid => #, + # # } + # + # @return [Hash] The collection of adapters + # + def self.adapters + @adapters ||= {} + end + + # Return the module with {Default::Records} interface implementation + # + # @api private + # + def records_mixin + adapter.const_get(:Records) + end + + # Return the module with {Default::Callbacks} interface implementation + # + # @api private + # + def callbacks_mixin + adapter.const_get(:Callbacks) + end + + # Return the module with {Default::Importing} interface implementation + # + # @api private + # + def importing_mixin + adapter.const_get(:Importing) + end + + # Returns the adapter module + # + # @api private + # + def adapter + @adapter ||= begin + self.class.adapters.find( lambda {[]} ) { |name, condition| condition.call(klass) }.first \ + || Elasticsearch::Model::Adapter::Default + end + end + + end + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/adapters/active_record.rb b/elasticsearch-model/lib/elasticsearch/model/adapters/active_record.rb new file mode 100644 index 0000000000..2d9bb53786 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/adapters/active_record.rb @@ -0,0 +1,114 @@ +module Elasticsearch + module Model + module Adapter + + # An adapter for ActiveRecord-based models + # + module ActiveRecord + + Adapter.register self, + lambda { |klass| !!defined?(::ActiveRecord::Base) && klass.respond_to?(:ancestors) && klass.ancestors.include?(::ActiveRecord::Base) } + + module Records + attr_writer :options + + def options + @options ||= {} + end + + # Returns an `ActiveRecord::Relation` instance + # + def records + sql_records = klass.where(klass.primary_key => ids) + sql_records = sql_records.includes(self.options[:includes]) if self.options[:includes] + + # Re-order records based on the order from Elasticsearch hits + # by redefining `to_a`, unless the user has called `order()` + # + sql_records.instance_exec(response.response['hits']['hits']) do |hits| + define_singleton_method :to_a do + if defined?(::ActiveRecord) && ::ActiveRecord::VERSION::MAJOR >= 4 + self.load + else + self.__send__(:exec_queries) + end + @records.sort_by { |record| hits.index { |hit| hit['_id'].to_s == record.id.to_s } } + end + end + + sql_records + end + + # Prevent clash with `ActiveSupport::Dependencies::Loadable` + # + def load + records.load + end + + # Intercept call to the `order` method, so we can ignore the order from Elasticsearch + # + def order(*args) + sql_records = records.__send__ :order, *args + + # Redefine the `to_a` method to the original one + # + sql_records.instance_exec do + define_singleton_method(:to_a) do + if defined?(::ActiveRecord) && ::ActiveRecord::VERSION::MAJOR >= 4 + self.load + else + self.__send__(:exec_queries) + end + @records + end + end + + sql_records + end + end + + module Callbacks + + # Handle index updates (creating, updating or deleting documents) + # when the model changes, by hooking into the lifecycle + # + # @see http://guides.rubyonrails.org/active_record_callbacks.html + # + def self.included(base) + base.class_eval do + after_commit lambda { __elasticsearch__.index_document }, on: :create + after_commit lambda { __elasticsearch__.update_document }, on: :update + after_commit lambda { __elasticsearch__.delete_document }, on: :destroy + end + end + end + + module Importing + + # Fetch batches of records from the database (used by the import method) + # + # + # @see http://api.rubyonrails.org/classes/ActiveRecord/Batches.html ActiveRecord::Batches.find_in_batches + # + def __find_in_batches(options={}, &block) + query = options.delete(:query) + named_scope = options.delete(:scope) + preprocess = options.delete(:preprocess) + + scope = self + scope = scope.__send__(named_scope) if named_scope + scope = scope.instance_exec(&query) if query + + scope.find_in_batches(options) do |batch| + yield (preprocess ? self.__send__(preprocess, batch) : batch) + end + end + + def __transform + lambda { |model| { index: { _id: model.id, data: model.__elasticsearch__.as_indexed_json } } } + end + end + end + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/adapters/default.rb b/elasticsearch-model/lib/elasticsearch/model/adapters/default.rb new file mode 100644 index 0000000000..e58cf4ceb3 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/adapters/default.rb @@ -0,0 +1,50 @@ +module Elasticsearch + module Model + module Adapter + + # The default adapter for models which haven't one registered + # + module Default + + # Module for implementing methods and logic related to fetching records from the database + # + module Records + + # Return the collection of records fetched from the database + # + # By default uses `MyModel#find[1, 2, 3]` + # + def records + klass.find(@ids) + end + end + + # Module for implementing methods and logic related to hooking into model lifecycle + # (e.g. to perform automatic index updates) + # + # @see http://api.rubyonrails.org/classes/ActiveModel/Callbacks.html + module Callbacks + # noop + end + + # Module for efficiently fetching records from the database to import them into the index + # + module Importing + + # @abstract Implement this method in your adapter + # + def __find_in_batches(options={}, &block) + raise NotImplemented, "Method not implemented for default adapter" + end + + # @abstract Implement this method in your adapter + # + def __transform + raise NotImplemented, "Method not implemented for default adapter" + end + end + + end + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/adapters/mongoid.rb b/elasticsearch-model/lib/elasticsearch/model/adapters/mongoid.rb new file mode 100644 index 0000000000..5117dbf58d --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/adapters/mongoid.rb @@ -0,0 +1,82 @@ +module Elasticsearch + module Model + module Adapter + + # An adapter for Mongoid-based models + # + # @see http://mongoid.org + # + module Mongoid + + Adapter.register self, + lambda { |klass| !!defined?(::Mongoid::Document) && klass.respond_to?(:ancestors) && klass.ancestors.include?(::Mongoid::Document) } + + module Records + + # Return a `Mongoid::Criteria` instance + # + def records + criteria = klass.where(:id.in => ids) + + criteria.instance_exec(response.response['hits']['hits']) do |hits| + define_singleton_method :to_a do + self.entries.sort_by { |e| hits.index { |hit| hit['_id'].to_s == e.id.to_s } } + end + end + + criteria + end + + # Intercept call to sorting methods, so we can ignore the order from Elasticsearch + # + %w| asc desc order_by |.each do |name| + define_method name do |*args| + criteria = records.__send__ name, *args + criteria.instance_exec do + define_singleton_method(:to_a) { self.entries } + end + + criteria + end + end + end + + module Callbacks + + # Handle index updates (creating, updating or deleting documents) + # when the model changes, by hooking into the lifecycle + # + # @see http://mongoid.org/en/mongoid/docs/callbacks.html + # + def self.included(base) + base.after_create { |document| document.__elasticsearch__.index_document } + base.after_update { |document| document.__elasticsearch__.update_document } + base.after_destroy { |document| document.__elasticsearch__.delete_document } + end + end + + module Importing + + # Fetch batches of records from the database + # + # @see https://github.com/mongoid/mongoid/issues/1334 + # @see https://github.com/karmi/retire/pull/724 + # + def __find_in_batches(options={}, &block) + options[:batch_size] ||= 1_000 + + all.no_timeout.each_slice(options[:batch_size]) do |items| + yield items + end + end + + def __transform + lambda {|a| { index: { _id: a.id.to_s, data: a.as_indexed_json } }} + end + end + + end + + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/adapters/multiple.rb b/elasticsearch-model/lib/elasticsearch/model/adapters/multiple.rb new file mode 100644 index 0000000000..9a0bc4e8eb --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/adapters/multiple.rb @@ -0,0 +1,112 @@ +module Elasticsearch + module Model + module Adapter + + # An adapter to be used for deserializing results from multiple models, + # retrieved through `Elasticsearch::Model.search` + # + # @see Elasticsearch::Model.search + # + module Multiple + Adapter.register self, lambda { |klass| klass.is_a? Multimodel } + + module Records + # Returns a collection of model instances, possibly of different classes (ActiveRecord, Mongoid, ...) + # + # @note The order of results in the Elasticsearch response is preserved + # + def records + records_by_type = __records_by_type + + records = response.response["hits"]["hits"].map do |hit| + records_by_type[ __type_for_hit(hit) ][ hit[:_id] ] + end + + records.compact + end + + # Returns the collection of records grouped by class based on `_type` + # + # Example: + # + # { + # Foo => {"1"=> # {"1"=> # ids) + when Elasticsearch::Model::Adapter::Mongoid.equal?(adapter) + klass.where(:id.in => ids) + else + klass.find(ids) + end + end + + # Returns the record IDs grouped by class based on type `_type` + # + # Example: + # + # { Foo => ["1"], Bar => ["1", "5"] } + # + # @api private + # + def __ids_by_type + ids_by_type = {} + + response.response["hits"]["hits"].each do |hit| + type = __type_for_hit(hit) + ids_by_type[type] ||= [] + ids_by_type[type] << hit[:_id] + end + ids_by_type + end + + # Returns the class of the model corresponding to a specific `hit` in Elasticsearch results + # + # @see Elasticsearch::Model::Registry + # + # @api private + # + def __type_for_hit(hit) + @@__types ||= {} + + @@__types[ "#{hit[:_index]}::#{hit[:_type]}" ] ||= begin + Registry.all.detect do |model| + model.index_name == hit[:_index] && model.document_type == hit[:_type] + end + end + end + + # Returns the adapter registered for a particular `klass` or `nil` if not available + # + # @api private + # + def __adapter_for_klass(klass) + Adapter.adapters.select { |name, checker| checker.call(klass) }.keys.first + end + end + end + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/callbacks.rb b/elasticsearch-model/lib/elasticsearch/model/callbacks.rb new file mode 100644 index 0000000000..1b72cb2a03 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/callbacks.rb @@ -0,0 +1,35 @@ +module Elasticsearch + module Model + + # Allows to automatically update index based on model changes, + # by hooking into the model lifecycle. + # + # @note A blocking HTTP request is done during the update process. + # If you need a more performant/resilient way of updating the index, + # consider adapting the callbacks behaviour, and use a background + # processing solution such as [Sidekiq](http://sidekiq.org) + # or [Resque](https://github.com/resque/resque). + # + module Callbacks + + # When included in a model, automatically injects the callback subscribers (`after_save`, etc) + # + # @example Automatically update Elasticsearch index when the model changes + # + # class Article + # include Elasticsearch::Model + # include Elasticsearch::Model::Callbacks + # end + # + # Article.first.update_attribute :title, 'Updated' + # # SQL (0.3ms) UPDATE "articles" SET "title" = ... + # # 2013-11-20 15:08:52 +0100: POST http://localhost:9200/articles/article/1/_update ... + # + def self.included(base) + adapter = Adapter.from_class(base) + base.__send__ :include, adapter.callbacks_mixin + end + + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/client.rb b/elasticsearch-model/lib/elasticsearch/model/client.rb new file mode 100644 index 0000000000..c1a9b4ed91 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/client.rb @@ -0,0 +1,61 @@ +module Elasticsearch + module Model + + # Contains an `Elasticsearch::Client` instance + # + module Client + + module ClassMethods + + # Get the client for a specific model class + # + # @example Get the client for `Article` and perform API request + # + # Article.client.cluster.health + # # => { "cluster_name" => "elasticsearch" ... } + # + def client client=nil + @client ||= Elasticsearch::Model.client + end + + # Set the client for a specific model class + # + # @example Configure the client for the `Article` model + # + # Article.client = Elasticsearch::Client.new host: 'http://api.server:8080' + # Article.search ... + # + def client=(client) + @client = client + end + end + + module InstanceMethods + + # Get or set the client for a specific model instance + # + # @example Get the client for a specific record and perform API request + # + # @article = Article.first + # @article.client.info + # # => { "name" => "Node-1", ... } + # + def client + @client ||= self.class.client + end + + # Set the client for a specific model instance + # + # @example Set the client for a specific record + # + # @article = Article.first + # @article.client = Elasticsearch::Client.new host: 'http://api.server:8080' + # + def client=(client) + @client = client + end + end + + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/ext/active_record.rb b/elasticsearch-model/lib/elasticsearch/model/ext/active_record.rb new file mode 100644 index 0000000000..ffa6cc385a --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/ext/active_record.rb @@ -0,0 +1,14 @@ +# Prevent `MyModel.inspect` failing with `ActiveRecord::ConnectionNotEstablished` +# (triggered by elasticsearch-model/lib/elasticsearch/model.rb:79:in `included') +# +ActiveRecord::Base.instance_eval do + class << self + def inspect_with_rescue + inspect_without_rescue + rescue ActiveRecord::ConnectionNotEstablished + "#{self}(no database connection)" + end + + alias_method_chain :inspect, :rescue + end +end if defined?(ActiveRecord) && ActiveRecord::VERSION::STRING < '4' diff --git a/elasticsearch-model/lib/elasticsearch/model/importing.rb b/elasticsearch-model/lib/elasticsearch/model/importing.rb new file mode 100644 index 0000000000..7c42545d2a --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/importing.rb @@ -0,0 +1,151 @@ +module Elasticsearch + module Model + + # Provides support for easily and efficiently importing large amounts of + # records from the including class into the index. + # + # @see ClassMethods#import + # + module Importing + + # When included in a model, adds the importing methods. + # + # @example Import all records from the `Article` model + # + # Article.import + # + # @see #import + # + def self.included(base) + base.__send__ :extend, ClassMethods + + adapter = Adapter.from_class(base) + base.__send__ :include, adapter.importing_mixin + base.__send__ :extend, adapter.importing_mixin + end + + module ClassMethods + + # Import all model records into the index + # + # The method will pick up correct strategy based on the `Importing` module + # defined in the corresponding adapter. + # + # @param options [Hash] Options passed to the underlying `__find_in_batches`method + # @param block [Proc] Optional block to evaluate for each batch + # + # @yield [Hash] Gives the Hash with the Elasticsearch response to the block + # + # @return [Fixnum] Number of errors encountered during importing + # + # @example Import all records into the index + # + # Article.import + # + # @example Set the batch size to 100 + # + # Article.import batch_size: 100 + # + # @example Process the response from Elasticsearch + # + # Article.import do |response| + # puts "Got " + response['items'].select { |i| i['index']['error'] }.size.to_s + " errors" + # end + # + # @example Delete and create the index with appropriate settings and mappings + # + # Article.import force: true + # + # @example Refresh the index after importing all batches + # + # Article.import refresh: true + # + # @example Import the records into a different index/type than the default one + # + # Article.import index: 'my-new-index', type: 'my-other-type' + # + # @example Pass an ActiveRecord scope to limit the imported records + # + # Article.import scope: 'published' + # + # @example Pass an ActiveRecord query to limit the imported records + # + # Article.import query: -> { where(author_id: author_id) } + # + # @example Transform records during the import with a lambda + # + # transform = lambda do |a| + # {index: {_id: a.id, _parent: a.author_id, data: a.__elasticsearch__.as_indexed_json}} + # end + # + # Article.import transform: transform + # + # @example Update the batch before yielding it + # + # class Article + # # ... + # def self.enrich(batch) + # batch.each do |item| + # item.metadata = MyAPI.get_metadata(item.id) + # end + # batch + # end + # end + # + # Article.import preprocess: :enrich + # + # @example Return an array of error elements instead of the number of errors, eg. + # to try importing these records again + # + # Article.import return: 'errors' + # + def import(options={}, &block) + errors = [] + refresh = options.delete(:refresh) || false + target_index = options.delete(:index) || index_name + target_type = options.delete(:type) || document_type + transform = options.delete(:transform) || __transform + return_value = options.delete(:return) || 'count' + + unless transform.respond_to?(:call) + raise ArgumentError, + "Pass an object responding to `call` as the :transform option, #{transform.class} given" + end + + if options.delete(:force) + self.create_index! force: true, index: target_index + elsif !self.index_exists? index: target_index + raise ArgumentError, + "#{target_index} does not exist to be imported into. Use create_index! or the :force option to create it." + end + + __find_in_batches(options) do |batch| + response = client.bulk \ + index: target_index, + type: target_type, + body: __batch_to_bulk(batch, transform) + + yield response if block_given? + + errors += response['items'].select { |k, v| k.values.first['error'] } + end + + self.refresh_index! if refresh + + case return_value + when 'errors' + errors + else + errors.size + end + end + + def __batch_to_bulk(batch, transform) + batch.map { |model| transform.call(model) } + end + end + + end + + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/indexing.rb b/elasticsearch-model/lib/elasticsearch/model/indexing.rb new file mode 100644 index 0000000000..9c90e9d823 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/indexing.rb @@ -0,0 +1,434 @@ +module Elasticsearch + module Model + + # Provides the necessary support to set up index options (mappings, settings) + # as well as instance methods to create, update or delete documents in the index. + # + # @see ClassMethods#settings + # @see ClassMethods#mapping + # + # @see InstanceMethods#index_document + # @see InstanceMethods#update_document + # @see InstanceMethods#delete_document + # + module Indexing + + # Wraps the [index settings](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/setup-configuration.html#configuration-index-settings) + # + class Settings + attr_accessor :settings + + def initialize(settings={}) + @settings = settings + end + + def to_hash + @settings + end + + def as_json(options={}) + to_hash + end + end + + # Wraps the [index mappings](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping.html) + # + class Mappings + attr_accessor :options, :type + + # @private + TYPES_WITH_EMBEDDED_PROPERTIES = %w(object nested) + + def initialize(type, options={}) + raise ArgumentError, "`type` is missing" if type.nil? + + @type = type + @options = options + @mapping = {} + end + + def indexes(name, options={}, &block) + @mapping[name] = options + + if block_given? + @mapping[name][:type] ||= 'object' + properties = TYPES_WITH_EMBEDDED_PROPERTIES.include?(@mapping[name][:type].to_s) ? :properties : :fields + + @mapping[name][properties] ||= {} + + previous = @mapping + begin + @mapping = @mapping[name][properties] + self.instance_eval(&block) + ensure + @mapping = previous + end + end + + # Set the type to `string` by default + @mapping[name][:type] ||= 'string' + + self + end + + def to_hash + { @type.to_sym => @options.merge( properties: @mapping ) } + end + + def as_json(options={}) + to_hash + end + end + + module ClassMethods + + # Defines mappings for the index + # + # @example Define mapping for model + # + # class Article + # mapping dynamic: 'strict' do + # indexes :foo do + # indexes :bar + # end + # indexes :baz + # end + # end + # + # Article.mapping.to_hash + # + # # => { :article => + # # { :dynamic => "strict", + # # :properties=> + # # { :foo => { + # # :type=>"object", + # # :properties => { + # # :bar => { :type => "string" } + # # } + # # } + # # }, + # # :baz => { :type=> "string" } + # # } + # # } + # + # @example Define index settings and mappings + # + # class Article + # settings number_of_shards: 1 do + # mappings do + # indexes :foo + # end + # end + # end + # + # @example Call the mapping method directly + # + # Article.mapping(dynamic: 'strict') { indexes :foo, type: 'long' } + # + # Article.mapping.to_hash + # + # # => {:article=>{:dynamic=>"strict", :properties=>{:foo=>{:type=>"long"}}}} + # + # The `mappings` and `settings` methods are accessible directly on the model class, + # when it doesn't already define them. Use the `__elasticsearch__` proxy otherwise. + # + def mapping(options={}, &block) + @mapping ||= Mappings.new(document_type, options) + + @mapping.options.update(options) unless options.empty? + + if block_given? + @mapping.instance_eval(&block) + return self + else + @mapping + end + end; alias_method :mappings, :mapping + + # Define settings for the index + # + # @example Define index settings + # + # Article.settings(index: { number_of_shards: 1 }) + # + # Article.settings.to_hash + # + # # => {:index=>{:number_of_shards=>1}} + # + # You can read settings from any object that responds to :read + # as long as its return value can be parsed as either YAML or JSON. + # + # @example Define index settings from YAML file + # + # # config/elasticsearch/articles.yml: + # # + # # index: + # # number_of_shards: 1 + # # + # + # Article.settings File.open("config/elasticsearch/articles.yml") + # + # Article.settings.to_hash + # + # # => { "index" => { "number_of_shards" => 1 } } + # + # + # @example Define index settings from JSON file + # + # # config/elasticsearch/articles.json: + # # + # # { "index": { "number_of_shards": 1 } } + # # + # + # Article.settings File.open("config/elasticsearch/articles.json") + # + # Article.settings.to_hash + # + # # => { "index" => { "number_of_shards" => 1 } } + # + def settings(settings={}, &block) + settings = YAML.load(settings.read) if settings.respond_to?(:read) + @settings ||= Settings.new(settings) + + @settings.settings.update(settings) unless settings.empty? + + if block_given? + self.instance_eval(&block) + return self + else + @settings + end + end + + def load_settings_from_io(settings) + YAML.load(settings.read) + end + + # Creates an index with correct name, automatically passing + # `settings` and `mappings` defined in the model + # + # @example Create an index for the `Article` model + # + # Article.__elasticsearch__.create_index! + # + # @example Forcefully create (delete first) an index for the `Article` model + # + # Article.__elasticsearch__.create_index! force: true + # + # @example Pass a specific index name + # + # Article.__elasticsearch__.create_index! index: 'my-index' + # + def create_index!(options={}) + target_index = options.delete(:index) || self.index_name + + delete_index!(options.merge index: target_index) if options[:force] + + unless index_exists?(index: target_index) + self.client.indices.create index: target_index, + body: { + settings: self.settings.to_hash, + mappings: self.mappings.to_hash } + end + end + + # Returns true if the index exists + # + # @example Check whether the model's index exists + # + # Article.__elasticsearch__.index_exists? + # + # @example Check whether a specific index exists + # + # Article.__elasticsearch__.index_exists? index: 'my-index' + # + def index_exists?(options={}) + target_index = options[:index] || self.index_name + + self.client.indices.exists(index: target_index) rescue false + end + + # Deletes the index with corresponding name + # + # @example Delete the index for the `Article` model + # + # Article.__elasticsearch__.delete_index! + # + # @example Pass a specific index name + # + # Article.__elasticsearch__.delete_index! index: 'my-index' + # + def delete_index!(options={}) + target_index = options.delete(:index) || self.index_name + + begin + self.client.indices.delete index: target_index + rescue Exception => e + if e.class.to_s =~ /NotFound/ && options[:force] + STDERR.puts "[!!!] Index does not exist (#{e.class})" + else + raise e + end + end + end + + # Performs the "refresh" operation for the index (useful e.g. in tests) + # + # @example Refresh the index for the `Article` model + # + # Article.__elasticsearch__.refresh_index! + # + # @example Pass a specific index name + # + # Article.__elasticsearch__.refresh_index! index: 'my-index' + # + # @see http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-refresh.html + # + def refresh_index!(options={}) + target_index = options.delete(:index) || self.index_name + + begin + self.client.indices.refresh index: target_index + rescue Exception => e + if e.class.to_s =~ /NotFound/ && options[:force] + STDERR.puts "[!!!] Index does not exist (#{e.class})" + else + raise e + end + end + end + end + + module InstanceMethods + + def self.included(base) + # Register callback for storing changed attributes for models + # which implement `before_save` and `changed_attributes` methods + # + # @note This is typically triggered only when the module would be + # included in the model directly, not within the proxy. + # + # @see #update_document + # + base.before_save do |instance| + instance.instance_variable_set(:@__changed_attributes, + Hash[ instance.changes.map { |key, value| [key, value.last] } ]) + end if base.respond_to?(:before_save) && base.instance_methods.include?(:changed_attributes) + end + + # Serializes the model instance into JSON (by calling `as_indexed_json`), + # and saves the document into the Elasticsearch index. + # + # @param options [Hash] Optional arguments for passing to the client + # + # @example Index a record + # + # @article.__elasticsearch__.index_document + # 2013-11-20 16:25:57 +0100: PUT http://localhost:9200/articles/article/1 ... + # + # @return [Hash] The response from Elasticsearch + # + # @see http://rubydoc.info/gems/elasticsearch-api/Elasticsearch/API/Actions:index + # + def index_document(options={}) + document = self.as_indexed_json + + client.index( + { index: index_name, + type: document_type, + id: self.id, + body: document }.merge(options) + ) + end + + # Deletes the model instance from the index + # + # @param options [Hash] Optional arguments for passing to the client + # + # @example Delete a record + # + # @article.__elasticsearch__.delete_document + # 2013-11-20 16:27:00 +0100: DELETE http://localhost:9200/articles/article/1 + # + # @return [Hash] The response from Elasticsearch + # + # @see http://rubydoc.info/gems/elasticsearch-api/Elasticsearch/API/Actions:delete + # + def delete_document(options={}) + client.delete( + { index: index_name, + type: document_type, + id: self.id }.merge(options) + ) + end + + # Tries to gather the changed attributes of a model instance + # (via [ActiveModel::Dirty](http://api.rubyonrails.org/classes/ActiveModel/Dirty.html)), + # performing a _partial_ update of the document. + # + # When the changed attributes are not available, performs full re-index of the record. + # + # See the {#update_document_attributes} method for updating specific attributes directly. + # + # @param options [Hash] Optional arguments for passing to the client + # + # @example Update a document corresponding to the record + # + # @article = Article.first + # @article.update_attribute :title, 'Updated' + # # SQL (0.3ms) UPDATE "articles" SET "title" = ?... + # + # @article.__elasticsearch__.update_document + # # 2013-11-20 17:00:05 +0100: POST http://localhost:9200/articles/article/1/_update ... + # # 2013-11-20 17:00:05 +0100: > {"doc":{"title":"Updated"}} + # + # @return [Hash] The response from Elasticsearch + # + # @see http://rubydoc.info/gems/elasticsearch-api/Elasticsearch/API/Actions:update + # + def update_document(options={}) + if changed_attributes = self.instance_variable_get(:@__changed_attributes) + attributes = if respond_to?(:as_indexed_json) + self.as_indexed_json.select { |k,v| changed_attributes.keys.map(&:to_s).include? k.to_s } + else + changed_attributes + end + + client.update( + { index: index_name, + type: document_type, + id: self.id, + body: { doc: attributes } }.merge(options) + ) + else + index_document(options) + end + end + + # Perform a _partial_ update of specific document attributes + # (without consideration for changed attributes as in {#update_document}) + # + # @param attributes [Hash] Attributes to be updated + # @param options [Hash] Optional arguments for passing to the client + # + # @example Update the `title` attribute + # + # @article = Article.first + # @article.title = "New title" + # @article.__elasticsearch__.update_document_attributes title: "New title" + # + # @return [Hash] The response from Elasticsearch + # + def update_document_attributes(attributes, options={}) + client.update( + { index: index_name, + type: document_type, + id: self.id, + body: { doc: attributes } }.merge(options) + ) + end + end + + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/multimodel.rb b/elasticsearch-model/lib/elasticsearch/model/multimodel.rb new file mode 100644 index 0000000000..8831d4fd09 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/multimodel.rb @@ -0,0 +1,83 @@ +module Elasticsearch + module Model + + # Keeps a global registry of classes that include `Elasticsearch::Model` + # + class Registry + def initialize + @models = [] + end + + # Returns the unique instance of the registry (Singleton) + # + # @api private + # + def self.__instance + @instance ||= new + end + + # Adds a model to the registry + # + def self.add(klass) + __instance.add(klass) + end + + # Returns an Array of registered models + # + def self.all + __instance.models + end + + # Adds a model to the registry + # + def add(klass) + @models << klass + end + + # Returns a copy of the registered models + # + def models + @models.dup + end + end + + # Wraps a collection of models when querying multiple indices + # + # @see Elasticsearch::Model.search + # + class Multimodel + attr_reader :models + + # @param models [Class] The list of models across which the search will be performed + # + def initialize(*models) + @models = models.flatten + @models = Model::Registry.all if @models.empty? + end + + # Get an Array of index names used for retrieving documents when doing a search across multiple models + # + # @return [Array] the list of index names used for retrieving documents + # + def index_name + models.map { |m| m.index_name } + end + + # Get an Array of document types used for retrieving documents when doing a search across multiple models + # + # @return [Array] the list of document types used for retrieving documents + # + def document_type + models.map { |m| m.document_type } + end + + # Get the client common for all models + # + # @return Elasticsearch::Transport::Client + # + def client + Elasticsearch::Model.client + end + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/naming.rb b/elasticsearch-model/lib/elasticsearch/model/naming.rb new file mode 100644 index 0000000000..ce510d2d47 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/naming.rb @@ -0,0 +1,122 @@ +module Elasticsearch + module Model + + # Provides methods for getting and setting index name and document type for the model + # + module Naming + + module ClassMethods + + # Get or set the name of the index + # + # @example Set the index name for the `Article` model + # + # class Article + # index_name "articles-#{Rails.env}" + # end + # + # @example Set the index name for the `Article` model and re-evaluate it on each call + # + # class Article + # index_name { "articles-#{Time.now.year}" } + # end + # + # @example Directly set the index name for the `Article` model + # + # Article.index_name "articles-#{Rails.env}" + # + # + def index_name name=nil, &block + if name || block_given? + return (@index_name = name || block) + end + + if @index_name.respond_to?(:call) + @index_name.call + else + @index_name || self.model_name.collection.gsub(/\//, '-') + end + end + + # Set the index name + # + # @see index_name + def index_name=(name) + @index_name = name + end + + # Get or set the document type + # + # @example Set the document type for the `Article` model + # + # class Article + # document_type "my-article" + # end + # + # @example Directly set the document type for the `Article` model + # + # Article.document_type "my-article" + # + def document_type name=nil + @document_type = name || @document_type || self.model_name.element + end + + + # Set the document type + # + # @see document_type + # + def document_type=(name) + @document_type = name + end + end + + module InstanceMethods + + # Get or set the index name for the model instance + # + # @example Set the index name for an instance of the `Article` model + # + # @article.index_name "articles-#{@article.user_id}" + # @article.__elasticsearch__.update_document + # + def index_name name=nil, &block + if name || block_given? + return (@index_name = name || block) + end + + if @index_name.respond_to?(:call) + @index_name.call + else + @index_name || self.class.index_name + end + end + + # Set the index name + # + # @see index_name + def index_name=(name) + @index_name = name + end + + # @example Set the document type for an instance of the `Article` model + # + # @article.document_type "my-article" + # @article.__elasticsearch__.update_document + # + def document_type name=nil + @document_type = name || @document_type || self.class.document_type + end + + # Set the document type + # + # @see document_type + # + def document_type=(name) + @document_type = name + end + end + + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/proxy.rb b/elasticsearch-model/lib/elasticsearch/model/proxy.rb new file mode 100644 index 0000000000..3e37f28ec3 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/proxy.rb @@ -0,0 +1,137 @@ +module Elasticsearch + module Model + + # This module provides a proxy interfacing between the including class and + # {Elasticsearch::Model}, preventing the pollution of the including class namespace. + # + # The only "gateway" between the model and Elasticsearch::Model is the + # `__elasticsearch__` class and instance method. + # + # The including class must be compatible with + # [ActiveModel](https://github.com/rails/rails/tree/master/activemodel). + # + # @example Include the {Elasticsearch::Model} module into an `Article` model + # + # class Article < ActiveRecord::Base + # include Elasticsearch::Model + # end + # + # Article.__elasticsearch__.respond_to?(:search) + # # => true + # + # article = Article.first + # + # article.respond_to? :index_document + # # => false + # + # article.__elasticsearch__.respond_to?(:index_document) + # # => true + # + module Proxy + + # Define the `__elasticsearch__` class and instance methods in the including class + # and register a callback for intercepting changes in the model. + # + # @note The callback is triggered only when `Elasticsearch::Model` is included in the + # module and the functionality is accessible via the proxy. + # + def self.included(base) + base.class_eval do + # {ClassMethodsProxy} instance, accessed as `MyModel.__elasticsearch__` + # + def self.__elasticsearch__ &block + @__elasticsearch__ ||= ClassMethodsProxy.new(self) + @__elasticsearch__.instance_eval(&block) if block_given? + @__elasticsearch__ + end + + # {InstanceMethodsProxy}, accessed as `@mymodel.__elasticsearch__` + # + def __elasticsearch__ &block + @__elasticsearch__ ||= InstanceMethodsProxy.new(self) + @__elasticsearch__.instance_eval(&block) if block_given? + @__elasticsearch__ + end + + # Register a callback for storing changed attributes for models which implement + # `before_save` and `changed_attributes` methods (when `Elasticsearch::Model` is included) + # + # @see http://api.rubyonrails.org/classes/ActiveModel/Dirty.html + # + before_save do |i| + changed_attr = i.__elasticsearch__.instance_variable_get(:@__changed_attributes) || {} + i.__elasticsearch__.instance_variable_set(:@__changed_attributes, + changed_attr.merge(Hash[ i.changes.map { |key, value| [key, value.last] } ])) + end if respond_to?(:before_save) && instance_methods.include?(:changed_attributes) + end + end + + # @overload dup + # + # Returns a copy of this object. Resets the __elasticsearch__ proxy so + # the duplicate will build its own proxy. + def initialize_dup(_) + @__elasticsearch__ = nil + super + end + + # Common module for the proxy classes + # + module Base + attr_reader :target + + def initialize(target) + @target = target + end + + # Delegate methods to `@target` + # + def method_missing(method_name, *arguments, &block) + target.respond_to?(method_name) ? target.__send__(method_name, *arguments, &block) : super + end + + # Respond to methods from `@target` + # + def respond_to?(method_name, include_private = false) + target.respond_to?(method_name) || super + end + + def inspect + "[PROXY] #{target.inspect}" + end + end + + # A proxy interfacing between Elasticsearch::Model class methods and model class methods + # + # TODO: Inherit from BasicObject and make Pry's `ls` command behave? + # + class ClassMethodsProxy + include Base + end + + # A proxy interfacing between Elasticsearch::Model instance methods and model instance methods + # + # TODO: Inherit from BasicObject and make Pry's `ls` command behave? + # + class InstanceMethodsProxy + include Base + + def klass + target.class + end + + def class + klass.__elasticsearch__ + end + + # Need to redefine `as_json` because we're not inheriting from `BasicObject`; + # see TODO note above. + # + def as_json(options={}) + target.as_json(options) + end + end + + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/response.rb b/elasticsearch-model/lib/elasticsearch/model/response.rb new file mode 100644 index 0000000000..fad3828b39 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/response.rb @@ -0,0 +1,83 @@ +module Elasticsearch + module Model + + # Contains modules and classes for wrapping the response from Elasticsearch + # + module Response + + # Encapsulate the response returned from the Elasticsearch client + # + # Implements Enumerable and forwards its methods to the {#results} object. + # + class Response + attr_reader :klass, :search, :response, + :took, :timed_out, :shards + + include Enumerable + + delegate :each, :empty?, :size, :slice, :[], :to_ary, to: :results + + def initialize(klass, search, options={}) + @klass = klass + @search = search + end + + # Returns the Elasticsearch response + # + # @return [Hash] + # + def response + @response ||= begin + Hashie::Mash.new(search.execute!) + end + end + + # Returns the collection of "hits" from Elasticsearch + # + # @return [Results] + # + def results + @results ||= Results.new(klass, self) + end + + # Returns the collection of records from the database + # + # @return [Records] + # + def records(options = {}) + @records ||= Records.new(klass, self, options) + end + + # Returns the "took" time + # + def took + response['took'] + end + + # Returns whether the response timed out + # + def timed_out + response['timed_out'] + end + + # Returns the statistics on shards + # + def shards + Hashie::Mash.new(response['_shards']) + end + + # Returns a Hashie::Mash of the aggregations + # + def aggregations + response['aggregations'] ? Hashie::Mash.new(response['aggregations']) : nil + end + + # Returns a Hashie::Mash of the suggestions + # + def suggestions + Suggestions.new(response['suggest']) + end + end + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/response/base.rb b/elasticsearch-model/lib/elasticsearch/model/response/base.rb new file mode 100644 index 0000000000..3bb8005b63 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/response/base.rb @@ -0,0 +1,44 @@ +module Elasticsearch + module Model + module Response + # Common funtionality for classes in the {Elasticsearch::Model::Response} module + # + module Base + attr_reader :klass, :response + + # @param klass [Class] The name of the model class + # @param response [Hash] The full response returned from Elasticsearch client + # @param options [Hash] Optional parameters + # + def initialize(klass, response, options={}) + @klass = klass + @response = response + end + + # @abstract Implement this method in specific class + # + def results + raise NotImplemented, "Implement this method in #{klass}" + end + + # @abstract Implement this method in specific class + # + def records + raise NotImplemented, "Implement this method in #{klass}" + end + + # Returns the total number of hits + # + def total + response.response['hits']['total'] + end + + # Returns the max_score + # + def max_score + response.response['hits']['max_score'] + end + end + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/response/pagination.rb b/elasticsearch-model/lib/elasticsearch/model/response/pagination.rb new file mode 100644 index 0000000000..c8e74b7934 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/response/pagination.rb @@ -0,0 +1,192 @@ +module Elasticsearch + module Model + module Response + + # Pagination for search results/records + # + module Pagination + # Allow models to be paginated with the "kaminari" gem [https://github.com/amatsuda/kaminari] + # + module Kaminari + def self.included(base) + # Include the Kaminari configuration and paging method in response + # + base.__send__ :include, ::Kaminari::ConfigurationMethods::ClassMethods + base.__send__ :include, ::Kaminari::PageScopeMethods + + # Include the Kaminari paging methods in results and records + # + Elasticsearch::Model::Response::Results.__send__ :include, ::Kaminari::ConfigurationMethods::ClassMethods + Elasticsearch::Model::Response::Results.__send__ :include, ::Kaminari::PageScopeMethods + Elasticsearch::Model::Response::Records.__send__ :include, ::Kaminari::PageScopeMethods + + Elasticsearch::Model::Response::Results.__send__ :delegate, :limit_value, :offset_value, :total_count, :max_pages, to: :response + Elasticsearch::Model::Response::Records.__send__ :delegate, :limit_value, :offset_value, :total_count, :max_pages, to: :response + + base.class_eval <<-RUBY, __FILE__, __LINE__ + 1 + # Define the `page` Kaminari method + # + def #{::Kaminari.config.page_method_name}(num=nil) + @results = nil + @records = nil + @response = nil + @page = [num.to_i, 1].max + @per_page ||= __default_per_page + + self.search.definition.update size: @per_page, + from: @per_page * (@page - 1) + + self + end + RUBY + end + + # Returns the current "limit" (`size`) value + # + def limit_value + case + when search.definition[:size] + search.definition[:size] + else + __default_per_page + end + end + + # Returns the current "offset" (`from`) value + # + def offset_value + case + when search.definition[:from] + search.definition[:from] + else + 0 + end + end + + # Set the "limit" (`size`) value + # + def limit(value) + return self if value.to_i <= 0 + @results = nil + @records = nil + @response = nil + @per_page = value.to_i + + search.definition.update :size => @per_page + search.definition.update :from => @per_page * (@page - 1) if @page + self + end + + # Set the "offset" (`from`) value + # + def offset(value) + return self if value.to_i < 0 + @results = nil + @records = nil + @response = nil + @page = nil + search.definition.update :from => value.to_i + self + end + + # Returns the total number of results + # + def total_count + results.total + end + + # Returns the models's `per_page` value or the default + # + # @api private + # + def __default_per_page + klass.respond_to?(:default_per_page) && klass.default_per_page || ::Kaminari.config.default_per_page + end + end + + # Allow models to be paginated with the "will_paginate" gem [https://github.com/mislav/will_paginate] + # + module WillPaginate + def self.included(base) + base.__send__ :include, ::WillPaginate::CollectionMethods + + # Include the paging methods in results and records + # + methods = [:current_page, :offset, :length, :per_page, :total_entries, :total_pages, :previous_page, :next_page, :out_of_bounds?] + Elasticsearch::Model::Response::Results.__send__ :delegate, *methods, to: :response + Elasticsearch::Model::Response::Records.__send__ :delegate, *methods, to: :response + end + + def offset + (current_page - 1) * per_page + end + + def length + search.definition[:size] + end + + # Main pagination method + # + # @example + # + # Article.search('foo').paginate(page: 1, per_page: 30) + # + def paginate(options) + param_name = options[:param_name] || :page + page = [options[param_name].to_i, 1].max + per_page = (options[:per_page] || __default_per_page).to_i + + search.definition.update size: per_page, + from: (page - 1) * per_page + self + end + + # Return the current page + # + def current_page + search.definition[:from] / per_page + 1 if search.definition[:from] && per_page + end + + # Pagination method + # + # @example + # + # Article.search('foo').page(2) + # + def page(num) + paginate(page: num, per_page: per_page) # shorthand + end + + # Return or set the "size" value + # + # @example + # + # Article.search('foo').per_page(15).page(2) + # + def per_page(num = nil) + if num.nil? + search.definition[:size] + else + paginate(page: current_page, per_page: num) # shorthand + end + end + + # Returns the total number of results + # + def total_entries + results.total + end + + # Returns the models's `per_page` value or the default + # + # @api private + # + def __default_per_page + klass.respond_to?(:per_page) && klass.per_page || ::WillPaginate.per_page + end + end + end + + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/response/records.rb b/elasticsearch-model/lib/elasticsearch/model/response/records.rb new file mode 100644 index 0000000000..4638ca6892 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/response/records.rb @@ -0,0 +1,73 @@ +module Elasticsearch + module Model + module Response + + # Encapsulates the collection of records returned from the database + # + # Implements Enumerable and forwards its methods to the {#records} object, + # which is provided by an {Elasticsearch::Model::Adapter::Adapter} implementation. + # + class Records + include Enumerable + + delegate :each, :empty?, :size, :slice, :[], :to_a, :to_ary, to: :records + + attr_accessor :options + + include Base + + # @see Base#initialize + # + def initialize(klass, response, options={}) + super + + # Include module provided by the adapter in the singleton class ("metaclass") + # + adapter = Adapter.from_class(klass) + metaclass = class << self; self; end + metaclass.__send__ :include, adapter.records_mixin + + self.options = options + self + end + + # Returns the hit IDs + # + def ids + response.response['hits']['hits'].map { |hit| hit['_id'] } + end + + # Returns the {Results} collection + # + def results + response.results + end + + # Yields [record, hit] pairs to the block + # + def each_with_hit(&block) + records.to_a.zip(results).each(&block) + end + + # Yields [record, hit] pairs and returns the result + # + def map_with_hit(&block) + records.to_a.zip(results).map(&block) + end + + # Delegate methods to `@records` + # + def method_missing(method_name, *arguments) + records.respond_to?(method_name) ? records.__send__(method_name, *arguments) : super + end + + # Respond to methods from `@records` + # + def respond_to?(method_name, include_private = false) + records.respond_to?(method_name) || super + end + + end + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/response/result.rb b/elasticsearch-model/lib/elasticsearch/model/response/result.rb new file mode 100644 index 0000000000..217723e8b9 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/response/result.rb @@ -0,0 +1,63 @@ +module Elasticsearch + module Model + module Response + + # Encapsulates the "hit" returned from the Elasticsearch client + # + # Wraps the raw Hash with in a `Hashie::Mash` instance, providing + # access to the Hash properties by calling Ruby methods. + # + # @see https://github.com/intridea/hashie + # + class Result + + # @param attributes [Hash] A Hash with document properties + # + def initialize(attributes={}) + @result = Hashie::Mash.new(attributes) + end + + # Return document `_id` as `id` + # + def id + @result['_id'] + end + + # Return document `_type` as `_type` + # + def type + @result['_type'] + end + + # Delegate methods to `@result` or `@result._source` + # + def method_missing(name, *arguments) + case + when name.to_s.end_with?('?') + @result.__send__(name, *arguments) || ( @result._source && @result._source.__send__(name, *arguments) ) + when @result.respond_to?(name) + @result.__send__ name, *arguments + when @result._source && @result._source.respond_to?(name) + @result._source.__send__ name, *arguments + else + super + end + end + + # Respond to methods from `@result` or `@result._source` + # + def respond_to?(method_name, include_private = false) + @result.respond_to?(method_name.to_sym) || \ + @result._source && @result._source.respond_to?(method_name.to_sym) || \ + super + end + + def as_json(options={}) + @result.as_json(options) + end + + # TODO: #to_s, #inspect, with support for Pry + end + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/response/results.rb b/elasticsearch-model/lib/elasticsearch/model/response/results.rb new file mode 100644 index 0000000000..006e66a46b --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/response/results.rb @@ -0,0 +1,31 @@ +module Elasticsearch + module Model + module Response + + # Encapsulates the collection of documents returned from Elasticsearch + # + # Implements Enumerable and forwards its methods to the {#results} object. + # + class Results + include Base + include Enumerable + + delegate :each, :empty?, :size, :slice, :[], :to_a, :to_ary, to: :results + + # @see Base#initialize + # + def initialize(klass, response, options={}) + super + end + + # Returns the {Results} collection + # + def results + # TODO: Configurable custom wrapper + response.response['hits']['hits'].map { |hit| Result.new(hit) } + end + + end + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/response/suggestions.rb b/elasticsearch-model/lib/elasticsearch/model/response/suggestions.rb new file mode 100644 index 0000000000..5088767cef --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/response/suggestions.rb @@ -0,0 +1,13 @@ +module Elasticsearch + module Model + module Response + + class Suggestions < Hashie::Mash + def terms + self.to_a.map { |k,v| v.first['options'] }.flatten.map {|v| v['text']}.uniq + end + end + + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/searching.rb b/elasticsearch-model/lib/elasticsearch/model/searching.rb new file mode 100644 index 0000000000..604657d5e0 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/searching.rb @@ -0,0 +1,109 @@ +module Elasticsearch + module Model + + # Contains functionality related to searching. + # + module Searching + + # Wraps a search request definition + # + class SearchRequest + attr_reader :klass, :definition, :options + + # @param klass [Class] The class of the model + # @param query_or_payload [String,Hash,Object] The search request definition + # (string, JSON, Hash, or object responding to `to_hash`) + # @param options [Hash] Optional parameters to be passed to the Elasticsearch client + # + def initialize(klass, query_or_payload, options={}) + @klass = klass + @options = options + + __index_name = options[:index] || klass.index_name + __document_type = options[:type] || klass.document_type + + case + # search query: ... + when query_or_payload.respond_to?(:to_hash) + body = query_or_payload.to_hash + + # search '{ "query" : ... }' + when query_or_payload.is_a?(String) && query_or_payload =~ /^\s*{/ + body = query_or_payload + + # search '...' + else + q = query_or_payload + end + + if body + @definition = { index: __index_name, type: __document_type, body: body }.update options + else + @definition = { index: __index_name, type: __document_type, q: q }.update options + end + end + + # Performs the request and returns the response from client + # + # @return [Hash] The response from Elasticsearch + # + def execute! + klass.client.search(@definition) + end + end + + module ClassMethods + + # Provides a `search` method for the model to easily search within an index/type + # corresponding to the model settings. + # + # @param query_or_payload [String,Hash,Object] The search request definition + # (string, JSON, Hash, or object responding to `to_hash`) + # @param options [Hash] Optional parameters to be passed to the Elasticsearch client + # + # @return [Elasticsearch::Model::Response::Response] + # + # @example Simple search in `Article` + # + # Article.search 'foo' + # + # @example Search using a search definition as a Hash + # + # response = Article.search \ + # query: { + # match: { + # title: 'foo' + # } + # }, + # highlight: { + # fields: { + # title: {} + # } + # }, + # size: 50 + # + # response.results.first.title + # # => "Foo" + # + # response.results.first.highlight.title + # # => ["Foo"] + # + # response.records.first.title + # # Article Load (0.2ms) SELECT "articles".* FROM "articles" WHERE "articles"."id" IN (1, 3) + # # => "Foo" + # + # @example Search using a search definition as a JSON string + # + # Article.search '{"query" : { "match_all" : {} }}' + # + def search(query_or_payload, options={}) + search = SearchRequest.new(self, query_or_payload, options) + + Response::Response.new(self, search) + end + + end + + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/serializing.rb b/elasticsearch-model/lib/elasticsearch/model/serializing.rb new file mode 100644 index 0000000000..659a58bb2a --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/serializing.rb @@ -0,0 +1,35 @@ +module Elasticsearch + module Model + + # Contains functionality for serializing model instances for the client + # + module Serializing + + module ClassMethods + end + + module InstanceMethods + + # Serialize the record as a Hash, to be passed to the client. + # + # Re-define this method to customize the serialization. + # + # @return [Hash] + # + # @example Return the model instance as a Hash + # + # Article.first.__elasticsearch__.as_indexed_json + # => {"title"=>"Foo"} + # + # @see Elasticsearch::Model::Indexing + # + def as_indexed_json(options={}) + # TODO: Play with the `MyModel.indexes` method -- reject non-mapped attributes, `:as` options, etc + self.as_json(options.merge root: false) + end + + end + + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/rails/version.rb b/elasticsearch-model/lib/elasticsearch/model/version.rb similarity index 77% rename from elasticsearch-model/lib/elasticsearch/rails/version.rb rename to elasticsearch-model/lib/elasticsearch/model/version.rb index 88b2dd7589..44cfdabeab 100644 --- a/elasticsearch-model/lib/elasticsearch/rails/version.rb +++ b/elasticsearch-model/lib/elasticsearch/model/version.rb @@ -1,5 +1,5 @@ module Elasticsearch - module Rails + module Model VERSION = "0.1.9" end end diff --git a/elasticsearch-model/lib/elasticsearch/rails.rb b/elasticsearch-model/lib/elasticsearch/rails.rb deleted file mode 100644 index f425f72763..0000000000 --- a/elasticsearch-model/lib/elasticsearch/rails.rb +++ /dev/null @@ -1,7 +0,0 @@ -require "elasticsearch/rails/version" - -module Elasticsearch - module Rails - # Your code goes here... - end -end diff --git a/elasticsearch-model/lib/elasticsearch/rails/instrumentation.rb b/elasticsearch-model/lib/elasticsearch/rails/instrumentation.rb deleted file mode 100644 index 081791ab55..0000000000 --- a/elasticsearch-model/lib/elasticsearch/rails/instrumentation.rb +++ /dev/null @@ -1,36 +0,0 @@ -require 'elasticsearch/rails/instrumentation/railtie' -require 'elasticsearch/rails/instrumentation/publishers' - -module Elasticsearch - module Rails - - # This module adds support for displaying statistics about search duration in the Rails application log - # by integrating with the `ActiveSupport::Notifications` framework and `ActionController` logger. - # - # == Usage - # - # Require the component in your `application.rb` file: - # - # require 'elasticsearch/rails/instrumentation' - # - # You should see an output like this in your application log in development environment: - # - # Article Search (321.3ms) { index: "articles", type: "article", body: { query: ... } } - # - # Also, the total duration of the request to Elasticsearch is displayed in the Rails request breakdown: - # - # Completed 200 OK in 615ms (Views: 230.9ms | ActiveRecord: 0.0ms | Elasticsearch: 321.3ms) - # - # @note The displayed duration includes the HTTP transfer -- the time it took Elasticsearch - # to process your request is available in the `response.took` property. - # - # @see Elasticsearch::Rails::Instrumentation::Publishers - # @see Elasticsearch::Rails::Instrumentation::Railtie - # - # @see http://api.rubyonrails.org/classes/ActiveSupport/Notifications.html - # - # - module Instrumentation - end - end -end diff --git a/elasticsearch-model/lib/elasticsearch/rails/instrumentation/controller_runtime.rb b/elasticsearch-model/lib/elasticsearch/rails/instrumentation/controller_runtime.rb deleted file mode 100644 index 461387c808..0000000000 --- a/elasticsearch-model/lib/elasticsearch/rails/instrumentation/controller_runtime.rb +++ /dev/null @@ -1,41 +0,0 @@ -require 'active_support/core_ext/module/attr_internal' - -module Elasticsearch - module Rails - module Instrumentation - - # Hooks into ActionController to display Elasticsearch runtime - # - # @see https://github.com/rails/rails/blob/master/activerecord/lib/active_record/railties/controller_runtime.rb - # - module ControllerRuntime - extend ActiveSupport::Concern - - protected - - attr_internal :elasticsearch_runtime - - def cleanup_view_runtime - elasticsearch_rt_before_render = Elasticsearch::Rails::Instrumentation::LogSubscriber.reset_runtime - runtime = super - elasticsearch_rt_after_render = Elasticsearch::Rails::Instrumentation::LogSubscriber.reset_runtime - self.elasticsearch_runtime = elasticsearch_rt_before_render + elasticsearch_rt_after_render - runtime - elasticsearch_rt_after_render - end - - def append_info_to_payload(payload) - super - payload[:elasticsearch_runtime] = (elasticsearch_runtime || 0) + Elasticsearch::Rails::Instrumentation::LogSubscriber.reset_runtime - end - - module ClassMethods - def log_process_action(payload) - messages, elasticsearch_runtime = super, payload[:elasticsearch_runtime] - messages << ("Elasticsearch: %.1fms" % elasticsearch_runtime.to_f) if elasticsearch_runtime - messages - end - end - end - end - end -end diff --git a/elasticsearch-model/lib/elasticsearch/rails/instrumentation/log_subscriber.rb b/elasticsearch-model/lib/elasticsearch/rails/instrumentation/log_subscriber.rb deleted file mode 100644 index c02bc07049..0000000000 --- a/elasticsearch-model/lib/elasticsearch/rails/instrumentation/log_subscriber.rb +++ /dev/null @@ -1,41 +0,0 @@ -module Elasticsearch - module Rails - module Instrumentation - - # A log subscriber to attach to Elasticsearch related events - # - # @see https://github.com/rails/rails/blob/master/activerecord/lib/active_record/log_subscriber.rb - # - class LogSubscriber < ActiveSupport::LogSubscriber - def self.runtime=(value) - Thread.current["elasticsearch_runtime"] = value - end - - def self.runtime - Thread.current["elasticsearch_runtime"] ||= 0 - end - - def self.reset_runtime - rt, self.runtime = runtime, 0 - rt - end - - # Intercept `search.elasticsearch` events, and display them in the Rails log - # - def search(event) - self.class.runtime += event.duration - return unless logger.debug? - - payload = event.payload - name = "#{payload[:klass]} #{payload[:name]} (#{event.duration.round(1)}ms)" - search = payload[:search].inspect.gsub(/:(\w+)=>/, '\1: ') - - debug %Q| #{color(name, GREEN, true)} #{colorize_logging ? "\e[2m#{search}\e[0m" : search}| - end - end - - end - end -end - -Elasticsearch::Rails::Instrumentation::LogSubscriber.attach_to :elasticsearch diff --git a/elasticsearch-model/lib/elasticsearch/rails/instrumentation/publishers.rb b/elasticsearch-model/lib/elasticsearch/rails/instrumentation/publishers.rb deleted file mode 100644 index e054d53712..0000000000 --- a/elasticsearch-model/lib/elasticsearch/rails/instrumentation/publishers.rb +++ /dev/null @@ -1,36 +0,0 @@ -module Elasticsearch - module Rails - module Instrumentation - module Publishers - - # Wraps the `SearchRequest` methods to perform the instrumentation - # - # @see SearchRequest#execute_with_instrumentation! - # @see http://api.rubyonrails.org/classes/ActiveSupport/Notifications.html - # - module SearchRequest - - def self.included(base) - base.class_eval do - unless method_defined?(:execute_without_instrumentation!) - alias_method :execute_without_instrumentation!, :execute! - alias_method :execute!, :execute_with_instrumentation! - end - end - end - - # Wrap `Search#execute!` and perform instrumentation - # - def execute_with_instrumentation! - ActiveSupport::Notifications.instrument "search.elasticsearch", - name: 'Search', - klass: (self.klass.is_a?(Elasticsearch::Model::Proxy::ClassMethodsProxy) ? self.klass.target.to_s : self.klass.to_s), - search: self.definition do - execute_without_instrumentation! - end - end - end - end - end - end -end diff --git a/elasticsearch-model/lib/elasticsearch/rails/instrumentation/railtie.rb b/elasticsearch-model/lib/elasticsearch/rails/instrumentation/railtie.rb deleted file mode 100644 index dbcd0fc389..0000000000 --- a/elasticsearch-model/lib/elasticsearch/rails/instrumentation/railtie.rb +++ /dev/null @@ -1,31 +0,0 @@ -module Elasticsearch - module Rails - module Instrumentation - - # Rails initializer class to require Elasticsearch::Rails::Instrumentation files, - # set up Elasticsearch::Model and hook into ActionController to display Elasticsearch-related duration - # - # @see http://edgeguides.rubyonrails.org/active_support_instrumentation.html - # - class Railtie < ::Rails::Railtie - initializer "elasticsearch.instrumentation" do |app| - require 'elasticsearch/rails/instrumentation/log_subscriber' - require 'elasticsearch/rails/instrumentation/controller_runtime' - - Elasticsearch::Model::Searching::SearchRequest.class_eval do - include Elasticsearch::Rails::Instrumentation::Publishers::SearchRequest - end if defined?(Elasticsearch::Model::Searching::SearchRequest) - - Elasticsearch::Persistence::Model::Find::SearchRequest.class_eval do - include Elasticsearch::Rails::Instrumentation::Publishers::SearchRequest - end if defined?(Elasticsearch::Persistence::Model::Find::SearchRequest) - - ActiveSupport.on_load(:action_controller) do - include Elasticsearch::Rails::Instrumentation::ControllerRuntime - end - end - end - - end - end -end diff --git a/elasticsearch-model/lib/elasticsearch/rails/lograge.rb b/elasticsearch-model/lib/elasticsearch/rails/lograge.rb deleted file mode 100644 index a8edd80848..0000000000 --- a/elasticsearch-model/lib/elasticsearch/rails/lograge.rb +++ /dev/null @@ -1,44 +0,0 @@ -module Elasticsearch - module Rails - module Lograge - - # Rails initializer class to require Elasticsearch::Rails::Instrumentation files, - # set up Elasticsearch::Model and add Lograge configuration to display Elasticsearch-related duration - # - # Require the component in your `application.rb` file and enable Lograge: - # - # require 'elasticsearch/rails/lograge' - # - # You should see the full duration of the request to Elasticsearch as part of each log event: - # - # method=GET path=/search ... status=200 duration=380.89 view=99.64 db=0.00 es=279.37 - # - # @see https://github.com/roidrage/lograge - # - class Railtie < ::Rails::Railtie - initializer "elasticsearch.lograge" do |app| - require 'elasticsearch/rails/instrumentation/publishers' - require 'elasticsearch/rails/instrumentation/log_subscriber' - require 'elasticsearch/rails/instrumentation/controller_runtime' - - Elasticsearch::Model::Searching::SearchRequest.class_eval do - include Elasticsearch::Rails::Instrumentation::Publishers::SearchRequest - end if defined?(Elasticsearch::Model::Searching::SearchRequest) - - Elasticsearch::Persistence::Model::Find::SearchRequest.class_eval do - include Elasticsearch::Rails::Instrumentation::Publishers::SearchRequest - end if defined?(Elasticsearch::Persistence::Model::Find::SearchRequest) - - ActiveSupport.on_load(:action_controller) do - include Elasticsearch::Rails::Instrumentation::ControllerRuntime - end - - config.lograge.custom_options = lambda do |event| - { es: event.payload[:elasticsearch_runtime].to_f.round(2) } - end - end - end - - end - end -end diff --git a/elasticsearch-model/lib/elasticsearch/rails/tasks/import.rb b/elasticsearch-model/lib/elasticsearch/rails/tasks/import.rb deleted file mode 100644 index bb2f9ff3d0..0000000000 --- a/elasticsearch-model/lib/elasticsearch/rails/tasks/import.rb +++ /dev/null @@ -1,112 +0,0 @@ -# A collection of Rake tasks to facilitate importing data from yout models into Elasticsearch. -# -# Add this e.g. into the `lib/tasks/elasticsearch.rake` file in your Rails application: -# -# require 'elasticsearch/rails/tasks/import' -# -# To import the records from your `Article` model, run: -# -# $ bundle exec rake environment elasticsearch:import:model CLASS='MyModel' -# -# Run this command to display usage instructions: -# -# $ bundle exec rake -D elasticsearch -# -STDOUT.sync = true -STDERR.sync = true - -begin; require 'ansi/progressbar'; rescue LoadError; end - -namespace :elasticsearch do - - task :import => 'import:model' - - namespace :import do - import_model_desc = <<-DESC.gsub(/ /, '') - Import data from your model (pass name as CLASS environment variable). - - $ rake environment elasticsearch:import:model CLASS='MyModel' - - Force rebuilding the index (delete and create): - $ rake environment elasticsearch:import:model CLASS='Article' FORCE=y - - Customize the batch size: - $ rake environment elasticsearch:import:model CLASS='Article' BATCH=100 - - Set target index name: - $ rake environment elasticsearch:import:model CLASS='Article' INDEX='articles-new' - - Pass an ActiveRecord scope to limit the imported records: - $ rake environment elasticsearch:import:model CLASS='Article' SCOPE='published' - DESC - desc import_model_desc - task :model do - if ENV['CLASS'].to_s == '' - puts '='*90, 'USAGE', '='*90, import_model_desc, "" - exit(1) - end - - klass = eval(ENV['CLASS'].to_s) - total = klass.count rescue nil - pbar = ANSI::Progressbar.new(klass.to_s, total) rescue nil - pbar.__send__ :show if pbar - - unless ENV['DEBUG'] - begin - klass.__elasticsearch__.client.transport.logger.level = Logger::WARN - rescue NoMethodError; end - begin - klass.__elasticsearch__.client.transport.tracer.level = Logger::WARN - rescue NoMethodError; end - end - - total_errors = klass.__elasticsearch__.import force: ENV.fetch('FORCE', false), - batch_size: ENV.fetch('BATCH', 1000).to_i, - index: ENV.fetch('INDEX', nil), - type: ENV.fetch('TYPE', nil), - scope: ENV.fetch('SCOPE', nil) do |response| - pbar.inc response['items'].size if pbar - STDERR.flush - STDOUT.flush - end - pbar.finish if pbar - - puts "[IMPORT] #{total_errors} errors occurred" unless total_errors.zero? - puts '[IMPORT] Done' - end - - desc <<-DESC.gsub(/ /, '') - Import all indices from `app/models` (or use DIR environment variable). - - $ rake environment elasticsearch:import:all DIR=app/models - DESC - task :all do - dir = ENV['DIR'].to_s != '' ? ENV['DIR'] : Rails.root.join("app/models") - - puts "[IMPORT] Loading models from: #{dir}" - Dir.glob(File.join("#{dir}/**/*.rb")).each do |path| - model_filename = path[/#{Regexp.escape(dir.to_s)}\/([^\.]+).rb/, 1] - - next if model_filename.match(/^concerns\//i) # Skip concerns/ folder - - begin - klass = model_filename.camelize.constantize - rescue NameError - require(path) ? retry : raise(RuntimeError, "Cannot load class '#{klass}'") - end - - # Skip if the class doesn't have Elasticsearch integration - next unless klass.respond_to?(:__elasticsearch__) - - puts "[IMPORT] Processing model: #{klass}..." - - ENV['CLASS'] = klass.to_s - Rake::Task["elasticsearch:import:model"].invoke - Rake::Task["elasticsearch:import:model"].reenable - puts - end - end - - end - -end diff --git a/elasticsearch-model/lib/rails/templates/01-basic.rb b/elasticsearch-model/lib/rails/templates/01-basic.rb deleted file mode 100644 index 129d5ffd75..0000000000 --- a/elasticsearch-model/lib/rails/templates/01-basic.rb +++ /dev/null @@ -1,335 +0,0 @@ -# ===================================================================================================== -# Template for generating a no-frills Rails application with support for Elasticsearch full-text search -# ===================================================================================================== -# -# This file creates a basic, fully working Rails application with support for Elasticsearch full-text -# search via the `elasticsearch-rails` gem; https://github.com/elasticsearch/elasticsearch-rails. -# -# Requirements: -# ------------- -# -# * Git -# * Ruby >= 1.9.3 -# * Rails >= 4 -# * Java >= 7 (for Elasticsearch) -# -# Usage: -# ------ -# -# $ rails new searchapp --skip --skip-bundle --template https://raw.github.com/elasticsearch/elasticsearch-rails/master/elasticsearch-rails/lib/rails/templates/01-basic.rb -# -# ===================================================================================================== - -require 'uri' -require 'net/http' - -at_exit do - pid = File.read("#{destination_root}/tmp/pids/elasticsearch.pid") rescue nil - if pid - say_status "Stop", "Elasticsearch", :yellow - run "kill #{pid}" - end -end - -run "touch tmp/.gitignore" - -append_to_file ".gitignore", "vendor/elasticsearch-1.0.1/\n" - -git :init -git add: "." -git commit: "-m 'Initial commit: Clean application'" - -# ----- Download Elasticsearch -------------------------------------------------------------------- - -unless (Net::HTTP.get(URI.parse('http://localhost:9200')) rescue false) - COMMAND = <<-COMMAND.gsub(/^ /, '') - curl -# -O "http://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.0.1.tar.gz" - tar -zxf elasticsearch-1.0.1.tar.gz - rm -f elasticsearch-1.0.1.tar.gz - ./elasticsearch-1.0.1/bin/elasticsearch -d -p #{destination_root}/tmp/pids/elasticsearch.pid - COMMAND - - puts "\n" - say_status "ERROR", "Elasticsearch not running!\n", :red - puts '-'*80 - say_status '', "It appears that Elasticsearch is not running on this machine." - say_status '', "Is it installed? Do you want me to install it for you with this command?\n\n" - COMMAND.each_line { |l| say_status '', "$ #{l}" } - puts - say_status '', "(To uninstall, just remove the generated application directory.)" - puts '-'*80, '' - - if yes?("Install Elasticsearch?", :bold) - puts - say_status "Install", "Elasticsearch", :yellow - - commands = COMMAND.split("\n") - exec = commands.pop - inside("vendor") do - commands.each { |command| run command } - run "(#{exec})" # Launch Elasticsearch in subshell - end - end -end unless ENV['RAILS_NO_ES_INSTALL'] - -# ----- Add README -------------------------------------------------------------------------------- - -puts -say_status "README", "Adding Readme...\n", :yellow -puts '-'*80, ''; sleep 0.25 - -remove_file 'README.rdoc' - -create_file 'README.rdoc', <<-README -= Ruby on Rails and Elasticsearch: Example application - -This application is an example of integrating the {Elasticsearch}[http://www.elasticsearch.org] -search engine with the {Ruby On Rails}[http://rubyonrails.org] web framework. - -It has been generated by application templates available at -https://github.com/elasticsearch/elasticsearch-rails/tree/master/elasticsearch-rails/lib/rails/templates. - -== [1] Basic - -The `basic` version provides a simple integration for a simple Rails model, `Article`, showing how -to include the search engine support in your model, automatically index changes to records, -and use a form to perform simple search require 'requests.' - -README - - -git add: "." -git commit: "-m '[01] Added README for the application'" - -# ----- Use Thin ---------------------------------------------------------------------------------- - -begin - require 'thin' - puts - say_status "Rubygems", "Adding Thin into Gemfile...\n", :yellow - puts '-'*80, ''; - - gem 'thin' -rescue LoadError -end - -# ----- Auxiliary gems ---------------------------------------------------------------------------- - -gem 'mocha', group: 'test', require: 'mocha/api' - -# ----- Remove CoffeeScript, Sass and "all that jazz" --------------------------------------------- - -comment_lines 'Gemfile', /gem 'coffee/ -comment_lines 'Gemfile', /gem 'sass/ -comment_lines 'Gemfile', /gem 'uglifier/ -uncomment_lines 'Gemfile', /gem 'therubyracer/ - -# ----- Add gems into Gemfile --------------------------------------------------------------------- - -puts -say_status "Rubygems", "Adding Elasticsearch libraries into Gemfile...\n", :yellow -puts '-'*80, ''; sleep 0.75 - -gem 'elasticsearch', git: 'git://github.com/elasticsearch/elasticsearch-ruby.git' -gem 'elasticsearch-model', git: 'git://github.com/elasticsearch/elasticsearch-rails.git' -gem 'elasticsearch-rails', git: 'git://github.com/elasticsearch/elasticsearch-rails.git' - - -git add: "Gemfile*" -git commit: "-m 'Added libraries into Gemfile'" - -# ----- Disable asset logging in development ------------------------------------------------------ - -puts -say_status "Application", "Disabling asset logging in development...\n", :yellow -puts '-'*80, ''; sleep 0.25 - -environment 'config.assets.logger = false', env: 'development' -gem 'quiet_assets', group: "development" - -git add: "Gemfile*" -git add: "config/" -git commit: "-m 'Disabled asset logging in development'" - -# ----- Install gems ------------------------------------------------------------------------------ - -puts -say_status "Rubygems", "Installing Rubygems...", :yellow -puts '-'*80, '' - -run "bundle install" - -# ----- Generate Article resource ----------------------------------------------------------------- - -puts -say_status "Model", "Generating the Article resource...", :yellow -puts '-'*80, ''; sleep 0.75 - -generate :scaffold, "Article title:string content:text published_on:date" -route "root to: 'articles#index'" -rake "db:migrate" - -git add: "." -git commit: "-m 'Added the generated Article resource'" - -# ----- Add Elasticsearch integration into the model ---------------------------------------------- - -puts -say_status "Model", "Adding search support into the Article model...", :yellow -puts '-'*80, ''; sleep 0.25 - -run "rm -f app/models/article.rb" -file 'app/models/article.rb', <<-CODE -class Article < ActiveRecord::Base - include Elasticsearch::Model - include Elasticsearch::Model::Callbacks - #{'attr_accessible :title, :content, :published_on' if Rails::VERSION::STRING < '4'} -end -CODE - -git commit: "-a -m 'Added Elasticsearch support into the Article model'" - -# ----- Add Elasticsearch integration into the interface ------------------------------------------ - -puts -say_status "Controller", "Adding controller action, route, and HTML for searching...", :yellow -puts '-'*80, ''; sleep 0.25 - -inject_into_file 'app/controllers/articles_controller.rb', before: %r|^\s*# GET /articles/1$| do - <<-CODE - - # GET /articles/search - def search - @articles = Article.search(params[:q]).records - - render action: "index" - end - - CODE -end - -inject_into_file 'app/views/articles/index.html.erb', after: %r{

Listing articles

}i do - <<-CODE - - -
- - <%= form_tag search_articles_path, method: 'get' do %> - <%= label_tag :query %> - <%= text_field_tag :q, params[:q] %> - <%= submit_tag :search %> - <% end %> - -
- CODE -end - -inject_into_file 'app/views/articles/index.html.erb', after: %r{<%= link_to 'New Article', new_article_path %>} do - <<-CODE - <%= link_to 'All Articles', articles_path if params[:q] %> - CODE -end - -gsub_file 'config/routes.rb', %r{resources :articles$}, <<-CODE -resources :articles do - collection { get :search } - end -CODE - -gsub_file "#{Rails::VERSION::STRING > '4' ? 'test/controllers' : 'test/functional'}/articles_controller_test.rb", %r{setup do.*?end}m, <<-CODE -setup do - @article = articles(:one) - - Article.__elasticsearch__.import - Article.__elasticsearch__.refresh_index! - end -CODE - -inject_into_file "#{Rails::VERSION::STRING > '4' ? 'test/controllers' : 'test/functional'}/articles_controller_test.rb", after: %r{test "should get index" do.*?end}m do - <<-CODE - - - test "should get search results" do - get :search, q: 'mystring' - assert_response :success - assert_not_nil assigns(:articles) - assert_equal 2, assigns(:articles).size - end - CODE -end - -git commit: "-a -m 'Added search form and controller action'" - -# ----- Seed the database ------------------------------------------------------------------------- - -puts -say_status "Database", "Seeding the database with data...", :yellow -puts '-'*80, ''; sleep 0.25 - -remove_file "db/seeds.rb" -create_file 'db/seeds.rb', %q{ -contents = [ -'Lorem ipsum dolor sit amet.', -'Consectetur adipisicing elit, sed do eiusmod tempor incididunt.', -'Labore et dolore magna aliqua.', -'Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris.', -'Excepteur sint occaecat cupidatat non proident.' -] - -puts "Deleting all articles..." -Article.delete_all - -unless ENV['COUNT'] - - puts "Creating articles..." - %w[ One Two Three Four Five ].each_with_index do |title, i| - Article.create title: title, content: contents[i], published_on: i.days.ago.utc - end - -else - - print "Generating articles..." - (1..ENV['COUNT'].to_i).each_with_index do |title, i| - Article.create title: "Title #{title}", content: 'Lorem ipsum dolor', published_on: i.days.ago.utc - print '.' if i % ENV['COUNT'].to_i/10 == 0 - end - puts "\n" - -end -} - -run "rails runner 'Article.__elasticsearch__.create_index! force: true'" -rake "db:seed" - -git add: "db/seeds.rb" -git commit: "-m 'Added the database seeding script'" - -# ----- Print Git log ----------------------------------------------------------------------------- - -puts -say_status "Git", "Details about the application:", :yellow -puts '-'*80, '' - -git tag: "basic" -git log: "--reverse --oneline" - -# ----- Start the application --------------------------------------------------------------------- - -unless ENV['RAILS_NO_SERVER_START'] - require 'net/http' - if (begin; Net::HTTP.get(URI('http://localhost:3000')); rescue Errno::ECONNREFUSED; false; rescue Exception; true; end) - puts "\n" - say_status "ERROR", "Some other application is running on port 3000!\n", :red - puts '-'*80 - - port = ask("Please provide free port:", :bold) - else - port = '3000' - end - - puts "", "="*80 - say_status "DONE", "\e[1mStarting the application.\e[0m", :yellow - puts "="*80, "" - - run "rails server --port=#{port}" -end diff --git a/elasticsearch-model/lib/rails/templates/02-pretty.rb b/elasticsearch-model/lib/rails/templates/02-pretty.rb deleted file mode 100644 index 7fd6e5048f..0000000000 --- a/elasticsearch-model/lib/rails/templates/02-pretty.rb +++ /dev/null @@ -1,311 +0,0 @@ -# $ rails new searchapp --skip --skip-bundle --template https://raw.github.com/elasticsearch/elasticsearch-rails/master/elasticsearch-rails/lib/rails/templates/02-pretty.rb - -unless File.read('README.rdoc').include? '== [1] Basic' - say_status "ERROR", "You have to run the 01-basic.rb template first.", :red - exit(1) -end - -puts -say_status "README", "Updating Readme...\n", :yellow -puts '-'*80, ''; sleep 0.25 - -append_to_file 'README.rdoc', <<-README - -== [2] Pretty - -The `pretty` template builds on the `basic` version and brings couple of improvements: - -* Using the [Bootstrap](http://getbootstrap.com) framework to enhance the visual style of the application -* Using an `Article.search` class method to customize the default search definition -* Highlighting matching phrases in search results -* Paginating results with Kaminari - -README - -git add: "README.rdoc" -git commit: "-m '[02] Updated the application README'" - -# ----- Update application.rb --------------------------------------------------------------------- - -puts -say_status "Rubygems", "Adding Rails logger integration...\n", :yellow -puts '-'*80, ''; sleep 0.25 - -insert_into_file 'config/application.rb', - "\n\nrequire 'elasticsearch/rails/instrumentation'", - after: /Bundler\.require.+$/ - -git add: "config/application.rb" -git commit: "-m 'Added the Rails logger integration to application.rb'" - -# ----- Add gems into Gemfile --------------------------------------------------------------------- - -puts -say_status "Rubygems", "Adding Rubygems into Gemfile...\n", :yellow -puts '-'*80, ''; sleep 0.25 - -# NOTE: Kaminari has to be loaded before Elasticsearch::Model so the callbacks are executed -# -insert_into_file 'Gemfile', <<-CODE, before: /gem ["']elasticsearch["'].+$/ - -# NOTE: Kaminari has to be loaded before Elasticsearch::Model so the callbacks are executed -gem 'kaminari' - -CODE - -run "bundle install" - -git add: "Gemfile*" -git commit: "-m 'Added the Kaminari gem'" - -# ----- Add `Article.search` class method --------------------------------------------------------- - -puts -say_status "Model", "Adding a `Article.search` class method...\n", :yellow -puts '-'*80, ''; sleep 0.5 - -insert_into_file 'app/models/article.rb', <<-CODE, after: 'include Elasticsearch::Model::Callbacks' - - - def self.search(query) - __elasticsearch__.search( - { - query: { - multi_match: { - query: query, - fields: ['title^10', 'content'] - } - }, - highlight: { - pre_tags: [''], - post_tags: [''], - fields: { - title: { number_of_fragments: 0 }, - content: { fragment_size: 25 } - } - } - } - ) - end -CODE - -insert_into_file "#{Rails::VERSION::STRING > '4' ? 'test/models' : 'test/unit' }/article_test.rb", <<-CODE, after: /class ArticleTest < ActiveSupport::TestCase$/ - - teardown do - Article.__elasticsearch__.unstub(:search) - end - -CODE - -gsub_file "#{Rails::VERSION::STRING > '4' ? 'test/models' : 'test/unit' }/article_test.rb", %r{# test "the truth" do.*?# end}m, <<-CODE - - test "has a search method delegating to __elasticsearch__" do - Article.__elasticsearch__.expects(:search).with do |definition| - assert_equal 'foo', definition[:query][:multi_match][:query] - true - end - - Article.search 'foo' - end -CODE - -git add: "app/models/article.rb" -git add: "test/**/article_test.rb" -git commit: "-m 'Added an `Article.search` method'" - -# ----- Add loading Bootstrap assets -------------------------------------------------------------- - -puts -say_status "Bootstrap", "Adding Bootstrap asset links into the 'application' layout...\n", :yellow -puts '-'*80, ''; sleep 0.5 - -gsub_file 'app/views/layouts/application.html.erb', %r{<%= yield %>}, <<-CODE unless File.read('app/views/layouts/application.html.erb').include?('class="container"') -
-<%= yield %> -
-CODE - -insert_into_file 'app/views/layouts/application.html.erb', <<-CODE, before: '' - - -CODE - -git commit: "-a -m 'Added loading Bootstrap assets in the application layout'" - -# ----- Customize the search form ----------------------------------------------------------------- - -puts -say_status "Bootstrap", "Customizing the index page...\n", :yellow -puts '-'*80, ''; sleep 0.5 - -gsub_file 'app/views/articles/index.html.erb', %r{<%= label_tag .* :search %>}m do |match| -<<-CODE -
- <%= text_field_tag :q, params[:q], class: 'form-control', placeholder: 'Search...' %> - - - - -
-CODE -end - -# ----- Customize the header ----------------------------------------------------------------- - -gsub_file 'app/views/articles/index.html.erb', %r{

Listing articles

} do |match| - "

<%= controller.action_name == 'search' ? 'Searching articles' : 'Listing articles' %>

" -end - -# ----- Customize the results listing ------------------------------------------------------------- - -gsub_file 'app/views/articles/index.html.erb', %r{} do |match| - '
' -end - -gsub_file 'app/views/articles/index.html.erb', %r{$} do |match| - "" -end - -gsub_file 'app/views/articles/index.html.erb', %r{$} do |match| - "" -end - -git commit: "-a -m 'Added highlighting for matches'" - -# ----- Paginate the results ---------------------------------------------------------------------- - -gsub_file 'app/controllers/articles_controller.rb', %r{@articles = Article.all} do |match| - "@articles = Article.page(params[:page])" -end - -gsub_file 'app/controllers/articles_controller.rb', %r{@articles = Article.search\(params\[\:q\]\).records} do |match| - "@articles = Article.search(params[:q]).page(params[:page]).records" -end - -insert_into_file 'app/views/articles/index.html.erb', after: '
<%= link_to [^%]+} do |match| - match.gsub!('', '') - match.include?("btn") ? match : (match + ", class: 'btn btn-default btn-xs'") -end - -gsub_file 'app/views/articles/index.html.erb', %r{
\s*(<\%= link_to 'New Article'.*)}m do |content| - replace = content.match(%r{
\s*(<\%= link_to 'New Article'.*)}m)[1] - <<-END.gsub(/^ /, '') -
- -

- #{replace} -

- END -end - -gsub_file 'app/views/articles/index.html.erb', %r{<%= link_to 'New Article',\s*new_article_path} do |match| - return match if match.include?('btn') - match + ", class: 'btn btn-primary btn-xs', style: 'color: #fff'" -end - -gsub_file 'app/views/articles/index.html.erb', %r{<%= link_to 'All Articles',\s*articles_path} do |match| - return match if match.include?('btn') - "\n " + match + ", class: 'btn btn-primary btn-xs', style: 'color: #fff'" -end - -git add: "app/views" -git commit: "-m 'Refactored the articles listing to use Bootstrap components'" - -# ----- Use highlighted excerpts in the listing --------------------------------------------------- - -gsub_file 'app/views/articles/index.html.erb', %r{<% @articles.each do \|article\| %>$} do |match| - "<% @articles.__send__ controller.action_name == 'search' ? :each_with_hit : :each do |article, hit| %>" -end - -gsub_file 'app/views/articles/index.html.erb', %r{		<%= article.title %><%= hit.try(:highlight).try(:title) ? hit.highlight.title.join.html_safe : article.title %><%= article.content %><%= hit.try(:highlight).try(:content) ? hit.highlight.content.join('…').html_safe : article.content %>
' do - <<-CODE.gsub(/^ /, '') - - -
- <%= paginate @articles %> -
- CODE -end - -generate "kaminari:views", "bootstrap2", "--force" - -gsub_file 'app/views/kaminari/_paginator.html.erb', %r{