diff --git a/elasticsearch-model/.gitignore b/elasticsearch-model/.gitignore new file mode 100644 index 0000000000..3934d7e554 --- /dev/null +++ b/elasticsearch-model/.gitignore @@ -0,0 +1,20 @@ +*.gem +*.rbc +.bundle +.config +.yardoc +Gemfile.lock +InstalledFiles +_yardoc +coverage +doc/ +lib/bundler/man +pkg +rdoc +spec/reports +test/tmp +test/version_tmp +tmp + +gemfiles/3.0.gemfile.lock +gemfiles/4.0.gemfile.lock diff --git a/elasticsearch-model/CHANGELOG.md b/elasticsearch-model/CHANGELOG.md new file mode 100644 index 0000000000..1b28033835 --- /dev/null +++ b/elasticsearch-model/CHANGELOG.md @@ -0,0 +1,74 @@ +## 0.1.9 + +* Added a `suggest` method to wrap the suggestions in response +* Added the `:includes` option to Adapter::ActiveRecord::Records for eagerly loading associated models +* Delegated `max_pages` method properly for Kaminari's `next_page` +* Fixed `#dup` behaviour for Elasticsearch::Model +* Fixed typos in the README and examples + +## 0.1.8 + +* Added "default per page" methods for pagination with multi model searches +* Added a convenience accessor for the `aggregations` part of response +* Added a full example with mapping for the completion suggester +* Added an integration test for paginating multiple models +* Added proper support for the new "multi_fields" in the mapping DSL +* Added the `no_timeout` option for `__find_in_batches` in the Mongoid adapter +* Added, that index settings can be loaded from any object that responds to `:read` +* Added, that index settings/mappings can be loaded from a YAML or JSON file +* Added, that String pagination parameters are converted to numbers +* Added, that empty block is not required for setting mapping options +* Added, that on MyModel#import, an exception is raised if the index does not exists +* Changed the Elasticsearch port in the Mongoid example to 9200 +* Cleaned up the tests for multiple fields/properties in mapping DSL +* Fixed a bug where continuous `#save` calls emptied the `@__changed_attributes` variable +* Fixed a buggy test introduced in #335 +* Fixed incorrect deserialization of records in the Multiple adapter +* Fixed incorrect examples and documentation +* Fixed unreliable order of returned results/records in the integration test for the multiple adapter +* Fixed, that `param_name` is used when paginating with WillPaginate +* Fixed the problem where `document_type` configuration was not propagated to mapping [6 months ago by Miguel Ferna +* Refactored the code in `__find_in_batches` to use Enumerable#each_slice +* Refactored the string queries in multiple_models_test.rb to avoid quote escaping + +## 0.1.7 + +* Improved examples and instructions in README and code annotations +* Prevented index methods to swallow all exceptions +* Added the `:validate` option to the `save` method for models +* Added support for searching across multiple models (elastic/elasticsearch-rails#345), + including documentation, examples and tests + +## 0.1.6 + +* Improved documentation +* Added dynamic getter/setter (block/proc) for `MyModel.index_name` +* Added the `update_document_attributes` method +* Added, that records to import can be limited by the `query` option + +## 0.1.5 + +* Improved documentation +* Fixes and improvements to the "will_paginate" integration +* Added a `:preprocess` option to the `import` method +* Changed, that attributes are fetched from `as_indexed_json` in the `update_document` method +* Added an option to the import method to return an array of error messages instead of just count +* Fixed many problems with dependency hell +* Fixed tests so they run on Ruby 2.2 + +## 0.1.2 + +* Properly delegate existence methods like `result.foo?` to `result._source.foo` +* Exception is raised when `type` is not passed to Mappings#new +* Allow passing an ActiveRecord scope to the `import` method +* Added, that `each_with_hit` and `map_with_hit` in `Elasticsearch::Model::Response::Records` call `to_a` +* Added support for [`will_paginate`](https://github.com/mislav/will_paginate) pagination library +* Added the ability to transform models during indexing +* Added explicit `type` and `id` methods to Response::Result, aliasing `_type` and `_id` + +## 0.1.1 + +* Improved documentation and tests +* Fixed Kaminari implementation bugs and inconsistencies + +## 0.1.0 (Initial Version) diff --git a/elasticsearch-model/Gemfile b/elasticsearch-model/Gemfile new file mode 100644 index 0000000000..a54f5084ea --- /dev/null +++ b/elasticsearch-model/Gemfile @@ -0,0 +1,4 @@ +source 'https://rubygems.org' + +# Specify your gem's dependencies in elasticsearch-model.gemspec +gemspec diff --git a/elasticsearch-model/LICENSE.txt b/elasticsearch-model/LICENSE.txt new file mode 100644 index 0000000000..7dc94b3e5a --- /dev/null +++ b/elasticsearch-model/LICENSE.txt @@ -0,0 +1,13 @@ +Copyright (c) 2014 Elasticsearch + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/elasticsearch-model/README.md b/elasticsearch-model/README.md new file mode 100644 index 0000000000..c219a46001 --- /dev/null +++ b/elasticsearch-model/README.md @@ -0,0 +1,720 @@ +# Elasticsearch::Model + +The `elasticsearch-model` library builds on top of the +the [`elasticsearch`](https://github.com/elasticsearch/elasticsearch-ruby) library. + +It aims to simplify integration of Ruby classes ("models"), commonly found +e.g. in [Ruby on Rails](http://rubyonrails.org) applications, with the +[Elasticsearch](http://www.elasticsearch.org) search and analytics engine. + +The library is compatible with Ruby 1.9.3 and higher. + +## Installation + +Install the package from [Rubygems](https://rubygems.org): + + gem install elasticsearch-model + +To use an unreleased version, either add it to your `Gemfile` for [Bundler](http://bundler.io): + + gem 'elasticsearch-model', git: 'git://github.com/elasticsearch/elasticsearch-rails.git' + +or install it from a source code checkout: + + git clone https://github.com/elasticsearch/elasticsearch-rails.git + cd elasticsearch-rails/elasticsearch-model + bundle install + rake install + + +## Usage + +Let's suppose you have an `Article` model: + +```ruby +require 'active_record' +ActiveRecord::Base.establish_connection( adapter: 'sqlite3', database: ":memory:" ) +ActiveRecord::Schema.define(version: 1) { create_table(:articles) { |t| t.string :title } } + +class Article < ActiveRecord::Base; end + +Article.create title: 'Quick brown fox' +Article.create title: 'Fast black dogs' +Article.create title: 'Swift green frogs' +``` + +### Setup + +To add the Elasticsearch integration for this model, require `elasticsearch/model` +and include the main module in your class: + +```ruby +require 'elasticsearch/model' + +class Article < ActiveRecord::Base + include Elasticsearch::Model +end +``` + +This will extend the model with functionality related to Elasticsearch. + +#### Feature Extraction Pattern + +Instead of including the `Elasticsearch::Model` module directly in your model, +you can include it in a "concern" or "trait" module, which is quite common pattern in Rails applications, +using e.g. `ActiveSupport::Concern` as the instrumentation: + +```ruby +# In: app/models/concerns/searchable.rb +# +module Searchable + extend ActiveSupport::Concern + + included do + include Elasticsearch::Model + + mapping do + # ... + end + + def self.search(query) + # ... + end + end +end + +# In: app/models/article.rb +# +class Article + include Searchable +end +``` + +#### The `__elasticsearch__` Proxy + +The `Elasticsearch::Model` module contains a big amount of class and instance methods to provide +all its functionality. To prevent polluting your model namespace, this functionality is primarily +available via the `__elasticsearch__` class and instance level proxy methods; +see the `Elasticsearch::Model::Proxy` class documentation for technical information. + +The module will include important methods, such as `search`, into the class or module only +when they haven't been defined already. Following two calls are thus functionally equivalent: + +```ruby +Article.__elasticsearch__.search 'fox' +Article.search 'fox' +``` + +See the `Elasticsearch::Model` module documentation for technical information. + +### The Elasticsearch client + +The module will set up a [client](https://github.com/elasticsearch/elasticsearch-ruby/tree/master/elasticsearch), +connected to `localhost:9200`, by default. You can access and use it as any other `Elasticsearch::Client`: + +```ruby +Article.__elasticsearch__.client.cluster.health +# => { "cluster_name"=>"elasticsearch", "status"=>"yellow", ... } +``` + +To use a client with different configuration, just set up a client for the model: + +```ruby +Article.__elasticsearch__.client = Elasticsearch::Client.new host: 'api.server.org' +``` + +Or configure the client for all models: + +```ruby +Elasticsearch::Model.client = Elasticsearch::Client.new log: true +``` + +You might want to do this during your application bootstrap process, e.g. in a Rails initializer. + +Please refer to the +[`elasticsearch-transport`](https://github.com/elasticsearch/elasticsearch-ruby/tree/master/elasticsearch-transport) +library documentation for all the configuration options, and to the +[`elasticsearch-api`](http://rubydoc.info/gems/elasticsearch-api) library documentation +for information about the Ruby client API. + +### Importing the data + +The first thing you'll want to do is importing your data into the index: + +```ruby +Article.import +# => 0 +``` + +It's possible to import only records from a specific `scope` or `query`, transform the batch with the `transform` +and `preprocess` options, or re-create the index by deleting it and creating it with correct mapping with the `force` option -- look for examples in the method documentation. + +No errors were reported during importing, so... let's search the index! + + +### Searching + +For starters, we can try the "simple" type of search: + +```ruby +response = Article.search 'fox dogs' + +response.took +# => 3 + +response.results.total +# => 2 + +response.results.first._score +# => 0.02250402 + +response.results.first._source.title +# => "Quick brown fox" +``` + +#### Search results + +The returned `response` object is a rich wrapper around the JSON returned from Elasticsearch, +providing access to response metadata and the actual results ("hits"). + +Each "hit" is wrapped in the `Result` class, and provides method access +to its properties via [`Hashie::Mash`](http://github.com/intridea/hashie). + +The `results` object supports the `Enumerable` interface: + +```ruby +response.results.map { |r| r._source.title } +# => ["Quick brown fox", "Fast black dogs"] + +response.results.select { |r| r.title =~ /^Q/ } +# => [#{"title"=>"Quick brown fox"}}>] +``` + +In fact, the `response` object will delegate `Enumerable` methods to `results`: + +```ruby +response.any? { |r| r.title =~ /fox|dog/ } +# => true +``` + +To use `Array`'s methods (including any _ActiveSupport_ extensions), just call `to_a` on the object: + +```ruby +response.to_a.last.title +# "Fast black dogs" +``` + +#### Search results as database records + +Instead of returning documents from Elasticsearch, the `records` method will return a collection +of model instances, fetched from the primary database, ordered by score: + +```ruby +response.records.to_a +# Article Load (0.3ms) SELECT "articles".* FROM "articles" WHERE "articles"."id" IN (1, 2) +# => [#
, #
] +``` + +The returned object is the genuine collection of model instances returned by your database, +i.e. `ActiveRecord::Relation` for ActiveRecord, or `Mongoid::Criteria` in case of MongoDB. + +This allows you to chain other methods on top of search results, as you would normally do: + +```ruby +response.records.where(title: 'Quick brown fox').to_a +# Article Load (0.2ms) SELECT "articles".* FROM "articles" WHERE "articles"."id" IN (1, 2) AND "articles"."title" = 'Quick brown fox' +# => [#
] + +response.records.records.class +# => ActiveRecord::Relation::ActiveRecord_Relation_Article +``` + +The ordering of the records by score will be preserved, unless you explicitly specify a different +order in your model query language: + +```ruby +response.records.order(:title).to_a +# Article Load (0.2ms) SELECT "articles".* FROM "articles" WHERE "articles"."id" IN (1, 2) ORDER BY "articles".title ASC +# => [#
, #
] +``` + +The `records` method returns the real instances of your model, which is useful when you want to access your +model methods -- at the expense of slowing down your application, of course. +In most cases, working with `results` coming from Elasticsearch is sufficient, and much faster. See the +[`elasticsearch-rails`](https://github.com/elasticsearch/elasticsearch-rails/tree/master/elasticsearch-rails) +library for more information about compatibility with the Ruby on Rails framework. + +When you want to access both the database `records` and search `results`, use the `each_with_hit` +(or `map_with_hit`) iterator: + +```ruby +response.records.each_with_hit { |record, hit| puts "* #{record.title}: #{hit._score}" } +# * Quick brown fox: 0.02250402 +# * Fast black dogs: 0.02250402 +``` + +#### Searching multiple models + +It is possible to search across multiple models with the module method: + +```ruby +Elasticsearch::Model.search('fox', [Article, Comment]).results.to_a.map(&:to_hash) +# => [ +# {"_index"=>"articles", "_type"=>"article", "_id"=>"1", "_score"=>0.35136628, "_source"=>...}, +# {"_index"=>"comments", "_type"=>"comment", "_id"=>"1", "_score"=>0.35136628, "_source"=>...} +# ] + +Elasticsearch::Model.search('fox', [Article, Comment]).records.to_a +# Article Load (0.3ms) SELECT "articles".* FROM "articles" WHERE "articles"."id" IN (1) +# Comment Load (0.2ms) SELECT "comments".* FROM "comments" WHERE "comments"."id" IN (1,5) +# => [#
, #, ...] +``` + +By default, all models which include the `Elasticsearch::Model` module are searched. + +NOTE: It is _not_ possible to chain other methods on top of the `records` object, since it + is a heterogenous collection, with models potentially backed by different databases. + +#### Pagination + +You can implement pagination with the `from` and `size` search parameters. However, search results +can be automatically paginated with the [`kaminari`](http://rubygems.org/gems/kaminari) or +[`will_paginate`](https://github.com/mislav/will_paginate) gems. +(The pagination gems must be added before the Elasticsearch gems in your Gemfile, +or loaded first in your application.) + +If Kaminari or WillPaginate is loaded, use the familiar paging methods: + +```ruby +response.page(2).results +response.page(2).records +``` + +In a Rails controller, use the the `params[:page]` parameter to paginate through results: + +```ruby +@articles = Article.search(params[:q]).page(params[:page]).records + +@articles.current_page +# => 2 +@articles.next_page +# => 3 +``` +To initialize and include the Kaminari pagination support manually: + +```ruby +Kaminari::Hooks.init +Elasticsearch::Model::Response::Response.__send__ :include, Elasticsearch::Model::Response::Pagination::Kaminari +``` + +#### The Elasticsearch DSL + +In most situation, you'll want to pass the search definition +in the Elasticsearch [domain-specific language](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html) to the client: + +```ruby +response = Article.search query: { match: { title: "Fox Dogs" } }, + highlight: { fields: { title: {} } } + +response.results.first.highlight.title +# ["Quick brown fox"] +``` + +You can pass any object which implements a `to_hash` method, or you can use your favourite JSON builder +to build the search definition as a JSON string: + +```ruby +require 'jbuilder' + +query = Jbuilder.encode do |json| + json.query do + json.match do + json.title do + json.query "fox dogs" + end + end + end +end + +response = Article.search query +response.results.first.title +# => "Quick brown fox" +``` + +### Index Configuration + +For proper search engine function, it's often necessary to configure the index properly. +The `Elasticsearch::Model` integration provides class methods to set up index settings and mappings. + +```ruby +class Article + settings index: { number_of_shards: 1 } do + mappings dynamic: 'false' do + indexes :title, analyzer: 'english', index_options: 'offsets' + end + end +end + +Article.mappings.to_hash +# => { +# :article => { +# :dynamic => "false", +# :properties => { +# :title => { +# :type => "string", +# :analyzer => "english", +# :index_options => "offsets" +# } +# } +# } +# } + +Article.settings.to_hash +# { :index => { :number_of_shards => 1 } } +``` + +You can use the defined settings and mappings to create an index with desired configuration: + +```ruby +Article.__elasticsearch__.client.indices.delete index: Article.index_name rescue nil +Article.__elasticsearch__.client.indices.create \ + index: Article.index_name, + body: { settings: Article.settings.to_hash, mappings: Article.mappings.to_hash } +``` + +There's a shortcut available for this common operation (convenient e.g. in tests): + +```ruby +Article.__elasticsearch__.create_index! force: true +Article.__elasticsearch__.refresh_index! +``` + +By default, index name and document type will be inferred from your class name, +you can set it explicitely, however: + +```ruby +class Article + index_name "articles-#{Rails.env}" + document_type "post" +end +``` + +### Updating the Documents in the Index + +Usually, we need to update the Elasticsearch index when records in the database are created, updated or deleted; +use the `index_document`, `update_document` and `delete_document` methods, respectively: + +```ruby +Article.first.__elasticsearch__.index_document +# => {"ok"=>true, ... "_version"=>2} +``` + +#### Automatic Callbacks + +You can automatically update the index whenever the record changes, by including +the `Elasticsearch::Model::Callbacks` module in your model: + +```ruby +class Article + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks +end + +Article.first.update_attribute :title, 'Updated!' + +Article.search('*').map { |r| r.title } +# => ["Updated!", "Lime green frogs", "Fast black dogs"] +``` + +The automatic callback on record update keeps track of changes in your model +(via [`ActiveModel::Dirty`](http://api.rubyonrails.org/classes/ActiveModel/Dirty.html)-compliant implementation), +and performs a _partial update_ when this support is available. + +The automatic callbacks are implemented in database adapters coming with `Elasticsearch::Model`. You can easily +implement your own adapter: please see the relevant chapter below. + +#### Custom Callbacks + +In case you would need more control of the indexing process, you can implement these callbacks yourself, +by hooking into `after_create`, `after_save`, `after_update` or `after_destroy` operations: + +```ruby +class Article + include Elasticsearch::Model + + after_save { logger.debug ["Updating document... ", index_document ].join } + after_destroy { logger.debug ["Deleting document... ", delete_document].join } +end +``` + +For ActiveRecord-based models, use the `after_commit` callback to protect +your data against inconsistencies caused by transaction rollbacks: + +```ruby +class Article < ActiveRecord::Base + include Elasticsearch::Model + + after_commit on: [:create] do + __elasticsearch__.index_document if self.published? + end + + after_commit on: [:update] do + __elasticsearch__.update_document if self.published? + end + + after_commit on: [:destroy] do + __elasticsearch__.delete_document if self.published? + end +end +``` + +#### Asynchronous Callbacks + +Of course, you're still performing an HTTP request during your database transaction, which is not optimal +for large-scale applications. A better option would be to process the index operations in background, +with a tool like [_Resque_](https://github.com/resque/resque) or [_Sidekiq_](https://github.com/mperham/sidekiq): + +```ruby +class Article + include Elasticsearch::Model + + after_save { Indexer.perform_async(:index, self.id) } + after_destroy { Indexer.perform_async(:delete, self.id) } +end +``` + +An example implementation of the `Indexer` worker class could look like this: + +```ruby +class Indexer + include Sidekiq::Worker + sidekiq_options queue: 'elasticsearch', retry: false + + Logger = Sidekiq.logger.level == Logger::DEBUG ? Sidekiq.logger : nil + Client = Elasticsearch::Client.new host: 'localhost:9200', logger: Logger + + def perform(operation, record_id) + logger.debug [operation, "ID: #{record_id}"] + + case operation.to_s + when /index/ + record = Article.find(record_id) + Client.index index: 'articles', type: 'article', id: record.id, body: record.as_indexed_json + when /delete/ + Client.delete index: 'articles', type: 'article', id: record_id + else raise ArgumentError, "Unknown operation '#{operation}'" + end + end +end +``` + +Start the _Sidekiq_ workers with `bundle exec sidekiq --queue elasticsearch --verbose` and +update a model: + +```ruby +Article.first.update_attribute :title, 'Updated' +``` + +You'll see the job being processed in the console where you started the _Sidekiq_ worker: + +``` +Indexer JID-eb7e2daf389a1e5e83697128 DEBUG: ["index", "ID: 7"] +Indexer JID-eb7e2daf389a1e5e83697128 INFO: PUT http://localhost:9200/articles/article/1 [status:200, request:0.004s, query:n/a] +Indexer JID-eb7e2daf389a1e5e83697128 DEBUG: > {"id":1,"title":"Updated", ...} +Indexer JID-eb7e2daf389a1e5e83697128 DEBUG: < {"ok":true,"_index":"articles","_type":"article","_id":"1","_version":6} +Indexer JID-eb7e2daf389a1e5e83697128 INFO: done: 0.006 sec +``` + +### Model Serialization + +By default, the model instance will be serialized to JSON using the `as_indexed_json` method, +which is defined automatically by the `Elasticsearch::Model::Serializing` module: + +```ruby +Article.first.__elasticsearch__.as_indexed_json +# => {"id"=>1, "title"=>"Quick brown fox"} +``` + +If you want to customize the serialization, just implement the `as_indexed_json` method yourself, +for instance with the [`as_json`](http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html#method-i-as_json) method: + +```ruby +class Article + include Elasticsearch::Model + + def as_indexed_json(options={}) + as_json(only: 'title') + end +end + +Article.first.as_indexed_json +# => {"title"=>"Quick brown fox"} +``` + +The re-defined method will be used in the indexing methods, such as `index_document`. + +Please note that in Rails 3, you need to either set `include_root_in_json: false`, or prevent adding +the "root" in the JSON representation with other means. + +#### Relationships and Associations + +When you have a more complicated structure/schema, you need to customize the `as_indexed_json` method - +or perform the indexing separately, on your own. +For example, let's have an `Article` model, which _has_many_ `Comment`s, +`Author`s and `Categories`. We might want to define the serialization like this: + +```ruby +def as_indexed_json(options={}) + self.as_json( + include: { categories: { only: :title}, + authors: { methods: [:full_name], only: [:full_name] }, + comments: { only: :text } + }) +end + +Article.first.as_indexed_json +# => { "id" => 1, +# "title" => "First Article", +# "created_at" => 2013-12-03 13:39:02 UTC, +# "updated_at" => 2013-12-03 13:39:02 UTC, +# "categories" => [ { "title" => "One" } ], +# "authors" => [ { "full_name" => "John Smith" } ], +# "comments" => [ { "text" => "First comment" } ] } +``` + +Of course, when you want to use the automatic indexing callbacks, you need to hook into the appropriate +_ActiveRecord_ callbacks -- please see the full example in `examples/activerecord_associations.rb`. + +### Other ActiveModel Frameworks + +The `Elasticsearch::Model` module is fully compatible with any ActiveModel-compatible model, such as _Mongoid_: + +```ruby +require 'mongoid' + +Mongoid.connect_to 'articles' + +class Article + include Mongoid::Document + + field :id, type: String + field :title, type: String + + attr_accessible :id, :title, :published_at + + include Elasticsearch::Model + + def as_indexed_json(options={}) + as_json(except: [:id, :_id]) + end +end + +Article.create id: '1', title: 'Quick brown fox' +Article.import + +response = Article.search 'fox'; +response.records.to_a +# MOPED: 127.0.0.1:27017 QUERY database=articles collection=articles selector={"_id"=>{"$in"=>["1"]}} ... +# => [#
] +``` + +Full examples for CouchBase, DataMapper, Mongoid, Ohm and Riak models can be found in the `examples` folder. + +### Adapters + +To support various "OxM" (object-relational- or object-document-mapper) implementations and frameworks, +the `Elasticsearch::Model` integration supports an "adapter" concept. + +An adapter provides implementations for common behaviour, such as fetching records from the database, +hooking into model callbacks for automatic index updates, or efficient bulk loading from the database. +The integration comes with adapters for _ActiveRecord_ and _Mongoid_ out of the box. + +Writing an adapter for your favourite framework is straightforward -- let's see +a simplified adapter for [_DataMapper_](http://datamapper.org): + +```ruby +module DataMapperAdapter + + # Implement the interface for fetching records + # + module Records + def records + klass.all(id: @ids) + end + + # ... + end +end + +# Register the adapter +# +Elasticsearch::Model::Adapter.register( + DataMapperAdapter, + lambda { |klass| defined?(::DataMapper::Resource) and klass.ancestors.include?(::DataMapper::Resource) } +) +``` + +Require the adapter and include `Elasticsearch::Model` in the class: + +```ruby +require 'datamapper_adapter' + +class Article + include DataMapper::Resource + include Elasticsearch::Model + + property :id, Serial + property :title, String +end +``` + +When accessing the `records` method of the response, for example, +the implementation from our adapter will be used now: + +```ruby +response = Article.search 'foo' + +response.records.to_a +# ~ (0.000057) SELECT "id", "title", "published_at" FROM "articles" WHERE "id" IN (3, 1) ORDER BY "id" +# => [#
, #
] + +response.records.records.class +# => DataMapper::Collection +``` + +More examples can be found in the `examples` folder. Please see the `Elasticsearch::Model::Adapter` +module and its submodules for technical information. + +## Development and Community + +For local development, clone the repository and run `bundle install`. See `rake -T` for a list of +available Rake tasks for running tests, generating documentation, starting a testing cluster, etc. + +Bug fixes and features must be covered by unit tests. + +Github's pull requests and issues are used to communicate, send bug reports and code contributions. + +To run all tests against a test Elasticsearch cluster, use a command like this: + +```bash +curl -# https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.0.0.RC1.tar.gz | tar xz -C tmp/ +SERVER=start TEST_CLUSTER_COMMAND=$PWD/tmp/elasticsearch-1.0.0.RC1/bin/elasticsearch bundle exec rake test:all +``` + +## License + +This software is licensed under the Apache 2 license, quoted below. + + Copyright (c) 2014 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/elasticsearch-model/Rakefile b/elasticsearch-model/Rakefile new file mode 100644 index 0000000000..2825f58b01 --- /dev/null +++ b/elasticsearch-model/Rakefile @@ -0,0 +1,61 @@ +require "bundler/gem_tasks" + +desc "Run unit tests" +task :default => 'test:unit' +task :test => 'test:unit' + +# ----- Test tasks ------------------------------------------------------------ + +require 'rake/testtask' +namespace :test do + task :ci_reporter do + ENV['CI_REPORTS'] ||= 'tmp/reports' + require 'ci/reporter/rake/minitest' + Rake::Task['ci:setup:minitest'].invoke + end + + Rake::TestTask.new(:unit) do |test| + Rake::Task['test:ci_reporter'].invoke if ENV['CI'] + test.libs << 'lib' << 'test' + test.test_files = FileList["test/unit/**/*_test.rb"] + # test.verbose = true + # test.warning = true + end + + Rake::TestTask.new(:run_integration) do |test| + Rake::Task['test:ci_reporter'].invoke if ENV['CI'] + test.libs << 'lib' << 'test' + test.test_files = FileList["test/integration/**/*_test.rb"] + end + + desc "Run integration tests against ActiveModel 3 and 4" + task :integration do + sh "BUNDLE_GEMFILE='#{File.expand_path('../gemfiles/3.0.gemfile', __FILE__)}' bundle exec rake test:run_integration" unless defined?(RUBY_VERSION) && RUBY_VERSION > '2.2' + sh "BUNDLE_GEMFILE='#{File.expand_path('../gemfiles/4.0.gemfile', __FILE__)}' bundle exec rake test:run_integration" + end + + desc "Run unit and integration tests" + task :all do + Rake::Task['test:ci_reporter'].invoke if ENV['CI'] + + Rake::Task['test:unit'].invoke + Rake::Task['test:integration'].invoke + end +end + +# ----- Documentation tasks --------------------------------------------------- + +require 'yard' +YARD::Rake::YardocTask.new(:doc) do |t| + t.options = %w| --embed-mixins --markup=markdown | +end + +# ----- Code analysis tasks --------------------------------------------------- + +if defined?(RUBY_VERSION) && RUBY_VERSION > '1.9' + require 'cane/rake_task' + Cane::RakeTask.new(:quality) do |cane| + cane.abc_max = 15 + cane.no_style = true + end +end diff --git a/elasticsearch-model/elasticsearch-model.gemspec b/elasticsearch-model/elasticsearch-model.gemspec new file mode 100644 index 0000000000..df9509f064 --- /dev/null +++ b/elasticsearch-model/elasticsearch-model.gemspec @@ -0,0 +1,57 @@ +# coding: utf-8 +lib = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'elasticsearch/model/version' + +Gem::Specification.new do |s| + s.name = "elasticsearch-model" + s.version = Elasticsearch::Model::VERSION + s.authors = ["Karel Minarik"] + s.email = ["karel.minarik@elasticsearch.org"] + s.description = "ActiveModel/Record integrations for Elasticsearch." + s.summary = "ActiveModel/Record integrations for Elasticsearch." + s.homepage = "https://github.com/elasticsearch/elasticsearch-rails/" + s.license = "Apache 2" + + s.files = `git ls-files`.split($/) + s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) } + s.test_files = s.files.grep(%r{^(test|spec|features)/}) + s.require_paths = ["lib"] + + s.extra_rdoc_files = [ "README.md", "LICENSE.txt" ] + s.rdoc_options = [ "--charset=UTF-8" ] + + s.required_ruby_version = ">= 1.9.3" + + s.add_dependency "elasticsearch", '> 0.4' + s.add_dependency "activesupport", '> 3' + s.add_dependency "hashie" + + s.add_development_dependency "bundler", "~> 1.3" + s.add_development_dependency "rake", "< 11.0" + + s.add_development_dependency "elasticsearch-extensions" + + s.add_development_dependency "sqlite3" + s.add_development_dependency "activemodel", "> 3.0" + + s.add_development_dependency "oj" + s.add_development_dependency "kaminari" + s.add_development_dependency "will_paginate" + + s.add_development_dependency "minitest", "~> 4.2" + s.add_development_dependency "test-unit" if defined?(RUBY_VERSION) && RUBY_VERSION > '2.2' + s.add_development_dependency "shoulda-context" + s.add_development_dependency "mocha" + s.add_development_dependency "turn" + s.add_development_dependency "yard" + s.add_development_dependency "ruby-prof" + s.add_development_dependency "pry" + s.add_development_dependency "ci_reporter", "~> 1.9" + + if defined?(RUBY_VERSION) && RUBY_VERSION > '1.9' + s.add_development_dependency "simplecov" + s.add_development_dependency "cane" + s.add_development_dependency "require-prof" + end +end diff --git a/elasticsearch-model/examples/activerecord_article.rb b/elasticsearch-model/examples/activerecord_article.rb new file mode 100644 index 0000000000..b18ee9c7bd --- /dev/null +++ b/elasticsearch-model/examples/activerecord_article.rb @@ -0,0 +1,77 @@ +# ActiveRecord and Elasticsearch +# ============================== +# +# https://github.com/rails/rails/tree/master/activerecord + +$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) + +require 'pry' +Pry.config.history.file = File.expand_path('../../tmp/elasticsearch_development.pry', __FILE__) + +require 'logger' +require 'ansi/core' +require 'active_record' +require 'kaminari' + +require 'elasticsearch/model' + +ActiveRecord::Base.logger = ActiveSupport::Logger.new(STDOUT) +ActiveRecord::Base.establish_connection( adapter: 'sqlite3', database: ":memory:" ) + +ActiveRecord::Schema.define(version: 1) do + create_table :articles do |t| + t.string :title + t.date :published_at + t.timestamps + end +end + +Kaminari::Hooks.init + +class Article < ActiveRecord::Base +end + +# Store data +# +Article.delete_all +Article.create title: 'Foo' +Article.create title: 'Bar' +Article.create title: 'Foo Foo' + +# Index data +# +client = Elasticsearch::Client.new log:true + +# client.indices.delete index: 'articles' rescue nil +# client.indices.create index: 'articles', body: { mappings: { article: { dynamic: 'strict' }, properties: {} } } + +client.indices.delete index: 'articles' rescue nil +client.bulk index: 'articles', + type: 'article', + body: Article.all.as_json.map { |a| { index: { _id: a.delete('id'), data: a } } }, + refresh: true + +# Extend the model with Elasticsearch support +# +Article.__send__ :include, Elasticsearch::Model +# Article.__send__ :include, Elasticsearch::Model::Callbacks + +# ActiveRecord::Base.logger.silence do +# 10_000.times do |i| +# Article.create title: "Foo #{i}" +# end +# end + +puts '', '-'*Pry::Terminal.width! + +Elasticsearch::Model.client = Elasticsearch::Client.new log: true + +response = Article.search 'foo'; + +p response.size +p response.results.size +p response.records.size + +Pry.start(binding, prompt: lambda { |obj, nest_level, _| '> ' }, + input: StringIO.new('response.records.to_a'), + quiet: true) diff --git a/elasticsearch-model/examples/activerecord_associations.rb b/elasticsearch-model/examples/activerecord_associations.rb new file mode 100644 index 0000000000..6143a03560 --- /dev/null +++ b/elasticsearch-model/examples/activerecord_associations.rb @@ -0,0 +1,177 @@ +# ActiveRecord associations and Elasticsearch +# =========================================== +# +# https://github.com/rails/rails/tree/master/activerecord +# http://guides.rubyonrails.org/association_basics.html +# +# Run me with: +# +# ruby -I lib examples/activerecord_associations.rb +# + +$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) + +require 'pry' +Pry.config.history.file = File.expand_path('../../tmp/elasticsearch_development.pry', __FILE__) + +require 'logger' +require 'ansi/core' +require 'active_record' + +require 'elasticsearch/model' + +ActiveRecord::Base.logger = ActiveSupport::Logger.new(STDOUT) +ActiveRecord::Base.establish_connection( adapter: 'sqlite3', database: ":memory:" ) + +# ----- Schema definition ------------------------------------------------------------------------- + +ActiveRecord::Schema.define(version: 1) do + create_table :categories do |t| + t.string :title + t.timestamps + end + + create_table :authors do |t| + t.string :first_name, :last_name + t.timestamps + end + + create_table :authorships do |t| + t.references :article + t.references :author + t.timestamps + end + + create_table :articles do |t| + t.string :title + t.timestamps + end + + create_table :articles_categories, id: false do |t| + t.references :article, :category + end + + create_table :comments do |t| + t.string :text + t.references :article + t.timestamps + end + add_index(:comments, :article_id) +end + +# ----- Elasticsearch client setup ---------------------------------------------------------------- + +Elasticsearch::Model.client = Elasticsearch::Client.new log: true +Elasticsearch::Model.client.transport.logger.formatter = proc { |s, d, p, m| "\e[32m#{m}\n\e[0m" } + +# ----- Search integration ------------------------------------------------------------------------ + +module Searchable + extend ActiveSupport::Concern + + included do + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks + + include Indexing + after_touch() { __elasticsearch__.index_document } + end + + module Indexing + + # Customize the JSON serialization for Elasticsearch + def as_indexed_json(options={}) + self.as_json( + include: { categories: { only: :title}, + authors: { methods: [:full_name], only: [:full_name] }, + comments: { only: :text } + }) + end + end +end + +# ----- Model definitions ------------------------------------------------------------------------- + +class Category < ActiveRecord::Base + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks + + has_and_belongs_to_many :articles +end + +class Author < ActiveRecord::Base + has_many :authorships + + after_update { self.authorships.each(&:touch) } + + def full_name + [first_name, last_name].compact.join(' ') + end +end + +class Authorship < ActiveRecord::Base + belongs_to :author + belongs_to :article, touch: true +end + +class Article < ActiveRecord::Base + include Searchable + + has_and_belongs_to_many :categories, after_add: [ lambda { |a,c| a.__elasticsearch__.index_document } ], + after_remove: [ lambda { |a,c| a.__elasticsearch__.index_document } ] + has_many :authorships + has_many :authors, through: :authorships + has_many :comments +end + +class Comment < ActiveRecord::Base + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks + + belongs_to :article, touch: true +end + +# ----- Insert data ------------------------------------------------------------------------------- + +# Create category +# +category = Category.create title: 'One' + +# Create author +# +author = Author.create first_name: 'John', last_name: 'Smith' + +# Create article + +article = Article.create title: 'First Article' + +# Assign category +# +article.categories << category + +# Assign author +# +article.authors << author + +# Add comment +# +article.comments.create text: 'First comment for article One' +article.comments.create text: 'Second comment for article One' + +Elasticsearch::Model.client.indices.refresh index: Elasticsearch::Model::Registry.all.map(&:index_name) + +puts "\n\e[1mArticles containing 'one':\e[0m", Article.search('one').records.to_a.map(&:inspect), "" + +puts "\n\e[1mModels containing 'one':\e[0m", Elasticsearch::Model.search('one').records.to_a.map(&:inspect), "" + +# Load model +# +article = Article.all.includes(:categories, :authors, :comments).first + +# ----- Pry --------------------------------------------------------------------------------------- + +puts '', '-'*Pry::Terminal.width! + +Pry.start(binding, prompt: lambda { |obj, nest_level, _| '> ' }, + input: StringIO.new("article.as_indexed_json\n"), + quiet: true) diff --git a/elasticsearch-model/examples/activerecord_mapping_completion.rb b/elasticsearch-model/examples/activerecord_mapping_completion.rb new file mode 100644 index 0000000000..46d986011c --- /dev/null +++ b/elasticsearch-model/examples/activerecord_mapping_completion.rb @@ -0,0 +1,69 @@ +require 'ansi' +require 'active_record' +require 'elasticsearch/model' + +ActiveRecord::Base.logger = ActiveSupport::Logger.new(STDOUT) +ActiveRecord::Base.establish_connection( adapter: 'sqlite3', database: ":memory:" ) + +ActiveRecord::Schema.define(version: 1) do + create_table :articles do |t| + t.string :title + t.date :published_at + t.timestamps + end +end + +class Article < ActiveRecord::Base + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks + + mapping do + indexes :title + indexes :title_suggest, type: 'completion', payloads: true + end + + def as_indexed_json(options={}) + as_json.merge \ + title_suggest: { + input: title, + output: title, + payload: { url: "/articles/#{id}" } + } + end +end + +Article.__elasticsearch__.client = Elasticsearch::Client.new log: true + +# Create index + +Article.__elasticsearch__.create_index! force: true + +# Store data + +Article.delete_all +Article.create title: 'Foo' +Article.create title: 'Bar' +Article.create title: 'Foo Foo' +Article.__elasticsearch__.refresh_index! + +# Search and suggest + +response_1 = Article.search 'foo'; + +puts "Article search:".ansi(:bold), + response_1.to_a.map { |d| "Title: #{d.title}" }.inspect.ansi(:bold, :yellow) + +response_2 = Article.__elasticsearch__.client.suggest \ + index: Article.index_name, + body: { + articles: { + text: 'foo', + completion: { field: 'title_suggest', size: 25 } + } + }; + +puts "Article suggest:".ansi(:bold), + response_2['articles'].first['options'].map { |d| "#{d['text']} -> #{d['payload']['url']}" }. + inspect.ansi(:bold, :green) + +require 'pry'; binding.pry; diff --git a/elasticsearch-model/examples/couchbase_article.rb b/elasticsearch-model/examples/couchbase_article.rb new file mode 100644 index 0000000000..57cc421b01 --- /dev/null +++ b/elasticsearch-model/examples/couchbase_article.rb @@ -0,0 +1,66 @@ +# Couchbase and Elasticsearch +# =========================== +# +# https://github.com/couchbase/couchbase-ruby-model + +$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) + +require 'pry' +Pry.config.history.file = File.expand_path('../../tmp/elasticsearch_development.pry', __FILE__) + +require 'logger' +require 'couchbase/model' + +require 'elasticsearch/model' + +# Documents are stored as JSON objects in Riak but have rich +# semantics, including validations and associations. +class Article < Couchbase::Model + attribute :title + attribute :published_at + + # view :all, :limit => 10, :descending => true + # TODO: Implement view a la + # bucket.save_design_doc <<-JSON + # { + # "_id": "_design/article", + # "language": "javascript", + # "views": { + # "all": { + # "map": "function(doc, meta) { emit(doc.id, doc.title); }" + # } + # } + # } + # JSON + +end + +# Extend the model with Elasticsearch support +# +Article.__send__ :extend, Elasticsearch::Model::Client::ClassMethods +Article.__send__ :extend, Elasticsearch::Model::Searching::ClassMethods +Article.__send__ :extend, Elasticsearch::Model::Naming::ClassMethods + +# Create documents in Riak +# +Article.create id: '1', title: 'Foo' rescue nil +Article.create id: '2', title: 'Bar' rescue nil +Article.create id: '3', title: 'Foo Foo' rescue nil + +# Index data into Elasticsearch +# +client = Elasticsearch::Client.new log:true + +client.indices.delete index: 'articles' rescue nil +client.bulk index: 'articles', + type: 'article', + body: Article.find(['1', '2', '3']).map { |a| + { index: { _id: a.id, data: a.attributes } } + }, + refresh: true + +response = Article.search 'foo', index: 'articles', type: 'article'; + +Pry.start(binding, prompt: lambda { |obj, nest_level, _| '> ' }, + input: StringIO.new('response.records.to_a'), + quiet: true) diff --git a/elasticsearch-model/examples/datamapper_article.rb b/elasticsearch-model/examples/datamapper_article.rb new file mode 100644 index 0000000000..383b3738f0 --- /dev/null +++ b/elasticsearch-model/examples/datamapper_article.rb @@ -0,0 +1,71 @@ +# DataMapper and Elasticsearch +# ============================ +# +# https://github.com/datamapper/dm-core +# https://github.com/datamapper/dm-active_model + + +$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) + +require 'pry' +Pry.config.history.file = File.expand_path('../../tmp/elasticsearch_development.pry', __FILE__) + +require 'logger' +require 'ansi/core' + +require 'data_mapper' +require 'dm-active_model' + +require 'active_support/all' + +require 'elasticsearch/model' + +DataMapper::Logger.new(STDOUT, :debug) +DataMapper.setup(:default, 'sqlite::memory:') + +class Article + include DataMapper::Resource + + property :id, Serial + property :title, String + property :published_at, DateTime +end + +DataMapper.auto_migrate! +DataMapper.finalize + +Article.create title: 'Foo' +Article.create title: 'Bar' +Article.create title: 'Foo Foo' + +# Extend the model with Elasticsearch support +# +Article.__send__ :include, Elasticsearch::Model + +# The DataMapper adapter +# +module DataMapperAdapter + + # Implement the interface for fetching records + # + module Records + def records + klass.all(id: @ids) + end + + # ... + end +end + +# Register the adapter +# +Elasticsearch::Model::Adapter.register( + DataMapperAdapter, + lambda { |klass| defined?(::DataMapper::Resource) and klass.ancestors.include?(::DataMapper::Resource) } +) + +response = Article.search 'foo'; + +Pry.start(binding, prompt: lambda { |obj, nest_level, _| '> ' }, + input: StringIO.new('response.records.to_a'), + quiet: true) diff --git a/elasticsearch-model/examples/mongoid_article.rb b/elasticsearch-model/examples/mongoid_article.rb new file mode 100644 index 0000000000..5cd12ca4fa --- /dev/null +++ b/elasticsearch-model/examples/mongoid_article.rb @@ -0,0 +1,68 @@ +# Mongoid and Elasticsearch +# ========================= +# +# http://mongoid.org/en/mongoid/index.html + +$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) + +require 'pry' +Pry.config.history.file = File.expand_path('../../tmp/elasticsearch_development.pry', __FILE__) + +require 'benchmark' +require 'logger' +require 'ansi/core' +require 'mongoid' + +require 'elasticsearch/model' +require 'elasticsearch/model/callbacks' + +Mongoid.logger.level = Logger::DEBUG +Moped.logger.level = Logger::DEBUG + +Mongoid.connect_to 'articles' + +Elasticsearch::Model.client = Elasticsearch::Client.new host: 'localhost:9200', log: true + +class Article + include Mongoid::Document + field :id, type: String + field :title, type: String + field :published_at, type: DateTime + attr_accessible :id, :title, :published_at if respond_to? :attr_accessible + + def as_indexed_json(options={}) + as_json(except: [:id, :_id]) + end +end + +# Extend the model with Elasticsearch support +# +Article.__send__ :include, Elasticsearch::Model +# Article.__send__ :include, Elasticsearch::Model::Callbacks + +# Store data +# +Article.delete_all +Article.create id: '1', title: 'Foo' +Article.create id: '2', title: 'Bar' +Article.create id: '3', title: 'Foo Foo' + +# Index data +# +client = Elasticsearch::Client.new host:'localhost:9200', log:true + +client.indices.delete index: 'articles' rescue nil +client.bulk index: 'articles', + type: 'article', + body: Article.all.map { |a| { index: { _id: a.id, data: a.attributes } } }, + refresh: true + +# puts Benchmark.realtime { 9_875.times { |i| Article.create title: "Foo #{i}" } } + +puts '', '-'*Pry::Terminal.width! + +response = Article.search 'foo'; + +Pry.start(binding, prompt: lambda { |obj, nest_level, _| '> ' }, + input: StringIO.new('response.records.to_a'), + quiet: true) diff --git a/elasticsearch-model/examples/ohm_article.rb b/elasticsearch-model/examples/ohm_article.rb new file mode 100644 index 0000000000..3145085e79 --- /dev/null +++ b/elasticsearch-model/examples/ohm_article.rb @@ -0,0 +1,70 @@ +# Ohm for Redis and Elasticsearch +# =============================== +# +# https://github.com/soveran/ohm#example + +$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) + +require 'pry' +Pry.config.history.file = File.expand_path('../../tmp/elasticsearch_development.pry', __FILE__) + +require 'logger' +require 'ansi/core' +require 'active_model' +require 'ohm' + +require 'elasticsearch/model' + +class Article < Ohm::Model + # Include JSON serialization from ActiveModel + include ActiveModel::Serializers::JSON + + attribute :title + attribute :published_at +end + +# Extend the model with Elasticsearch support +# +Article.__send__ :include, Elasticsearch::Model + +# Register a custom adapter +# +module Elasticsearch + module Model + module Adapter + module Ohm + Adapter.register self, + lambda { |klass| defined?(::Ohm::Model) and klass.ancestors.include?(::Ohm::Model) } + module Records + def records + klass.fetch(@ids) + end + end + end + end + end +end + +# Configure the Elasticsearch client to log operations +# +Elasticsearch::Model.client = Elasticsearch::Client.new log: true + +puts '', '-'*Pry::Terminal.width! + +Article.all.map { |a| a.delete } +Article.create id: '1', title: 'Foo' +Article.create id: '2', title: 'Bar' +Article.create id: '3', title: 'Foo Foo' + +Article.__elasticsearch__.client.indices.delete index: 'articles' rescue nil +Article.__elasticsearch__.client.bulk index: 'articles', + type: 'article', + body: Article.all.map { |a| { index: { _id: a.id, data: a.attributes } } }, + refresh: true + + +response = Article.search 'foo', index: 'articles', type: 'article'; + +Pry.start(binding, prompt: lambda { |obj, nest_level, _| '> ' }, + input: StringIO.new('response.records.to_a'), + quiet: true) diff --git a/elasticsearch-model/examples/riak_article.rb b/elasticsearch-model/examples/riak_article.rb new file mode 100644 index 0000000000..8013cda7ea --- /dev/null +++ b/elasticsearch-model/examples/riak_article.rb @@ -0,0 +1,52 @@ +# Riak and Elasticsearch +# ====================== +# +# https://github.com/basho-labs/ripple + +$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) + +require 'pry' +Pry.config.history.file = File.expand_path('../../tmp/elasticsearch_development.pry', __FILE__) + +require 'logger' +require 'ripple' + +require 'elasticsearch/model' + +# Documents are stored as JSON objects in Riak but have rich +# semantics, including validations and associations. +class Article + include Ripple::Document + + property :title, String + property :published_at, Time, :default => proc { Time.now } +end + +# Extend the model with Elasticsearch support +# +Article.__send__ :include, Elasticsearch::Model + +# Create documents in Riak +# +Article.destroy_all +Article.create id: '1', title: 'Foo' +Article.create id: '2', title: 'Bar' +Article.create id: '3', title: 'Foo Foo' + +# Index data into Elasticsearch +# +client = Elasticsearch::Client.new log:true + +client.indices.delete index: 'articles' rescue nil +client.bulk index: 'articles', + type: 'article', + body: Article.all.map { |a| + { index: { _id: a.key, data: JSON.parse(a.robject.raw_data) } } + }.as_json, + refresh: true + +response = Article.search 'foo'; + +Pry.start(binding, prompt: lambda { |obj, nest_level, _| '> ' }, + input: StringIO.new('response.records.to_a'), + quiet: true) diff --git a/elasticsearch-model/gemfiles/3.0.gemfile b/elasticsearch-model/gemfiles/3.0.gemfile new file mode 100644 index 0000000000..23cbdf53d5 --- /dev/null +++ b/elasticsearch-model/gemfiles/3.0.gemfile @@ -0,0 +1,13 @@ +# Usage: +# +# $ BUNDLE_GEMFILE=./gemfiles/3.0.gemfile bundle install +# $ BUNDLE_GEMFILE=./gemfiles/3.0.gemfile bundle exec rake test:integration + +source 'https://rubygems.org' + +gemspec path: '../' + +gem 'activemodel', '>= 3.0' +gem 'activerecord', '~> 3.2' +gem 'mongoid', '>= 3.0' +gem 'sqlite3' diff --git a/elasticsearch-model/gemfiles/4.0.gemfile b/elasticsearch-model/gemfiles/4.0.gemfile new file mode 100644 index 0000000000..89044bb19e --- /dev/null +++ b/elasticsearch-model/gemfiles/4.0.gemfile @@ -0,0 +1,12 @@ +# Usage: +# +# $ BUNDLE_GEMFILE=./gemfiles/4.0.gemfile bundle install +# $ BUNDLE_GEMFILE=./gemfiles/4.0.gemfile bundle exec rake test:integration + +source 'https://rubygems.org' + +gemspec path: '../' + +gem 'activemodel', '~> 4' +gem 'activerecord', '~> 4' +gem 'sqlite3' diff --git a/elasticsearch-model/lib/elasticsearch/model.rb b/elasticsearch-model/lib/elasticsearch/model.rb new file mode 100644 index 0000000000..9d2b93da51 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model.rb @@ -0,0 +1,188 @@ +require 'elasticsearch' + +require 'hashie' + +require 'active_support/core_ext/module/delegation' + +require 'elasticsearch/model/version' + +require 'elasticsearch/model/client' + +require 'elasticsearch/model/multimodel' + +require 'elasticsearch/model/adapter' +require 'elasticsearch/model/adapters/default' +require 'elasticsearch/model/adapters/active_record' +require 'elasticsearch/model/adapters/mongoid' +require 'elasticsearch/model/adapters/multiple' + +require 'elasticsearch/model/importing' +require 'elasticsearch/model/indexing' +require 'elasticsearch/model/naming' +require 'elasticsearch/model/serializing' +require 'elasticsearch/model/searching' +require 'elasticsearch/model/callbacks' + +require 'elasticsearch/model/proxy' + +require 'elasticsearch/model/response' +require 'elasticsearch/model/response/base' +require 'elasticsearch/model/response/result' +require 'elasticsearch/model/response/results' +require 'elasticsearch/model/response/records' +require 'elasticsearch/model/response/pagination' +require 'elasticsearch/model/response/suggestions' + +require 'elasticsearch/model/ext/active_record' + +case +when defined?(::Kaminari) + Elasticsearch::Model::Response::Response.__send__ :include, Elasticsearch::Model::Response::Pagination::Kaminari +when defined?(::WillPaginate) + Elasticsearch::Model::Response::Response.__send__ :include, Elasticsearch::Model::Response::Pagination::WillPaginate +end + +module Elasticsearch + + # Elasticsearch integration for Ruby models + # ========================================= + # + # `Elasticsearch::Model` contains modules for integrating the Elasticsearch search and analytical engine + # with ActiveModel-based classes, or models, for the Ruby programming language. + # + # It facilitates importing your data into an index, automatically updating it when a record changes, + # searching the specific index, setting up the index mapping or the model JSON serialization. + # + # When the `Elasticsearch::Model` module is included in your class, it automatically extends it + # with the functionality; see {Elasticsearch::Model.included}. Most methods are available via + # the `__elasticsearch__` class and instance method proxies. + # + # It is possible to include/extend the model with the corresponding + # modules directly, if that is desired: + # + # MyModel.__send__ :extend, Elasticsearch::Model::Client::ClassMethods + # MyModel.__send__ :include, Elasticsearch::Model::Client::InstanceMethods + # MyModel.__send__ :extend, Elasticsearch::Model::Searching::ClassMethods + # # ... + # + module Model + METHODS = [:search, :mapping, :mappings, :settings, :index_name, :document_type, :import] + + # Adds the `Elasticsearch::Model` functionality to the including class. + # + # * Creates the `__elasticsearch__` class and instance methods, pointing to the proxy object + # * Includes the necessary modules in the proxy classes + # * Sets up delegation for crucial methods such as `search`, etc. + # + # @example Include the module in the `Article` model definition + # + # class Article < ActiveRecord::Base + # include Elasticsearch::Model + # end + # + # @example Inject the module into the `Article` model during run time + # + # Article.__send__ :include, Elasticsearch::Model + # + # + def self.included(base) + base.class_eval do + include Elasticsearch::Model::Proxy + + Elasticsearch::Model::Proxy::ClassMethodsProxy.class_eval do + include Elasticsearch::Model::Client::ClassMethods + include Elasticsearch::Model::Naming::ClassMethods + include Elasticsearch::Model::Indexing::ClassMethods + include Elasticsearch::Model::Searching::ClassMethods + end + + Elasticsearch::Model::Proxy::InstanceMethodsProxy.class_eval do + include Elasticsearch::Model::Client::InstanceMethods + include Elasticsearch::Model::Naming::InstanceMethods + include Elasticsearch::Model::Indexing::InstanceMethods + include Elasticsearch::Model::Serializing::InstanceMethods + end + + Elasticsearch::Model::Proxy::InstanceMethodsProxy.class_eval <<-CODE, __FILE__, __LINE__ + 1 + def as_indexed_json(options={}) + target.respond_to?(:as_indexed_json) ? target.__send__(:as_indexed_json, options) : super + end + CODE + + # Delegate important methods to the `__elasticsearch__` proxy, unless they are defined already + # + class << self + METHODS.each do |method| + delegate method, to: :__elasticsearch__ unless self.public_instance_methods.include?(method) + end + end + + # Mix the importing module into the proxy + # + self.__elasticsearch__.class_eval do + include Elasticsearch::Model::Importing::ClassMethods + include Adapter.from_class(base).importing_mixin + end + + # Add to the registry if it's a class (and not in intermediate module) + Registry.add(base) if base.is_a?(Class) + end + end + + module ClassMethods + + # Get the client common for all models + # + # @example Get the client + # + # Elasticsearch::Model.client + # => # + # + def client + @client ||= Elasticsearch::Client.new + end + + # Set the client for all models + # + # @example Configure (set) the client for all models + # + # Elasticsearch::Model.client = Elasticsearch::Client.new host: 'http://localhost:9200', tracer: true + # => # + # + # @note You have to set the client before you call Elasticsearch methods on the model, + # or set it directly on the model; see {Elasticsearch::Model::Client::ClassMethods#client} + # + def client=(client) + @client = client + end + + # Search across multiple models + # + # By default, all models which include the `Elasticsearch::Model` module are searched + # + # @param query_or_payload [String,Hash,Object] The search request definition + # (string, JSON, Hash, or object responding to `to_hash`) + # @param models [Array] The Array of Model objects to search + # @param options [Hash] Optional parameters to be passed to the Elasticsearch client + # + # @return [Elasticsearch::Model::Response::Response] + # + # @example Search across specific models + # + # Elasticsearch::Model.search('foo', [Author, Article]) + # + # @example Search across all models which include the `Elasticsearch::Model` module + # + # Elasticsearch::Model.search('foo') + # + def search(query_or_payload, models=[], options={}) + models = Multimodel.new(models) + request = Searching::SearchRequest.new(models, query_or_payload, options) + Response::Response.new(models, request) + end + end + extend ClassMethods + + class NotImplemented < NoMethodError; end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/adapter.rb b/elasticsearch-model/lib/elasticsearch/model/adapter.rb new file mode 100644 index 0000000000..3a25e5d97b --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/adapter.rb @@ -0,0 +1,145 @@ +module Elasticsearch + module Model + + # Contains an adapter which provides OxM-specific implementations for common behaviour: + # + # * {Adapter::Adapter#records_mixin Fetching records from the database} + # * {Adapter::Adapter#callbacks_mixin Model callbacks for automatic index updates} + # * {Adapter::Adapter#importing_mixin Efficient bulk loading from the database} + # + # @see Elasticsearch::Model::Adapter::Default + # @see Elasticsearch::Model::Adapter::ActiveRecord + # @see Elasticsearch::Model::Adapter::Mongoid + # + module Adapter + + # Returns an adapter based on the Ruby class passed + # + # @example Create an adapter for an ActiveRecord-based model + # + # class Article < ActiveRecord::Base; end + # + # myadapter = Elasticsearch::Model::Adapter.from_class(Article) + # myadapter.adapter + # # => Elasticsearch::Model::Adapter::ActiveRecord + # + # @see Adapter.adapters The list of included adapters + # @see Adapter.register Register a custom adapter + # + def from_class(klass) + Adapter.new(klass) + end; module_function :from_class + + # Returns registered adapters + # + # @see ::Elasticsearch::Model::Adapter::Adapter.adapters + # + def adapters + Adapter.adapters + end; module_function :adapters + + # Registers an adapter + # + # @see ::Elasticsearch::Model::Adapter::Adapter.register + # + def register(name, condition) + Adapter.register(name, condition) + end; module_function :register + + # Contains an adapter for specific OxM or architecture. + # + class Adapter + attr_reader :klass + + def initialize(klass) + @klass = klass + end + + # Registers an adapter for specific condition + # + # @param name [Module] The module containing the implemented interface + # @param condition [Proc] An object with a `call` method which is evaluated in {.adapter} + # + # @example Register an adapter for DataMapper + # + # module DataMapperAdapter + # + # # Implement the interface for fetching records + # # + # module Records + # def records + # klass.all(id: @ids) + # end + # + # # ... + # end + # end + # + # # Register the adapter + # # + # Elasticsearch::Model::Adapter.register( + # DataMapperAdapter, + # lambda { |klass| + # defined?(::DataMapper::Resource) and klass.ancestors.include?(::DataMapper::Resource) + # } + # ) + # + def self.register(name, condition) + self.adapters[name] = condition + end + + # Return the collection of registered adapters + # + # @example Return the currently registered adapters + # + # Elasticsearch::Model::Adapter.adapters + # # => { + # # Elasticsearch::Model::Adapter::ActiveRecord => #, + # # Elasticsearch::Model::Adapter::Mongoid => #, + # # } + # + # @return [Hash] The collection of adapters + # + def self.adapters + @adapters ||= {} + end + + # Return the module with {Default::Records} interface implementation + # + # @api private + # + def records_mixin + adapter.const_get(:Records) + end + + # Return the module with {Default::Callbacks} interface implementation + # + # @api private + # + def callbacks_mixin + adapter.const_get(:Callbacks) + end + + # Return the module with {Default::Importing} interface implementation + # + # @api private + # + def importing_mixin + adapter.const_get(:Importing) + end + + # Returns the adapter module + # + # @api private + # + def adapter + @adapter ||= begin + self.class.adapters.find( lambda {[]} ) { |name, condition| condition.call(klass) }.first \ + || Elasticsearch::Model::Adapter::Default + end + end + + end + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/adapters/active_record.rb b/elasticsearch-model/lib/elasticsearch/model/adapters/active_record.rb new file mode 100644 index 0000000000..2d9bb53786 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/adapters/active_record.rb @@ -0,0 +1,114 @@ +module Elasticsearch + module Model + module Adapter + + # An adapter for ActiveRecord-based models + # + module ActiveRecord + + Adapter.register self, + lambda { |klass| !!defined?(::ActiveRecord::Base) && klass.respond_to?(:ancestors) && klass.ancestors.include?(::ActiveRecord::Base) } + + module Records + attr_writer :options + + def options + @options ||= {} + end + + # Returns an `ActiveRecord::Relation` instance + # + def records + sql_records = klass.where(klass.primary_key => ids) + sql_records = sql_records.includes(self.options[:includes]) if self.options[:includes] + + # Re-order records based on the order from Elasticsearch hits + # by redefining `to_a`, unless the user has called `order()` + # + sql_records.instance_exec(response.response['hits']['hits']) do |hits| + define_singleton_method :to_a do + if defined?(::ActiveRecord) && ::ActiveRecord::VERSION::MAJOR >= 4 + self.load + else + self.__send__(:exec_queries) + end + @records.sort_by { |record| hits.index { |hit| hit['_id'].to_s == record.id.to_s } } + end + end + + sql_records + end + + # Prevent clash with `ActiveSupport::Dependencies::Loadable` + # + def load + records.load + end + + # Intercept call to the `order` method, so we can ignore the order from Elasticsearch + # + def order(*args) + sql_records = records.__send__ :order, *args + + # Redefine the `to_a` method to the original one + # + sql_records.instance_exec do + define_singleton_method(:to_a) do + if defined?(::ActiveRecord) && ::ActiveRecord::VERSION::MAJOR >= 4 + self.load + else + self.__send__(:exec_queries) + end + @records + end + end + + sql_records + end + end + + module Callbacks + + # Handle index updates (creating, updating or deleting documents) + # when the model changes, by hooking into the lifecycle + # + # @see http://guides.rubyonrails.org/active_record_callbacks.html + # + def self.included(base) + base.class_eval do + after_commit lambda { __elasticsearch__.index_document }, on: :create + after_commit lambda { __elasticsearch__.update_document }, on: :update + after_commit lambda { __elasticsearch__.delete_document }, on: :destroy + end + end + end + + module Importing + + # Fetch batches of records from the database (used by the import method) + # + # + # @see http://api.rubyonrails.org/classes/ActiveRecord/Batches.html ActiveRecord::Batches.find_in_batches + # + def __find_in_batches(options={}, &block) + query = options.delete(:query) + named_scope = options.delete(:scope) + preprocess = options.delete(:preprocess) + + scope = self + scope = scope.__send__(named_scope) if named_scope + scope = scope.instance_exec(&query) if query + + scope.find_in_batches(options) do |batch| + yield (preprocess ? self.__send__(preprocess, batch) : batch) + end + end + + def __transform + lambda { |model| { index: { _id: model.id, data: model.__elasticsearch__.as_indexed_json } } } + end + end + end + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/adapters/default.rb b/elasticsearch-model/lib/elasticsearch/model/adapters/default.rb new file mode 100644 index 0000000000..e58cf4ceb3 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/adapters/default.rb @@ -0,0 +1,50 @@ +module Elasticsearch + module Model + module Adapter + + # The default adapter for models which haven't one registered + # + module Default + + # Module for implementing methods and logic related to fetching records from the database + # + module Records + + # Return the collection of records fetched from the database + # + # By default uses `MyModel#find[1, 2, 3]` + # + def records + klass.find(@ids) + end + end + + # Module for implementing methods and logic related to hooking into model lifecycle + # (e.g. to perform automatic index updates) + # + # @see http://api.rubyonrails.org/classes/ActiveModel/Callbacks.html + module Callbacks + # noop + end + + # Module for efficiently fetching records from the database to import them into the index + # + module Importing + + # @abstract Implement this method in your adapter + # + def __find_in_batches(options={}, &block) + raise NotImplemented, "Method not implemented for default adapter" + end + + # @abstract Implement this method in your adapter + # + def __transform + raise NotImplemented, "Method not implemented for default adapter" + end + end + + end + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/adapters/mongoid.rb b/elasticsearch-model/lib/elasticsearch/model/adapters/mongoid.rb new file mode 100644 index 0000000000..5117dbf58d --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/adapters/mongoid.rb @@ -0,0 +1,82 @@ +module Elasticsearch + module Model + module Adapter + + # An adapter for Mongoid-based models + # + # @see http://mongoid.org + # + module Mongoid + + Adapter.register self, + lambda { |klass| !!defined?(::Mongoid::Document) && klass.respond_to?(:ancestors) && klass.ancestors.include?(::Mongoid::Document) } + + module Records + + # Return a `Mongoid::Criteria` instance + # + def records + criteria = klass.where(:id.in => ids) + + criteria.instance_exec(response.response['hits']['hits']) do |hits| + define_singleton_method :to_a do + self.entries.sort_by { |e| hits.index { |hit| hit['_id'].to_s == e.id.to_s } } + end + end + + criteria + end + + # Intercept call to sorting methods, so we can ignore the order from Elasticsearch + # + %w| asc desc order_by |.each do |name| + define_method name do |*args| + criteria = records.__send__ name, *args + criteria.instance_exec do + define_singleton_method(:to_a) { self.entries } + end + + criteria + end + end + end + + module Callbacks + + # Handle index updates (creating, updating or deleting documents) + # when the model changes, by hooking into the lifecycle + # + # @see http://mongoid.org/en/mongoid/docs/callbacks.html + # + def self.included(base) + base.after_create { |document| document.__elasticsearch__.index_document } + base.after_update { |document| document.__elasticsearch__.update_document } + base.after_destroy { |document| document.__elasticsearch__.delete_document } + end + end + + module Importing + + # Fetch batches of records from the database + # + # @see https://github.com/mongoid/mongoid/issues/1334 + # @see https://github.com/karmi/retire/pull/724 + # + def __find_in_batches(options={}, &block) + options[:batch_size] ||= 1_000 + + all.no_timeout.each_slice(options[:batch_size]) do |items| + yield items + end + end + + def __transform + lambda {|a| { index: { _id: a.id.to_s, data: a.as_indexed_json } }} + end + end + + end + + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/adapters/multiple.rb b/elasticsearch-model/lib/elasticsearch/model/adapters/multiple.rb new file mode 100644 index 0000000000..9a0bc4e8eb --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/adapters/multiple.rb @@ -0,0 +1,112 @@ +module Elasticsearch + module Model + module Adapter + + # An adapter to be used for deserializing results from multiple models, + # retrieved through `Elasticsearch::Model.search` + # + # @see Elasticsearch::Model.search + # + module Multiple + Adapter.register self, lambda { |klass| klass.is_a? Multimodel } + + module Records + # Returns a collection of model instances, possibly of different classes (ActiveRecord, Mongoid, ...) + # + # @note The order of results in the Elasticsearch response is preserved + # + def records + records_by_type = __records_by_type + + records = response.response["hits"]["hits"].map do |hit| + records_by_type[ __type_for_hit(hit) ][ hit[:_id] ] + end + + records.compact + end + + # Returns the collection of records grouped by class based on `_type` + # + # Example: + # + # { + # Foo => {"1"=> # {"1"=> # ids) + when Elasticsearch::Model::Adapter::Mongoid.equal?(adapter) + klass.where(:id.in => ids) + else + klass.find(ids) + end + end + + # Returns the record IDs grouped by class based on type `_type` + # + # Example: + # + # { Foo => ["1"], Bar => ["1", "5"] } + # + # @api private + # + def __ids_by_type + ids_by_type = {} + + response.response["hits"]["hits"].each do |hit| + type = __type_for_hit(hit) + ids_by_type[type] ||= [] + ids_by_type[type] << hit[:_id] + end + ids_by_type + end + + # Returns the class of the model corresponding to a specific `hit` in Elasticsearch results + # + # @see Elasticsearch::Model::Registry + # + # @api private + # + def __type_for_hit(hit) + @@__types ||= {} + + @@__types[ "#{hit[:_index]}::#{hit[:_type]}" ] ||= begin + Registry.all.detect do |model| + model.index_name == hit[:_index] && model.document_type == hit[:_type] + end + end + end + + # Returns the adapter registered for a particular `klass` or `nil` if not available + # + # @api private + # + def __adapter_for_klass(klass) + Adapter.adapters.select { |name, checker| checker.call(klass) }.keys.first + end + end + end + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/callbacks.rb b/elasticsearch-model/lib/elasticsearch/model/callbacks.rb new file mode 100644 index 0000000000..1b72cb2a03 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/callbacks.rb @@ -0,0 +1,35 @@ +module Elasticsearch + module Model + + # Allows to automatically update index based on model changes, + # by hooking into the model lifecycle. + # + # @note A blocking HTTP request is done during the update process. + # If you need a more performant/resilient way of updating the index, + # consider adapting the callbacks behaviour, and use a background + # processing solution such as [Sidekiq](http://sidekiq.org) + # or [Resque](https://github.com/resque/resque). + # + module Callbacks + + # When included in a model, automatically injects the callback subscribers (`after_save`, etc) + # + # @example Automatically update Elasticsearch index when the model changes + # + # class Article + # include Elasticsearch::Model + # include Elasticsearch::Model::Callbacks + # end + # + # Article.first.update_attribute :title, 'Updated' + # # SQL (0.3ms) UPDATE "articles" SET "title" = ... + # # 2013-11-20 15:08:52 +0100: POST http://localhost:9200/articles/article/1/_update ... + # + def self.included(base) + adapter = Adapter.from_class(base) + base.__send__ :include, adapter.callbacks_mixin + end + + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/client.rb b/elasticsearch-model/lib/elasticsearch/model/client.rb new file mode 100644 index 0000000000..c1a9b4ed91 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/client.rb @@ -0,0 +1,61 @@ +module Elasticsearch + module Model + + # Contains an `Elasticsearch::Client` instance + # + module Client + + module ClassMethods + + # Get the client for a specific model class + # + # @example Get the client for `Article` and perform API request + # + # Article.client.cluster.health + # # => { "cluster_name" => "elasticsearch" ... } + # + def client client=nil + @client ||= Elasticsearch::Model.client + end + + # Set the client for a specific model class + # + # @example Configure the client for the `Article` model + # + # Article.client = Elasticsearch::Client.new host: 'http://api.server:8080' + # Article.search ... + # + def client=(client) + @client = client + end + end + + module InstanceMethods + + # Get or set the client for a specific model instance + # + # @example Get the client for a specific record and perform API request + # + # @article = Article.first + # @article.client.info + # # => { "name" => "Node-1", ... } + # + def client + @client ||= self.class.client + end + + # Set the client for a specific model instance + # + # @example Set the client for a specific record + # + # @article = Article.first + # @article.client = Elasticsearch::Client.new host: 'http://api.server:8080' + # + def client=(client) + @client = client + end + end + + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/ext/active_record.rb b/elasticsearch-model/lib/elasticsearch/model/ext/active_record.rb new file mode 100644 index 0000000000..ffa6cc385a --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/ext/active_record.rb @@ -0,0 +1,14 @@ +# Prevent `MyModel.inspect` failing with `ActiveRecord::ConnectionNotEstablished` +# (triggered by elasticsearch-model/lib/elasticsearch/model.rb:79:in `included') +# +ActiveRecord::Base.instance_eval do + class << self + def inspect_with_rescue + inspect_without_rescue + rescue ActiveRecord::ConnectionNotEstablished + "#{self}(no database connection)" + end + + alias_method_chain :inspect, :rescue + end +end if defined?(ActiveRecord) && ActiveRecord::VERSION::STRING < '4' diff --git a/elasticsearch-model/lib/elasticsearch/model/importing.rb b/elasticsearch-model/lib/elasticsearch/model/importing.rb new file mode 100644 index 0000000000..7c42545d2a --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/importing.rb @@ -0,0 +1,151 @@ +module Elasticsearch + module Model + + # Provides support for easily and efficiently importing large amounts of + # records from the including class into the index. + # + # @see ClassMethods#import + # + module Importing + + # When included in a model, adds the importing methods. + # + # @example Import all records from the `Article` model + # + # Article.import + # + # @see #import + # + def self.included(base) + base.__send__ :extend, ClassMethods + + adapter = Adapter.from_class(base) + base.__send__ :include, adapter.importing_mixin + base.__send__ :extend, adapter.importing_mixin + end + + module ClassMethods + + # Import all model records into the index + # + # The method will pick up correct strategy based on the `Importing` module + # defined in the corresponding adapter. + # + # @param options [Hash] Options passed to the underlying `__find_in_batches`method + # @param block [Proc] Optional block to evaluate for each batch + # + # @yield [Hash] Gives the Hash with the Elasticsearch response to the block + # + # @return [Fixnum] Number of errors encountered during importing + # + # @example Import all records into the index + # + # Article.import + # + # @example Set the batch size to 100 + # + # Article.import batch_size: 100 + # + # @example Process the response from Elasticsearch + # + # Article.import do |response| + # puts "Got " + response['items'].select { |i| i['index']['error'] }.size.to_s + " errors" + # end + # + # @example Delete and create the index with appropriate settings and mappings + # + # Article.import force: true + # + # @example Refresh the index after importing all batches + # + # Article.import refresh: true + # + # @example Import the records into a different index/type than the default one + # + # Article.import index: 'my-new-index', type: 'my-other-type' + # + # @example Pass an ActiveRecord scope to limit the imported records + # + # Article.import scope: 'published' + # + # @example Pass an ActiveRecord query to limit the imported records + # + # Article.import query: -> { where(author_id: author_id) } + # + # @example Transform records during the import with a lambda + # + # transform = lambda do |a| + # {index: {_id: a.id, _parent: a.author_id, data: a.__elasticsearch__.as_indexed_json}} + # end + # + # Article.import transform: transform + # + # @example Update the batch before yielding it + # + # class Article + # # ... + # def self.enrich(batch) + # batch.each do |item| + # item.metadata = MyAPI.get_metadata(item.id) + # end + # batch + # end + # end + # + # Article.import preprocess: :enrich + # + # @example Return an array of error elements instead of the number of errors, eg. + # to try importing these records again + # + # Article.import return: 'errors' + # + def import(options={}, &block) + errors = [] + refresh = options.delete(:refresh) || false + target_index = options.delete(:index) || index_name + target_type = options.delete(:type) || document_type + transform = options.delete(:transform) || __transform + return_value = options.delete(:return) || 'count' + + unless transform.respond_to?(:call) + raise ArgumentError, + "Pass an object responding to `call` as the :transform option, #{transform.class} given" + end + + if options.delete(:force) + self.create_index! force: true, index: target_index + elsif !self.index_exists? index: target_index + raise ArgumentError, + "#{target_index} does not exist to be imported into. Use create_index! or the :force option to create it." + end + + __find_in_batches(options) do |batch| + response = client.bulk \ + index: target_index, + type: target_type, + body: __batch_to_bulk(batch, transform) + + yield response if block_given? + + errors += response['items'].select { |k, v| k.values.first['error'] } + end + + self.refresh_index! if refresh + + case return_value + when 'errors' + errors + else + errors.size + end + end + + def __batch_to_bulk(batch, transform) + batch.map { |model| transform.call(model) } + end + end + + end + + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/indexing.rb b/elasticsearch-model/lib/elasticsearch/model/indexing.rb new file mode 100644 index 0000000000..9c90e9d823 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/indexing.rb @@ -0,0 +1,434 @@ +module Elasticsearch + module Model + + # Provides the necessary support to set up index options (mappings, settings) + # as well as instance methods to create, update or delete documents in the index. + # + # @see ClassMethods#settings + # @see ClassMethods#mapping + # + # @see InstanceMethods#index_document + # @see InstanceMethods#update_document + # @see InstanceMethods#delete_document + # + module Indexing + + # Wraps the [index settings](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/setup-configuration.html#configuration-index-settings) + # + class Settings + attr_accessor :settings + + def initialize(settings={}) + @settings = settings + end + + def to_hash + @settings + end + + def as_json(options={}) + to_hash + end + end + + # Wraps the [index mappings](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping.html) + # + class Mappings + attr_accessor :options, :type + + # @private + TYPES_WITH_EMBEDDED_PROPERTIES = %w(object nested) + + def initialize(type, options={}) + raise ArgumentError, "`type` is missing" if type.nil? + + @type = type + @options = options + @mapping = {} + end + + def indexes(name, options={}, &block) + @mapping[name] = options + + if block_given? + @mapping[name][:type] ||= 'object' + properties = TYPES_WITH_EMBEDDED_PROPERTIES.include?(@mapping[name][:type].to_s) ? :properties : :fields + + @mapping[name][properties] ||= {} + + previous = @mapping + begin + @mapping = @mapping[name][properties] + self.instance_eval(&block) + ensure + @mapping = previous + end + end + + # Set the type to `string` by default + @mapping[name][:type] ||= 'string' + + self + end + + def to_hash + { @type.to_sym => @options.merge( properties: @mapping ) } + end + + def as_json(options={}) + to_hash + end + end + + module ClassMethods + + # Defines mappings for the index + # + # @example Define mapping for model + # + # class Article + # mapping dynamic: 'strict' do + # indexes :foo do + # indexes :bar + # end + # indexes :baz + # end + # end + # + # Article.mapping.to_hash + # + # # => { :article => + # # { :dynamic => "strict", + # # :properties=> + # # { :foo => { + # # :type=>"object", + # # :properties => { + # # :bar => { :type => "string" } + # # } + # # } + # # }, + # # :baz => { :type=> "string" } + # # } + # # } + # + # @example Define index settings and mappings + # + # class Article + # settings number_of_shards: 1 do + # mappings do + # indexes :foo + # end + # end + # end + # + # @example Call the mapping method directly + # + # Article.mapping(dynamic: 'strict') { indexes :foo, type: 'long' } + # + # Article.mapping.to_hash + # + # # => {:article=>{:dynamic=>"strict", :properties=>{:foo=>{:type=>"long"}}}} + # + # The `mappings` and `settings` methods are accessible directly on the model class, + # when it doesn't already define them. Use the `__elasticsearch__` proxy otherwise. + # + def mapping(options={}, &block) + @mapping ||= Mappings.new(document_type, options) + + @mapping.options.update(options) unless options.empty? + + if block_given? + @mapping.instance_eval(&block) + return self + else + @mapping + end + end; alias_method :mappings, :mapping + + # Define settings for the index + # + # @example Define index settings + # + # Article.settings(index: { number_of_shards: 1 }) + # + # Article.settings.to_hash + # + # # => {:index=>{:number_of_shards=>1}} + # + # You can read settings from any object that responds to :read + # as long as its return value can be parsed as either YAML or JSON. + # + # @example Define index settings from YAML file + # + # # config/elasticsearch/articles.yml: + # # + # # index: + # # number_of_shards: 1 + # # + # + # Article.settings File.open("config/elasticsearch/articles.yml") + # + # Article.settings.to_hash + # + # # => { "index" => { "number_of_shards" => 1 } } + # + # + # @example Define index settings from JSON file + # + # # config/elasticsearch/articles.json: + # # + # # { "index": { "number_of_shards": 1 } } + # # + # + # Article.settings File.open("config/elasticsearch/articles.json") + # + # Article.settings.to_hash + # + # # => { "index" => { "number_of_shards" => 1 } } + # + def settings(settings={}, &block) + settings = YAML.load(settings.read) if settings.respond_to?(:read) + @settings ||= Settings.new(settings) + + @settings.settings.update(settings) unless settings.empty? + + if block_given? + self.instance_eval(&block) + return self + else + @settings + end + end + + def load_settings_from_io(settings) + YAML.load(settings.read) + end + + # Creates an index with correct name, automatically passing + # `settings` and `mappings` defined in the model + # + # @example Create an index for the `Article` model + # + # Article.__elasticsearch__.create_index! + # + # @example Forcefully create (delete first) an index for the `Article` model + # + # Article.__elasticsearch__.create_index! force: true + # + # @example Pass a specific index name + # + # Article.__elasticsearch__.create_index! index: 'my-index' + # + def create_index!(options={}) + target_index = options.delete(:index) || self.index_name + + delete_index!(options.merge index: target_index) if options[:force] + + unless index_exists?(index: target_index) + self.client.indices.create index: target_index, + body: { + settings: self.settings.to_hash, + mappings: self.mappings.to_hash } + end + end + + # Returns true if the index exists + # + # @example Check whether the model's index exists + # + # Article.__elasticsearch__.index_exists? + # + # @example Check whether a specific index exists + # + # Article.__elasticsearch__.index_exists? index: 'my-index' + # + def index_exists?(options={}) + target_index = options[:index] || self.index_name + + self.client.indices.exists(index: target_index) rescue false + end + + # Deletes the index with corresponding name + # + # @example Delete the index for the `Article` model + # + # Article.__elasticsearch__.delete_index! + # + # @example Pass a specific index name + # + # Article.__elasticsearch__.delete_index! index: 'my-index' + # + def delete_index!(options={}) + target_index = options.delete(:index) || self.index_name + + begin + self.client.indices.delete index: target_index + rescue Exception => e + if e.class.to_s =~ /NotFound/ && options[:force] + STDERR.puts "[!!!] Index does not exist (#{e.class})" + else + raise e + end + end + end + + # Performs the "refresh" operation for the index (useful e.g. in tests) + # + # @example Refresh the index for the `Article` model + # + # Article.__elasticsearch__.refresh_index! + # + # @example Pass a specific index name + # + # Article.__elasticsearch__.refresh_index! index: 'my-index' + # + # @see http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-refresh.html + # + def refresh_index!(options={}) + target_index = options.delete(:index) || self.index_name + + begin + self.client.indices.refresh index: target_index + rescue Exception => e + if e.class.to_s =~ /NotFound/ && options[:force] + STDERR.puts "[!!!] Index does not exist (#{e.class})" + else + raise e + end + end + end + end + + module InstanceMethods + + def self.included(base) + # Register callback for storing changed attributes for models + # which implement `before_save` and `changed_attributes` methods + # + # @note This is typically triggered only when the module would be + # included in the model directly, not within the proxy. + # + # @see #update_document + # + base.before_save do |instance| + instance.instance_variable_set(:@__changed_attributes, + Hash[ instance.changes.map { |key, value| [key, value.last] } ]) + end if base.respond_to?(:before_save) && base.instance_methods.include?(:changed_attributes) + end + + # Serializes the model instance into JSON (by calling `as_indexed_json`), + # and saves the document into the Elasticsearch index. + # + # @param options [Hash] Optional arguments for passing to the client + # + # @example Index a record + # + # @article.__elasticsearch__.index_document + # 2013-11-20 16:25:57 +0100: PUT http://localhost:9200/articles/article/1 ... + # + # @return [Hash] The response from Elasticsearch + # + # @see http://rubydoc.info/gems/elasticsearch-api/Elasticsearch/API/Actions:index + # + def index_document(options={}) + document = self.as_indexed_json + + client.index( + { index: index_name, + type: document_type, + id: self.id, + body: document }.merge(options) + ) + end + + # Deletes the model instance from the index + # + # @param options [Hash] Optional arguments for passing to the client + # + # @example Delete a record + # + # @article.__elasticsearch__.delete_document + # 2013-11-20 16:27:00 +0100: DELETE http://localhost:9200/articles/article/1 + # + # @return [Hash] The response from Elasticsearch + # + # @see http://rubydoc.info/gems/elasticsearch-api/Elasticsearch/API/Actions:delete + # + def delete_document(options={}) + client.delete( + { index: index_name, + type: document_type, + id: self.id }.merge(options) + ) + end + + # Tries to gather the changed attributes of a model instance + # (via [ActiveModel::Dirty](http://api.rubyonrails.org/classes/ActiveModel/Dirty.html)), + # performing a _partial_ update of the document. + # + # When the changed attributes are not available, performs full re-index of the record. + # + # See the {#update_document_attributes} method for updating specific attributes directly. + # + # @param options [Hash] Optional arguments for passing to the client + # + # @example Update a document corresponding to the record + # + # @article = Article.first + # @article.update_attribute :title, 'Updated' + # # SQL (0.3ms) UPDATE "articles" SET "title" = ?... + # + # @article.__elasticsearch__.update_document + # # 2013-11-20 17:00:05 +0100: POST http://localhost:9200/articles/article/1/_update ... + # # 2013-11-20 17:00:05 +0100: > {"doc":{"title":"Updated"}} + # + # @return [Hash] The response from Elasticsearch + # + # @see http://rubydoc.info/gems/elasticsearch-api/Elasticsearch/API/Actions:update + # + def update_document(options={}) + if changed_attributes = self.instance_variable_get(:@__changed_attributes) + attributes = if respond_to?(:as_indexed_json) + self.as_indexed_json.select { |k,v| changed_attributes.keys.map(&:to_s).include? k.to_s } + else + changed_attributes + end + + client.update( + { index: index_name, + type: document_type, + id: self.id, + body: { doc: attributes } }.merge(options) + ) + else + index_document(options) + end + end + + # Perform a _partial_ update of specific document attributes + # (without consideration for changed attributes as in {#update_document}) + # + # @param attributes [Hash] Attributes to be updated + # @param options [Hash] Optional arguments for passing to the client + # + # @example Update the `title` attribute + # + # @article = Article.first + # @article.title = "New title" + # @article.__elasticsearch__.update_document_attributes title: "New title" + # + # @return [Hash] The response from Elasticsearch + # + def update_document_attributes(attributes, options={}) + client.update( + { index: index_name, + type: document_type, + id: self.id, + body: { doc: attributes } }.merge(options) + ) + end + end + + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/multimodel.rb b/elasticsearch-model/lib/elasticsearch/model/multimodel.rb new file mode 100644 index 0000000000..8831d4fd09 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/multimodel.rb @@ -0,0 +1,83 @@ +module Elasticsearch + module Model + + # Keeps a global registry of classes that include `Elasticsearch::Model` + # + class Registry + def initialize + @models = [] + end + + # Returns the unique instance of the registry (Singleton) + # + # @api private + # + def self.__instance + @instance ||= new + end + + # Adds a model to the registry + # + def self.add(klass) + __instance.add(klass) + end + + # Returns an Array of registered models + # + def self.all + __instance.models + end + + # Adds a model to the registry + # + def add(klass) + @models << klass + end + + # Returns a copy of the registered models + # + def models + @models.dup + end + end + + # Wraps a collection of models when querying multiple indices + # + # @see Elasticsearch::Model.search + # + class Multimodel + attr_reader :models + + # @param models [Class] The list of models across which the search will be performed + # + def initialize(*models) + @models = models.flatten + @models = Model::Registry.all if @models.empty? + end + + # Get an Array of index names used for retrieving documents when doing a search across multiple models + # + # @return [Array] the list of index names used for retrieving documents + # + def index_name + models.map { |m| m.index_name } + end + + # Get an Array of document types used for retrieving documents when doing a search across multiple models + # + # @return [Array] the list of document types used for retrieving documents + # + def document_type + models.map { |m| m.document_type } + end + + # Get the client common for all models + # + # @return Elasticsearch::Transport::Client + # + def client + Elasticsearch::Model.client + end + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/naming.rb b/elasticsearch-model/lib/elasticsearch/model/naming.rb new file mode 100644 index 0000000000..ce510d2d47 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/naming.rb @@ -0,0 +1,122 @@ +module Elasticsearch + module Model + + # Provides methods for getting and setting index name and document type for the model + # + module Naming + + module ClassMethods + + # Get or set the name of the index + # + # @example Set the index name for the `Article` model + # + # class Article + # index_name "articles-#{Rails.env}" + # end + # + # @example Set the index name for the `Article` model and re-evaluate it on each call + # + # class Article + # index_name { "articles-#{Time.now.year}" } + # end + # + # @example Directly set the index name for the `Article` model + # + # Article.index_name "articles-#{Rails.env}" + # + # + def index_name name=nil, &block + if name || block_given? + return (@index_name = name || block) + end + + if @index_name.respond_to?(:call) + @index_name.call + else + @index_name || self.model_name.collection.gsub(/\//, '-') + end + end + + # Set the index name + # + # @see index_name + def index_name=(name) + @index_name = name + end + + # Get or set the document type + # + # @example Set the document type for the `Article` model + # + # class Article + # document_type "my-article" + # end + # + # @example Directly set the document type for the `Article` model + # + # Article.document_type "my-article" + # + def document_type name=nil + @document_type = name || @document_type || self.model_name.element + end + + + # Set the document type + # + # @see document_type + # + def document_type=(name) + @document_type = name + end + end + + module InstanceMethods + + # Get or set the index name for the model instance + # + # @example Set the index name for an instance of the `Article` model + # + # @article.index_name "articles-#{@article.user_id}" + # @article.__elasticsearch__.update_document + # + def index_name name=nil, &block + if name || block_given? + return (@index_name = name || block) + end + + if @index_name.respond_to?(:call) + @index_name.call + else + @index_name || self.class.index_name + end + end + + # Set the index name + # + # @see index_name + def index_name=(name) + @index_name = name + end + + # @example Set the document type for an instance of the `Article` model + # + # @article.document_type "my-article" + # @article.__elasticsearch__.update_document + # + def document_type name=nil + @document_type = name || @document_type || self.class.document_type + end + + # Set the document type + # + # @see document_type + # + def document_type=(name) + @document_type = name + end + end + + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/proxy.rb b/elasticsearch-model/lib/elasticsearch/model/proxy.rb new file mode 100644 index 0000000000..3e37f28ec3 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/proxy.rb @@ -0,0 +1,137 @@ +module Elasticsearch + module Model + + # This module provides a proxy interfacing between the including class and + # {Elasticsearch::Model}, preventing the pollution of the including class namespace. + # + # The only "gateway" between the model and Elasticsearch::Model is the + # `__elasticsearch__` class and instance method. + # + # The including class must be compatible with + # [ActiveModel](https://github.com/rails/rails/tree/master/activemodel). + # + # @example Include the {Elasticsearch::Model} module into an `Article` model + # + # class Article < ActiveRecord::Base + # include Elasticsearch::Model + # end + # + # Article.__elasticsearch__.respond_to?(:search) + # # => true + # + # article = Article.first + # + # article.respond_to? :index_document + # # => false + # + # article.__elasticsearch__.respond_to?(:index_document) + # # => true + # + module Proxy + + # Define the `__elasticsearch__` class and instance methods in the including class + # and register a callback for intercepting changes in the model. + # + # @note The callback is triggered only when `Elasticsearch::Model` is included in the + # module and the functionality is accessible via the proxy. + # + def self.included(base) + base.class_eval do + # {ClassMethodsProxy} instance, accessed as `MyModel.__elasticsearch__` + # + def self.__elasticsearch__ &block + @__elasticsearch__ ||= ClassMethodsProxy.new(self) + @__elasticsearch__.instance_eval(&block) if block_given? + @__elasticsearch__ + end + + # {InstanceMethodsProxy}, accessed as `@mymodel.__elasticsearch__` + # + def __elasticsearch__ &block + @__elasticsearch__ ||= InstanceMethodsProxy.new(self) + @__elasticsearch__.instance_eval(&block) if block_given? + @__elasticsearch__ + end + + # Register a callback for storing changed attributes for models which implement + # `before_save` and `changed_attributes` methods (when `Elasticsearch::Model` is included) + # + # @see http://api.rubyonrails.org/classes/ActiveModel/Dirty.html + # + before_save do |i| + changed_attr = i.__elasticsearch__.instance_variable_get(:@__changed_attributes) || {} + i.__elasticsearch__.instance_variable_set(:@__changed_attributes, + changed_attr.merge(Hash[ i.changes.map { |key, value| [key, value.last] } ])) + end if respond_to?(:before_save) && instance_methods.include?(:changed_attributes) + end + end + + # @overload dup + # + # Returns a copy of this object. Resets the __elasticsearch__ proxy so + # the duplicate will build its own proxy. + def initialize_dup(_) + @__elasticsearch__ = nil + super + end + + # Common module for the proxy classes + # + module Base + attr_reader :target + + def initialize(target) + @target = target + end + + # Delegate methods to `@target` + # + def method_missing(method_name, *arguments, &block) + target.respond_to?(method_name) ? target.__send__(method_name, *arguments, &block) : super + end + + # Respond to methods from `@target` + # + def respond_to?(method_name, include_private = false) + target.respond_to?(method_name) || super + end + + def inspect + "[PROXY] #{target.inspect}" + end + end + + # A proxy interfacing between Elasticsearch::Model class methods and model class methods + # + # TODO: Inherit from BasicObject and make Pry's `ls` command behave? + # + class ClassMethodsProxy + include Base + end + + # A proxy interfacing between Elasticsearch::Model instance methods and model instance methods + # + # TODO: Inherit from BasicObject and make Pry's `ls` command behave? + # + class InstanceMethodsProxy + include Base + + def klass + target.class + end + + def class + klass.__elasticsearch__ + end + + # Need to redefine `as_json` because we're not inheriting from `BasicObject`; + # see TODO note above. + # + def as_json(options={}) + target.as_json(options) + end + end + + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/response.rb b/elasticsearch-model/lib/elasticsearch/model/response.rb new file mode 100644 index 0000000000..fad3828b39 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/response.rb @@ -0,0 +1,83 @@ +module Elasticsearch + module Model + + # Contains modules and classes for wrapping the response from Elasticsearch + # + module Response + + # Encapsulate the response returned from the Elasticsearch client + # + # Implements Enumerable and forwards its methods to the {#results} object. + # + class Response + attr_reader :klass, :search, :response, + :took, :timed_out, :shards + + include Enumerable + + delegate :each, :empty?, :size, :slice, :[], :to_ary, to: :results + + def initialize(klass, search, options={}) + @klass = klass + @search = search + end + + # Returns the Elasticsearch response + # + # @return [Hash] + # + def response + @response ||= begin + Hashie::Mash.new(search.execute!) + end + end + + # Returns the collection of "hits" from Elasticsearch + # + # @return [Results] + # + def results + @results ||= Results.new(klass, self) + end + + # Returns the collection of records from the database + # + # @return [Records] + # + def records(options = {}) + @records ||= Records.new(klass, self, options) + end + + # Returns the "took" time + # + def took + response['took'] + end + + # Returns whether the response timed out + # + def timed_out + response['timed_out'] + end + + # Returns the statistics on shards + # + def shards + Hashie::Mash.new(response['_shards']) + end + + # Returns a Hashie::Mash of the aggregations + # + def aggregations + response['aggregations'] ? Hashie::Mash.new(response['aggregations']) : nil + end + + # Returns a Hashie::Mash of the suggestions + # + def suggestions + Suggestions.new(response['suggest']) + end + end + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/response/base.rb b/elasticsearch-model/lib/elasticsearch/model/response/base.rb new file mode 100644 index 0000000000..3bb8005b63 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/response/base.rb @@ -0,0 +1,44 @@ +module Elasticsearch + module Model + module Response + # Common funtionality for classes in the {Elasticsearch::Model::Response} module + # + module Base + attr_reader :klass, :response + + # @param klass [Class] The name of the model class + # @param response [Hash] The full response returned from Elasticsearch client + # @param options [Hash] Optional parameters + # + def initialize(klass, response, options={}) + @klass = klass + @response = response + end + + # @abstract Implement this method in specific class + # + def results + raise NotImplemented, "Implement this method in #{klass}" + end + + # @abstract Implement this method in specific class + # + def records + raise NotImplemented, "Implement this method in #{klass}" + end + + # Returns the total number of hits + # + def total + response.response['hits']['total'] + end + + # Returns the max_score + # + def max_score + response.response['hits']['max_score'] + end + end + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/response/pagination.rb b/elasticsearch-model/lib/elasticsearch/model/response/pagination.rb new file mode 100644 index 0000000000..c8e74b7934 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/response/pagination.rb @@ -0,0 +1,192 @@ +module Elasticsearch + module Model + module Response + + # Pagination for search results/records + # + module Pagination + # Allow models to be paginated with the "kaminari" gem [https://github.com/amatsuda/kaminari] + # + module Kaminari + def self.included(base) + # Include the Kaminari configuration and paging method in response + # + base.__send__ :include, ::Kaminari::ConfigurationMethods::ClassMethods + base.__send__ :include, ::Kaminari::PageScopeMethods + + # Include the Kaminari paging methods in results and records + # + Elasticsearch::Model::Response::Results.__send__ :include, ::Kaminari::ConfigurationMethods::ClassMethods + Elasticsearch::Model::Response::Results.__send__ :include, ::Kaminari::PageScopeMethods + Elasticsearch::Model::Response::Records.__send__ :include, ::Kaminari::PageScopeMethods + + Elasticsearch::Model::Response::Results.__send__ :delegate, :limit_value, :offset_value, :total_count, :max_pages, to: :response + Elasticsearch::Model::Response::Records.__send__ :delegate, :limit_value, :offset_value, :total_count, :max_pages, to: :response + + base.class_eval <<-RUBY, __FILE__, __LINE__ + 1 + # Define the `page` Kaminari method + # + def #{::Kaminari.config.page_method_name}(num=nil) + @results = nil + @records = nil + @response = nil + @page = [num.to_i, 1].max + @per_page ||= __default_per_page + + self.search.definition.update size: @per_page, + from: @per_page * (@page - 1) + + self + end + RUBY + end + + # Returns the current "limit" (`size`) value + # + def limit_value + case + when search.definition[:size] + search.definition[:size] + else + __default_per_page + end + end + + # Returns the current "offset" (`from`) value + # + def offset_value + case + when search.definition[:from] + search.definition[:from] + else + 0 + end + end + + # Set the "limit" (`size`) value + # + def limit(value) + return self if value.to_i <= 0 + @results = nil + @records = nil + @response = nil + @per_page = value.to_i + + search.definition.update :size => @per_page + search.definition.update :from => @per_page * (@page - 1) if @page + self + end + + # Set the "offset" (`from`) value + # + def offset(value) + return self if value.to_i < 0 + @results = nil + @records = nil + @response = nil + @page = nil + search.definition.update :from => value.to_i + self + end + + # Returns the total number of results + # + def total_count + results.total + end + + # Returns the models's `per_page` value or the default + # + # @api private + # + def __default_per_page + klass.respond_to?(:default_per_page) && klass.default_per_page || ::Kaminari.config.default_per_page + end + end + + # Allow models to be paginated with the "will_paginate" gem [https://github.com/mislav/will_paginate] + # + module WillPaginate + def self.included(base) + base.__send__ :include, ::WillPaginate::CollectionMethods + + # Include the paging methods in results and records + # + methods = [:current_page, :offset, :length, :per_page, :total_entries, :total_pages, :previous_page, :next_page, :out_of_bounds?] + Elasticsearch::Model::Response::Results.__send__ :delegate, *methods, to: :response + Elasticsearch::Model::Response::Records.__send__ :delegate, *methods, to: :response + end + + def offset + (current_page - 1) * per_page + end + + def length + search.definition[:size] + end + + # Main pagination method + # + # @example + # + # Article.search('foo').paginate(page: 1, per_page: 30) + # + def paginate(options) + param_name = options[:param_name] || :page + page = [options[param_name].to_i, 1].max + per_page = (options[:per_page] || __default_per_page).to_i + + search.definition.update size: per_page, + from: (page - 1) * per_page + self + end + + # Return the current page + # + def current_page + search.definition[:from] / per_page + 1 if search.definition[:from] && per_page + end + + # Pagination method + # + # @example + # + # Article.search('foo').page(2) + # + def page(num) + paginate(page: num, per_page: per_page) # shorthand + end + + # Return or set the "size" value + # + # @example + # + # Article.search('foo').per_page(15).page(2) + # + def per_page(num = nil) + if num.nil? + search.definition[:size] + else + paginate(page: current_page, per_page: num) # shorthand + end + end + + # Returns the total number of results + # + def total_entries + results.total + end + + # Returns the models's `per_page` value or the default + # + # @api private + # + def __default_per_page + klass.respond_to?(:per_page) && klass.per_page || ::WillPaginate.per_page + end + end + end + + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/response/records.rb b/elasticsearch-model/lib/elasticsearch/model/response/records.rb new file mode 100644 index 0000000000..4638ca6892 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/response/records.rb @@ -0,0 +1,73 @@ +module Elasticsearch + module Model + module Response + + # Encapsulates the collection of records returned from the database + # + # Implements Enumerable and forwards its methods to the {#records} object, + # which is provided by an {Elasticsearch::Model::Adapter::Adapter} implementation. + # + class Records + include Enumerable + + delegate :each, :empty?, :size, :slice, :[], :to_a, :to_ary, to: :records + + attr_accessor :options + + include Base + + # @see Base#initialize + # + def initialize(klass, response, options={}) + super + + # Include module provided by the adapter in the singleton class ("metaclass") + # + adapter = Adapter.from_class(klass) + metaclass = class << self; self; end + metaclass.__send__ :include, adapter.records_mixin + + self.options = options + self + end + + # Returns the hit IDs + # + def ids + response.response['hits']['hits'].map { |hit| hit['_id'] } + end + + # Returns the {Results} collection + # + def results + response.results + end + + # Yields [record, hit] pairs to the block + # + def each_with_hit(&block) + records.to_a.zip(results).each(&block) + end + + # Yields [record, hit] pairs and returns the result + # + def map_with_hit(&block) + records.to_a.zip(results).map(&block) + end + + # Delegate methods to `@records` + # + def method_missing(method_name, *arguments) + records.respond_to?(method_name) ? records.__send__(method_name, *arguments) : super + end + + # Respond to methods from `@records` + # + def respond_to?(method_name, include_private = false) + records.respond_to?(method_name) || super + end + + end + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/response/result.rb b/elasticsearch-model/lib/elasticsearch/model/response/result.rb new file mode 100644 index 0000000000..217723e8b9 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/response/result.rb @@ -0,0 +1,63 @@ +module Elasticsearch + module Model + module Response + + # Encapsulates the "hit" returned from the Elasticsearch client + # + # Wraps the raw Hash with in a `Hashie::Mash` instance, providing + # access to the Hash properties by calling Ruby methods. + # + # @see https://github.com/intridea/hashie + # + class Result + + # @param attributes [Hash] A Hash with document properties + # + def initialize(attributes={}) + @result = Hashie::Mash.new(attributes) + end + + # Return document `_id` as `id` + # + def id + @result['_id'] + end + + # Return document `_type` as `_type` + # + def type + @result['_type'] + end + + # Delegate methods to `@result` or `@result._source` + # + def method_missing(name, *arguments) + case + when name.to_s.end_with?('?') + @result.__send__(name, *arguments) || ( @result._source && @result._source.__send__(name, *arguments) ) + when @result.respond_to?(name) + @result.__send__ name, *arguments + when @result._source && @result._source.respond_to?(name) + @result._source.__send__ name, *arguments + else + super + end + end + + # Respond to methods from `@result` or `@result._source` + # + def respond_to?(method_name, include_private = false) + @result.respond_to?(method_name.to_sym) || \ + @result._source && @result._source.respond_to?(method_name.to_sym) || \ + super + end + + def as_json(options={}) + @result.as_json(options) + end + + # TODO: #to_s, #inspect, with support for Pry + end + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/response/results.rb b/elasticsearch-model/lib/elasticsearch/model/response/results.rb new file mode 100644 index 0000000000..006e66a46b --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/response/results.rb @@ -0,0 +1,31 @@ +module Elasticsearch + module Model + module Response + + # Encapsulates the collection of documents returned from Elasticsearch + # + # Implements Enumerable and forwards its methods to the {#results} object. + # + class Results + include Base + include Enumerable + + delegate :each, :empty?, :size, :slice, :[], :to_a, :to_ary, to: :results + + # @see Base#initialize + # + def initialize(klass, response, options={}) + super + end + + # Returns the {Results} collection + # + def results + # TODO: Configurable custom wrapper + response.response['hits']['hits'].map { |hit| Result.new(hit) } + end + + end + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/response/suggestions.rb b/elasticsearch-model/lib/elasticsearch/model/response/suggestions.rb new file mode 100644 index 0000000000..5088767cef --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/response/suggestions.rb @@ -0,0 +1,13 @@ +module Elasticsearch + module Model + module Response + + class Suggestions < Hashie::Mash + def terms + self.to_a.map { |k,v| v.first['options'] }.flatten.map {|v| v['text']}.uniq + end + end + + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/searching.rb b/elasticsearch-model/lib/elasticsearch/model/searching.rb new file mode 100644 index 0000000000..604657d5e0 --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/searching.rb @@ -0,0 +1,109 @@ +module Elasticsearch + module Model + + # Contains functionality related to searching. + # + module Searching + + # Wraps a search request definition + # + class SearchRequest + attr_reader :klass, :definition, :options + + # @param klass [Class] The class of the model + # @param query_or_payload [String,Hash,Object] The search request definition + # (string, JSON, Hash, or object responding to `to_hash`) + # @param options [Hash] Optional parameters to be passed to the Elasticsearch client + # + def initialize(klass, query_or_payload, options={}) + @klass = klass + @options = options + + __index_name = options[:index] || klass.index_name + __document_type = options[:type] || klass.document_type + + case + # search query: ... + when query_or_payload.respond_to?(:to_hash) + body = query_or_payload.to_hash + + # search '{ "query" : ... }' + when query_or_payload.is_a?(String) && query_or_payload =~ /^\s*{/ + body = query_or_payload + + # search '...' + else + q = query_or_payload + end + + if body + @definition = { index: __index_name, type: __document_type, body: body }.update options + else + @definition = { index: __index_name, type: __document_type, q: q }.update options + end + end + + # Performs the request and returns the response from client + # + # @return [Hash] The response from Elasticsearch + # + def execute! + klass.client.search(@definition) + end + end + + module ClassMethods + + # Provides a `search` method for the model to easily search within an index/type + # corresponding to the model settings. + # + # @param query_or_payload [String,Hash,Object] The search request definition + # (string, JSON, Hash, or object responding to `to_hash`) + # @param options [Hash] Optional parameters to be passed to the Elasticsearch client + # + # @return [Elasticsearch::Model::Response::Response] + # + # @example Simple search in `Article` + # + # Article.search 'foo' + # + # @example Search using a search definition as a Hash + # + # response = Article.search \ + # query: { + # match: { + # title: 'foo' + # } + # }, + # highlight: { + # fields: { + # title: {} + # } + # }, + # size: 50 + # + # response.results.first.title + # # => "Foo" + # + # response.results.first.highlight.title + # # => ["Foo"] + # + # response.records.first.title + # # Article Load (0.2ms) SELECT "articles".* FROM "articles" WHERE "articles"."id" IN (1, 3) + # # => "Foo" + # + # @example Search using a search definition as a JSON string + # + # Article.search '{"query" : { "match_all" : {} }}' + # + def search(query_or_payload, options={}) + search = SearchRequest.new(self, query_or_payload, options) + + Response::Response.new(self, search) + end + + end + + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/serializing.rb b/elasticsearch-model/lib/elasticsearch/model/serializing.rb new file mode 100644 index 0000000000..659a58bb2a --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/serializing.rb @@ -0,0 +1,35 @@ +module Elasticsearch + module Model + + # Contains functionality for serializing model instances for the client + # + module Serializing + + module ClassMethods + end + + module InstanceMethods + + # Serialize the record as a Hash, to be passed to the client. + # + # Re-define this method to customize the serialization. + # + # @return [Hash] + # + # @example Return the model instance as a Hash + # + # Article.first.__elasticsearch__.as_indexed_json + # => {"title"=>"Foo"} + # + # @see Elasticsearch::Model::Indexing + # + def as_indexed_json(options={}) + # TODO: Play with the `MyModel.indexes` method -- reject non-mapped attributes, `:as` options, etc + self.as_json(options.merge root: false) + end + + end + + end + end +end diff --git a/elasticsearch-model/lib/elasticsearch/model/version.rb b/elasticsearch-model/lib/elasticsearch/model/version.rb new file mode 100644 index 0000000000..44cfdabeab --- /dev/null +++ b/elasticsearch-model/lib/elasticsearch/model/version.rb @@ -0,0 +1,5 @@ +module Elasticsearch + module Model + VERSION = "0.1.9" + end +end diff --git a/elasticsearch-model/test/integration/active_record_associations_parent_child.rb b/elasticsearch-model/test/integration/active_record_associations_parent_child.rb new file mode 100644 index 0000000000..39be1144b3 --- /dev/null +++ b/elasticsearch-model/test/integration/active_record_associations_parent_child.rb @@ -0,0 +1,139 @@ +require 'test_helper' +require 'active_record' + +class Question < ActiveRecord::Base + include Elasticsearch::Model + + has_many :answers, dependent: :destroy + + index_name 'questions_and_answers' + + mapping do + indexes :title + indexes :text + indexes :author + end + + after_commit lambda { __elasticsearch__.index_document }, on: :create + after_commit lambda { __elasticsearch__.update_document }, on: :update + after_commit lambda { __elasticsearch__.delete_document }, on: :destroy +end + +class Answer < ActiveRecord::Base + include Elasticsearch::Model + + belongs_to :question + + index_name 'questions_and_answers' + + mapping _parent: { type: 'question', required: true } do + indexes :text + indexes :author + end + + after_commit lambda { __elasticsearch__.index_document(parent: question_id) }, on: :create + after_commit lambda { __elasticsearch__.update_document(parent: question_id) }, on: :update + after_commit lambda { __elasticsearch__.delete_document(parent: question_id) }, on: :destroy +end + +module ParentChildSearchable + INDEX_NAME = 'questions_and_answers' + + def create_index!(options={}) + client = Question.__elasticsearch__.client + client.indices.delete index: INDEX_NAME rescue nil if options[:force] + + settings = Question.settings.to_hash.merge Answer.settings.to_hash + mappings = Question.mappings.to_hash.merge Answer.mappings.to_hash + + client.indices.create index: INDEX_NAME, + body: { + settings: settings.to_hash, + mappings: mappings.to_hash } + end + + extend self +end + +module Elasticsearch + module Model + class ActiveRecordAssociationsParentChildIntegrationTest < Elasticsearch::Test::IntegrationTestCase + + context "ActiveRecord associations with parent/child modelling" do + setup do + ActiveRecord::Schema.define(version: 1) do + create_table :questions do |t| + t.string :title + t.text :text + t.string :author + t.timestamps + end + create_table :answers do |t| + t.text :text + t.string :author + t.references :question + t.timestamps + end and add_index(:answers, :question_id) + end + + Question.delete_all + ParentChildSearchable.create_index! force: true + + q_1 = Question.create! title: 'First Question', author: 'John' + q_2 = Question.create! title: 'Second Question', author: 'Jody' + + q_1.answers.create! text: 'Lorem Ipsum', author: 'Adam' + q_1.answers.create! text: 'Dolor Sit', author: 'Ryan' + + q_2.answers.create! text: 'Amet Et', author: 'John' + + Question.__elasticsearch__.refresh_index! + end + + should "find questions by matching answers" do + response = Question.search( + { query: { + has_child: { + type: 'answer', + query: { + match: { + author: 'john' + } + } + } + } + }) + + assert_equal 'Second Question', response.records.first.title + end + + should "find answers for matching questions" do + response = Answer.search( + { query: { + has_parent: { + parent_type: 'question', + query: { + match: { + author: 'john' + } + } + } + } + }) + + assert_same_elements ['Adam', 'Ryan'], response.records.map(&:author) + end + + should "delete answers when the question is deleted" do + Question.where(title: 'First Question').each(&:destroy) + Question.__elasticsearch__.refresh_index! + + response = Answer.search query: { match_all: {} } + + assert_equal 1, response.results.total + end + end + + end + end +end diff --git a/elasticsearch-model/test/integration/active_record_associations_test.rb b/elasticsearch-model/test/integration/active_record_associations_test.rb new file mode 100644 index 0000000000..af67ad889c --- /dev/null +++ b/elasticsearch-model/test/integration/active_record_associations_test.rb @@ -0,0 +1,326 @@ +require 'test_helper' +require 'active_record' + +module Elasticsearch + module Model + class ActiveRecordAssociationsIntegrationTest < Elasticsearch::Test::IntegrationTestCase + + context "ActiveRecord associations" do + setup do + + # ----- Schema definition --------------------------------------------------------------- + + ActiveRecord::Schema.define(version: 1) do + create_table :categories do |t| + t.string :title + t.timestamps + end + + create_table :categories_posts, id: false do |t| + t.references :post, :category + end + + create_table :authors do |t| + t.string :first_name, :last_name + t.timestamps + end + + create_table :authorships do |t| + t.string :first_name, :last_name + t.references :post + t.references :author + t.timestamps + end + + create_table :comments do |t| + t.string :text + t.string :author + t.references :post + t.timestamps + end and add_index(:comments, :post_id) + + create_table :posts do |t| + t.string :title + t.text :text + t.boolean :published + t.timestamps + end + end + + # ----- Models definition ------------------------------------------------------------------------- + + class Category < ActiveRecord::Base + has_and_belongs_to_many :posts + end + + class Author < ActiveRecord::Base + has_many :authorships + + def full_name + [first_name, last_name].compact.join(' ') + end + end + + class Authorship < ActiveRecord::Base + belongs_to :author + belongs_to :post, touch: true + end + + class Comment < ActiveRecord::Base + belongs_to :post, touch: true + end + + class Post < ActiveRecord::Base + has_and_belongs_to_many :categories, after_add: [ lambda { |a,c| a.__elasticsearch__.index_document } ], + after_remove: [ lambda { |a,c| a.__elasticsearch__.index_document } ] + has_many :authorships + has_many :authors, through: :authorships + has_many :comments + end + + # ----- Search integration via Concern module ----------------------------------------------------- + + module Searchable + extend ActiveSupport::Concern + + included do + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks + + # Set up the mapping + # + settings index: { number_of_shards: 1, number_of_replicas: 0 } do + mapping do + indexes :title, analyzer: 'snowball' + indexes :created_at, type: 'date' + + indexes :authors do + indexes :first_name + indexes :last_name + indexes :full_name, type: 'multi_field' do + indexes :full_name + indexes :raw, analyzer: 'keyword' + end + end + + indexes :categories, analyzer: 'keyword' + + indexes :comments, type: 'nested' do + indexes :text + indexes :author + end + end + end + + # Customize the JSON serialization for Elasticsearch + # + def as_indexed_json(options={}) + { + title: title, + text: text, + categories: categories.map(&:title), + authors: authors.as_json(methods: [:full_name], only: [:full_name, :first_name, :last_name]), + comments: comments.as_json(only: [:text, :author]) + } + end + + # Update document in the index after touch + # + after_touch() { __elasticsearch__.index_document } + end + end + + # Include the search integration + # + Post.__send__ :include, Searchable + Comment.__send__ :include, Elasticsearch::Model + Comment.__send__ :include, Elasticsearch::Model::Callbacks + + # ----- Reset the indices ----------------------------------------------------------------- + + Post.delete_all + Post.__elasticsearch__.create_index! force: true + + Comment.delete_all + Comment.__elasticsearch__.create_index! force: true + end + + should "index and find a document" do + Post.create! title: 'Test' + Post.create! title: 'Testing Coding' + Post.create! title: 'Coding' + Post.__elasticsearch__.refresh_index! + + response = Post.search('title:test') + + assert_equal 2, response.results.size + assert_equal 2, response.records.size + + assert_equal 'Test', response.results.first.title + assert_equal 'Test', response.records.first.title + end + + should "reindex a document after categories are changed" do + # Create categories + category_a = Category.where(title: "One").first_or_create! + category_b = Category.where(title: "Two").first_or_create! + + # Create post + post = Post.create! title: "First Post", text: "This is the first post..." + + # Assign categories + post.categories = [category_a, category_b] + + Post.__elasticsearch__.refresh_index! + + query = { query: { + filtered: { + query: { + multi_match: { + fields: ['title'], + query: 'first' + } + }, + filter: { + terms: { + categories: ['One'] + } + } + } + } + } + + response = Post.search query + + assert_equal 1, response.results.size + assert_equal 1, response.records.size + + # Remove category "One" + post.categories = [category_b] + + Post.__elasticsearch__.refresh_index! + response = Post.search query + + assert_equal 0, response.results.size + assert_equal 0, response.records.size + end + + should "reindex a document after authors are changed" do + # Create authors + author_a = Author.where(first_name: "John", last_name: "Smith").first_or_create! + author_b = Author.where(first_name: "Mary", last_name: "Smith").first_or_create! + author_c = Author.where(first_name: "Kobe", last_name: "Griss").first_or_create! + + # Create posts + post_1 = Post.create! title: "First Post", text: "This is the first post..." + post_2 = Post.create! title: "Second Post", text: "This is the second post..." + post_3 = Post.create! title: "Third Post", text: "This is the third post..." + + # Assign authors + post_1.authors = [author_a, author_b] + post_2.authors = [author_a] + post_3.authors = [author_c] + + Post.__elasticsearch__.refresh_index! + + response = Post.search 'authors.full_name:john' + + assert_equal 2, response.results.size + assert_equal 2, response.records.size + + post_3.authors << author_a + + Post.__elasticsearch__.refresh_index! + + response = Post.search 'authors.full_name:john' + + assert_equal 3, response.results.size + assert_equal 3, response.records.size + end if defined?(::ActiveRecord) && ::ActiveRecord::VERSION::MAJOR >= 4 + + should "reindex a document after comments are added" do + # Create posts + post_1 = Post.create! title: "First Post", text: "This is the first post..." + post_2 = Post.create! title: "Second Post", text: "This is the second post..." + + # Add comments + post_1.comments.create! author: 'John', text: 'Excellent' + post_1.comments.create! author: 'Abby', text: 'Good' + + post_2.comments.create! author: 'John', text: 'Terrible' + + Post.__elasticsearch__.refresh_index! + + response = Post.search 'comments.author:john AND comments.text:good' + assert_equal 0, response.results.size + + # Add comment + post_1.comments.create! author: 'John', text: 'Or rather just good...' + + Post.__elasticsearch__.refresh_index! + + response = Post.search 'comments.author:john AND comments.text:good' + assert_equal 0, response.results.size + + response = Post.search \ + query: { + nested: { + path: 'comments', + query: { + bool: { + must: [ + { match: { 'comments.author' => 'john' } }, + { match: { 'comments.text' => 'good' } } + ] + } + } + } + } + + assert_equal 1, response.results.size + end if defined?(::ActiveRecord) && ::ActiveRecord::VERSION::MAJOR >= 4 + + should "reindex a document after Post#touch" do + # Create categories + category_a = Category.where(title: "One").first_or_create! + + # Create post + post = Post.create! title: "First Post", text: "This is the first post..." + + # Assign category + post.categories << category_a + + Post.__elasticsearch__.refresh_index! + + assert_equal 1, Post.search('categories:One').size + + # Update category + category_a.update_attribute :title, "Updated" + + # Trigger touch on posts in category + category_a.posts.each { |p| p.touch } + + Post.__elasticsearch__.refresh_index! + + assert_equal 0, Post.search('categories:One').size + assert_equal 1, Post.search('categories:Updated').size + end + + should "eagerly load associated records" do + post_1 = Post.create(title: 'One') + post_2 = Post.create(title: 'Two') + post_1.comments.create text: 'First comment' + post_1.comments.create text: 'Second comment' + + Comment.__elasticsearch__.refresh_index! + + records = Comment.search('first').records(includes: :post) + + assert records.first.association(:post).loaded?, "The associated Post should be eagerly loaded" + assert_equal 'One', records.first.post.title + end + end + + end + end +end diff --git a/elasticsearch-model/test/integration/active_record_basic_test.rb b/elasticsearch-model/test/integration/active_record_basic_test.rb new file mode 100644 index 0000000000..e6ca97d6dc --- /dev/null +++ b/elasticsearch-model/test/integration/active_record_basic_test.rb @@ -0,0 +1,234 @@ +require 'test_helper' +require 'active_record' + +puts "ActiveRecord #{ActiveRecord::VERSION::STRING}", '-'*80 + +module Elasticsearch + module Model + class ActiveRecordBasicIntegrationTest < Elasticsearch::Test::IntegrationTestCase + context "ActiveRecord basic integration" do + setup do + ActiveRecord::Schema.define(:version => 1) do + create_table :articles do |t| + t.string :title + t.string :body + t.datetime :created_at, :default => 'NOW()' + end + end + + class ::Article < ActiveRecord::Base + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks + + settings index: { number_of_shards: 1, number_of_replicas: 0 } do + mapping do + indexes :title, type: 'string', analyzer: 'snowball' + indexes :body, type: 'string' + indexes :created_at, type: 'date' + end + end + + def as_indexed_json(options = {}) + attributes + .symbolize_keys + .slice(:title, :body, :created_at) + .merge(suggest_title: title) + end + end + + Article.delete_all + Article.__elasticsearch__.create_index! force: true + + ::Article.create! title: 'Test', body: '' + ::Article.create! title: 'Testing Coding', body: '' + ::Article.create! title: 'Coding', body: '' + + Article.__elasticsearch__.refresh_index! + end + + should "index and find a document" do + response = Article.search('title:test') + + assert response.any?, "Response should not be empty: #{response.to_a.inspect}" + + assert_equal 2, response.results.size + assert_equal 2, response.records.size + + assert_instance_of Elasticsearch::Model::Response::Result, response.results.first + assert_instance_of Article, response.records.first + + assert_equal 'Test', response.results.first.title + assert_equal 'Test', response.records.first.title + end + + should "provide access to result" do + response = Article.search query: { match: { title: 'test' } }, highlight: { fields: { title: {} } } + + assert_equal 'Test', response.results.first.title + + assert_equal true, response.results.first.title? + assert_equal false, response.results.first.boo? + + assert_equal true, response.results.first.highlight? + assert_equal true, response.results.first.highlight.title? + assert_equal false, response.results.first.highlight.boo? + end + + should "iterate over results" do + response = Article.search('title:test') + + assert_equal ['1', '2'], response.results.map(&:_id) + assert_equal [1, 2], response.records.map(&:id) + end + + should "return _id and _type as #id and #type" do + response = Article.search('title:test') + + assert_equal '1', response.results.first.id + assert_equal 'article', response.results.first.type + end + + should "access results from records" do + response = Article.search('title:test') + + response.records.each_with_hit do |r, h| + assert_not_nil h._score + assert_not_nil h._source.title + end + end + + should "preserve the search results order for records" do + response = Article.search('title:code') + + response.records.each_with_hit do |r, h| + assert_equal h._id, r.id.to_s + end + + response.records.map_with_hit do |r, h| + assert_equal h._id, r.id.to_s + end + end + + should "remove document from index on destroy" do + article = Article.first + + article.destroy + assert_equal 2, Article.count + + Article.__elasticsearch__.refresh_index! + + response = Article.search 'title:test' + + assert_equal 1, response.results.size + assert_equal 1, response.records.size + end + + should "index updates to the document" do + article = Article.first + + article.title = 'Writing' + article.save + + Article.__elasticsearch__.refresh_index! + + response = Article.search 'title:write' + + assert_equal 1, response.results.size + assert_equal 1, response.records.size + end + + should "update specific attributes" do + article = Article.first + + response = Article.search 'title:special' + + assert_equal 0, response.results.size + assert_equal 0, response.records.size + + article.__elasticsearch__.update_document_attributes title: 'special' + + Article.__elasticsearch__.refresh_index! + + response = Article.search 'title:special' + + assert_equal 1, response.results.size + assert_equal 1, response.records.size + end + + should "update document when save is called multiple times in a transaction" do + article = Article.first + response = Article.search 'body:dummy' + + assert_equal 0, response.results.size + assert_equal 0, response.records.size + + ActiveRecord::Base.transaction do + article.body = 'dummy' + article.save + + article.title = 'special' + article.save + end + + article.__elasticsearch__.update_document + Article.__elasticsearch__.refresh_index! + + response = Article.search 'body:dummy' + assert_equal 1, response.results.size + assert_equal 1, response.records.size + end + + should "return results for a DSL search" do + response = Article.search query: { match: { title: { query: 'test' } } } + + assert_equal 2, response.results.size + assert_equal 2, response.records.size + end + + should "return a paged collection" do + response = Article.search query: { match: { title: { query: 'test' } } }, + size: 2, + from: 1 + + assert_equal 1, response.results.size + assert_equal 1, response.records.size + + assert_equal 'Testing Coding', response.results.first.title + assert_equal 'Testing Coding', response.records.first.title + end + + should "allow chaining SQL commands on response.records" do + response = Article.search query: { match: { title: { query: 'test' } } } + + assert_equal 2, response.records.size + assert_equal 1, response.records.where(title: 'Test').size + assert_equal 'Test', response.records.where(title: 'Test').first.title + end + + should "allow ordering response.records in SQL" do + response = Article.search query: { match: { title: { query: 'test' } } } + + if defined?(::ActiveRecord) && ::ActiveRecord::VERSION::MAJOR >= 4 + assert_equal 'Testing Coding', response.records.order(title: :desc).first.title + else + assert_equal 'Testing Coding', response.records.order('title DESC').first.title + end + end + + should "allow dot access to response" do + response = Article.search query: { match: { title: { query: 'test' } } }, + aggregations: { dates: { date_histogram: { field: 'created_at', interval: 'hour' } } }, + suggest: { text: 'tezt', title: { term: { field: 'title', suggest_mode: 'always' } } } + + response.response.respond_to?(:aggregations) + assert_equal 2, response.aggregations.dates.buckets.first.doc_count + + response.response.respond_to?(:suggest) + assert_equal 1, response.suggestions.title.first.options.size + assert_equal ['test'], response.suggestions.terms + end + end + + end + end +end diff --git a/elasticsearch-model/test/integration/active_record_custom_serialization_test.rb b/elasticsearch-model/test/integration/active_record_custom_serialization_test.rb new file mode 100644 index 0000000000..03eb9a4410 --- /dev/null +++ b/elasticsearch-model/test/integration/active_record_custom_serialization_test.rb @@ -0,0 +1,62 @@ +require 'test_helper' +require 'active_record' + +module Elasticsearch + module Model + class ActiveRecordCustomSerializationTest < Elasticsearch::Test::IntegrationTestCase + context "ActiveRecord model with custom JSON serialization" do + setup do + class ::ArticleWithCustomSerialization < ActiveRecord::Base + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks + + mapping do + indexes :title + end + + def as_indexed_json(options={}) + # as_json(options.merge root: false).slice('title') + { title: self.title } + end + end + + ActiveRecord::Schema.define(:version => 1) do + create_table ArticleWithCustomSerialization.table_name do |t| + t.string :title + t.string :status + end + end + + ArticleWithCustomSerialization.delete_all + ArticleWithCustomSerialization.__elasticsearch__.create_index! force: true + end + + should "index only the title attribute when creating" do + ArticleWithCustomSerialization.create! title: 'Test', status: 'green' + + a = ArticleWithCustomSerialization.__elasticsearch__.client.get \ + index: 'article_with_custom_serializations', + type: 'article_with_custom_serialization', + id: '1' + + assert_equal( { 'title' => 'Test' }, a['_source'] ) + end + + should "index only the title attribute when updating" do + ArticleWithCustomSerialization.create! title: 'Test', status: 'green' + + article = ArticleWithCustomSerialization.first + article.update_attributes title: 'UPDATED', status: 'red' + + a = ArticleWithCustomSerialization.__elasticsearch__.client.get \ + index: 'article_with_custom_serializations', + type: 'article_with_custom_serialization', + id: '1' + + assert_equal( { 'title' => 'UPDATED' }, a['_source'] ) + end + end + + end + end +end diff --git a/elasticsearch-model/test/integration/active_record_import_test.rb b/elasticsearch-model/test/integration/active_record_import_test.rb new file mode 100644 index 0000000000..8cc7448cc3 --- /dev/null +++ b/elasticsearch-model/test/integration/active_record_import_test.rb @@ -0,0 +1,109 @@ +require 'test_helper' +require 'active_record' + +module Elasticsearch + module Model + class ActiveRecordImportIntegrationTest < Elasticsearch::Test::IntegrationTestCase + context "ActiveRecord importing" do + setup do + ActiveRecord::Schema.define(:version => 1) do + create_table :import_articles do |t| + t.string :title + t.integer :views + t.string :numeric # For the sake of invalid data sent to Elasticsearch + t.datetime :created_at, :default => 'NOW()' + end + end + + class ::ImportArticle < ActiveRecord::Base + include Elasticsearch::Model + + scope :popular, -> { where('views >= 50') } + + mapping do + indexes :title, type: 'string' + indexes :views, type: 'integer' + indexes :numeric, type: 'integer' + indexes :created_at, type: 'date' + end + end + + ImportArticle.delete_all + ImportArticle.__elasticsearch__.create_index! force: true + ImportArticle.__elasticsearch__.client.cluster.health wait_for_status: 'yellow' + + 100.times { |i| ImportArticle.create! title: "Test #{i}", views: i } + end + + should "import all the documents" do + assert_equal 100, ImportArticle.count + + ImportArticle.__elasticsearch__.refresh_index! + assert_equal 0, ImportArticle.search('*').results.total + + batches = 0 + errors = ImportArticle.import(batch_size: 10) do |response| + batches += 1 + end + + assert_equal 0, errors + assert_equal 10, batches + + ImportArticle.__elasticsearch__.refresh_index! + assert_equal 100, ImportArticle.search('*').results.total + end + + should "import only documents from a specific scope" do + assert_equal 100, ImportArticle.count + + assert_equal 0, ImportArticle.import(scope: 'popular') + + ImportArticle.__elasticsearch__.refresh_index! + assert_equal 50, ImportArticle.search('*').results.total + end + + should "import only documents from a specific query" do + assert_equal 100, ImportArticle.count + + assert_equal 0, ImportArticle.import(query: -> { where('views >= 30') }) + + ImportArticle.__elasticsearch__.refresh_index! + assert_equal 70, ImportArticle.search('*').results.total + end + + should "report and not store/index invalid documents" do + ImportArticle.create! title: "Test INVALID", numeric: "INVALID" + + assert_equal 101, ImportArticle.count + + ImportArticle.__elasticsearch__.refresh_index! + assert_equal 0, ImportArticle.search('*').results.total + + batches = 0 + errors = ImportArticle.__elasticsearch__.import(batch_size: 10) do |response| + batches += 1 + end + + assert_equal 1, errors + assert_equal 11, batches + + ImportArticle.__elasticsearch__.refresh_index! + assert_equal 100, ImportArticle.search('*').results.total + end + + should "transform documents with the option" do + assert_equal 100, ImportArticle.count + + assert_equal 0, ImportArticle.import( transform: ->(a) {{ index: { data: { name: a.title, foo: 'BAR' } }}} ) + + ImportArticle.__elasticsearch__.refresh_index! + assert_contains ImportArticle.search('*').results.first._source.keys, 'name' + assert_contains ImportArticle.search('*').results.first._source.keys, 'foo' + assert_equal 100, ImportArticle.search('test').results.total + assert_equal 100, ImportArticle.search('bar').results.total + end + end + + end + end +end diff --git a/elasticsearch-model/test/integration/active_record_namespaced_model_test.rb b/elasticsearch-model/test/integration/active_record_namespaced_model_test.rb new file mode 100644 index 0000000000..be047f4e7a --- /dev/null +++ b/elasticsearch-model/test/integration/active_record_namespaced_model_test.rb @@ -0,0 +1,49 @@ +require 'test_helper' +require 'active_record' + +module Elasticsearch + module Model + class ActiveRecordNamespacedModelIntegrationTest < Elasticsearch::Test::IntegrationTestCase + context "Namespaced ActiveRecord model integration" do + setup do + ActiveRecord::Schema.define(:version => 1) do + create_table :articles do |t| + t.string :title + end + end + + module ::MyNamespace + class Article < ActiveRecord::Base + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks + + mapping { indexes :title } + end + end + + MyNamespace::Article.delete_all + MyNamespace::Article.__elasticsearch__.create_index! force: true + + MyNamespace::Article.create! title: 'Test' + + MyNamespace::Article.__elasticsearch__.refresh_index! + end + + should "have proper index name and document type" do + assert_equal "my_namespace-articles", MyNamespace::Article.index_name + assert_equal "article", MyNamespace::Article.document_type + end + + should "save document into index on save and find it" do + response = MyNamespace::Article.search 'title:test' + + assert response.any?, "No results returned: #{response.inspect}" + assert_equal 1, response.size + + assert_equal 'Test', response.results.first.title + end + end + + end + end +end diff --git a/elasticsearch-model/test/integration/active_record_pagination_test.rb b/elasticsearch-model/test/integration/active_record_pagination_test.rb new file mode 100644 index 0000000000..e1d6fefb11 --- /dev/null +++ b/elasticsearch-model/test/integration/active_record_pagination_test.rb @@ -0,0 +1,145 @@ +require 'test_helper' +require 'active_record' + +module Elasticsearch + module Model + class ActiveRecordPaginationTest < Elasticsearch::Test::IntegrationTestCase + context "ActiveRecord pagination" do + setup do + class ::ArticleForPagination < ActiveRecord::Base + include Elasticsearch::Model + + scope :published, -> { where(published: true) } + + settings index: { number_of_shards: 1, number_of_replicas: 0 } do + mapping do + indexes :title, type: 'string', analyzer: 'snowball' + indexes :created_at, type: 'date' + end + end + end + + ActiveRecord::Schema.define(:version => 1) do + create_table ::ArticleForPagination.table_name do |t| + t.string :title + t.datetime :created_at, :default => 'NOW()' + t.boolean :published + end + end + + Kaminari::Hooks.init + + ArticleForPagination.delete_all + ArticleForPagination.__elasticsearch__.create_index! force: true + + 68.times do |i| + ::ArticleForPagination.create! title: "Test #{i}", published: (i % 2 == 0) + end + + ArticleForPagination.import + ArticleForPagination.__elasticsearch__.refresh_index! + end + + should "be on the first page by default" do + records = ArticleForPagination.search('title:test').page(1).records + + assert_equal 25, records.size + assert_equal 1, records.current_page + assert_equal nil, records.prev_page + assert_equal 2, records.next_page + assert_equal 3, records.total_pages + + assert records.first_page?, "Should be the first page" + assert ! records.last_page?, "Should NOT be the last page" + assert ! records.out_of_range?, "Should NOT be out of range" + end + + should "load next page" do + records = ArticleForPagination.search('title:test').page(2).records + + assert_equal 25, records.size + assert_equal 2, records.current_page + assert_equal 1, records.prev_page + assert_equal 3, records.next_page + assert_equal 3, records.total_pages + + assert ! records.first_page?, "Should NOT be the first page" + assert ! records.last_page?, "Should NOT be the last page" + assert ! records.out_of_range?, "Should NOT be out of range" + end + + should "load last page" do + records = ArticleForPagination.search('title:test').page(3).records + + assert_equal 18, records.size + assert_equal 3, records.current_page + assert_equal 2, records.prev_page + assert_equal nil, records.next_page + assert_equal 3, records.total_pages + + assert ! records.first_page?, "Should NOT be the first page" + assert records.last_page?, "Should be the last page" + assert ! records.out_of_range?, "Should NOT be out of range" + end + + should "not load invalid page" do + records = ArticleForPagination.search('title:test').page(6).records + + assert_equal 0, records.size + assert_equal 6, records.current_page + assert_equal 5, records.prev_page + assert_equal nil, records.next_page + assert_equal 3, records.total_pages + + assert ! records.first_page?, "Should NOT be the first page" + assert records.last_page?, "Should be the last page" + assert records.out_of_range?, "Should be out of range" + end + + should "be combined with scopes" do + records = ArticleForPagination.search('title:test').page(2).records.published + assert records.all? { |r| r.published? } + assert_equal 12, records.size + end + + should "respect sort" do + search = ArticleForPagination.search({ query: { match: { title: 'test' } }, sort: [ { id: 'desc' } ] }) + + records = search.page(2).records + assert_equal 43, records.first.id # 68 - 25 = 42 + + records = search.page(3).records + assert_equal 18, records.first.id # 68 - (2 * 25) = 18 + + records = search.page(2).per(5).records + assert_equal 63, records.first.id # 68 - 5 = 63 + end + + should "set the limit per request" do + records = ArticleForPagination.search('title:test').limit(50).page(2).records + + assert_equal 18, records.size + assert_equal 2, records.current_page + assert_equal 1, records.prev_page + assert_equal nil, records.next_page + assert_equal 2, records.total_pages + + assert records.last_page?, "Should be the last page" + end + + context "with specific model settings" do + teardown do + ArticleForPagination.instance_variable_set(:@_default_per_page, nil) + end + + should "respect paginates_per" do + ArticleForPagination.paginates_per 50 + + assert_equal 50, ArticleForPagination.search('*').page(1).records.size + end + end + end + + end + end +end diff --git a/elasticsearch-model/test/integration/dynamic_index_name_test.rb b/elasticsearch-model/test/integration/dynamic_index_name_test.rb new file mode 100755 index 0000000000..a71633c6ae --- /dev/null +++ b/elasticsearch-model/test/integration/dynamic_index_name_test.rb @@ -0,0 +1,47 @@ +require 'test_helper' +require 'active_record' + +module Elasticsearch + module Model + class DynamicIndexNameTest < Elasticsearch::Test::IntegrationTestCase + context "Dynamic index name" do + setup do + class ::ArticleWithDynamicIndexName < ActiveRecord::Base + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks + + def self.counter=(value) + @counter = 0 + end + + def self.counter + (@counter ||= 0) && @counter += 1 + end + + mapping { indexes :title } + index_name { "articles-#{counter}" } + end + + ::ActiveRecord::Schema.define(:version => 1) do + create_table ::ArticleWithDynamicIndexName.table_name do |t| + t.string :title + end + end + + ::ArticleWithDynamicIndexName.counter = 0 + end + + should 'evaluate the index_name value' do + assert_equal ArticleWithDynamicIndexName.index_name, "articles-1" + end + + should 're-evaluate the index_name value each time' do + assert_equal ArticleWithDynamicIndexName.index_name, "articles-1" + assert_equal ArticleWithDynamicIndexName.index_name, "articles-2" + assert_equal ArticleWithDynamicIndexName.index_name, "articles-3" + end + end + + end + end +end diff --git a/elasticsearch-model/test/integration/mongoid_basic_test.rb b/elasticsearch-model/test/integration/mongoid_basic_test.rb new file mode 100644 index 0000000000..e370bd82aa --- /dev/null +++ b/elasticsearch-model/test/integration/mongoid_basic_test.rb @@ -0,0 +1,177 @@ +require 'test_helper' + +Mongo.setup! + +if Mongo.available? + Mongo.connect_to 'mongoid_articles' + + module Elasticsearch + module Model + class MongoidBasicIntegrationTest < Elasticsearch::Test::IntegrationTestCase + + class ::MongoidArticle + include Mongoid::Document + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks + + field :id, type: String + field :title, type: String + attr_accessible :title if respond_to? :attr_accessible + + settings index: { number_of_shards: 1, number_of_replicas: 0 } do + mapping do + indexes :title, type: 'string', analyzer: 'snowball' + indexes :created_at, type: 'date' + end + end + + def as_indexed_json(options={}) + as_json(except: [:id, :_id]) + end + end + + context "Mongoid integration" do + setup do + Elasticsearch::Model::Adapter.register \ + Elasticsearch::Model::Adapter::Mongoid, + lambda { |klass| !!defined?(::Mongoid::Document) && klass.respond_to?(:ancestors) && klass.ancestors.include?(::Mongoid::Document) } + + MongoidArticle.__elasticsearch__.create_index! force: true + + MongoidArticle.delete_all + + MongoidArticle.create! title: 'Test' + MongoidArticle.create! title: 'Testing Coding' + MongoidArticle.create! title: 'Coding' + + MongoidArticle.__elasticsearch__.refresh_index! + MongoidArticle.__elasticsearch__.client.cluster.health wait_for_status: 'yellow' + end + + should "index and find a document" do + response = MongoidArticle.search('title:test') + + assert response.any? + + assert_equal 2, response.results.size + assert_equal 2, response.records.size + + assert_instance_of Elasticsearch::Model::Response::Result, response.results.first + assert_instance_of MongoidArticle, response.records.first + + assert_equal 'Test', response.results.first.title + assert_equal 'Test', response.records.first.title + end + + should "iterate over results" do + response = MongoidArticle.search('title:test') + + assert_equal ['Test', 'Testing Coding'], response.results.map(&:title) + assert_equal ['Test', 'Testing Coding'], response.records.map(&:title) + end + + should "access results from records" do + response = MongoidArticle.search('title:test') + + response.records.each_with_hit do |r, h| + assert_not_nil h._score + assert_not_nil h._source.title + end + end + + should "preserve the search results order for records" do + response = MongoidArticle.search('title:code') + + response.records.each_with_hit do |r, h| + assert_equal h._id, r.id.to_s + end + + response.records.map_with_hit do |r, h| + assert_equal h._id, r.id.to_s + end + end + + should "remove document from index on destroy" do + article = MongoidArticle.first + + article.destroy + assert_equal 2, MongoidArticle.count + + MongoidArticle.__elasticsearch__.refresh_index! + + response = MongoidArticle.search 'title:test' + + assert_equal 1, response.results.size + assert_equal 1, response.records.size + end + + should "index updates to the document" do + article = MongoidArticle.first + + article.title = 'Writing' + article.save + + MongoidArticle.__elasticsearch__.refresh_index! + + response = MongoidArticle.search 'title:write' + + assert_equal 1, response.results.size + assert_equal 1, response.records.size + end + + should "return results for a DSL search" do + response = MongoidArticle.search query: { match: { title: { query: 'test' } } } + + assert_equal 2, response.results.size + assert_equal 2, response.records.size + end + + should "return a paged collection" do + response = MongoidArticle.search query: { match: { title: { query: 'test' } } }, + size: 2, + from: 1 + + assert_equal 1, response.results.size + assert_equal 1, response.records.size + + assert_equal 'Testing Coding', response.results.first.title + assert_equal 'Testing Coding', response.records.first.title + end + + + context "importing" do + setup do + MongoidArticle.delete_all + 97.times { |i| MongoidArticle.create! title: "Test #{i}" } + MongoidArticle.__elasticsearch__.create_index! force: true + MongoidArticle.__elasticsearch__.client.cluster.health wait_for_status: 'yellow' + end + + should "import all the documents" do + assert_equal 97, MongoidArticle.count + + MongoidArticle.__elasticsearch__.refresh_index! + assert_equal 0, MongoidArticle.search('*').results.total + + batches = 0 + errors = MongoidArticle.import(batch_size: 10) do |response| + batches += 1 + end + + assert_equal 0, errors + assert_equal 10, batches + + MongoidArticle.__elasticsearch__.refresh_index! + assert_equal 97, MongoidArticle.search('*').results.total + + response = MongoidArticle.search('test') + assert response.results.any?, "Search has not returned results: #{response.to_a}" + end + end + end + + end + end + end + +end diff --git a/elasticsearch-model/test/integration/multiple_models_test.rb b/elasticsearch-model/test/integration/multiple_models_test.rb new file mode 100644 index 0000000000..7d0bf7b6b6 --- /dev/null +++ b/elasticsearch-model/test/integration/multiple_models_test.rb @@ -0,0 +1,172 @@ +require 'test_helper' +require 'active_record' + +Mongo.setup! + +module Elasticsearch + module Model + class MultipleModelsIntegration < Elasticsearch::Test::IntegrationTestCase + context "Multiple models" do + setup do + ActiveRecord::Schema.define(:version => 1) do + create_table :episodes do |t| + t.string :name + t.datetime :created_at, :default => 'NOW()' + end + + create_table :series do |t| + t.string :name + t.datetime :created_at, :default => 'NOW()' + end + end + + module ::NameSearch + extend ActiveSupport::Concern + + included do + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks + + settings index: {number_of_shards: 1, number_of_replicas: 0} do + mapping do + indexes :name, type: 'string', analyzer: 'snowball' + indexes :created_at, type: 'date' + end + end + end + end + + class ::Episode < ActiveRecord::Base + include NameSearch + end + + class ::Series < ActiveRecord::Base + include NameSearch + end + + [::Episode, ::Series].each do |model| + model.delete_all + model.__elasticsearch__.create_index! force: true + model.create name: "The #{model.name}" + model.create name: "A great #{model.name}" + model.create name: "The greatest #{model.name}" + model.__elasticsearch__.refresh_index! + end + + end + + should "find matching documents across multiple models" do + response = Elasticsearch::Model.search(%q<"The greatest Episode"^2 OR "The greatest Series">, [Series, Episode]) + + assert response.any?, "Response should not be empty: #{response.to_a.inspect}" + + assert_equal 2, response.results.size + assert_equal 2, response.records.size + + assert_instance_of Elasticsearch::Model::Response::Result, response.results.first + assert_instance_of Episode, response.records.first + assert_instance_of Series, response.records.last + + assert_equal 'The greatest Episode', response.results[0].name + assert_equal 'The greatest Episode', response.records[0].name + + assert_equal 'The greatest Series', response.results[1].name + assert_equal 'The greatest Series', response.records[1].name + end + + should "provide access to results" do + response = Elasticsearch::Model.search(%q<"A great Episode"^2 OR "A great Series">, [Series, Episode]) + + assert_equal 'A great Episode', response.results[0].name + assert_equal true, response.results[0].name? + assert_equal false, response.results[0].boo? + + assert_equal 'A great Series', response.results[1].name + assert_equal true, response.results[1].name? + assert_equal false, response.results[1].boo? + end + + should "only retrieve records for existing results" do + ::Series.find_by_name("The greatest Series").delete + ::Series.__elasticsearch__.refresh_index! + response = Elasticsearch::Model.search(%q<"The greatest Episode"^2 OR "The greatest Series">, [Series, Episode]) + + assert response.any?, "Response should not be empty: #{response.to_a.inspect}" + + assert_equal 2, response.results.size + assert_equal 1, response.records.size + + assert_instance_of Elasticsearch::Model::Response::Result, response.results.first + assert_instance_of Episode, response.records.first + + assert_equal 'The greatest Episode', response.results[0].name + assert_equal 'The greatest Episode', response.records[0].name + end + + should "paginate the results" do + response = Elasticsearch::Model.search('series OR episode', [Series, Episode]) + + assert_equal 3, response.page(1).per(3).results.size + assert_equal 3, response.page(2).per(3).results.size + assert_equal 0, response.page(3).per(3).results.size + end + + if Mongo.available? + Mongo.connect_to 'mongoid_collections' + + context "Across mongoid models" do + setup do + class ::Image + include Mongoid::Document + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks + + field :name, type: String + attr_accessible :name if respond_to? :attr_accessible + + settings index: {number_of_shards: 1, number_of_replicas: 0} do + mapping do + indexes :name, type: 'string', analyzer: 'snowball' + indexes :created_at, type: 'date' + end + end + + def as_indexed_json(options={}) + as_json(except: [:_id]) + end + end + + Image.delete_all + Image.__elasticsearch__.create_index! force: true + Image.create! name: "The Image" + Image.create! name: "A great Image" + Image.create! name: "The greatest Image" + Image.__elasticsearch__.refresh_index! + Image.__elasticsearch__.client.cluster.health wait_for_status: 'yellow' + end + + should "find matching documents across multiple models" do + response = Elasticsearch::Model.search(%q<"greatest Episode" OR "greatest Image"^2>, [Episode, Image]) + + assert response.any?, "Response should not be empty: #{response.to_a.inspect}" + + assert_equal 2, response.results.size + assert_equal 2, response.records.size + + assert_instance_of Elasticsearch::Model::Response::Result, response.results.first + assert_instance_of Image, response.records.first + assert_instance_of Episode, response.records.last + + assert_equal 'The greatest Image', response.results[0].name + assert_equal 'The greatest Image', response.records[0].name + + assert_equal 'The greatest Episode', response.results[1].name + assert_equal 'The greatest Episode', response.records[1].name + end + end + end + + end + end + end +end diff --git a/elasticsearch-model/test/support/model.json b/elasticsearch-model/test/support/model.json new file mode 100644 index 0000000000..fcf3a64730 --- /dev/null +++ b/elasticsearch-model/test/support/model.json @@ -0,0 +1 @@ +{ "baz": "qux" } diff --git a/elasticsearch-model/test/support/model.yml b/elasticsearch-model/test/support/model.yml new file mode 100644 index 0000000000..ba8ca60f34 --- /dev/null +++ b/elasticsearch-model/test/support/model.yml @@ -0,0 +1,2 @@ +baz: + 'qux' diff --git a/elasticsearch-model/test/test_helper.rb b/elasticsearch-model/test/test_helper.rb new file mode 100644 index 0000000000..ff3a6d9354 --- /dev/null +++ b/elasticsearch-model/test/test_helper.rb @@ -0,0 +1,93 @@ +RUBY_1_8 = defined?(RUBY_VERSION) && RUBY_VERSION < '1.9' + +exit(0) if RUBY_1_8 + +require 'simplecov' and SimpleCov.start { add_filter "/test|test_/" } if ENV["COVERAGE"] + +# Register `at_exit` handler for integration tests shutdown. +# MUST be called before requiring `test/unit`. +at_exit { Elasticsearch::Test::IntegrationTestCase.__run_at_exit_hooks } + +puts '-'*80 + +if defined?(RUBY_VERSION) && RUBY_VERSION > '2.2' + require 'test-unit' + require 'mocha/test_unit' +else + require 'minitest/autorun' + require 'mocha/mini_test' +end + +require 'shoulda-context' + +require 'turn' unless ENV["TM_FILEPATH"] || ENV["NOTURN"] || defined?(RUBY_VERSION) && RUBY_VERSION > '2.2' + +require 'ansi' +require 'oj' + +require 'active_model' + +require 'kaminari' + +require 'elasticsearch/model' + +require 'elasticsearch/extensions/test/cluster' +require 'elasticsearch/extensions/test/startup_shutdown' + +module Elasticsearch + module Test + class IntegrationTestCase < ::Test::Unit::TestCase + extend Elasticsearch::Extensions::Test::StartupShutdown + + startup { Elasticsearch::Extensions::Test::Cluster.start(nodes: 1) if ENV['SERVER'] and not Elasticsearch::Extensions::Test::Cluster.running? } + shutdown { Elasticsearch::Extensions::Test::Cluster.stop if ENV['SERVER'] && started? } + context "IntegrationTest" do; should "noop on Ruby 1.8" do; end; end if RUBY_1_8 + + def setup + ActiveRecord::Base.establish_connection( :adapter => 'sqlite3', :database => ":memory:" ) + logger = ::Logger.new(STDERR) + logger.formatter = lambda { |s, d, p, m| "\e[2;36m#{m}\e[0m\n" } + ActiveRecord::Base.logger = logger unless ENV['QUIET'] + + ActiveRecord::LogSubscriber.colorize_logging = false + ActiveRecord::Migration.verbose = false + + tracer = ::Logger.new(STDERR) + tracer.formatter = lambda { |s, d, p, m| "#{m.gsub(/^.*$/) { |n| ' ' + n }.ansi(:faint)}\n" } + + Elasticsearch::Model.client = Elasticsearch::Client.new host: "localhost:#{(ENV['TEST_CLUSTER_PORT'] || 9250)}", + tracer: (ENV['QUIET'] ? nil : tracer) + end + end + end +end + +class Mongo + def self.setup! + begin + require 'mongoid' + session = Moped::Connection.new("localhost", 27017, 0.5) + session.connect + ENV['MONGODB_AVAILABLE'] = 'yes' + rescue LoadError, Moped::Errors::ConnectionFailure => e + $stderr.puts "MongoDB not installed or running: #{e}" + end + end + + def self.available? + !!ENV['MONGODB_AVAILABLE'] + end + + def self.connect_to(source) + $stderr.puts "Mongoid #{Mongoid::VERSION}", '-'*80 + + logger = ::Logger.new($stderr) + logger.formatter = lambda { |s, d, p, m| " #{m.ansi(:faint, :cyan)}\n" } + logger.level = ::Logger::DEBUG + + Mongoid.logger = logger unless ENV['QUIET'] + Moped.logger = logger unless ENV['QUIET'] + + Mongoid.connect_to source + end +end diff --git a/elasticsearch-model/test/unit/adapter_active_record_test.rb b/elasticsearch-model/test/unit/adapter_active_record_test.rb new file mode 100644 index 0000000000..335e3bd10e --- /dev/null +++ b/elasticsearch-model/test/unit/adapter_active_record_test.rb @@ -0,0 +1,157 @@ +require 'test_helper' + +class Elasticsearch::Model::AdapterActiveRecordTest < Test::Unit::TestCase + context "Adapter ActiveRecord module: " do + class ::DummyClassForActiveRecord + RESPONSE = Struct.new('DummyActiveRecordResponse') do + def response + { 'hits' => {'hits' => [ {'_id' => 2}, {'_id' => 1} ]} } + end + end.new + + def response + RESPONSE + end + + def ids + [2, 1] + end + end + + RESPONSE = { 'hits' => { 'total' => 123, 'max_score' => 456, 'hits' => [] } } + + setup do + @records = [ stub(id: 1, inspect: ''), stub(id: 2, inspect: '') ] + @records.stubs(:load).returns(true) + @records.stubs(:exec_queries).returns(true) + end + + should "have the register condition" do + assert_not_nil Elasticsearch::Model::Adapter.adapters[Elasticsearch::Model::Adapter::ActiveRecord] + assert_equal false, Elasticsearch::Model::Adapter.adapters[Elasticsearch::Model::Adapter::ActiveRecord].call(DummyClassForActiveRecord) + end + + context "Records" do + setup do + DummyClassForActiveRecord.__send__ :include, Elasticsearch::Model::Adapter::ActiveRecord::Records + end + + should "have the implementation" do + assert_instance_of Module, Elasticsearch::Model::Adapter::ActiveRecord::Records + + instance = DummyClassForActiveRecord.new + instance.expects(:klass).returns(mock('class', primary_key: :some_key, where: @records)).at_least_once + + assert_equal @records, instance.records + end + + should "load the records" do + instance = DummyClassForActiveRecord.new + instance.expects(:records).returns(@records) + instance.load + end + + should "load the records with its submodels when using :includes" do + klass = mock('class', primary_key: :some_key, where: @records) + @records.expects(:includes).with([:submodel]).at_least_once + + instance = DummyClassForActiveRecord.new + instance.expects(:klass).returns(klass).at_least_once + instance.options[:includes] = [:submodel] + instance.records + end + + should "reorder the records based on hits order" do + @records.instance_variable_set(:@records, @records) + + instance = DummyClassForActiveRecord.new + instance.expects(:klass).returns(mock('class', primary_key: :some_key, where: @records)).at_least_once + + assert_equal [1, 2], @records. to_a.map(&:id) + assert_equal [2, 1], instance.records.to_a.map(&:id) + end + + should "not reorder records when SQL order is present" do + @records.instance_variable_set(:@records, @records) + + instance = DummyClassForActiveRecord.new + instance.expects(:klass).returns(stub('class', primary_key: :some_key, where: @records)).at_least_once + instance.records.expects(:order).returns(@records) + + assert_equal [2, 1], instance.records. to_a.map(&:id) + assert_equal [1, 2], instance.order(:foo).to_a.map(&:id) + end + end + + context "Callbacks" do + should "register hooks for automatically updating the index" do + DummyClassForActiveRecord.expects(:after_commit).times(3) + + Elasticsearch::Model::Adapter::ActiveRecord::Callbacks.included(DummyClassForActiveRecord) + end + end + + context "Importing" do + setup do + DummyClassForActiveRecord.__send__ :extend, Elasticsearch::Model::Adapter::ActiveRecord::Importing + end + + should "raise an exception when passing an invalid scope" do + assert_raise NoMethodError do + DummyClassForActiveRecord.__find_in_batches(scope: :not_found_method) do; end + end + end + + should "implement the __find_in_batches method" do + DummyClassForActiveRecord.expects(:find_in_batches).returns([]) + DummyClassForActiveRecord.__find_in_batches do; end + end + + should "limit the relation to a specific scope" do + DummyClassForActiveRecord.expects(:find_in_batches).returns([]) + DummyClassForActiveRecord.expects(:published).returns(DummyClassForActiveRecord) + + DummyClassForActiveRecord.__find_in_batches(scope: :published) do; end + end + + should "limit the relation to a specific query" do + DummyClassForActiveRecord.expects(:find_in_batches).returns([]) + DummyClassForActiveRecord.expects(:where).returns(DummyClassForActiveRecord) + + DummyClassForActiveRecord.__find_in_batches(query: -> { where(color: "red") }) do; end + end + + should "preprocess the batch if option provided" do + class << DummyClassForActiveRecord + # Updates/transforms the batch while fetching it from the database + # (eg. with information from an external system) + # + def update_batch(batch) + batch.collect { |b| b.to_s + '!' } + end + end + + DummyClassForActiveRecord.expects(:__find_in_batches).returns( [:a, :b] ) + + DummyClassForActiveRecord.__find_in_batches(preprocess: :update_batch) do |batch| + assert_same_elements ["a!", "b!"], batch + end + end + + context "when transforming models" do + setup do + @transform = DummyClassForActiveRecord.__transform + end + + should "provide an object that responds to #call" do + assert_respond_to @transform, :call + end + + should "provide default transformation" do + model = mock("model", id: 1, __elasticsearch__: stub(as_indexed_json: {})) + assert_equal @transform.call(model), { index: { _id: 1, data: {} } } + end + end + end + end +end diff --git a/elasticsearch-model/test/unit/adapter_default_test.rb b/elasticsearch-model/test/unit/adapter_default_test.rb new file mode 100644 index 0000000000..48edd205d5 --- /dev/null +++ b/elasticsearch-model/test/unit/adapter_default_test.rb @@ -0,0 +1,41 @@ +require 'test_helper' + +class Elasticsearch::Model::AdapterDefaultTest < Test::Unit::TestCase + context "Adapter default module" do + class ::DummyClassForDefaultAdapter; end + + should "have the default Records implementation" do + assert_instance_of Module, Elasticsearch::Model::Adapter::Default::Records + + DummyClassForDefaultAdapter.__send__ :include, Elasticsearch::Model::Adapter::Default::Records + + instance = DummyClassForDefaultAdapter.new + klass = mock('class', find: [1]) + instance.expects(:klass).returns(klass) + instance.records + end + + should "have the default Callbacks implementation" do + assert_instance_of Module, Elasticsearch::Model::Adapter::Default::Callbacks + end + + context "concerning abstract methods" do + setup do + DummyClassForDefaultAdapter.__send__ :include, Elasticsearch::Model::Adapter::Default::Importing + end + + should "have the default Importing implementation" do + assert_raise Elasticsearch::Model::NotImplemented do + DummyClassForDefaultAdapter.new.__find_in_batches + end + end + + should "have the default transform implementation" do + assert_raise Elasticsearch::Model::NotImplemented do + DummyClassForDefaultAdapter.new.__transform + end + end + end + + end +end diff --git a/elasticsearch-model/test/unit/adapter_mongoid_test.rb b/elasticsearch-model/test/unit/adapter_mongoid_test.rb new file mode 100644 index 0000000000..ca9b0d20bc --- /dev/null +++ b/elasticsearch-model/test/unit/adapter_mongoid_test.rb @@ -0,0 +1,104 @@ +require 'test_helper' + +class Elasticsearch::Model::AdapterMongoidTest < Test::Unit::TestCase + context "Adapter Mongoid module: " do + class ::DummyClassForMongoid + RESPONSE = Struct.new('DummyMongoidResponse') do + def response + { 'hits' => {'hits' => [ {'_id' => 2}, {'_id' => 1} ]} } + end + end.new + + def response + RESPONSE + end + + def ids + [2, 1] + end + end + + setup do + @records = [ stub(id: 1, inspect: ''), stub(id: 2, inspect: '') ] + ::Symbol.any_instance.stubs(:in).returns(@records) + end + + should "have the register condition" do + assert_not_nil Elasticsearch::Model::Adapter.adapters[Elasticsearch::Model::Adapter::Mongoid] + assert_equal false, Elasticsearch::Model::Adapter.adapters[Elasticsearch::Model::Adapter::Mongoid].call(DummyClassForMongoid) + end + + context "Records" do + setup do + DummyClassForMongoid.__send__ :include, Elasticsearch::Model::Adapter::Mongoid::Records + end + + should "have the implementation" do + assert_instance_of Module, Elasticsearch::Model::Adapter::Mongoid::Records + + instance = DummyClassForMongoid.new + instance.expects(:klass).returns(mock('class', where: @records)) + + assert_equal @records, instance.records + end + + should "reorder the records based on hits order" do + @records.instance_variable_set(:@records, @records) + + instance = DummyClassForMongoid.new + instance.expects(:klass).returns(mock('class', where: @records)) + + assert_equal [1, 2], @records. to_a.map(&:id) + assert_equal [2, 1], instance.records.to_a.map(&:id) + end + + should "not reorder records when SQL order is present" do + @records.instance_variable_set(:@records, @records) + + instance = DummyClassForMongoid.new + instance.expects(:klass).returns(stub('class', where: @records)).at_least_once + instance.records.expects(:asc).returns(@records) + + assert_equal [2, 1], instance.records.to_a.map(&:id) + assert_equal [1, 2], instance.asc.to_a.map(&:id) + end + end + + context "Callbacks" do + should "register hooks for automatically updating the index" do + DummyClassForMongoid.expects(:after_create) + DummyClassForMongoid.expects(:after_update) + DummyClassForMongoid.expects(:after_destroy) + + Elasticsearch::Model::Adapter::Mongoid::Callbacks.included(DummyClassForMongoid) + end + end + + context "Importing" do + should "implement the __find_in_batches method" do + relation = mock() + relation.stubs(:no_timeout).returns([]) + DummyClassForMongoid.expects(:all).returns(relation) + + DummyClassForMongoid.__send__ :extend, Elasticsearch::Model::Adapter::Mongoid::Importing + DummyClassForMongoid.__find_in_batches do; end + end + + context "when transforming models" do + setup do + @transform = DummyClassForMongoid.__transform + end + + should "provide an object that responds to #call" do + assert_respond_to @transform, :call + end + + should "provide basic transformation" do + model = mock("model", id: 1, as_indexed_json: {}) + assert_equal @transform.call(model), { index: { _id: "1", data: {} } } + end + end + end + + end +end diff --git a/elasticsearch-model/test/unit/adapter_multiple_test.rb b/elasticsearch-model/test/unit/adapter_multiple_test.rb new file mode 100644 index 0000000000..b848286fbd --- /dev/null +++ b/elasticsearch-model/test/unit/adapter_multiple_test.rb @@ -0,0 +1,106 @@ +require 'test_helper' + +class Elasticsearch::Model::MultipleTest < Test::Unit::TestCase + + context "Adapter for multiple models" do + + class ::DummyOne + include Elasticsearch::Model + + index_name 'dummy' + document_type 'dummy_one' + + def self.find(ids) + ids.map { |id| new(id) } + end + + attr_reader :id + + def initialize(id) + @id = id.to_i + end + end + + module ::Namespace + class DummyTwo + include Elasticsearch::Model + + index_name 'dummy' + document_type 'dummy_two' + + def self.find(ids) + ids.map { |id| new(id) } + end + + attr_reader :id + + def initialize(id) + @id = id.to_i + end + end + end + + class ::DummyTwo + include Elasticsearch::Model + + index_name 'other_index' + document_type 'dummy_two' + + def self.find(ids) + ids.map { |id| new(id) } + end + + attr_reader :id + + def initialize(id) + @id = id.to_i + end + end + + HITS = [{_index: 'dummy', + _type: 'dummy_two', + _id: '2', + }, { + _index: 'dummy', + _type: 'dummy_one', + _id: '2', + }, { + _index: 'other_index', + _type: 'dummy_two', + _id: '1', + }, { + _index: 'dummy', + _type: 'dummy_two', + _id: '1', + }, { + _index: 'dummy', + _type: 'dummy_one', + _id: '3'}] + + setup do + @multimodel = Elasticsearch::Model::Multimodel.new(DummyOne, DummyTwo, Namespace::DummyTwo) + end + + context "when returning records" do + setup do + @multimodel.class.send :include, Elasticsearch::Model::Adapter::Multiple::Records + @multimodel.expects(:response).at_least_once.returns(stub(response: { 'hits' => { 'hits' => HITS } })) + end + + should "keep the order from response" do + assert_instance_of Module, Elasticsearch::Model::Adapter::Multiple::Records + records = @multimodel.records + + assert_equal 5, records.count + + assert_kind_of ::Namespace::DummyTwo, records[0] + assert_kind_of ::DummyOne, records[1] + assert_kind_of ::DummyTwo, records[2] + assert_kind_of ::Namespace::DummyTwo, records[3] + assert_kind_of ::DummyOne, records[4] + + assert_equal [2, 2, 1, 1, 3], records.map(&:id) + end + end + end +end diff --git a/elasticsearch-model/test/unit/adapter_test.rb b/elasticsearch-model/test/unit/adapter_test.rb new file mode 100644 index 0000000000..71b4e7cea3 --- /dev/null +++ b/elasticsearch-model/test/unit/adapter_test.rb @@ -0,0 +1,69 @@ +require 'test_helper' + +class Elasticsearch::Model::AdapterTest < Test::Unit::TestCase + context "Adapter module" do + class ::DummyAdapterClass; end + class ::DummyAdapterClassWithAdapter; end + class ::DummyAdapter + Records = Module.new + Callbacks = Module.new + Importing = Module.new + end + + should "return an Adapter instance" do + assert_instance_of Elasticsearch::Model::Adapter::Adapter, + Elasticsearch::Model::Adapter.from_class(DummyAdapterClass) + end + + should "return a list of adapters" do + Elasticsearch::Model::Adapter::Adapter.expects(:adapters) + Elasticsearch::Model::Adapter.adapters + end + + should "register an adapter" do + begin + Elasticsearch::Model::Adapter::Adapter.expects(:register) + Elasticsearch::Model::Adapter.register(:foo, lambda { |c| false }) + ensure + Elasticsearch::Model::Adapter::Adapter.instance_variable_set(:@adapters, {}) + end + end + end + + context "Adapter class" do + should "register an adapter" do + begin + Elasticsearch::Model::Adapter::Adapter.register(:foo, lambda { |c| false }) + assert Elasticsearch::Model::Adapter::Adapter.adapters[:foo] + ensure + Elasticsearch::Model::Adapter::Adapter.instance_variable_set(:@adapters, {}) + end + end + + should "return the default adapter" do + adapter = Elasticsearch::Model::Adapter::Adapter.new(DummyAdapterClass) + assert_equal Elasticsearch::Model::Adapter::Default, adapter.adapter + end + + should "return a specific adapter" do + Elasticsearch::Model::Adapter::Adapter.register(DummyAdapter, + lambda { |c| c == DummyAdapterClassWithAdapter }) + + adapter = Elasticsearch::Model::Adapter::Adapter.new(DummyAdapterClassWithAdapter) + assert_equal DummyAdapter, adapter.adapter + end + + should "return the modules" do + assert_nothing_raised do + Elasticsearch::Model::Adapter::Adapter.register(DummyAdapter, + lambda { |c| c == DummyAdapterClassWithAdapter }) + + adapter = Elasticsearch::Model::Adapter::Adapter.new(DummyAdapterClassWithAdapter) + + assert_instance_of Module, adapter.records_mixin + assert_instance_of Module, adapter.callbacks_mixin + assert_instance_of Module, adapter.importing_mixin + end + end + end +end diff --git a/elasticsearch-model/test/unit/callbacks_test.rb b/elasticsearch-model/test/unit/callbacks_test.rb new file mode 100644 index 0000000000..95617a414e --- /dev/null +++ b/elasticsearch-model/test/unit/callbacks_test.rb @@ -0,0 +1,31 @@ +require 'test_helper' + +class Elasticsearch::Model::CallbacksTest < Test::Unit::TestCase + context "Callbacks module" do + class ::DummyCallbacksModel + end + + module DummyCallbacksAdapter + module CallbacksMixin + end + + def callbacks_mixin + CallbacksMixin + end; module_function :callbacks_mixin + end + + should "include the callbacks mixin from adapter" do + Elasticsearch::Model::Adapter.expects(:from_class) + .with(DummyCallbacksModel) + .returns(DummyCallbacksAdapter) + + ::DummyCallbacksModel.expects(:__send__).with do |method, parameter| + assert_equal :include, method + assert_equal DummyCallbacksAdapter::CallbacksMixin, parameter + true + end + + Elasticsearch::Model::Callbacks.included(DummyCallbacksModel) + end + end +end diff --git a/elasticsearch-model/test/unit/client_test.rb b/elasticsearch-model/test/unit/client_test.rb new file mode 100644 index 0000000000..315a3ab44a --- /dev/null +++ b/elasticsearch-model/test/unit/client_test.rb @@ -0,0 +1,27 @@ +require 'test_helper' + +class Elasticsearch::Model::ClientTest < Test::Unit::TestCase + context "Client module" do + class ::DummyClientModel + extend Elasticsearch::Model::Client::ClassMethods + include Elasticsearch::Model::Client::InstanceMethods + end + + should "have the default client method" do + assert_instance_of Elasticsearch::Transport::Client, DummyClientModel.client + assert_instance_of Elasticsearch::Transport::Client, DummyClientModel.new.client + end + + should "set the client for the model" do + DummyClientModel.client = 'foobar' + assert_equal 'foobar', DummyClientModel.client + assert_equal 'foobar', DummyClientModel.new.client + end + + should "set the client for a model instance" do + instance = DummyClientModel.new + instance.client = 'moobam' + assert_equal 'moobam', instance.client + end + end +end diff --git a/elasticsearch-model/test/unit/importing_test.rb b/elasticsearch-model/test/unit/importing_test.rb new file mode 100644 index 0000000000..6f739acecc --- /dev/null +++ b/elasticsearch-model/test/unit/importing_test.rb @@ -0,0 +1,203 @@ +require 'test_helper' + +class Elasticsearch::Model::ImportingTest < Test::Unit::TestCase + context "Importing module" do + class ::DummyImportingModel + end + + module ::DummyImportingAdapter + module ImportingMixin + def __find_in_batches(options={}, &block) + yield if block_given? + end + def __transform + lambda {|a|} + end + end + + def importing_mixin + ImportingMixin + end; module_function :importing_mixin + end + + should "include methods from the module and adapter" do + Elasticsearch::Model::Adapter.expects(:from_class) + .with(DummyImportingModel) + .returns(DummyImportingAdapter) + + DummyImportingModel.__send__ :include, Elasticsearch::Model::Importing + + assert_respond_to DummyImportingModel, :import + assert_respond_to DummyImportingModel, :__find_in_batches + end + + should "call the client when importing" do + Elasticsearch::Model::Adapter.expects(:from_class) + .with(DummyImportingModel) + .returns(DummyImportingAdapter) + + DummyImportingModel.__send__ :include, Elasticsearch::Model::Importing + + client = mock('client') + client.expects(:bulk).returns({'items' => []}) + + DummyImportingModel.expects(:client).returns(client) + DummyImportingModel.expects(:index_name).returns('foo') + DummyImportingModel.expects(:document_type).returns('foo') + DummyImportingModel.stubs(:index_exists?).returns(true) + DummyImportingModel.stubs(:__batch_to_bulk) + assert_equal 0, DummyImportingModel.import + end + + should "return the number of errors" do + Elasticsearch::Model::Adapter.expects(:from_class) + .with(DummyImportingModel) + .returns(DummyImportingAdapter) + + DummyImportingModel.__send__ :include, Elasticsearch::Model::Importing + + client = mock('client') + client.expects(:bulk).returns({'items' => [ {'index' => {}}, {'index' => {'error' => 'FAILED'}} ]}) + + DummyImportingModel.stubs(:client).returns(client) + DummyImportingModel.stubs(:index_name).returns('foo') + DummyImportingModel.stubs(:document_type).returns('foo') + DummyImportingModel.stubs(:index_exists?).returns(true) + DummyImportingModel.stubs(:__batch_to_bulk) + + assert_equal 1, DummyImportingModel.import + end + + should "return an array of error elements" do + Elasticsearch::Model::Adapter.expects(:from_class) + .with(DummyImportingModel) + .returns(DummyImportingAdapter) + + DummyImportingModel.__send__ :include, Elasticsearch::Model::Importing + + client = mock('client') + client.expects(:bulk).returns({'items' => [ {'index' => {}}, {'index' => {'error' => 'FAILED'}} ]}) + + DummyImportingModel.stubs(:client).returns(client) + DummyImportingModel.stubs(:index_name).returns('foo') + DummyImportingModel.stubs(:document_type).returns('foo') + DummyImportingModel.stubs(:index_exists?).returns(true) + DummyImportingModel.stubs(:__batch_to_bulk) + + assert_equal [{'index' => {'error' => 'FAILED'}}], DummyImportingModel.import(return: 'errors') + end + + should "yield the response" do + Elasticsearch::Model::Adapter.expects(:from_class) + .with(DummyImportingModel) + .returns(DummyImportingAdapter) + + DummyImportingModel.__send__ :include, Elasticsearch::Model::Importing + + client = mock('client') + client.expects(:bulk).returns({'items' => [ {'index' => {}}, {'index' => {'error' => 'FAILED'}} ]}) + + DummyImportingModel.stubs(:client).returns(client) + DummyImportingModel.stubs(:index_name).returns('foo') + DummyImportingModel.stubs(:document_type).returns('foo') + DummyImportingModel.stubs(:index_exists?).returns(true) + DummyImportingModel.stubs(:__batch_to_bulk) + + DummyImportingModel.import do |response| + assert_equal 2, response['items'].size + end + end + + context "when the index does not exist" do + should "raise an exception" do + Elasticsearch::Model::Adapter.expects(:from_class) + .with(DummyImportingModel) + .returns(DummyImportingAdapter) + + DummyImportingModel.__send__ :include, Elasticsearch::Model::Importing + + DummyImportingModel.expects(:index_name).returns('foo') + DummyImportingModel.expects(:document_type).returns('foo') + DummyImportingModel.expects(:index_exists?).returns(false) + + assert_raise ArgumentError do + DummyImportingModel.import + end + end + end + + context "with the force option" do + should "delete and create the index" do + DummyImportingModel.expects(:__find_in_batches).with do |options| + assert_equal 'bar', options[:foo] + assert_nil options[:force] + true + end + + DummyImportingModel.expects(:create_index!).with do |options| + assert_equal true, options[:force] + true + end + + DummyImportingModel.expects(:index_name).returns('foo') + DummyImportingModel.expects(:document_type).returns('foo') + + DummyImportingModel.import force: true, foo: 'bar' + end + end + + should "allow passing a different index / type" do + Elasticsearch::Model::Adapter.expects(:from_class) + .with(DummyImportingModel) + .returns(DummyImportingAdapter) + + DummyImportingModel.__send__ :include, Elasticsearch::Model::Importing + + client = mock('client') + + client + .expects(:bulk) + .with do |options| + assert_equal 'my-new-index', options[:index] + assert_equal 'my-other-type', options[:type] + true + end + .returns({'items' => [ {'index' => {} }]}) + + DummyImportingModel.stubs(:client).returns(client) + DummyImportingModel.stubs(:index_exists?).returns(true) + DummyImportingModel.stubs(:__batch_to_bulk) + + DummyImportingModel.import index: 'my-new-index', type: 'my-other-type' + end + + should "use the default transform from adapter" do + client = mock('client', bulk: {'items' => []}) + transform = lambda {|a|} + + DummyImportingModel.stubs(:client).returns(client) + DummyImportingModel.stubs(:index_exists?).returns(true) + DummyImportingModel.expects(:__transform).returns(transform) + DummyImportingModel.expects(:__batch_to_bulk).with(anything, transform) + + DummyImportingModel.import index: 'foo', type: 'bar' + end + + should "use the transformer from options" do + client = mock('client', bulk: {'items' => []}) + transform = lambda {|a|} + + DummyImportingModel.stubs(:client).returns(client) + DummyImportingModel.stubs(:index_exists?).returns(true) + DummyImportingModel.expects(:__batch_to_bulk).with(anything, transform) + + DummyImportingModel.import index: 'foo', type: 'bar', transform: transform + end + + should "raise an ArgumentError if transform doesn't respond to the call method" do + assert_raise ArgumentError do + DummyImportingModel.import index: 'foo', type: 'bar', transform: "not_callable" + end + end + end +end diff --git a/elasticsearch-model/test/unit/indexing_test.rb b/elasticsearch-model/test/unit/indexing_test.rb new file mode 100644 index 0000000000..a52603a1ec --- /dev/null +++ b/elasticsearch-model/test/unit/indexing_test.rb @@ -0,0 +1,650 @@ +require 'test_helper' + +class Elasticsearch::Model::IndexingTest < Test::Unit::TestCase + context "Indexing module: " do + class ::DummyIndexingModel + extend ActiveModel::Naming + extend Elasticsearch::Model::Naming::ClassMethods + extend Elasticsearch::Model::Indexing::ClassMethods + + def self.foo + 'bar' + end + end + + class NotFound < Exception; end + + context "Settings class" do + should "be convertible to hash" do + hash = { foo: 'bar' } + settings = Elasticsearch::Model::Indexing::Settings.new hash + assert_equal hash, settings.to_hash + assert_equal settings.to_hash, settings.as_json + end + end + + context "Settings method" do + should "initialize the index settings" do + assert_instance_of Elasticsearch::Model::Indexing::Settings, DummyIndexingModel.settings + end + + should "update and return the index settings from a hash" do + DummyIndexingModel.settings foo: 'boo' + DummyIndexingModel.settings bar: 'bam' + + assert_equal( {foo: 'boo', bar: 'bam'}, DummyIndexingModel.settings.to_hash) + end + + should "update and return the index settings from a yml file" do + DummyIndexingModel.settings File.open("test/support/model.yml") + DummyIndexingModel.settings bar: 'bam' + + assert_equal( {foo: 'boo', bar: 'bam', 'baz' => 'qux'}, DummyIndexingModel.settings.to_hash) + end + + should "update and return the index settings from a json file" do + DummyIndexingModel.settings File.open("test/support/model.json") + DummyIndexingModel.settings bar: 'bam' + + assert_equal( {foo: 'boo', bar: 'bam', 'baz' => 'qux'}, DummyIndexingModel.settings.to_hash) + end + + should "evaluate the block" do + DummyIndexingModel.expects(:foo) + + DummyIndexingModel.settings do + foo + end + end + end + + context "Mappings class" do + should "initialize the index mappings" do + assert_instance_of Elasticsearch::Model::Indexing::Mappings, DummyIndexingModel.mappings + end + + should "raise an exception when not passed type" do + assert_raise ArgumentError do + Elasticsearch::Model::Indexing::Mappings.new + end + end + + should "be convertible to hash" do + mappings = Elasticsearch::Model::Indexing::Mappings.new :mytype, { foo: 'bar' } + assert_equal( { :mytype => { foo: 'bar', :properties => {} } }, mappings.to_hash ) + assert_equal mappings.to_hash, mappings.as_json + end + + should "define properties" do + mappings = Elasticsearch::Model::Indexing::Mappings.new :mytype + assert_respond_to mappings, :indexes + + mappings.indexes :foo, { type: 'boolean', include_in_all: false } + assert_equal 'boolean', mappings.to_hash[:mytype][:properties][:foo][:type] + end + + should "define type as string by default" do + mappings = Elasticsearch::Model::Indexing::Mappings.new :mytype + + mappings.indexes :bar, {} + assert_equal 'string', mappings.to_hash[:mytype][:properties][:bar][:type] + end + + should "define multiple fields" do + mappings = Elasticsearch::Model::Indexing::Mappings.new :mytype + + mappings.indexes :foo_1, type: 'string' do + indexes :raw, analyzer: 'keyword' + end + + mappings.indexes :foo_2, type: 'multi_field' do + indexes :raw, analyzer: 'keyword' + end + + assert_equal 'string', mappings.to_hash[:mytype][:properties][:foo_1][:type] + assert_equal 'string', mappings.to_hash[:mytype][:properties][:foo_1][:fields][:raw][:type] + assert_equal 'keyword', mappings.to_hash[:mytype][:properties][:foo_1][:fields][:raw][:analyzer] + assert_nil mappings.to_hash[:mytype][:properties][:foo_1][:properties] + + assert_equal 'multi_field', mappings.to_hash[:mytype][:properties][:foo_2][:type] + assert_equal 'string', mappings.to_hash[:mytype][:properties][:foo_2][:fields][:raw][:type] + assert_equal 'keyword', mappings.to_hash[:mytype][:properties][:foo_2][:fields][:raw][:analyzer] + assert_nil mappings.to_hash[:mytype][:properties][:foo_2][:properties] + end + + should "define embedded properties" do + mappings = Elasticsearch::Model::Indexing::Mappings.new :mytype + + mappings.indexes :foo do + indexes :bar + end + + mappings.indexes :foo_object, type: 'object' do + indexes :bar + end + + mappings.indexes :foo_nested, type: 'nested' do + indexes :bar + end + + mappings.indexes :foo_nested_as_symbol, type: :nested do + indexes :bar + end + + # Object is the default when `type` is missing and there's a block passed + # + assert_equal 'object', mappings.to_hash[:mytype][:properties][:foo][:type] + assert_equal 'string', mappings.to_hash[:mytype][:properties][:foo][:properties][:bar][:type] + assert_nil mappings.to_hash[:mytype][:properties][:foo][:fields] + + assert_equal 'object', mappings.to_hash[:mytype][:properties][:foo_object][:type] + assert_equal 'string', mappings.to_hash[:mytype][:properties][:foo_object][:properties][:bar][:type] + assert_nil mappings.to_hash[:mytype][:properties][:foo_object][:fields] + + assert_equal 'nested', mappings.to_hash[:mytype][:properties][:foo_nested][:type] + assert_equal 'string', mappings.to_hash[:mytype][:properties][:foo_nested][:properties][:bar][:type] + assert_nil mappings.to_hash[:mytype][:properties][:foo_nested][:fields] + + assert_equal :nested, mappings.to_hash[:mytype][:properties][:foo_nested_as_symbol][:type] + assert_not_nil mappings.to_hash[:mytype][:properties][:foo_nested_as_symbol][:properties] + assert_nil mappings.to_hash[:mytype][:properties][:foo_nested_as_symbol][:fields] + end + end + + context "Mappings method" do + should "initialize the index mappings" do + assert_instance_of Elasticsearch::Model::Indexing::Mappings, DummyIndexingModel.mappings + end + + should "update and return the index mappings" do + DummyIndexingModel.mappings foo: 'boo' + DummyIndexingModel.mappings bar: 'bam' + assert_equal( { dummy_indexing_model: { foo: "boo", bar: "bam", properties: {} } }, + DummyIndexingModel.mappings.to_hash ) + end + + should "evaluate the block" do + DummyIndexingModel.mappings.expects(:indexes).with(:foo).returns(true) + + DummyIndexingModel.mappings do + indexes :foo + end + end + end + + context "Instance methods" do + class ::DummyIndexingModelWithCallbacks + extend Elasticsearch::Model::Indexing::ClassMethods + include Elasticsearch::Model::Indexing::InstanceMethods + + def self.before_save(&block) + (@callbacks ||= {})[block.hash] = block + end + + def changed_attributes; [:foo]; end + + def changes + {:foo => ['One', 'Two']} + end + end + + class ::DummyIndexingModelWithCallbacksAndCustomAsIndexedJson + extend Elasticsearch::Model::Indexing::ClassMethods + include Elasticsearch::Model::Indexing::InstanceMethods + + def self.before_save(&block) + (@callbacks ||= {})[block.hash] = block + end + + def changed_attributes; [:foo, :bar]; end + + def changes + {:foo => ['A', 'B'], :bar => ['C', 'D']} + end + + def as_indexed_json(options={}) + { :foo => 'B' } + end + end + + should "register before_save callback when included" do + ::DummyIndexingModelWithCallbacks.expects(:before_save).returns(true) + ::DummyIndexingModelWithCallbacks.__send__ :include, Elasticsearch::Model::Indexing::InstanceMethods + end + + should "set the @__changed_attributes variable before save" do + instance = ::DummyIndexingModelWithCallbacks.new + instance.expects(:instance_variable_set).with do |name, value| + assert_equal :@__changed_attributes, name + assert_equal({foo: 'Two'}, value) + true + end + + ::DummyIndexingModelWithCallbacks.__send__ :include, Elasticsearch::Model::Indexing::InstanceMethods + + ::DummyIndexingModelWithCallbacks.instance_variable_get(:@callbacks).each do |n,b| + instance.instance_eval(&b) + end + end + + should "have the index_document method" do + client = mock('client') + instance = ::DummyIndexingModelWithCallbacks.new + + client.expects(:index).with do |payload| + assert_equal 'foo', payload[:index] + assert_equal 'bar', payload[:type] + assert_equal '1', payload[:id] + assert_equal 'JSON', payload[:body] + true + end + + instance.expects(:client).returns(client) + instance.expects(:as_indexed_json).returns('JSON') + instance.expects(:index_name).returns('foo') + instance.expects(:document_type).returns('bar') + instance.expects(:id).returns('1') + + instance.index_document + end + + should "pass extra options to the index_document method to client.index" do + client = mock('client') + instance = ::DummyIndexingModelWithCallbacks.new + + client.expects(:index).with do |payload| + assert_equal 'A', payload[:parent] + true + end + + instance.expects(:client).returns(client) + instance.expects(:as_indexed_json).returns('JSON') + instance.expects(:index_name).returns('foo') + instance.expects(:document_type).returns('bar') + instance.expects(:id).returns('1') + + instance.index_document(parent: 'A') + end + + should "have the delete_document method" do + client = mock('client') + instance = ::DummyIndexingModelWithCallbacks.new + + client.expects(:delete).with do |payload| + assert_equal 'foo', payload[:index] + assert_equal 'bar', payload[:type] + assert_equal '1', payload[:id] + true + end + + instance.expects(:client).returns(client) + instance.expects(:index_name).returns('foo') + instance.expects(:document_type).returns('bar') + instance.expects(:id).returns('1') + + instance.delete_document() + end + + should "pass extra options to the delete_document method to client.delete" do + client = mock('client') + instance = ::DummyIndexingModelWithCallbacks.new + + client.expects(:delete).with do |payload| + assert_equal 'A', payload[:parent] + true + end + + instance.expects(:client).returns(client) + instance.expects(:id).returns('1') + instance.expects(:index_name).returns('foo') + instance.expects(:document_type).returns('bar') + + instance.delete_document(parent: 'A') + end + + should "update the document by re-indexing when no changes are present" do + client = mock('client') + instance = ::DummyIndexingModelWithCallbacks.new + + # Reset the fake `changes` + instance.instance_variable_set(:@__changed_attributes, nil) + + instance.expects(:index_document) + instance.update_document + end + + should "update the document by partial update when changes are present" do + client = mock('client') + instance = ::DummyIndexingModelWithCallbacks.new + + # Set the fake `changes` hash + instance.instance_variable_set(:@__changed_attributes, {foo: 'bar'}) + + client.expects(:update).with do |payload| + assert_equal 'foo', payload[:index] + assert_equal 'bar', payload[:type] + assert_equal '1', payload[:id] + assert_equal({foo: 'bar'}, payload[:body][:doc]) + true + end + + instance.expects(:client).returns(client) + instance.expects(:index_name).returns('foo') + instance.expects(:document_type).returns('bar') + instance.expects(:id).returns('1') + + instance.update_document + end + + should "exclude attributes not contained in custom as_indexed_json during partial update" do + client = mock('client') + instance = ::DummyIndexingModelWithCallbacksAndCustomAsIndexedJson.new + + # Set the fake `changes` hash + instance.instance_variable_set(:@__changed_attributes, {'foo' => 'B', 'bar' => 'D' }) + + client.expects(:update).with do |payload| + assert_equal({:foo => 'B'}, payload[:body][:doc]) + true + end + + instance.expects(:client).returns(client) + instance.expects(:index_name).returns('foo') + instance.expects(:document_type).returns('bar') + instance.expects(:id).returns('1') + + instance.update_document + end + + should "get attributes from as_indexed_json during partial update" do + client = mock('client') + instance = ::DummyIndexingModelWithCallbacksAndCustomAsIndexedJson.new + + instance.instance_variable_set(:@__changed_attributes, { 'foo' => { 'bar' => 'BAR'} }) + # Overload as_indexed_json + instance.expects(:as_indexed_json).returns({ 'foo' => 'BAR' }) + + client.expects(:update).with do |payload| + assert_equal({'foo' => 'BAR'}, payload[:body][:doc]) + true + end + + instance.expects(:client).returns(client) + instance.expects(:index_name).returns('foo') + instance.expects(:document_type).returns('bar') + instance.expects(:id).returns('1') + + instance.update_document + end + + should "update only the specific attributes" do + client = mock('client') + instance = ::DummyIndexingModelWithCallbacks.new + + # Set the fake `changes` hash + instance.instance_variable_set(:@__changed_attributes, {author: 'john'}) + + client.expects(:update).with do |payload| + assert_equal 'foo', payload[:index] + assert_equal 'bar', payload[:type] + assert_equal '1', payload[:id] + assert_equal({title: 'green'}, payload[:body][:doc]) + true + end + + instance.expects(:client).returns(client) + instance.expects(:index_name).returns('foo') + instance.expects(:document_type).returns('bar') + instance.expects(:id).returns('1') + + instance.update_document_attributes title: "green" + end + + should "pass options to the update_document_attributes method" do + client = mock('client') + instance = ::DummyIndexingModelWithCallbacks.new + + client.expects(:update).with do |payload| + assert_equal 'foo', payload[:index] + assert_equal 'bar', payload[:type] + assert_equal '1', payload[:id] + assert_equal({title: 'green'}, payload[:body][:doc]) + assert_equal true, payload[:refresh] + true + end + + instance.expects(:client).returns(client) + instance.expects(:index_name).returns('foo') + instance.expects(:document_type).returns('bar') + instance.expects(:id).returns('1') + + instance.update_document_attributes( { title: "green" }, { refresh: true } ) + end + end + + context "Checking for index existence" do + context "the index exists" do + should "return true" do + indices = mock('indices', exists: true) + client = stub('client', indices: indices) + + DummyIndexingModelForRecreate.expects(:client).returns(client).at_least_once + + assert_equal true, DummyIndexingModelForRecreate.index_exists? + end + end + + context "the index does not exists" do + should "return false" do + indices = mock('indices', exists: false) + client = stub('client', indices: indices) + + DummyIndexingModelForRecreate.expects(:client).returns(client).at_least_once + + assert_equal false, DummyIndexingModelForRecreate.index_exists? + end + end + + context "the indices raises" do + should "return false" do + client = stub('client') + client.expects(:indices).raises(StandardError) + + DummyIndexingModelForRecreate.expects(:client).returns(client).at_least_once + + assert_equal false, DummyIndexingModelForRecreate.index_exists? + end + end + + context "the indices raises" do + should "return false" do + indices = stub('indices') + client = stub('client') + client.expects(:indices).returns(indices) + + indices.expects(:exists).raises(StandardError) + + DummyIndexingModelForRecreate.expects(:client).returns(client).at_least_once + + assert_equal false, DummyIndexingModelForRecreate.index_exists? + end + end + end + + context "Re-creating the index" do + class ::DummyIndexingModelForRecreate + extend ActiveModel::Naming + extend Elasticsearch::Model::Naming::ClassMethods + extend Elasticsearch::Model::Indexing::ClassMethods + + settings index: { number_of_shards: 1 } do + mappings do + indexes :foo, analyzer: 'keyword' + end + end + end + + should "delete the index without raising exception when the index is not found" do + client = stub('client') + indices = stub('indices') + client.stubs(:indices).returns(indices) + + indices.expects(:delete).returns({}).then.raises(NotFound).at_least_once + + DummyIndexingModelForRecreate.expects(:client).returns(client).at_least_once + + assert_nothing_raised { DummyIndexingModelForRecreate.delete_index! force: true } + end + + should "raise an exception without the force option" do + client = stub('client') + indices = stub('indices') + client.stubs(:indices).returns(indices) + + indices.expects(:delete).raises(NotFound) + + DummyIndexingModelForRecreate.expects(:client).returns(client) + + assert_raise(NotFound) { DummyIndexingModelForRecreate.delete_index! } + end + + should "raise a regular exception when deleting the index" do + client = stub('client') + + indices = stub('indices') + indices.expects(:delete).raises(Exception) + client.stubs(:indices).returns(indices) + + DummyIndexingModelForRecreate.expects(:client).returns(client) + + assert_raise(Exception) { DummyIndexingModelForRecreate.delete_index! force: true } + end + + should "create the index with correct settings and mappings when it doesn't exist" do + client = stub('client') + indices = stub('indices') + client.stubs(:indices).returns(indices) + + indices.expects(:create).with do |payload| + assert_equal 'dummy_indexing_model_for_recreates', payload[:index] + assert_equal 1, payload[:body][:settings][:index][:number_of_shards] + assert_equal 'keyword', payload[:body][:mappings][:dummy_indexing_model_for_recreate][:properties][:foo][:analyzer] + true + end.returns({}) + + DummyIndexingModelForRecreate.expects(:index_exists?).returns(false) + DummyIndexingModelForRecreate.expects(:client).returns(client).at_least_once + + assert_nothing_raised { DummyIndexingModelForRecreate.create_index! } + end + + should "not create the index when it exists" do + client = stub('client') + indices = stub('indices') + client.stubs(:indices).returns(indices) + + indices.expects(:create).never + + DummyIndexingModelForRecreate.expects(:index_exists?).returns(true) + DummyIndexingModelForRecreate.expects(:client).returns(client).never + + assert_nothing_raised { DummyIndexingModelForRecreate.create_index! } + end + + should "raise exception during index creation" do + client = stub('client') + indices = stub('indices') + client.stubs(:indices).returns(indices) + + indices.expects(:delete).returns({}) + indices.expects(:create).raises(Exception).at_least_once + + DummyIndexingModelForRecreate.expects(:index_exists?).returns(false) + DummyIndexingModelForRecreate.expects(:client).returns(client).at_least_once + + assert_raise(Exception) { DummyIndexingModelForRecreate.create_index! force: true } + end + + should "delete the index first with the force option" do + client = stub('client') + indices = stub('indices') + client.stubs(:indices).returns(indices) + + indices.expects(:delete).returns({}) + indices.expects(:create).returns({}).at_least_once + + DummyIndexingModelForRecreate.expects(:index_exists?).returns(false) + DummyIndexingModelForRecreate.expects(:client).returns(client).at_least_once + + assert_nothing_raised do + DummyIndexingModelForRecreate.create_index! force: true + end + end + + should "refresh the index without raising exception with the force option" do + client = stub('client') + indices = stub('indices') + client.stubs(:indices).returns(indices) + + indices.expects(:refresh).returns({}).then.raises(NotFound).at_least_once + + DummyIndexingModelForRecreate.expects(:client).returns(client).at_least_once + + assert_nothing_raised { DummyIndexingModelForRecreate.refresh_index! force: true } + end + + should "raise a regular exception when refreshing the index" do + client = stub('client') + indices = stub('indices') + client.stubs(:indices).returns(indices) + + indices.expects(:refresh).returns({}).then.raises(Exception).at_least_once + + DummyIndexingModelForRecreate.expects(:client).returns(client).at_least_once + + assert_nothing_raised { DummyIndexingModelForRecreate.refresh_index! force: true } + end + + context "with a custom index name" do + setup do + @client = stub('client') + @indices = stub('indices') + @client.stubs(:indices).returns(@indices) + DummyIndexingModelForRecreate.expects(:client).returns(@client).at_least_once + end + + should "create the custom index" do + @indices.expects(:create).with do |arguments| + assert_equal 'custom-foo', arguments[:index] + true + end + DummyIndexingModelForRecreate.expects(:index_exists?).with do |arguments| + assert_equal 'custom-foo', arguments[:index] + true + end + + DummyIndexingModelForRecreate.create_index! index: 'custom-foo' + end + + should "delete the custom index" do + @indices.expects(:delete).with do |arguments| + assert_equal 'custom-foo', arguments[:index] + true + end + + DummyIndexingModelForRecreate.delete_index! index: 'custom-foo' + end + + should "refresh the custom index" do + @indices.expects(:refresh).with do |arguments| + assert_equal 'custom-foo', arguments[:index] + true + end + + DummyIndexingModelForRecreate.refresh_index! index: 'custom-foo' + end + end + end + + end +end diff --git a/elasticsearch-model/test/unit/module_test.rb b/elasticsearch-model/test/unit/module_test.rb new file mode 100644 index 0000000000..a429b3d11f --- /dev/null +++ b/elasticsearch-model/test/unit/module_test.rb @@ -0,0 +1,57 @@ +require 'test_helper' + +class Elasticsearch::Model::ModuleTest < Test::Unit::TestCase + context "The main module" do + + context "client" do + should "have a default" do + client = Elasticsearch::Model.client + assert_not_nil client + assert_instance_of Elasticsearch::Transport::Client, client + end + + should "be settable" do + begin + Elasticsearch::Model.client = "Foobar" + assert_equal "Foobar", Elasticsearch::Model.client + ensure + Elasticsearch::Model.client = nil + end + end + end + + context "when included in module/class, " do + class ::DummyIncludingModel; end + class ::DummyIncludingModelWithSearchMethodDefined + def self.search(query, options={}) + "SEARCH" + end + end + + should "include and set up the proxy" do + DummyIncludingModel.__send__ :include, Elasticsearch::Model + + assert_respond_to DummyIncludingModel, :__elasticsearch__ + assert_respond_to DummyIncludingModel.new, :__elasticsearch__ + end + + should "delegate important methods to the proxy" do + DummyIncludingModel.__send__ :include, Elasticsearch::Model + + assert_respond_to DummyIncludingModel, :search + assert_respond_to DummyIncludingModel, :mappings + assert_respond_to DummyIncludingModel, :settings + assert_respond_to DummyIncludingModel, :index_name + assert_respond_to DummyIncludingModel, :document_type + assert_respond_to DummyIncludingModel, :import + end + + should "not override existing method" do + DummyIncludingModelWithSearchMethodDefined.__send__ :include, Elasticsearch::Model + + assert_equal 'SEARCH', DummyIncludingModelWithSearchMethodDefined.search('foo') + end + end + + end +end diff --git a/elasticsearch-model/test/unit/multimodel_test.rb b/elasticsearch-model/test/unit/multimodel_test.rb new file mode 100644 index 0000000000..89e88f7a16 --- /dev/null +++ b/elasticsearch-model/test/unit/multimodel_test.rb @@ -0,0 +1,38 @@ +require 'test_helper' + +class Elasticsearch::Model::MultimodelTest < Test::Unit::TestCase + + context "Multimodel class" do + setup do + title = stub('Foo', index_name: 'foo_index', document_type: 'foo') + series = stub('Bar', index_name: 'bar_index', document_type: 'bar') + @multimodel = Elasticsearch::Model::Multimodel.new(title, series) + end + + should "have an index_name" do + assert_equal ['foo_index', 'bar_index'], @multimodel.index_name + end + + should "have a document_type" do + assert_equal ['foo', 'bar'], @multimodel.document_type + end + + should "have a client" do + assert_equal Elasticsearch::Model.client, @multimodel.client + end + + should "include models in the registry" do + class ::JustAModel + include Elasticsearch::Model + end + + class ::JustAnotherModel + include Elasticsearch::Model + end + + multimodel = Elasticsearch::Model::Multimodel.new + assert multimodel.models.include?(::JustAModel) + assert multimodel.models.include?(::JustAnotherModel) + end + end +end diff --git a/elasticsearch-model/test/unit/naming_test.rb b/elasticsearch-model/test/unit/naming_test.rb new file mode 100644 index 0000000000..424adf7cc5 --- /dev/null +++ b/elasticsearch-model/test/unit/naming_test.rb @@ -0,0 +1,103 @@ +require 'test_helper' + +class Elasticsearch::Model::NamingTest < Test::Unit::TestCase + context "Naming module" do + class ::DummyNamingModel + extend ActiveModel::Naming + + extend Elasticsearch::Model::Naming::ClassMethods + include Elasticsearch::Model::Naming::InstanceMethods + end + + module ::MyNamespace + class DummyNamingModelInNamespace + extend ActiveModel::Naming + + extend Elasticsearch::Model::Naming::ClassMethods + include Elasticsearch::Model::Naming::InstanceMethods + end + end + + should "return the default index_name" do + assert_equal 'dummy_naming_models', DummyNamingModel.index_name + assert_equal 'dummy_naming_models', DummyNamingModel.new.index_name + end + + should "return the sanitized default index_name for namespaced model" do + assert_equal 'my_namespace-dummy_naming_model_in_namespaces', ::MyNamespace::DummyNamingModelInNamespace.index_name + assert_equal 'my_namespace-dummy_naming_model_in_namespaces', ::MyNamespace::DummyNamingModelInNamespace.new.index_name + end + + should "return the default document_type" do + assert_equal 'dummy_naming_model', DummyNamingModel.document_type + assert_equal 'dummy_naming_model', DummyNamingModel.new.document_type + end + + should "set and return the index_name" do + DummyNamingModel.index_name 'foobar' + assert_equal 'foobar', DummyNamingModel.index_name + + d = DummyNamingModel.new + d.index_name 'foobar_d' + assert_equal 'foobar_d', d.index_name + + modifier = 'r' + d.index_name Proc.new{ "foobar_#{modifier}" } + assert_equal 'foobar_r', d.index_name + + modifier = 'z' + assert_equal 'foobar_z', d.index_name + + modifier = 'f' + d.index_name { "foobar_#{modifier}" } + assert_equal 'foobar_f', d.index_name + + modifier = 't' + assert_equal 'foobar_t', d.index_name + end + + should "set the index_name with setter" do + DummyNamingModel.index_name = 'foobar_index_S' + assert_equal 'foobar_index_S', DummyNamingModel.index_name + + d = DummyNamingModel.new + d.index_name = 'foobar_index_s' + assert_equal 'foobar_index_s', d.index_name + + assert_equal 'foobar_index_S', DummyNamingModel.index_name + + modifier2 = 'y' + DummyNamingModel.index_name = Proc.new{ "foobar_index_#{modifier2}" } + assert_equal 'foobar_index_y', DummyNamingModel.index_name + + modifier = 'r' + d.index_name = Proc.new{ "foobar_index_#{modifier}" } + assert_equal 'foobar_index_r', d.index_name + + modifier = 'z' + assert_equal 'foobar_index_z', d.index_name + + assert_equal 'foobar_index_y', DummyNamingModel.index_name + end + + should "set and return the document_type" do + DummyNamingModel.document_type 'foobar' + assert_equal 'foobar', DummyNamingModel.document_type + + d = DummyNamingModel.new + d.document_type 'foobar_d' + assert_equal 'foobar_d', d.document_type + end + + should "set the document_type with setter" do + DummyNamingModel.document_type = 'foobar_type_S' + assert_equal 'foobar_type_S', DummyNamingModel.document_type + + d = DummyNamingModel.new + d.document_type = 'foobar_type_s' + assert_equal 'foobar_type_s', d.document_type + + assert_equal 'foobar_type_S', DummyNamingModel.document_type + end + end +end diff --git a/elasticsearch-model/test/unit/proxy_test.rb b/elasticsearch-model/test/unit/proxy_test.rb new file mode 100644 index 0000000000..d7299f884b --- /dev/null +++ b/elasticsearch-model/test/unit/proxy_test.rb @@ -0,0 +1,100 @@ +require 'test_helper' + +class Elasticsearch::Model::SearchTest < Test::Unit::TestCase + context "Searching module" do + class ::DummyProxyModel + include Elasticsearch::Model::Proxy + + def self.foo + 'classy foo' + end + + def bar + 'insta barr' + end + + def as_json(options) + {foo: 'bar'} + end + end + + class ::DummyProxyModelWithCallbacks + def self.before_save(&block) + (@callbacks ||= {})[block.hash] = block + end + + def changed_attributes; [:foo]; end + + def changes + {:foo => ['One', 'Two']} + end + end + + should "setup the class proxy method" do + assert_respond_to DummyProxyModel, :__elasticsearch__ + end + + should "setup the instance proxy method" do + assert_respond_to DummyProxyModel.new, :__elasticsearch__ + end + + should "register the hook for before_save callback" do + ::DummyProxyModelWithCallbacks.expects(:before_save).returns(true) + DummyProxyModelWithCallbacks.__send__ :include, Elasticsearch::Model::Proxy + end + + should "set the @__changed_attributes variable before save" do + instance = ::DummyProxyModelWithCallbacks.new + instance.__elasticsearch__.expects(:instance_variable_set).with do |name, value| + assert_equal :@__changed_attributes, name + assert_equal({foo: 'Two'}, value) + true + end + + ::DummyProxyModelWithCallbacks.__send__ :include, Elasticsearch::Model::Proxy + + ::DummyProxyModelWithCallbacks.instance_variable_get(:@callbacks).each do |n,b| + instance.instance_eval(&b) + end + end + + should "delegate methods to the target" do + assert_respond_to DummyProxyModel.__elasticsearch__, :foo + assert_respond_to DummyProxyModel.new.__elasticsearch__, :bar + + assert_raise(NoMethodError) { DummyProxyModel.__elasticsearch__.xoxo } + assert_raise(NoMethodError) { DummyProxyModel.new.__elasticsearch__.xoxo } + + assert_equal 'classy foo', DummyProxyModel.__elasticsearch__.foo + assert_equal 'insta barr', DummyProxyModel.new.__elasticsearch__.bar + end + + should "reset the proxy target for duplicates" do + model = DummyProxyModel.new + model_target = model.__elasticsearch__.target + duplicate = model.dup + duplicate_target = duplicate.__elasticsearch__.target + + assert_not_equal model, duplicate + assert_equal model, model_target + assert_equal duplicate, duplicate_target + end + + should "return the proxy class from instance proxy" do + assert_equal Elasticsearch::Model::Proxy::ClassMethodsProxy, DummyProxyModel.new.__elasticsearch__.class.class + end + + should "return the origin class from instance proxy" do + assert_equal DummyProxyModel, DummyProxyModel.new.__elasticsearch__.klass + end + + should "delegate as_json from the proxy to target" do + assert_equal({foo: 'bar'}, DummyProxyModel.new.__elasticsearch__.as_json) + end + + should "have inspect method indicating the proxy" do + assert_match /PROXY/, DummyProxyModel.__elasticsearch__.inspect + assert_match /PROXY/, DummyProxyModel.new.__elasticsearch__.inspect + end + end +end diff --git a/elasticsearch-model/test/unit/response_base_test.rb b/elasticsearch-model/test/unit/response_base_test.rb new file mode 100644 index 0000000000..aa9b4244d6 --- /dev/null +++ b/elasticsearch-model/test/unit/response_base_test.rb @@ -0,0 +1,40 @@ +require 'test_helper' + +class Elasticsearch::Model::BaseTest < Test::Unit::TestCase + context "Response base module" do + class OriginClass + def self.index_name; 'foo'; end + def self.document_type; 'bar'; end + end + + class DummyBaseClass + include Elasticsearch::Model::Response::Base + end + + RESPONSE = { 'hits' => { 'total' => 123, 'max_score' => 456, 'hits' => [] } } + + setup do + @search = Elasticsearch::Model::Searching::SearchRequest.new OriginClass, '*' + @response = Elasticsearch::Model::Response::Response.new OriginClass, @search + @search.stubs(:execute!).returns(RESPONSE) + end + + should "access klass, response, total and max_score" do + r = DummyBaseClass.new OriginClass, @response + + assert_equal OriginClass, r.klass + assert_equal @response, r.response + assert_equal RESPONSE, r.response.response + assert_equal 123, r.total + assert_equal 456, r.max_score + end + + should "have abstract methods results and records" do + r = DummyBaseClass.new OriginClass, @response + + assert_raise(Elasticsearch::Model::NotImplemented) { |e| r.results } + assert_raise(Elasticsearch::Model::NotImplemented) { |e| r.records } + end + + end +end diff --git a/elasticsearch-model/test/unit/response_pagination_kaminari_test.rb b/elasticsearch-model/test/unit/response_pagination_kaminari_test.rb new file mode 100644 index 0000000000..1fc9b2f3c0 --- /dev/null +++ b/elasticsearch-model/test/unit/response_pagination_kaminari_test.rb @@ -0,0 +1,433 @@ +require 'test_helper' + +class Elasticsearch::Model::ResponsePaginationKaminariTest < Test::Unit::TestCase + class ModelClass + include ::Kaminari::ConfigurationMethods + + def self.index_name; 'foo'; end + def self.document_type; 'bar'; end + end + + RESPONSE = { 'took' => '5', 'timed_out' => false, '_shards' => {'one' => 'OK'}, + 'hits' => { 'total' => 100, 'hits' => (1..100).to_a.map { |i| { _id: i } } } } + + context "Response pagination" do + + setup do + @search = Elasticsearch::Model::Searching::SearchRequest.new ModelClass, '*' + @response = Elasticsearch::Model::Response::Response.new ModelClass, @search, RESPONSE + @response.klass.stubs(:client).returns mock('client') + end + + should "have pagination methods" do + assert_respond_to @response, :page + assert_respond_to @response, :limit_value + assert_respond_to @response, :offset_value + assert_respond_to @response, :limit + assert_respond_to @response, :offset + assert_respond_to @response, :total_count + end + + context "#page method" do + should "advance the from/size" do + @response.klass.client + .expects(:search) + .with do |definition| + assert_equal 25, definition[:from] + assert_equal 25, definition[:size] + true + end + .returns(RESPONSE) + + assert_nil @response.search.definition[:from] + assert_nil @response.search.definition[:size] + + @response.page(2).to_a + assert_equal 25, @response.search.definition[:from] + assert_equal 25, @response.search.definition[:size] + end + + should "advance the from/size further" do + @response.klass.client + .expects(:search) + .with do |definition| + assert_equal 75, definition[:from] + assert_equal 25, definition[:size] + true + end + .returns(RESPONSE) + + @response.page(4).to_a + assert_equal 75, @response.search.definition[:from] + assert_equal 25, @response.search.definition[:size] + end + end + + context "limit/offset readers" do + should "return the default" do + assert_equal Kaminari.config.default_per_page, @response.limit_value + assert_equal 0, @response.offset_value + end + + should "return the value from URL parameters" do + search = Elasticsearch::Model::Searching::SearchRequest.new ModelClass, '*', size: 10, from: 50 + @response = Elasticsearch::Model::Response::Response.new ModelClass, search, RESPONSE + + assert_equal 10, @response.limit_value + assert_equal 50, @response.offset_value + end + + should "ignore the value from request body" do + search = Elasticsearch::Model::Searching::SearchRequest.new ModelClass, + { query: { match_all: {} }, from: 333, size: 999 } + @response = Elasticsearch::Model::Response::Response.new ModelClass, search, RESPONSE + + assert_equal Kaminari.config.default_per_page, @response.limit_value + assert_equal 0, @response.offset_value + end + end + + context "limit setter" do + setup do + @response.records + @response.results + end + + should "set the values" do + @response.limit(35) + assert_equal 35, @response.search.definition[:size] + end + + should "reset the variables" do + @response.limit(35) + + assert_nil @response.instance_variable_get(:@response) + assert_nil @response.instance_variable_get(:@records) + assert_nil @response.instance_variable_get(:@results) + end + + should 'coerce string parameters' do + @response.limit("35") + assert_equal 35, @response.search.definition[:size] + end + + should 'ignore invalid string parameters' do + @response.limit(35) + @response.limit("asdf") + assert_equal 35, @response.search.definition[:size] + end + end + + context "with the page() and limit() methods" do + setup do + @response.records + @response.results + end + + should "set the values" do + @response.page(3).limit(35) + assert_equal 35, @response.search.definition[:size] + assert_equal 70, @response.search.definition[:from] + end + + should "set the values when limit is called first" do + @response.limit(35).page(3) + assert_equal 35, @response.search.definition[:size] + assert_equal 70, @response.search.definition[:from] + end + + should "reset the instance variables" do + @response.page(3).limit(35) + + assert_nil @response.instance_variable_get(:@response) + assert_nil @response.instance_variable_get(:@records) + assert_nil @response.instance_variable_get(:@results) + end + end + + context "offset setter" do + setup do + @response.records + @response.results + end + + should "set the values" do + @response.offset(15) + assert_equal 15, @response.search.definition[:from] + end + + should "reset the variables" do + @response.offset(35) + + assert_nil @response.instance_variable_get(:@response) + assert_nil @response.instance_variable_get(:@records) + assert_nil @response.instance_variable_get(:@results) + end + + should 'coerce string parameters' do + @response.offset("35") + assert_equal 35, @response.search.definition[:from] + end + + should 'coerce invalid string parameters' do + @response.offset(35) + @response.offset("asdf") + assert_equal 0, @response.search.definition[:from] + end + end + + context "total" do + should "return the number of hits" do + @response.expects(:results).returns(mock('results', total: 100)) + assert_equal 100, @response.total_count + end + end + + context "results" do + setup do + @search.stubs(:execute!).returns RESPONSE + end + + should "return current page and total count" do + assert_equal 1, @response.page(1).results.current_page + assert_equal 100, @response.results.total_count + + assert_equal 5, @response.page(5).results.current_page + end + + should "return previous page and next page" do + assert_equal nil, @response.page(1).results.prev_page + assert_equal 2, @response.page(1).results.next_page + + assert_equal 3, @response.page(4).results.prev_page + assert_equal nil, @response.page(4).results.next_page + + assert_equal 2, @response.page(3).results.prev_page + assert_equal 4, @response.page(3).results.next_page + end + end + + context "records" do + setup do + @search.stubs(:execute!).returns RESPONSE + end + + should "return current page and total count" do + assert_equal 1, @response.page(1).records.current_page + assert_equal 100, @response.records.total_count + + assert_equal 5, @response.page(5).records.current_page + end + + should "return previous page and next page" do + assert_equal nil, @response.page(1).records.prev_page + assert_equal 2, @response.page(1).records.next_page + + assert_equal 3, @response.page(4).records.prev_page + assert_equal nil, @response.page(4).records.next_page + + assert_equal 2, @response.page(3).records.prev_page + assert_equal 4, @response.page(3).records.next_page + end + end + end + + context "Multimodel response pagination" do + setup do + @multimodel = Elasticsearch::Model::Multimodel.new(ModelClass) + @search = Elasticsearch::Model::Searching::SearchRequest.new @multimodel, '*' + @response = Elasticsearch::Model::Response::Response.new @multimodel, @search, RESPONSE + @response.klass.stubs(:client).returns mock('client') + end + + should "have pagination methods" do + assert_respond_to @response, :page + assert_respond_to @response, :limit_value + assert_respond_to @response, :offset_value + assert_respond_to @response, :limit + assert_respond_to @response, :offset + assert_respond_to @response, :total_count + end + + context "#page method" do + should "advance the from/size" do + @response.klass.client + .expects(:search) + .with do |definition| + assert_equal 25, definition[:from] + assert_equal 25, definition[:size] + true + end + .returns(RESPONSE) + + assert_nil @response.search.definition[:from] + assert_nil @response.search.definition[:size] + + @response.page(2).to_a + assert_equal 25, @response.search.definition[:from] + assert_equal 25, @response.search.definition[:size] + end + + should "advance the from/size further" do + @response.klass.client + .expects(:search) + .with do |definition| + assert_equal 75, definition[:from] + assert_equal 25, definition[:size] + true + end + .returns(RESPONSE) + + @response.page(4).to_a + assert_equal 75, @response.search.definition[:from] + assert_equal 25, @response.search.definition[:size] + end + end + + context "limit/offset readers" do + should "return the default" do + assert_equal Kaminari.config.default_per_page, @response.limit_value + assert_equal 0, @response.offset_value + end + + should "return the value from URL parameters" do + search = Elasticsearch::Model::Searching::SearchRequest.new ModelClass, '*', size: 10, from: 50 + @response = Elasticsearch::Model::Response::Response.new ModelClass, search, RESPONSE + + assert_equal 10, @response.limit_value + assert_equal 50, @response.offset_value + end + + should "ignore the value from request body" do + search = Elasticsearch::Model::Searching::SearchRequest.new ModelClass, + { query: { match_all: {} }, from: 333, size: 999 } + @response = Elasticsearch::Model::Response::Response.new ModelClass, search, RESPONSE + + assert_equal Kaminari.config.default_per_page, @response.limit_value + assert_equal 0, @response.offset_value + end + end + + context "limit setter" do + setup do + @response.records + @response.results + end + + should "set the values" do + @response.limit(35) + assert_equal 35, @response.search.definition[:size] + end + + should "reset the variables" do + @response.limit(35) + + assert_nil @response.instance_variable_get(:@response) + assert_nil @response.instance_variable_get(:@records) + assert_nil @response.instance_variable_get(:@results) + end + end + + context "with the page() and limit() methods" do + setup do + @response.records + @response.results + end + + should "set the values" do + @response.page(3).limit(35) + assert_equal 35, @response.search.definition[:size] + assert_equal 70, @response.search.definition[:from] + end + + should "set the values when limit is called first" do + @response.limit(35).page(3) + assert_equal 35, @response.search.definition[:size] + assert_equal 70, @response.search.definition[:from] + end + + should "reset the instance variables" do + @response.page(3).limit(35) + + assert_nil @response.instance_variable_get(:@response) + assert_nil @response.instance_variable_get(:@records) + assert_nil @response.instance_variable_get(:@results) + end + end + + context "offset setter" do + setup do + @response.records + @response.results + end + + should "set the values" do + @response.offset(15) + assert_equal 15, @response.search.definition[:from] + end + + should "reset the variables" do + @response.offset(35) + + assert_nil @response.instance_variable_get(:@response) + assert_nil @response.instance_variable_get(:@records) + assert_nil @response.instance_variable_get(:@results) + end + end + + context "total" do + should "return the number of hits" do + @response.expects(:results).returns(mock('results', total: 100)) + assert_equal 100, @response.total_count + end + end + + context "results" do + setup do + @search.stubs(:execute!).returns RESPONSE + end + + should "return current page and total count" do + assert_equal 1, @response.page(1).results.current_page + assert_equal 100, @response.results.total_count + + assert_equal 5, @response.page(5).results.current_page + end + + should "return previous page and next page" do + assert_equal nil, @response.page(1).results.prev_page + assert_equal 2, @response.page(1).results.next_page + + assert_equal 3, @response.page(4).results.prev_page + assert_equal nil, @response.page(4).results.next_page + + assert_equal 2, @response.page(3).results.prev_page + assert_equal 4, @response.page(3).results.next_page + end + end + + context "records" do + setup do + @search.stubs(:execute!).returns RESPONSE + end + + should "return current page and total count" do + assert_equal 1, @response.page(1).records.current_page + assert_equal 100, @response.records.total_count + + assert_equal 5, @response.page(5).records.current_page + end + + should "return previous page and next page" do + assert_equal nil, @response.page(1).records.prev_page + assert_equal 2, @response.page(1).records.next_page + + assert_equal 3, @response.page(4).records.prev_page + assert_equal nil, @response.page(4).records.next_page + + assert_equal 2, @response.page(3).records.prev_page + assert_equal 4, @response.page(3).records.next_page + end + end + end +end diff --git a/elasticsearch-model/test/unit/response_pagination_will_paginate_test.rb b/elasticsearch-model/test/unit/response_pagination_will_paginate_test.rb new file mode 100644 index 0000000000..6c93835256 --- /dev/null +++ b/elasticsearch-model/test/unit/response_pagination_will_paginate_test.rb @@ -0,0 +1,398 @@ +require 'test_helper' +require 'will_paginate' +require 'will_paginate/collection' + +class Elasticsearch::Model::ResponsePaginationWillPaginateTest < Test::Unit::TestCase + class ModelClass + def self.index_name; 'foo'; end + def self.document_type; 'bar'; end + + # WillPaginate adds this method to models (see WillPaginate::PerPage module) + def self.per_page + 33 + end + end + + # Subsclass Response so we can include WillPaginate module without conflicts with Kaminari. + class WillPaginateResponse < Elasticsearch::Model::Response::Response + include Elasticsearch::Model::Response::Pagination::WillPaginate + end + + RESPONSE = { 'took' => '5', 'timed_out' => false, '_shards' => {'one' => 'OK'}, + 'hits' => { 'total' => 100, 'hits' => (1..100).to_a.map { |i| { _id: i } } } } + + context "Response pagination" do + + setup do + @search = Elasticsearch::Model::Searching::SearchRequest.new ModelClass, '*' + @response = WillPaginateResponse.new ModelClass, @search, RESPONSE + @response.klass.stubs(:client).returns mock('client') + + @expected_methods = [ + # methods needed by WillPaginate::CollectionMethods + :current_page, + :offset, + :per_page, + :total_entries, + :length, + + # methods defined by WillPaginate::CollectionMethods + :total_pages, + :previous_page, + :next_page, + :out_of_bounds?, + ] + end + + should "have pagination methods" do + assert_respond_to @response, :paginate + + @expected_methods.each do |method| + assert_respond_to @response, method + end + end + + context "response.results" do + should "have pagination methods" do + @expected_methods.each do |method| + assert_respond_to @response.results, method + end + end + end + + context "response.records" do + should "have pagination methods" do + @expected_methods.each do |method| + @response.klass.stubs(:find).returns([]) + assert_respond_to @response.records, method + end + end + end + + context "#offset method" do + should "calculate offset using current_page and per_page" do + @response.per_page(3).page(3) + assert_equal 6, @response.offset + end + end + context "#length method" do + should "return count of paginated results" do + @response.per_page(3).page(3) + assert_equal 3, @response.length + end + end + + context "#paginate method" do + should "set from/size using defaults" do + @response.klass.client + .expects(:search) + .with do |definition| + assert_equal 0, definition[:from] + assert_equal 33, definition[:size] + true + end + .returns(RESPONSE) + + assert_nil @response.search.definition[:from] + assert_nil @response.search.definition[:size] + + @response.paginate(page: nil).to_a + assert_equal 0, @response.search.definition[:from] + assert_equal 33, @response.search.definition[:size] + end + + should "set from/size using default per_page" do + @response.klass.client + .expects(:search) + .with do |definition| + assert_equal 33, definition[:from] + assert_equal 33, definition[:size] + true + end + .returns(RESPONSE) + + assert_nil @response.search.definition[:from] + assert_nil @response.search.definition[:size] + + @response.paginate(page: 2).to_a + assert_equal 33, @response.search.definition[:from] + assert_equal 33, @response.search.definition[:size] + end + + should "set from/size using custom page and per_page" do + @response.klass.client + .expects(:search) + .with do |definition| + assert_equal 18, definition[:from] + assert_equal 9, definition[:size] + true + end + .returns(RESPONSE) + + assert_nil @response.search.definition[:from] + assert_nil @response.search.definition[:size] + + @response.paginate(page: 3, per_page: 9).to_a + assert_equal 18, @response.search.definition[:from] + assert_equal 9, @response.search.definition[:size] + end + + should "search for first page if specified page is < 1" do + @response.klass.client + .expects(:search) + .with do |definition| + assert_equal 0, definition[:from] + assert_equal 33, definition[:size] + true + end + .returns(RESPONSE) + + assert_nil @response.search.definition[:from] + assert_nil @response.search.definition[:size] + + @response.paginate(page: "-1").to_a + assert_equal 0, @response.search.definition[:from] + assert_equal 33, @response.search.definition[:size] + end + + should "use the param_name" do + @response.klass.client + .expects(:search) + .with do |definition| + assert_equal 10, definition[:from] + true + end + .returns(RESPONSE) + + @response.paginate(my_page: 2, per_page: 10, param_name: :my_page).to_a + end + end + + context "#page and #per_page shorthand methods" do + should "set from/size using default per_page" do + @response.page(5) + assert_equal 132, @response.search.definition[:from] + assert_equal 33, @response.search.definition[:size] + end + + should "set from/size when calling #page then #per_page" do + @response.page(5).per_page(3) + assert_equal 12, @response.search.definition[:from] + assert_equal 3, @response.search.definition[:size] + end + + should "set from/size when calling #per_page then #page" do + @response.per_page(3).page(5) + assert_equal 12, @response.search.definition[:from] + assert_equal 3, @response.search.definition[:size] + end + end + + context "#current_page method" do + should "return 1 by default" do + @response.paginate({}) + assert_equal 1, @response.current_page + end + + should "return current page number" do + @response.paginate(page: 3, per_page: 9) + assert_equal 3, @response.current_page + end + + should "return nil if not pagination set" do + assert_equal nil, @response.current_page + end + end + + context "#per_page method" do + should "return value set in paginate call" do + @response.paginate(per_page: 8) + assert_equal 8, @response.per_page + end + end + + context "#total_entries method" do + should "return total from response" do + @response.expects(:results).returns(mock('results', total: 100)) + assert_equal 100, @response.total_entries + end + end + end + + context "Multimodel response pagination" do + setup do + @multimodel = Elasticsearch::Model::Multimodel.new ModelClass + @search = Elasticsearch::Model::Searching::SearchRequest.new @multimodel, '*' + @response = WillPaginateResponse.new @multimodel, @search, RESPONSE + @response.klass.stubs(:client).returns mock('client') + + @expected_methods = [ + # methods needed by WillPaginate::CollectionMethods + :current_page, + :offset, + :per_page, + :total_entries, + :length, + + # methods defined by WillPaginate::CollectionMethods + :total_pages, + :previous_page, + :next_page, + :out_of_bounds?, + ] + end + + should "have pagination methods" do + assert_respond_to @response, :paginate + + @expected_methods.each do |method| + assert_respond_to @response, method + end + end + + context "response.results" do + should "have pagination methods" do + @expected_methods.each do |method| + assert_respond_to @response.results, method + end + end + end + + context "#offset method" do + should "calculate offset using current_page and per_page" do + @response.per_page(3).page(3) + assert_equal 6, @response.offset + end + end + context "#length method" do + should "return count of paginated results" do + @response.per_page(3).page(3) + assert_equal 3, @response.length + end + end + + context "#paginate method" do + should "set from/size using WillPaginate defaults, ignoring aggregated models configuration" do + @response.klass.client + .expects(:search) + .with do |definition| + assert_equal 0, definition[:from] + assert_equal ::WillPaginate.per_page, definition[:size] + true + end + .returns(RESPONSE) + + assert_nil @response.search.definition[:from] + assert_nil @response.search.definition[:size] + + @response.paginate(page: nil).to_a + assert_equal 0, @response.search.definition[:from] + assert_equal ::WillPaginate.per_page, @response.search.definition[:size] + end + + should "set from/size using default per_page, ignoring aggregated models' configuration" do + @response.klass.client + .expects(:search) + .with do |definition| + assert_equal ::WillPaginate.per_page, definition[:from] + assert_equal ::WillPaginate.per_page, definition[:size] + true + end + .returns(RESPONSE) + + assert_nil @response.search.definition[:from] + assert_nil @response.search.definition[:size] + + @response.paginate(page: 2).to_a + assert_equal ::WillPaginate.per_page, @response.search.definition[:from] + assert_equal ::WillPaginate.per_page, @response.search.definition[:size] + end + + should "set from/size using custom page and per_page" do + @response.klass.client + .expects(:search) + .with do |definition| + assert_equal 18, definition[:from] + assert_equal 9, definition[:size] + true + end + .returns(RESPONSE) + + assert_nil @response.search.definition[:from] + assert_nil @response.search.definition[:size] + + @response.paginate(page: 3, per_page: 9).to_a + assert_equal 18, @response.search.definition[:from] + assert_equal 9, @response.search.definition[:size] + end + + should "search for first page if specified page is < 1" do + @response.klass.client + .expects(:search) + .with do |definition| + assert_equal 0, definition[:from] + assert_equal ::WillPaginate.per_page, definition[:size] + true + end + .returns(RESPONSE) + + assert_nil @response.search.definition[:from] + assert_nil @response.search.definition[:size] + + @response.paginate(page: "-1").to_a + assert_equal 0, @response.search.definition[:from] + assert_equal ::WillPaginate.per_page, @response.search.definition[:size] + end + end + + context "#page and #per_page shorthand methods" do + should "set from/size using default per_page" do + @response.page(5) + assert_equal 120, @response.search.definition[:from] + assert_equal ::WillPaginate.per_page, @response.search.definition[:size] + end + + should "set from/size when calling #page then #per_page" do + @response.page(5).per_page(3) + assert_equal 12, @response.search.definition[:from] + assert_equal 3, @response.search.definition[:size] + end + + should "set from/size when calling #per_page then #page" do + @response.per_page(3).page(5) + assert_equal 12, @response.search.definition[:from] + assert_equal 3, @response.search.definition[:size] + end + end + + context "#current_page method" do + should "return 1 by default" do + @response.paginate({}) + assert_equal 1, @response.current_page + end + + should "return current page number" do + @response.paginate(page: 3, per_page: 9) + assert_equal 3, @response.current_page + end + + should "return nil if not pagination set" do + assert_equal nil, @response.current_page + end + end + + context "#per_page method" do + should "return value set in paginate call" do + @response.paginate(per_page: 8) + assert_equal 8, @response.per_page + end + end + + context "#total_entries method" do + should "return total from response" do + @response.expects(:results).returns(mock('results', total: 100)) + assert_equal 100, @response.total_entries + end + end + end +end diff --git a/elasticsearch-model/test/unit/response_records_test.rb b/elasticsearch-model/test/unit/response_records_test.rb new file mode 100644 index 0000000000..8a78255d7c --- /dev/null +++ b/elasticsearch-model/test/unit/response_records_test.rb @@ -0,0 +1,91 @@ +require 'test_helper' + +class Elasticsearch::Model::RecordsTest < Test::Unit::TestCase + context "Response records" do + class DummyCollection + include Enumerable + + def each(&block); ['FOO'].each(&block); end + def size; ['FOO'].size; end + def empty?; ['FOO'].empty?; end + def foo; 'BAR'; end + end + + class DummyModel + def self.index_name; 'foo'; end + def self.document_type; 'bar'; end + + def self.find(*args) + DummyCollection.new + end + end + + RESPONSE = { 'hits' => { 'total' => 123, 'max_score' => 456, 'hits' => [{'_id' => '1', 'foo' => 'bar'}] } } + RESULTS = Elasticsearch::Model::Response::Results.new DummyModel, RESPONSE + + setup do + search = Elasticsearch::Model::Searching::SearchRequest.new DummyModel, '*' + search.stubs(:execute!).returns RESPONSE + + response = Elasticsearch::Model::Response::Response.new DummyModel, search + @records = Elasticsearch::Model::Response::Records.new DummyModel, response + end + + should "access the records" do + assert_respond_to @records, :records + assert_equal 1, @records.records.size + assert_equal 'FOO', @records.records.first + end + + should "delegate Enumerable methods to records" do + assert ! @records.empty? + assert_equal 'FOO', @records.first + end + + should "delegate methods to records" do + assert_respond_to @records, :foo + assert_equal 'BAR', @records.foo + end + + should "have each_with_hit method" do + @records.each_with_hit do |record, hit| + assert_equal 'FOO', record + assert_equal 'bar', hit.foo + end + end + + should "have map_with_hit method" do + assert_equal ['FOO---bar'], @records.map_with_hit { |record, hit| "#{record}---#{hit.foo}" } + end + + should "return the IDs" do + assert_equal ['1'], @records.ids + end + + context "with adapter" do + module DummyAdapter + module RecordsMixin + def records + ['FOOBAR'] + end + end + + def records_mixin + RecordsMixin + end; module_function :records_mixin + end + + should "delegate the records method to the adapter" do + Elasticsearch::Model::Adapter.expects(:from_class) + .with(DummyModel) + .returns(DummyAdapter) + + @records = Elasticsearch::Model::Response::Records.new DummyModel, + RESPONSE + + assert_equal ['FOOBAR'], @records.records + end + end + + end +end diff --git a/elasticsearch-model/test/unit/response_result_test.rb b/elasticsearch-model/test/unit/response_result_test.rb new file mode 100644 index 0000000000..ff78d25790 --- /dev/null +++ b/elasticsearch-model/test/unit/response_result_test.rb @@ -0,0 +1,90 @@ +require 'test_helper' + +class Elasticsearch::Model::ResultTest < Test::Unit::TestCase + context "Response result" do + + should "have method access to properties" do + result = Elasticsearch::Model::Response::Result.new foo: 'bar', bar: { bam: 'baz' } + + assert_respond_to result, :foo + assert_respond_to result, :bar + + assert_equal 'bar', result.foo + assert_equal 'baz', result.bar.bam + + assert_raise(NoMethodError) { result.xoxo } + end + + should "return _id as #id" do + result = Elasticsearch::Model::Response::Result.new foo: 'bar', _id: 42, _source: { id: 12 } + + assert_equal 42, result.id + assert_equal 12, result._source.id + end + + should "return _type as #type" do + result = Elasticsearch::Model::Response::Result.new foo: 'bar', _type: 'baz', _source: { type: 'BAM' } + + assert_equal 'baz', result.type + assert_equal 'BAM', result._source.type + end + + should "delegate method calls to `_source` when available" do + result = Elasticsearch::Model::Response::Result.new foo: 'bar', _source: { bar: 'baz' } + + assert_respond_to result, :foo + assert_respond_to result, :_source + assert_respond_to result, :bar + + assert_equal 'bar', result.foo + assert_equal 'baz', result._source.bar + assert_equal 'baz', result.bar + end + + should "delegate existence method calls to `_source`" do + result = Elasticsearch::Model::Response::Result.new foo: 'bar', _source: { bar: { bam: 'baz' } } + + assert_respond_to result._source, :bar? + assert_respond_to result, :bar? + + assert_equal true, result._source.bar? + assert_equal true, result.bar? + assert_equal false, result.boo? + + assert_equal true, result.bar.bam? + assert_equal false, result.bar.boo? + end + + should "delegate methods to @result" do + result = Elasticsearch::Model::Response::Result.new foo: 'bar' + + assert_equal 'bar', result.foo + assert_equal 'bar', result.fetch('foo') + assert_equal 'moo', result.fetch('NOT_EXIST', 'moo') + assert_equal ['foo'], result.keys + + assert_respond_to result, :to_hash + assert_equal({'foo' => 'bar'}, result.to_hash) + + assert_raise(NoMethodError) { result.does_not_exist } + end + + should "delegate existence method calls to @result" do + result = Elasticsearch::Model::Response::Result.new foo: 'bar', _source: { bar: 'bam' } + assert_respond_to result, :foo? + + assert_equal true, result.foo? + assert_equal false, result.boo? + assert_equal false, result._source.foo? + assert_equal false, result._source.boo? + end + + should "delegate as_json to @result even when ActiveSupport changed half of Ruby" do + require 'active_support/json/encoding' + result = Elasticsearch::Model::Response::Result.new foo: 'bar' + + result.instance_variable_get(:@result).expects(:as_json) + result.as_json(except: 'foo') + end + end +end diff --git a/elasticsearch-model/test/unit/response_results_test.rb b/elasticsearch-model/test/unit/response_results_test.rb new file mode 100644 index 0000000000..e97539ecdc --- /dev/null +++ b/elasticsearch-model/test/unit/response_results_test.rb @@ -0,0 +1,31 @@ +require 'test_helper' + +class Elasticsearch::Model::ResultsTest < Test::Unit::TestCase + context "Response results" do + class OriginClass + def self.index_name; 'foo'; end + def self.document_type; 'bar'; end + end + + RESPONSE = { 'hits' => { 'total' => 123, 'max_score' => 456, 'hits' => [{'foo' => 'bar'}] } } + + setup do + @search = Elasticsearch::Model::Searching::SearchRequest.new OriginClass, '*' + @response = Elasticsearch::Model::Response::Response.new OriginClass, @search + @results = Elasticsearch::Model::Response::Results.new OriginClass, @response + @search.stubs(:execute!).returns(RESPONSE) + end + + should "access the results" do + assert_respond_to @results, :results + assert_equal 1, @results.results.size + assert_equal 'bar', @results.results.first.foo + end + + should "delegate Enumerable methods to results" do + assert ! @results.empty? + assert_equal 'bar', @results.first.foo + end + + end +end diff --git a/elasticsearch-model/test/unit/response_test.rb b/elasticsearch-model/test/unit/response_test.rb new file mode 100644 index 0000000000..71cfb2d6d9 --- /dev/null +++ b/elasticsearch-model/test/unit/response_test.rb @@ -0,0 +1,104 @@ +require 'test_helper' + +class Elasticsearch::Model::ResponseTest < Test::Unit::TestCase + context "Response" do + class OriginClass + def self.index_name; 'foo'; end + def self.document_type; 'bar'; end + end + + RESPONSE = { 'took' => '5', 'timed_out' => false, '_shards' => {'one' => 'OK'}, 'hits' => { 'hits' => [] }, + 'aggregations' => {'foo' => {'bar' => 10}}, + 'suggest' => {'my_suggest' => [ { 'text' => 'foo', 'options' => [ { 'text' => 'Foo', 'score' => 2.0 }, { 'text' => 'Bar', 'score' => 1.0 } ] } ]}} + + setup do + @search = Elasticsearch::Model::Searching::SearchRequest.new OriginClass, '*' + @search.stubs(:execute!).returns(RESPONSE) + end + + should "access klass, response, took, timed_out, shards" do + response = Elasticsearch::Model::Response::Response.new OriginClass, @search + + assert_equal OriginClass, response.klass + assert_equal @search, response.search + assert_equal RESPONSE, response.response + assert_equal '5', response.took + assert_equal false, response.timed_out + assert_equal 'OK', response.shards.one + end + + should "wrap the raw Hash response in Hashie::Mash" do + @search = Elasticsearch::Model::Searching::SearchRequest.new OriginClass, '*' + @search.stubs(:execute!).returns({'hits' => { 'hits' => [] }, 'aggregations' => { 'dates' => 'FOO' }}) + + response = Elasticsearch::Model::Response::Response.new OriginClass, @search + + assert_respond_to response.response, :aggregations + assert_equal 'FOO', response.response.aggregations.dates + end + + should "load and access the results" do + @search.expects(:execute!).returns(RESPONSE) + + response = Elasticsearch::Model::Response::Response.new OriginClass, @search + assert_instance_of Elasticsearch::Model::Response::Results, response.results + assert_equal 0, response.size + end + + should "load and access the records" do + @search.expects(:execute!).returns(RESPONSE) + + response = Elasticsearch::Model::Response::Response.new OriginClass, @search + assert_instance_of Elasticsearch::Model::Response::Records, response.records + assert_equal 0, response.size + end + + should "delegate Enumerable methods to results" do + @search.expects(:execute!).returns(RESPONSE) + + response = Elasticsearch::Model::Response::Response.new OriginClass, @search + assert response.empty? + end + + should "be initialized lazily" do + @search.expects(:execute!).never + + Elasticsearch::Model::Response::Response.new OriginClass, @search + end + + should "access the aggregations" do + @search.expects(:execute!).returns(RESPONSE) + + response = Elasticsearch::Model::Response::Response.new OriginClass, @search + assert_respond_to response, :aggregations + assert_kind_of Hashie::Mash, response.aggregations.foo + assert_equal 10, response.aggregations.foo.bar + end + + should "access the suggest" do + @search.expects(:execute!).returns(RESPONSE) + + response = Elasticsearch::Model::Response::Response.new OriginClass, @search + + assert_respond_to response, :suggestions + assert_kind_of Hashie::Mash, response.suggestions + assert_equal 'Foo', response.suggestions.my_suggest.first.options.first.text + end + + should "return array of terms from the suggestions" do + @search.expects(:execute!).returns(RESPONSE) + response = Elasticsearch::Model::Response::Response.new OriginClass, @search + + assert_not_empty response.suggestions + assert_equal [ 'Foo', 'Bar' ], response.suggestions.terms + end + + should "return empty array as suggest terms when there are no suggestions" do + @search.expects(:execute!).returns({}) + response = Elasticsearch::Model::Response::Response.new OriginClass, @search + + assert_empty response.suggestions + assert_equal [], response.suggestions.terms + end + end +end diff --git a/elasticsearch-model/test/unit/searching_search_request_test.rb b/elasticsearch-model/test/unit/searching_search_request_test.rb new file mode 100644 index 0000000000..b2e84aecce --- /dev/null +++ b/elasticsearch-model/test/unit/searching_search_request_test.rb @@ -0,0 +1,78 @@ +require 'test_helper' + +class Elasticsearch::Model::SearchRequestTest < Test::Unit::TestCase + context "SearchRequest class" do + class ::DummySearchingModel + extend Elasticsearch::Model::Searching::ClassMethods + + def self.index_name; 'foo'; end + def self.document_type; 'bar'; end + + end + + setup do + @client = mock('client') + DummySearchingModel.stubs(:client).returns(@client) + end + + should "pass the search definition as a simple query" do + @client.expects(:search).with do |params| + assert_equal 'foo', params[:q] + true + end + .returns({}) + + s = Elasticsearch::Model::Searching::SearchRequest.new ::DummySearchingModel, 'foo' + s.execute! + end + + should "pass the search definition as a Hash" do + @client.expects(:search).with do |params| + assert_equal( {foo: 'bar'}, params[:body] ) + true + end + .returns({}) + + s = Elasticsearch::Model::Searching::SearchRequest.new ::DummySearchingModel, foo: 'bar' + s.execute! + end + + should "pass the search definition as a JSON string" do + @client.expects(:search).with do |params| + assert_equal( '{"foo":"bar"}', params[:body] ) + true + end + .returns({}) + + s = Elasticsearch::Model::Searching::SearchRequest.new ::DummySearchingModel, '{"foo":"bar"}' + s.execute! + end + + should "pass the search definition as an object which responds to to_hash" do + class MySpecialQueryBuilder + def to_hash; {foo: 'bar'}; end + end + + @client.expects(:search).with do |params| + assert_equal( {foo: 'bar'}, params[:body] ) + true + end + .returns({}) + + s = Elasticsearch::Model::Searching::SearchRequest.new ::DummySearchingModel, MySpecialQueryBuilder.new + s.execute! + end + + should "pass the options to the client" do + @client.expects(:search).with do |params| + assert_equal 'foo', params[:q] + assert_equal 15, params[:size] + true + end + .returns({}) + + s = Elasticsearch::Model::Searching::SearchRequest.new ::DummySearchingModel, 'foo', size: 15 + s.execute! + end + end +end diff --git a/elasticsearch-model/test/unit/searching_test.rb b/elasticsearch-model/test/unit/searching_test.rb new file mode 100644 index 0000000000..f6cb78136f --- /dev/null +++ b/elasticsearch-model/test/unit/searching_test.rb @@ -0,0 +1,41 @@ +require 'test_helper' + +class Elasticsearch::Model::SearchingTest < Test::Unit::TestCase + context "Searching module" do + class ::DummySearchingModel + extend Elasticsearch::Model::Searching::ClassMethods + + def self.index_name; 'foo'; end + def self.document_type; 'bar'; end + end + + setup do + @client = mock('client') + DummySearchingModel.stubs(:client).returns(@client) + end + + should "have the search method" do + assert_respond_to DummySearchingModel, :search + end + + should "initialize the search object" do + Elasticsearch::Model::Searching::SearchRequest + .expects(:new).with do |klass, query, options| + assert_equal DummySearchingModel, klass + assert_equal 'foo', query + assert_equal({default_operator: 'AND'}, options) + true + end + .returns( stub('search') ) + + DummySearchingModel.search 'foo', default_operator: 'AND' + end + + should "not execute the search" do + Elasticsearch::Model::Searching::SearchRequest + .expects(:new).returns( mock('search').expects(:execute!).never ) + + DummySearchingModel.search 'foo' + end + end +end diff --git a/elasticsearch-model/test/unit/serializing_test.rb b/elasticsearch-model/test/unit/serializing_test.rb new file mode 100644 index 0000000000..201329257c --- /dev/null +++ b/elasticsearch-model/test/unit/serializing_test.rb @@ -0,0 +1,17 @@ +require 'test_helper' + +class Elasticsearch::Model::SerializingTest < Test::Unit::TestCase + context "Serializing module" do + class DummyClass + include Elasticsearch::Model::Serializing::InstanceMethods + + def as_json(options={}) + 'HASH' + end + end + + should "delegate to as_json by default" do + assert_equal 'HASH', DummyClass.new.as_indexed_json + end + end +end diff --git a/snowplow-tracker/LICENSE-2.0.txt b/snowplow-tracker/LICENSE-2.0.txt deleted file mode 100644 index 7a4a3ea242..0000000000 --- a/snowplow-tracker/LICENSE-2.0.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/snowplow-tracker/README.md b/snowplow-tracker/README.md deleted file mode 100644 index dac689f899..0000000000 --- a/snowplow-tracker/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# Ruby Analytics for Snowplow -[![Gem Version](https://badge.fury.io/rb/snowplow-tracker.svg)](http://badge.fury.io/rb/snowplow-tracker) -[![Build Status](https://travis-ci.org/snowplow/snowplow-ruby-tracker.png?branch=master)](https://travis-ci.org/snowplow/snowplow-ruby-tracker) -[![Code Climate](https://codeclimate.com/github/snowplow/snowplow-ruby-tracker.png)](https://codeclimate.com/github/snowplow/snowplow-ruby-tracker) -[![Coverage Status](https://coveralls.io/repos/snowplow/snowplow-ruby-tracker/badge.png)](https://coveralls.io/r/snowplow/snowplow-ruby-tracker) -[![License][license-image]][license] - -## Overview - -Add analytics to your Ruby and Rails apps and gems with the **[Snowplow] [snowplow]** event tracker for **[Ruby] [ruby]**. - -With this tracker you can collect event data from your **[Ruby] [ruby]** applications, **[Ruby on Rails] [rails]** web applications and **[Ruby gems] [rubygems]**. - -## Quickstart - -Assuming git, **[Vagrant] [vagrant-install]** and **[VirtualBox] [virtualbox-install]** installed: - -```bash - host$ git clone https://github.com/snowplow/snowplow-ruby-tracker.git - host$ cd snowplow-ruby-tracker - host$ vagrant up && vagrant ssh -guest$ cd /vagrant -guest$ gem install bundler -guest$ bundle install -guest$ rspec -``` - -## Publishing - -```bash - host$ vagrant push -``` - -## Find out more - -| Technical Docs | Setup Guide | Roadmap | Contributing | -|---------------------------------|---------------------------|-------------------------|-----------------------------------| -| ![i1] [techdocs-image] | ![i2] [setup-image] | ![i3] [roadmap-image] | ![i4] [contributing-image] | -| **[Technical Docs] [techdocs]** | **[Setup Guide] [setup]** | **[Roadmap] [roadmap]** | **[Contributing] [contributing]** | - -## Copyright and license - -The Snowplow Ruby Tracker is copyright 2013-2016 Snowplow Analytics Ltd. - -Licensed under the **[Apache License, Version 2.0] [license]** (the "License"); -you may not use this software except in compliance with the License. - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -[license-image]: http://img.shields.io/badge/license-Apache--2-blue.svg?style=flat -[license]: http://www.apache.org/licenses/LICENSE-2.0 - -[ruby]: https://www.ruby-lang.org/en/ -[rails]: http://rubyonrails.org/ -[rubygems]: https://rubygems.org/ - -[snowplow]: http://snowplowanalytics.com - -[vagrant-install]: http://docs.vagrantup.com/v2/installation/index.html -[virtualbox-install]: https://www.virtualbox.org/wiki/Downloads - -[techdocs-image]: https://d3i6fms1cm1j0i.cloudfront.net/github/images/techdocs.png -[setup-image]: https://d3i6fms1cm1j0i.cloudfront.net/github/images/setup.png -[roadmap-image]: https://d3i6fms1cm1j0i.cloudfront.net/github/images/roadmap.png -[contributing-image]: https://d3i6fms1cm1j0i.cloudfront.net/github/images/contributing.png - -[techdocs]: https://github.com/snowplow/snowplow/wiki/Ruby-Tracker -[setup]: https://github.com/snowplow/snowplow/wiki/Ruby-Tracker-Setup -[roadmap]: https://github.com/snowplow/snowplow/wiki/Ruby-Tracker-Roadmap -[contributing]: https://github.com/snowplow/snowplow/wiki/Ruby-Tracker-Contributing diff --git a/snowplow-tracker/lib/snowplow-tracker.rb b/snowplow-tracker/lib/snowplow-tracker.rb deleted file mode 100644 index a08defef22..0000000000 --- a/snowplow-tracker/lib/snowplow-tracker.rb +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (c) 2013-2014 Snowplow Analytics Ltd. All rights reserved. -# -# This program is licensed to you under the Apache License Version 2.0, -# and you may not use this file except in compliance with the Apache License Version 2.0. -# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the Apache License Version 2.0 is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. - -# Author:: Alex Dean, Fred Blundun (mailto:snowplow-user@googlegroups.com) -# Copyright:: Copyright (c) 2013-2014 Snowplow Analytics Ltd -# License:: Apache License Version 2.0 - -require 'snowplow-tracker/contracts.rb' -require 'snowplow-tracker/version.rb' -require 'snowplow-tracker/self_describing_json.rb' -require 'snowplow-tracker/payload.rb' -require 'snowplow-tracker/subject.rb' -require 'snowplow-tracker/emitters.rb' -require 'snowplow-tracker/timestamp.rb' -require 'snowplow-tracker/tracker.rb' - diff --git a/snowplow-tracker/lib/snowplow-tracker/contracts.rb b/snowplow-tracker/lib/snowplow-tracker/contracts.rb deleted file mode 100644 index 0ce2907b24..0000000000 --- a/snowplow-tracker/lib/snowplow-tracker/contracts.rb +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) 2013-2014 Snowplow Analytics Ltd. All rights reserved. -# -# This program is licensed to you under the Apache License Version 2.0, -# and you may not use this file except in compliance with the Apache License Version 2.0. -# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the Apache License Version 2.0 is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. - -# Author:: Alex Dean, Fred Blundun (mailto:support@snowplowanalytics.com) -# Copyright:: Copyright (c) 2013-2014 Snowplow Analytics Ltd -# License:: Apache License Version 2.0 - -require 'contracts' - -module SnowplowTracker - - ORIGINAL_FAILURE_CALLBACK = Contract.method(:failure_callback) - - def self.disable_contracts - Contract.define_singleton_method(:failure_callback) {|data| true} - end - - def self.enable_contracts - Contract.define_singleton_method(:failure_callback, ORIGINAL_FAILURE_CALLBACK) - end -end diff --git a/snowplow-tracker/lib/snowplow-tracker/emitters.rb b/snowplow-tracker/lib/snowplow-tracker/emitters.rb deleted file mode 100644 index 09c75d199e..0000000000 --- a/snowplow-tracker/lib/snowplow-tracker/emitters.rb +++ /dev/null @@ -1,280 +0,0 @@ -# Copyright (c) 2013-2014 Snowplow Analytics Ltd. All rights reserved. -# -# This program is licensed to you under the Apache License Version 2.0, -# and you may not use this file except in compliance with the Apache License Version 2.0. -# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the Apache License Version 2.0 is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. - -# Author:: Alex Dean, Fred Blundun (mailto:support@snowplowanalytics.com) -# Copyright:: Copyright (c) 2013-2014 Snowplow Analytics Ltd -# License:: Apache License Version 2.0 - -require 'net/https' -require 'set' -require 'logger' -require 'contracts' - -module SnowplowTracker - - LOGGER = Logger.new(STDERR) - LOGGER.level = Logger::INFO - - class Emitter - - include Contracts - - @@ConfigHash = ({ - :protocol => Maybe[Or['http', 'https']], - :port => Maybe[Num], - :method => Maybe[Or['get', 'post']], - :buffer_size => Maybe[Num], - :on_success => Maybe[Func[Num => Any]], - :on_failure => Maybe[Func[Num, Hash => Any]], - :thread_count => Maybe[Num] - }) - - @@StrictConfigHash = And[@@ConfigHash, lambda { |x| - x.class == Hash and Set.new(x.keys).subset? Set.new(@@ConfigHash.keys) - }] - - @@DefaultConfig = { - :protocol => 'http', - :method => 'get' - } - - Contract String, @@StrictConfigHash => lambda { |x| x.is_a? Emitter } - def initialize(endpoint, config={}) - config = @@DefaultConfig.merge(config) - @lock = Monitor.new - @collector_uri = as_collector_uri(endpoint, config[:protocol], config[:port], config[:method]) - @buffer = [] - if not config[:buffer_size].nil? - @buffer_size = config[:buffer_size] - elsif config[:method] == 'get' - @buffer_size = 1 - else - @buffer_size = 10 - end - @method = config[:method] - @on_success = config[:on_success] - @on_failure = config[:on_failure] - LOGGER.info("#{self.class} initialized with endpoint #{@collector_uri}") - - self - end - - # Build the collector URI from the configuration hash - # - Contract String, String, Maybe[Num], String => String - def as_collector_uri(endpoint, protocol, port, method) - port_string = port == nil ? '' : ":#{port.to_s}" - path = method == 'get' ? '/i' : '/com.snowplowanalytics.snowplow/tp2' - - "#{protocol}://#{endpoint}#{port_string}#{path}" - end - - # Add an event to the buffer and flush it if maximum size has been reached - # - Contract Hash => nil - def input(payload) - payload.each { |k,v| payload[k] = v.to_s} - @lock.synchronize do - @buffer.push(payload) - if @buffer.size >= @buffer_size - flush - end - end - - nil - end - - # Flush the buffer - # - Contract Bool => nil - def flush(async=true) - @lock.synchronize do - send_requests(@buffer) - @buffer = [] - end - nil - end - - # Send all events in the buffer to the collector - # - Contract ArrayOf[Hash] => nil - def send_requests(evts) - if evts.size < 1 - LOGGER.info("Skipping sending events since buffer is empty") - return - end - LOGGER.info("Attempting to send #{evts.size} request#{evts.size == 1 ? '' : 's'}") - - evts.each do |event| - event['stm'] = (Time.now.to_f * 1000).to_i.to_s # add the sent timestamp, overwrite if already exists - end - - if @method == 'post' - post_succeeded = false - begin - request = http_post(SelfDescribingJson.new( - 'iglu:com.snowplowanalytics.snowplow/payload_data/jsonschema/1-0-4', - evts - ).to_json) - post_succeeded = is_good_status_code(request.code) - rescue StandardError => se - LOGGER.warn(se) - end - if post_succeeded - unless @on_success.nil? - @on_success.call(evts.size) - end - else - unless @on_failure.nil? - @on_failure.call(0, evts) - end - end - - elsif @method == 'get' - success_count = 0 - unsent_requests = [] - evts.each do |evt| - get_succeeded = false - begin - request = http_get(evt) - get_succeeded = is_good_status_code(request.code) - rescue StandardError => se - LOGGER.warn(se) - end - if get_succeeded - success_count += 1 - else - unsent_requests << evt - end - end - if unsent_requests.size == 0 - unless @on_success.nil? - @on_success.call(success_count) - end - else - unless @on_failure.nil? - @on_failure.call(success_count, unsent_requests) - end - end - end - - nil - end - - # Send a GET request - # - Contract Hash => lambda { |x| x.is_a? Net::HTTPResponse } - def http_get(payload) - destination = URI(@collector_uri + '?' + URI.encode_www_form(payload)) - LOGGER.info("Sending GET request to #{@collector_uri}...") - LOGGER.debug("Payload: #{payload}") - http = Net::HTTP.new(destination.host, destination.port) - request = Net::HTTP::Get.new(destination.request_uri) - if destination.scheme == 'https' - http.use_ssl = true - end - response = http.request(request) - LOGGER.add(is_good_status_code(response.code) ? Logger::INFO : Logger::WARN) { - "GET request to #{@collector_uri} finished with status code #{response.code}" - } - - response - end - - # Send a POST request - # - Contract Hash => lambda { |x| x.is_a? Net::HTTPResponse } - def http_post(payload) - LOGGER.info("Sending POST request to #{@collector_uri}...") - LOGGER.debug("Payload: #{payload}") - destination = URI(@collector_uri) - http = Net::HTTP.new(destination.host, destination.port) - request = Net::HTTP::Post.new(destination.request_uri) - if destination.scheme == 'https' - http.use_ssl = true - end - request.body = payload.to_json - request.set_content_type('application/json; charset=utf-8') - response = http.request(request) - LOGGER.add(is_good_status_code(response.code) ? Logger::INFO : Logger::WARN) { - "POST request to #{@collector_uri} finished with status code #{response.code}" - } - - response - end - - # Only 2xx and 3xx status codes are considered successes - # - Contract String => Bool - def is_good_status_code(status_code) - status_code.to_i >= 200 && status_code.to_i < 400 - end - - private :as_collector_uri, - :http_get, - :http_post - - end - - - class AsyncEmitter < Emitter - - Contract String, @@StrictConfigHash => lambda { |x| x.is_a? Emitter } - def initialize(endpoint, config={}) - @queue = Queue.new() - # @all_processed_condition and @results_unprocessed are used to emulate Python's Queue.task_done() - @queue.extend(MonitorMixin) - @all_processed_condition = @queue.new_cond - @results_unprocessed = 0 - (config[:thread_count] || 1).times do - t = Thread.new do - consume - end - end - super(endpoint, config) - end - - def consume - loop do - work_unit = @queue.pop - send_requests(work_unit) - @queue.synchronize do - @results_unprocessed -= 1 - @all_processed_condition.broadcast - end - end - end - - # Flush the buffer - # If async is false, block until the queue is empty - # - def flush(async=true) - loop do - @lock.synchronize do - @queue.synchronize do - @results_unprocessed += 1 - end - @queue << @buffer - @buffer = [] - end - if not async - LOGGER.info('Starting synchronous flush') - @queue.synchronize do - @all_processed_condition.wait_while { @results_unprocessed > 0 } - LOGGER.info('Finished synchronous flush') - end - end - break if @buffer.size < 1 - end - end - end - -end diff --git a/snowplow-tracker/lib/snowplow-tracker/payload.rb b/snowplow-tracker/lib/snowplow-tracker/payload.rb deleted file mode 100644 index 383f525269..0000000000 --- a/snowplow-tracker/lib/snowplow-tracker/payload.rb +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright (c) 2013-2014 Snowplow Analytics Ltd. All rights reserved. -# -# This program is licensed to you under the Apache License Version 2.0, -# and you may not use this file except in compliance with the Apache License Version 2.0. -# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the Apache License Version 2.0 is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. - -# Author:: Alex Dean, Fred Blundun (mailto:support@snowplowanalytics.com) -# Copyright:: Copyright (c) 2013-2014 Snowplow Analytics Ltd -# License:: Apache License Version 2.0 - -require 'base64' -require 'json' -require 'net/http' -require 'contracts' - -module SnowplowTracker - - class Payload - - include Contracts - - attr_reader :context - - Contract nil => Payload - def initialize - @context = {} - self - end - - # Add a single name-value pair to @context - # - Contract String, Or[String, Bool, Num, nil] => Or[String, Bool, Num, nil] - def add(name, value) - if value != "" and not value.nil? - @context[name] = value - end - end - - # Add each name-value pair in dict to @context - # - Contract Hash => Hash - def add_dict(dict) - for f in dict - self.add(f[0], f[1]) - end - end - - # Stringify a JSON and add it to @context - # - Contract Maybe[Hash], Bool, String, String => Maybe[String] - def add_json(dict, encode_base64, type_when_encoded, type_when_not_encoded) - - if dict.nil? - return - end - - dict_string = JSON.generate(dict) - - if encode_base64 - self.add(type_when_encoded, Base64.strict_encode64(dict_string)) - else - self.add(type_when_not_encoded, dict_string) - end - - end - - end -end diff --git a/snowplow-tracker/lib/snowplow-tracker/self_describing_json.rb b/snowplow-tracker/lib/snowplow-tracker/self_describing_json.rb deleted file mode 100644 index 7b917c1b00..0000000000 --- a/snowplow-tracker/lib/snowplow-tracker/self_describing_json.rb +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2013-2014 Snowplow Analytics Ltd. All rights reserved. -# -# This program is licensed to you under the Apache License Version 2.0, -# and you may not use this file except in compliance with the Apache License Version 2.0. -# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the Apache License Version 2.0 is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. - -# Author:: Alex Dean, Fred Blundun (mailto:support@snowplowanalytics.com) -# Copyright:: Copyright (c) 2013-2014 Snowplow Analytics Ltd -# License:: Apache License Version 2.0 - -module SnowplowTracker - - class SelfDescribingJson - - def initialize(schema, data) - @schema = schema - @data = data - end - - def to_json - { - :schema => @schema, - :data => @data - } - end - - end - -end diff --git a/snowplow-tracker/lib/snowplow-tracker/subject.rb b/snowplow-tracker/lib/snowplow-tracker/subject.rb deleted file mode 100644 index 09d2bdfb60..0000000000 --- a/snowplow-tracker/lib/snowplow-tracker/subject.rb +++ /dev/null @@ -1,139 +0,0 @@ -# Copyright (c) 2013-2014 Snowplow Analytics Ltd. All rights reserved. -# -# This program is licensed to you under the Apache License Version 2.0, -# and you may not use this file except in compliance with the Apache License Version 2.0. -# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the Apache License Version 2.0 is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. - -# Author:: Alex Dean, Fred Blundun (mailto:support@snowplowanalytics.com) -# Copyright:: Copyright (c) 2013-2014 Snowplow Analytics Ltd -# License:: Apache License Version 2.0 - -require 'contracts' - -module SnowplowTracker - - class Subject - - include Contracts - - @@default_platform = 'srv' - @@supported_platforms = ['pc', 'tv', 'mob', 'cnsl', 'iot'] - - attr_reader :standard_nv_pairs - - Contract None => Subject - def initialize - @standard_nv_pairs = {"p" => @@default_platform} - self - end - - # Specify the platform - # - Contract String => Subject - def set_platform(value) - if @@supported_platforms.include?(value) - @standard_nv_pairs['p'] = value - else - raise "#{value} is not a supported platform" - end - - self - end - - # Set the business-defined user ID for a user - # - Contract String => Subject - def set_user_id(user_id) - @standard_nv_pairs['uid'] = user_id - self - end - - # Set fingerprint for the user - # - Contract Num => Subject - def set_fingerprint(fingerprint) - @standard_nv_pairs['fp'] = fingerprint - self - end - - # Set the screen resolution for a device - # - Contract Num, Num => Subject - def set_screen_resolution(width, height) - @standard_nv_pairs['res'] = "#{width}x#{height}" - self - end - - # Set the dimensions of the current viewport - # - Contract Num, Num => Subject - def set_viewport(width, height) - @standard_nv_pairs['vp'] = "#{width}x#{height}" - self - end - - # Set the color depth of the device in bits per pixel - # - Contract Num => Subject - def set_color_depth(depth) - @standard_nv_pairs['cd'] = depth - self - end - - # Set the timezone field - # - Contract String => Subject - def set_timezone(timezone) - @standard_nv_pairs['tz'] = timezone - self - end - - # Set the language field - # - Contract String => Subject - def set_lang(lang) - @standard_nv_pairs['lang'] = lang - self - end - - # Set the domain user ID - # - Contract String => Subject - def set_domain_user_id(duid) - @standard_nv_pairs['duid'] = duid - self - end - - # Set the IP address field - # - Contract String => Subject - def set_ip_address(ip) - @standard_nv_pairs['ip'] = ip - self - end - - # Set the user agent - # - Contract String => Subject - def set_useragent(ua) - @standard_nv_pairs['ua'] = ua - self - end - - # Set the network user ID field - # This overwrites the nuid field set by the collector - # - Contract String => Subject - def set_network_user_id(nuid) - @standard_nv_pairs['tnuid'] = nuid - self - end - - end - -end diff --git a/snowplow-tracker/lib/snowplow-tracker/timestamp.rb b/snowplow-tracker/lib/snowplow-tracker/timestamp.rb deleted file mode 100644 index d81a12850c..0000000000 --- a/snowplow-tracker/lib/snowplow-tracker/timestamp.rb +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 2016 Snowplow Analytics Ltd. All rights reserved. -# -# This program is licensed to you under the Apache License Version 2.0, -# and you may not use this file except in compliance with the Apache License Version 2.0. -# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the Apache License Version 2.0 is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. - -# Author:: Alex Dean, Fred Blundun, Ed Lewis (mailto:support@snowplowanalytics.com) -# Copyright:: Copyright (c) 2016 Snowplow Analytics Ltd -# License:: Apache License Version 2.0 - -module SnowplowTracker - - class Timestamp - - attr_reader :type - attr_reader :value - - def initialize(type, value) - @type = type - @value = value - end - - end - - class TrueTimestamp < Timestamp - - def initialize(value) - super 'ttm', value - end - - end - - class DeviceTimestamp < Timestamp - - def initialize(value) - super 'dtm', value - end - - end - -end \ No newline at end of file diff --git a/snowplow-tracker/lib/snowplow-tracker/tracker.rb b/snowplow-tracker/lib/snowplow-tracker/tracker.rb deleted file mode 100644 index f73dcef505..0000000000 --- a/snowplow-tracker/lib/snowplow-tracker/tracker.rb +++ /dev/null @@ -1,371 +0,0 @@ -# Copyright (c) 2013-2014 Snowplow Analytics Ltd. All rights reserved. -# -# This program is licensed to you under the Apache License Version 2.0, -# and you may not use this file except in compliance with the Apache License Version 2.0. -# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the Apache License Version 2.0 is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. - -# Author:: Alex Dean, Fred Blundun (mailto:support@snowplowanalytics.com) -# Copyright:: Copyright (c) 2013-2014 Snowplow Analytics Ltd -# License:: Apache License Version 2.0 - -require 'contracts' -require 'securerandom' -require 'set' - -module SnowplowTracker - - class Tracker - - include Contracts - - @@EmitterInput = Or[lambda {|x| x.is_a? Emitter}, ArrayOf[lambda {|x| x.is_a? Emitter}]] - - @@required_transaction_keys = Set.new(%w(order_id total_value)) - @@recognised_transaction_keys = Set.new(%w(order_id total_value affiliation tax_value shipping city state country currency)) - - @@Transaction = lambda { |x| - return false unless x.class == Hash - transaction_keys = Set.new(x.keys) - @@required_transaction_keys.subset? transaction_keys and - transaction_keys.subset? @@recognised_transaction_keys - } - - @@required_item_keys = Set.new(%w(sku price quantity)) - @@recognised_item_keys = Set.new(%w(sku price quantity name category context)) - - @@Item = lambda { |x| - return false unless x.class == Hash - item_keys = Set.new(x.keys) - @@required_item_keys.subset? item_keys and - item_keys.subset? @@recognised_item_keys - } - - @@required_augmented_item_keys = Set.new(%w(sku price quantity tstamp order_id)) - @@recognised_augmented_item_keys = Set.new(%w(sku price quantity name category context tstamp order_id currency)) - - @@AugmentedItem = lambda { |x| - return false unless x.class == Hash - augmented_item_keys = Set.new(x.keys) - @@required_augmented_item_keys.subset? augmented_item_keys and - augmented_item_keys.subset? @@recognised_augmented_item_keys - } - - @@ContextsInput = ArrayOf[SelfDescribingJson] - - @@version = TRACKER_VERSION - @@default_encode_base64 = true - - @@base_schema_path = "iglu:com.snowplowanalytics.snowplow" - @@schema_tag = "jsonschema" - @@context_schema = "#{@@base_schema_path}/contexts/#{@@schema_tag}/1-0-1" - @@unstruct_event_schema = "#{@@base_schema_path}/unstruct_event/#{@@schema_tag}/1-0-0" - - Contract @@EmitterInput, Maybe[Subject], Maybe[String], Maybe[String], Bool => Tracker - def initialize(emitters, subject=nil, namespace=nil, app_id=nil, encode_base64=@@default_encode_base64) - @emitters = Array(emitters) - if subject.nil? - @subject = Subject.new - else - @subject = subject - end - @standard_nv_pairs = { - 'tna' => namespace, - 'tv' => @@version, - 'aid' => app_id - } - @config = { - 'encode_base64' => encode_base64 - } - - self - end - - # Call subject methods from tracker instance - # - Subject.instance_methods(false).each do |name| - define_method name, ->(*splat) do - @subject.method(name.to_sym).call(*splat) - - self - end - end - - # Generates a type-4 UUID to identify this event - Contract nil => String - def get_event_id() - SecureRandom.uuid - end - - # Generates the timestamp (in milliseconds) to be attached to each event - # - Contract nil => Num - def get_timestamp - (Time.now.to_f * 1000).to_i - end - - # Builds a self-describing JSON from an array of custom contexts - # - Contract @@ContextsInput => Hash - def build_context(context) - SelfDescribingJson.new( - @@context_schema, - context.map {|c| c.to_json} - ).to_json - end - - # Tracking methods - - # Attaches all the fields in @standard_nv_pairs to the request - # Only attaches the context vendor if the event has a custom context - # - Contract Payload => nil - def track(pb) - pb.add_dict(@subject.standard_nv_pairs) - pb.add_dict(@standard_nv_pairs) - pb.add('eid', get_event_id()) - @emitters.each{ |emitter| emitter.input(pb.context)} - - nil - end - - # Log a visit to this page with an inserted device timestamp - # - Contract String, Maybe[String], Maybe[String], Maybe[@@ContextsInput], Maybe[Num] => Tracker - def track_page_view(page_url, page_title=nil, referrer=nil, context=nil, tstamp=nil) - if tstamp.nil? - tstamp = get_timestamp - end - - track_page_view(page_url, page_title, referrer, context, DeviceTimestamp.new(tstamp)) - end - - # Log a visit to this page - # - Contract String, Maybe[String], Maybe[String], Maybe[@@ContextsInput], SnowplowTracker::Timestamp => Tracker - def track_page_view(page_url, page_title=nil, referrer=nil, context=nil, tstamp=nil) - pb = Payload.new - pb.add('e', 'pv') - pb.add('url', page_url) - pb.add('page', page_title) - pb.add('refr', referrer) - - unless context.nil? - pb.add_json(build_context(context), @config['encode_base64'], 'cx', 'co') - end - - pb.add(tstamp.type, tstamp.value) - - track(pb) - - self - end - - # Track a single item within an ecommerce transaction - # Not part of the public API - # - Contract @@AugmentedItem => self - def track_ecommerce_transaction_item(argmap) - pb = Payload.new - pb.add('e', 'ti') - pb.add('ti_id', argmap['order_id']) - pb.add('ti_sk', argmap['sku']) - pb.add('ti_pr', argmap['price']) - pb.add('ti_qu', argmap['quantity']) - pb.add('ti_nm', argmap['name']) - pb.add('ti_ca', argmap['category']) - pb.add('ti_cu', argmap['currency']) - unless argmap['context'].nil? - pb.add_json(build_context(argmap['context']), @config['encode_base64'], 'cx', 'co') - end - pb.add(argmap['tstamp'].type, argmap['tstamp'].value) - track(pb) - - self - end - - # Track an ecommerce transaction and all the items in it - # Set the timestamp as the device timestamp - Contract @@Transaction, ArrayOf[@@Item], Maybe[@@ContextsInput], Maybe[Num] => Tracker - def track_ecommerce_transaction(transaction, - items, - context=nil, - tstamp=nil) - if tstamp.nil? - tstamp = get_timestamp - end - - track_ecommerce_transaction(transaction, items, context, DeviceTimestamp.new(tstamp)) - end - - # Track an ecommerce transaction and all the items in it - # - Contract @@Transaction, ArrayOf[@@Item], Maybe[@@ContextsInput], Timestamp => Tracker - def track_ecommerce_transaction(transaction, items, - context=nil, tstamp=nil) - pb = Payload.new - pb.add('e', 'tr') - pb.add('tr_id', transaction['order_id']) - pb.add('tr_tt', transaction['total_value']) - pb.add('tr_af', transaction['affiliation']) - pb.add('tr_tx', transaction['tax_value']) - pb.add('tr_sh', transaction['shipping']) - pb.add('tr_ci', transaction['city']) - pb.add('tr_st', transaction['state']) - pb.add('tr_co', transaction['country']) - pb.add('tr_cu', transaction['currency']) - unless context.nil? - pb.add_json(build_context(context), @config['encode_base64'], 'cx', 'co') - end - - pb.add(tstamp.type, tstamp.value) - - track(pb) - - for item in items - item['tstamp'] = tstamp - item['order_id'] = transaction['order_id'] - item['currency'] = transaction['currency'] - track_ecommerce_transaction_item(item) - end - - self - end - - # Track a structured event - # set the timestamp to the device timestamp - Contract String, String, Maybe[String], Maybe[String], Maybe[Num], Maybe[@@ContextsInput], Maybe[Num] => Tracker - def track_struct_event(category, action, label=nil, property=nil, value=nil, context=nil, tstamp=nil) - if tstamp.nil? - tstamp = get_timestamp - end - - track_struct_event(category, action, label, property, value, context, DeviceTimestamp.new(tstamp)) - end - # Track a structured event - # - Contract String, String, Maybe[String], Maybe[String], Maybe[Num], Maybe[@@ContextsInput], Timestamp => Tracker - def track_struct_event(category, action, label=nil, property=nil, value=nil, context=nil, tstamp=nil) - pb = Payload.new - pb.add('e', 'se') - pb.add('se_ca', category) - pb.add('se_ac', action) - pb.add('se_la', label) - pb.add('se_pr', property) - pb.add('se_va', value) - unless context.nil? - pb.add_json(build_context(context), @config['encode_base64'], 'cx', 'co') - end - - pb.add(tstamp.type, tstamp.value) - track(pb) - - self - end - - # Track a screen view event - # - Contract Maybe[String], Maybe[String], Maybe[@@ContextsInput], Or[Timestamp, Num, nil] => Tracker - def track_screen_view(name=nil, id=nil, context=nil, tstamp=nil) - screen_view_properties = {} - unless name.nil? - screen_view_properties['name'] = name - end - unless id.nil? - screen_view_properties['id'] = id - end - screen_view_schema = "#{@@base_schema_path}/screen_view/#{@@schema_tag}/1-0-0" - - event_json = SelfDescribingJson.new(screen_view_schema, screen_view_properties) - - self.track_unstruct_event(event_json, context, tstamp) - - self - end - - # Better name for track unstruct event - # - Contract SelfDescribingJson, Maybe[@@ContextsInput], Timestamp => Tracker - def track_self_describing_event(event_json, context=nil, tstamp=nil) - track_unstruct_event(event_json, context, tstamp) - end - - # Better name for track unstruct event - # set the timestamp to the device timestamp - Contract SelfDescribingJson, Maybe[@@ContextsInput], Maybe[Num] => Tracker - def track_self_describing_event(event_json, context=nil, tstamp=nil) - track_unstruct_event(event_json, context, tstamp) - end - - # Track an unstructured event - # set the timestamp to the device timstamp - Contract SelfDescribingJson, Maybe[@@ContextsInput], Maybe[Num] => Tracker - def track_unstruct_event(event_json, context=nil, tstamp=nil) - if tstamp.nil? - tstamp = get_timestamp - end - - track_unstruct_event(event_json, context, DeviceTimestamp.new(tstamp)) - end - - # Track an unstructured event - # - Contract SelfDescribingJson, Maybe[@@ContextsInput], Timestamp => Tracker - def track_unstruct_event(event_json, context=nil, tstamp=nil) - pb = Payload.new - pb.add('e', 'ue') - - envelope = SelfDescribingJson.new(@@unstruct_event_schema, event_json.to_json) - - pb.add_json(envelope.to_json, @config['encode_base64'], 'ue_px', 'ue_pr') - - unless context.nil? - pb.add_json(build_context(context), @config['encode_base64'], 'cx', 'co') - end - - pb.add(tstamp.type, tstamp.value) - - track(pb) - - self - end - - # Flush all events stored in all emitters - # - Contract Bool => Tracker - def flush(async=false) - @emitters.each do |emitter| - emitter.flush(async) - end - - self - end - - # Set the subject of the events fired by the tracker - # - Contract Subject => Tracker - def set_subject(subject) - @subject = subject - self - end - - # Add a new emitter - # - Contract Emitter => Tracker - def add_emitter(emitter) - @emitters.push(emitter) - self - end - - private :get_timestamp, - :build_context, - :track, - :track_ecommerce_transaction_item - - end - -end diff --git a/snowplow-tracker/lib/snowplow-tracker/version.rb b/snowplow-tracker/lib/snowplow-tracker/version.rb deleted file mode 100644 index 18bde7bf60..0000000000 --- a/snowplow-tracker/lib/snowplow-tracker/version.rb +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) 2013-2014 Snowplow Analytics Ltd. All rights reserved. -# -# This program is licensed to you under the Apache License Version 2.0, -# and you may not use this file except in compliance with the Apache License Version 2.0. -# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the Apache License Version 2.0 is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. - -# Author:: Alex Dean, Fred Blundun (mailto:support@snowplowanalytics.com) -# Copyright:: Copyright (c) 2013-2014 Snowplow Analytics Ltd -# License:: Apache License Version 2.0 - -module SnowplowTracker - VERSION = '0.6.1' - TRACKER_VERSION = "rb-#{VERSION}" -end diff --git a/snowplow-tracker/snowplow-tracker.gemspec b/snowplow-tracker/snowplow-tracker.gemspec deleted file mode 100644 index c30cb26829..0000000000 --- a/snowplow-tracker/snowplow-tracker.gemspec +++ /dev/null @@ -1,41 +0,0 @@ -######################################################### -# This file has been automatically generated by gem2tgz # -######################################################### -# -*- encoding: utf-8 -*- -# stub: snowplow-tracker 0.6.1 ruby lib - -Gem::Specification.new do |s| - s.name = "snowplow-tracker".freeze - s.version = "0.6.1" - - s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version= - s.require_paths = ["lib".freeze] - s.authors = ["Alexander Dean".freeze, "Fred Blundun".freeze] - s.date = "2016-12-26" - s.description = "With this tracker you can collect event data from your Ruby applications, Ruby on Rails web applications and Ruby gems.".freeze - s.email = "support@snowplowanalytics.com".freeze - s.files = ["LICENSE-2.0.txt".freeze, "README.md".freeze, "lib/snowplow-tracker.rb".freeze, "lib/snowplow-tracker/contracts.rb".freeze, "lib/snowplow-tracker/emitters.rb".freeze, "lib/snowplow-tracker/payload.rb".freeze, "lib/snowplow-tracker/self_describing_json.rb".freeze, "lib/snowplow-tracker/subject.rb".freeze, "lib/snowplow-tracker/timestamp.rb".freeze, "lib/snowplow-tracker/tracker.rb".freeze, "lib/snowplow-tracker/version.rb".freeze] - s.homepage = "http://github.com/snowplow/snowplow-ruby-tracker".freeze - s.licenses = ["Apache License 2.0".freeze] - s.required_ruby_version = Gem::Requirement.new(">= 2.0.0".freeze) - s.rubygems_version = "2.5.2.1".freeze - s.summary = "Ruby Analytics for Snowplow".freeze - - if s.respond_to? :specification_version then - s.specification_version = 4 - - if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then - s.add_runtime_dependency(%q.freeze, ["<= 0.11", "~> 0.7"]) - s.add_development_dependency(%q.freeze, ["~> 2.14.1"]) - s.add_development_dependency(%q.freeze, ["~> 1.17.4"]) - else - s.add_dependency(%q.freeze, ["<= 0.11", "~> 0.7"]) - s.add_dependency(%q.freeze, ["~> 2.14.1"]) - s.add_dependency(%q.freeze, ["~> 1.17.4"]) - end - else - s.add_dependency(%q.freeze, ["<= 0.11", "~> 0.7"]) - s.add_dependency(%q.freeze, ["~> 2.14.1"]) - s.add_dependency(%q.freeze, ["~> 1.17.4"]) - end -end