diff --git a/elasticsearch-rails/.gitignore b/elasticsearch-rails/.gitignore new file mode 100644 index 0000000000..d87d4be66f --- /dev/null +++ b/elasticsearch-rails/.gitignore @@ -0,0 +1,17 @@ +*.gem +*.rbc +.bundle +.config +.yardoc +Gemfile.lock +InstalledFiles +_yardoc +coverage +doc/ +lib/bundler/man +pkg +rdoc +spec/reports +test/tmp +test/version_tmp +tmp diff --git a/elasticsearch-rails/CHANGELOG.md b/elasticsearch-rails/CHANGELOG.md new file mode 100644 index 0000000000..5bb4e13bd3 --- /dev/null +++ b/elasticsearch-rails/CHANGELOG.md @@ -0,0 +1,44 @@ +## 0.1.9 + +* Added checks for proper launch order and other updates to the example application templates +* Updated the example application to work with Elasticsearch 2.x +* Used the `suggest` method instead of `response['suggest']` in the application template + +## 0.1.8 + +* Added an example application template that loads settings from a file +* Added missing require in the seeds.rb file for the expert template +* Fixed double include of the aliased method (execute_without_instrumentation) +* Fixed the error when getting the search_controller_test.rb asset in `03-expert.rb` template +* Updated URLs for getting raw assets from Github in the `03-expert.rb` template + +## 0.1.7 + +* Updated dependencies for the gem and example applications +* Fixed various small errors in the `01-basic.rb` template +* Fixed error when inserting the Kaminari gem into Gemfile in the 02-pretty.rb template +* Fixed incorrect regex for adding Rails instrumentation into the application.rb in the `02-pretty.rb` template +* Fixed other small errors in the `02-pretty.rb` template +* Improved and added tests for the generated application from the `02-pretty.rb` template +* Added the `04-dsl.rb` template which uses the `elasticsearch-dsl` gem to build the search definition + +## 0.1.6 + +* Fixed errors in templates for the Rails example applications +* Fixed errors in the importing Rake task +* Refactored and updated the instrumentation support to allow integration with `Persistence::Model` + +## 0.1.5 + +* Fixed an exception when no suggestions were returned in the `03-expert` example application template + +## 0.1.2 + +* Allow passing an ActiveRecord scope to the importing Rake task + +## 0.1.1 + +* Improved the Rake tasks +* Improved the example application templates + +## 0.1.0 (Initial Version) diff --git a/elasticsearch-rails/Gemfile b/elasticsearch-rails/Gemfile new file mode 100644 index 0000000000..1aeec6c9a0 --- /dev/null +++ b/elasticsearch-rails/Gemfile @@ -0,0 +1,9 @@ +source 'https://rubygems.org' + +# Specify your gem's dependencies in elasticsearch-rails.gemspec +gemspec + +# TODO: Figure out how to specify dependency on local elasticsearch-model without endless "Resolving dependencies" +# if File.exists? File.expand_path("../../elasticsearch-model", __FILE__) +# gem 'elasticsearch-model', :path => File.expand_path("../../elasticsearch-model", __FILE__), :require => true +# end diff --git a/elasticsearch-rails/LICENSE.txt b/elasticsearch-rails/LICENSE.txt new file mode 100644 index 0000000000..7dc94b3e5a --- /dev/null +++ b/elasticsearch-rails/LICENSE.txt @@ -0,0 +1,13 @@ +Copyright (c) 2014 Elasticsearch + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/elasticsearch-rails/README.md b/elasticsearch-rails/README.md new file mode 100644 index 0000000000..4760549385 --- /dev/null +++ b/elasticsearch-rails/README.md @@ -0,0 +1,132 @@ +# Elasticsearch::Rails + +The `elasticsearch-rails` library is a companion for the +the [`elasticsearch-model`](https://github.com/elasticsearch/elasticsearch-rails/tree/master/elasticsearch-model) +library, providing features suitable for Ruby on Rails applications. + +The library is compatible with Ruby 1.9.3 and higher. + +## Installation + +Install the package from [Rubygems](https://rubygems.org): + + gem install elasticsearch-rails + +To use an unreleased version, either add it to your `Gemfile` for [Bundler](http://bundler.io): + + gem 'elasticsearch-rails', git: 'git://github.com/elasticsearch/elasticsearch-rails.git' + +or install it from a source code checkout: + + git clone https://github.com/elasticsearch/elasticsearch-rails.git + cd elasticsearch-rails/elasticsearch-rails + bundle install + rake install + +## Features + +### Rake Tasks + +To facilitate importing data from your models into Elasticsearch, require the task definition in your application, +eg. in the `lib/tasks/elasticsearch.rake` file: + +```ruby +require 'elasticsearch/rails/tasks/import' +``` + +To import the records from your `Article` model, run: + +```bash +$ bundle exec rake environment elasticsearch:import:model CLASS='Article' +``` + +To limit the imported records to a certain +ActiveRecord [scope](http://guides.rubyonrails.org/active_record_querying.html#scopes), +pass it to the task: + +```bash +$ bundle exec rake environment elasticsearch:import:model CLASS='Article' SCOPE='published' +``` + +Run this command to display usage instructions: + +```bash +$ bundle exec rake -D elasticsearch +``` + +### ActiveSupport Instrumentation + +To display information about the search request (duration, search definition) during development, +and to include the information in the Rails log file, require the component in your `application.rb` file: + +```ruby +require 'elasticsearch/rails/instrumentation' +``` + +You should see an output like this in your application log in development environment: + + Article Search (321.3ms) { index: "articles", type: "article", body: { query: ... } } + +Also, the total duration of the request to Elasticsearch is displayed in the Rails request breakdown: + + Completed 200 OK in 615ms (Views: 230.9ms | ActiveRecord: 0.0ms | Elasticsearch: 321.3ms) + +There's a special component for the [Lograge](https://github.com/roidrage/lograge) logger. +Require the component in your `application.rb` file (and set `config.lograge.enabled`): + +```ruby +require 'elasticsearch/rails/lograge' +``` + +You should see the duration of the request to Elasticsearch as part of each log event: + + method=GET path=/search ... status=200 duration=380.89 view=99.64 db=0.00 es=279.37 + +### Rails Application Templates + +You can generate a fully working example Ruby on Rails application, with an `Article` model and a search form, +to play with (it even downloads _Elasticsearch_ itself, generates the application skeleton and leaves you with +a _Git_ repository to explore the steps and the code) with the +[`01-basic.rb`](https://github.com/elasticsearch/elasticsearch-rails/blob/master/elasticsearch-rails/lib/rails/templates/01-basic.rb) template: + +```bash +rails new searchapp --skip --skip-bundle --template https://raw.github.com/elasticsearch/elasticsearch-rails/master/elasticsearch-rails/lib/rails/templates/01-basic.rb +``` + +Run the same command again, in the same folder, with the +[`02-pretty`](https://github.com/elasticsearch/elasticsearch-rails/blob/master/elasticsearch-rails/lib/rails/templates/02-pretty.rb) +template to add features such as a custom `Article.search` method, result highlighting and +[_Bootstrap_](http://getbootstrap.com) integration: + +```bash +rails new searchapp --skip --skip-bundle --template https://raw.github.com/elasticsearch/elasticsearch-rails/master/elasticsearch-rails/lib/rails/templates/02-pretty.rb +``` + +Run the same command with the [`03-expert.rb`](https://github.com/elasticsearch/elasticsearch-rails/blob/master/elasticsearch-rails/lib/rails/templates/03-expert.rb) +template to refactor the application into a more complex use case, +with couple of hundreds of The New York Times articles as the example content. +The template will extract the Elasticsearch integration into a `Searchable` "concern" module, +define complex mapping, custom serialization, implement faceted navigation and suggestions as a part of +a complex query, and add a _Sidekiq_-based worker for updating the index in the background. + +```bash +rails new searchapp --skip --skip-bundle --template https://raw.github.com/elasticsearch/elasticsearch-rails/master/elasticsearch-rails/lib/rails/templates/03-expert.rb +``` + +## License + +This software is licensed under the Apache 2 license, quoted below. + + Copyright (c) 2014 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/elasticsearch-rails/Rakefile b/elasticsearch-rails/Rakefile new file mode 100644 index 0000000000..3cf581a919 --- /dev/null +++ b/elasticsearch-rails/Rakefile @@ -0,0 +1,53 @@ +require "bundler/gem_tasks" + +desc "Run unit tests" +task :default => 'test:unit' +task :test => 'test:unit' + +# ----- Test tasks ------------------------------------------------------------ + +require 'rake/testtask' +namespace :test do + task :ci_reporter do + ENV['CI_REPORTS'] ||= 'tmp/reports' + require 'ci/reporter/rake/minitest' + Rake::Task['ci:setup:minitest'].invoke + end + + Rake::TestTask.new(:unit) do |test| + Rake::Task['test:ci_reporter'].invoke if ENV['CI'] + test.libs << 'lib' << 'test' + test.test_files = FileList["test/unit/**/*_test.rb"] + # test.verbose = true + # test.warning = true + end + + Rake::TestTask.new(:integration) do |test| + Rake::Task['test:ci_reporter'].invoke if ENV['CI'] + test.libs << 'lib' << 'test' + test.test_files = FileList["test/integration/**/*_test.rb"] + end + + Rake::TestTask.new(:all) do |test| + Rake::Task['test:ci_reporter'].invoke if ENV['CI'] + test.libs << 'lib' << 'test' + test.test_files = FileList["test/unit/**/*_test.rb", "test/integration/**/*_test.rb"] + end +end + +# ----- Documentation tasks --------------------------------------------------- + +require 'yard' +YARD::Rake::YardocTask.new(:doc) do |t| + t.options = %w| --embed-mixins --markup=markdown | +end + +# ----- Code analysis tasks --------------------------------------------------- + +if defined?(RUBY_VERSION) && RUBY_VERSION > '1.9' + require 'cane/rake_task' + Cane::RakeTask.new(:quality) do |cane| + cane.abc_max = 15 + cane.no_style = true + end +end diff --git a/elasticsearch-rails/elasticsearch-rails.gemspec b/elasticsearch-rails/elasticsearch-rails.gemspec new file mode 100644 index 0000000000..14dee1ddf7 --- /dev/null +++ b/elasticsearch-rails/elasticsearch-rails.gemspec @@ -0,0 +1,52 @@ +# coding: utf-8 +lib = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'elasticsearch/rails/version' + +Gem::Specification.new do |s| + s.name = "elasticsearch-rails" + s.version = Elasticsearch::Rails::VERSION + s.authors = ["Karel Minarik"] + s.email = ["karel.minarik@elasticsearch.org"] + s.description = "Ruby on Rails integrations for Elasticsearch." + s.summary = "Ruby on Rails integrations for Elasticsearch." + s.homepage = "https://github.com/elasticsearch/elasticsearch-rails/" + s.license = "Apache 2" + + s.files = `git ls-files`.split($/) + s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) } + s.test_files = s.files.grep(%r{^(test|spec|features)/}) + s.require_paths = ["lib"] + + s.extra_rdoc_files = [ "README.md", "LICENSE.txt" ] + s.rdoc_options = [ "--charset=UTF-8" ] + + s.required_ruby_version = ">= 1.9.3" + + s.add_development_dependency "bundler", "~> 1.3" + s.add_development_dependency "rake", "< 11.0" + + s.add_development_dependency "elasticsearch-extensions" + s.add_development_dependency "elasticsearch-model" + + s.add_development_dependency "oj" + s.add_development_dependency "rails", ">= 3.1" + + s.add_development_dependency "lograge" + + s.add_development_dependency "minitest", "~> 4.2" + s.add_development_dependency "test-unit" if defined?(RUBY_VERSION) && RUBY_VERSION > '2.2' + s.add_development_dependency "shoulda-context" + s.add_development_dependency "mocha" + s.add_development_dependency "turn" + s.add_development_dependency "yard" + s.add_development_dependency "ruby-prof" + s.add_development_dependency "pry" + s.add_development_dependency "ci_reporter", "~> 1.9" + + if defined?(RUBY_VERSION) && RUBY_VERSION > '1.9' + s.add_development_dependency "simplecov" + s.add_development_dependency "cane" + s.add_development_dependency "require-prof" + end +end diff --git a/elasticsearch-rails/lib/elasticsearch/rails.rb b/elasticsearch-rails/lib/elasticsearch/rails.rb new file mode 100644 index 0000000000..f425f72763 --- /dev/null +++ b/elasticsearch-rails/lib/elasticsearch/rails.rb @@ -0,0 +1,7 @@ +require "elasticsearch/rails/version" + +module Elasticsearch + module Rails + # Your code goes here... + end +end diff --git a/elasticsearch-rails/lib/elasticsearch/rails/instrumentation.rb b/elasticsearch-rails/lib/elasticsearch/rails/instrumentation.rb new file mode 100644 index 0000000000..081791ab55 --- /dev/null +++ b/elasticsearch-rails/lib/elasticsearch/rails/instrumentation.rb @@ -0,0 +1,36 @@ +require 'elasticsearch/rails/instrumentation/railtie' +require 'elasticsearch/rails/instrumentation/publishers' + +module Elasticsearch + module Rails + + # This module adds support for displaying statistics about search duration in the Rails application log + # by integrating with the `ActiveSupport::Notifications` framework and `ActionController` logger. + # + # == Usage + # + # Require the component in your `application.rb` file: + # + # require 'elasticsearch/rails/instrumentation' + # + # You should see an output like this in your application log in development environment: + # + # Article Search (321.3ms) { index: "articles", type: "article", body: { query: ... } } + # + # Also, the total duration of the request to Elasticsearch is displayed in the Rails request breakdown: + # + # Completed 200 OK in 615ms (Views: 230.9ms | ActiveRecord: 0.0ms | Elasticsearch: 321.3ms) + # + # @note The displayed duration includes the HTTP transfer -- the time it took Elasticsearch + # to process your request is available in the `response.took` property. + # + # @see Elasticsearch::Rails::Instrumentation::Publishers + # @see Elasticsearch::Rails::Instrumentation::Railtie + # + # @see http://api.rubyonrails.org/classes/ActiveSupport/Notifications.html + # + # + module Instrumentation + end + end +end diff --git a/elasticsearch-rails/lib/elasticsearch/rails/instrumentation/controller_runtime.rb b/elasticsearch-rails/lib/elasticsearch/rails/instrumentation/controller_runtime.rb new file mode 100644 index 0000000000..461387c808 --- /dev/null +++ b/elasticsearch-rails/lib/elasticsearch/rails/instrumentation/controller_runtime.rb @@ -0,0 +1,41 @@ +require 'active_support/core_ext/module/attr_internal' + +module Elasticsearch + module Rails + module Instrumentation + + # Hooks into ActionController to display Elasticsearch runtime + # + # @see https://github.com/rails/rails/blob/master/activerecord/lib/active_record/railties/controller_runtime.rb + # + module ControllerRuntime + extend ActiveSupport::Concern + + protected + + attr_internal :elasticsearch_runtime + + def cleanup_view_runtime + elasticsearch_rt_before_render = Elasticsearch::Rails::Instrumentation::LogSubscriber.reset_runtime + runtime = super + elasticsearch_rt_after_render = Elasticsearch::Rails::Instrumentation::LogSubscriber.reset_runtime + self.elasticsearch_runtime = elasticsearch_rt_before_render + elasticsearch_rt_after_render + runtime - elasticsearch_rt_after_render + end + + def append_info_to_payload(payload) + super + payload[:elasticsearch_runtime] = (elasticsearch_runtime || 0) + Elasticsearch::Rails::Instrumentation::LogSubscriber.reset_runtime + end + + module ClassMethods + def log_process_action(payload) + messages, elasticsearch_runtime = super, payload[:elasticsearch_runtime] + messages << ("Elasticsearch: %.1fms" % elasticsearch_runtime.to_f) if elasticsearch_runtime + messages + end + end + end + end + end +end diff --git a/elasticsearch-rails/lib/elasticsearch/rails/instrumentation/log_subscriber.rb b/elasticsearch-rails/lib/elasticsearch/rails/instrumentation/log_subscriber.rb new file mode 100644 index 0000000000..c02bc07049 --- /dev/null +++ b/elasticsearch-rails/lib/elasticsearch/rails/instrumentation/log_subscriber.rb @@ -0,0 +1,41 @@ +module Elasticsearch + module Rails + module Instrumentation + + # A log subscriber to attach to Elasticsearch related events + # + # @see https://github.com/rails/rails/blob/master/activerecord/lib/active_record/log_subscriber.rb + # + class LogSubscriber < ActiveSupport::LogSubscriber + def self.runtime=(value) + Thread.current["elasticsearch_runtime"] = value + end + + def self.runtime + Thread.current["elasticsearch_runtime"] ||= 0 + end + + def self.reset_runtime + rt, self.runtime = runtime, 0 + rt + end + + # Intercept `search.elasticsearch` events, and display them in the Rails log + # + def search(event) + self.class.runtime += event.duration + return unless logger.debug? + + payload = event.payload + name = "#{payload[:klass]} #{payload[:name]} (#{event.duration.round(1)}ms)" + search = payload[:search].inspect.gsub(/:(\w+)=>/, '\1: ') + + debug %Q| #{color(name, GREEN, true)} #{colorize_logging ? "\e[2m#{search}\e[0m" : search}| + end + end + + end + end +end + +Elasticsearch::Rails::Instrumentation::LogSubscriber.attach_to :elasticsearch diff --git a/elasticsearch-rails/lib/elasticsearch/rails/instrumentation/publishers.rb b/elasticsearch-rails/lib/elasticsearch/rails/instrumentation/publishers.rb new file mode 100644 index 0000000000..e054d53712 --- /dev/null +++ b/elasticsearch-rails/lib/elasticsearch/rails/instrumentation/publishers.rb @@ -0,0 +1,36 @@ +module Elasticsearch + module Rails + module Instrumentation + module Publishers + + # Wraps the `SearchRequest` methods to perform the instrumentation + # + # @see SearchRequest#execute_with_instrumentation! + # @see http://api.rubyonrails.org/classes/ActiveSupport/Notifications.html + # + module SearchRequest + + def self.included(base) + base.class_eval do + unless method_defined?(:execute_without_instrumentation!) + alias_method :execute_without_instrumentation!, :execute! + alias_method :execute!, :execute_with_instrumentation! + end + end + end + + # Wrap `Search#execute!` and perform instrumentation + # + def execute_with_instrumentation! + ActiveSupport::Notifications.instrument "search.elasticsearch", + name: 'Search', + klass: (self.klass.is_a?(Elasticsearch::Model::Proxy::ClassMethodsProxy) ? self.klass.target.to_s : self.klass.to_s), + search: self.definition do + execute_without_instrumentation! + end + end + end + end + end + end +end diff --git a/elasticsearch-rails/lib/elasticsearch/rails/instrumentation/railtie.rb b/elasticsearch-rails/lib/elasticsearch/rails/instrumentation/railtie.rb new file mode 100644 index 0000000000..dbcd0fc389 --- /dev/null +++ b/elasticsearch-rails/lib/elasticsearch/rails/instrumentation/railtie.rb @@ -0,0 +1,31 @@ +module Elasticsearch + module Rails + module Instrumentation + + # Rails initializer class to require Elasticsearch::Rails::Instrumentation files, + # set up Elasticsearch::Model and hook into ActionController to display Elasticsearch-related duration + # + # @see http://edgeguides.rubyonrails.org/active_support_instrumentation.html + # + class Railtie < ::Rails::Railtie + initializer "elasticsearch.instrumentation" do |app| + require 'elasticsearch/rails/instrumentation/log_subscriber' + require 'elasticsearch/rails/instrumentation/controller_runtime' + + Elasticsearch::Model::Searching::SearchRequest.class_eval do + include Elasticsearch::Rails::Instrumentation::Publishers::SearchRequest + end if defined?(Elasticsearch::Model::Searching::SearchRequest) + + Elasticsearch::Persistence::Model::Find::SearchRequest.class_eval do + include Elasticsearch::Rails::Instrumentation::Publishers::SearchRequest + end if defined?(Elasticsearch::Persistence::Model::Find::SearchRequest) + + ActiveSupport.on_load(:action_controller) do + include Elasticsearch::Rails::Instrumentation::ControllerRuntime + end + end + end + + end + end +end diff --git a/elasticsearch-rails/lib/elasticsearch/rails/lograge.rb b/elasticsearch-rails/lib/elasticsearch/rails/lograge.rb new file mode 100644 index 0000000000..a8edd80848 --- /dev/null +++ b/elasticsearch-rails/lib/elasticsearch/rails/lograge.rb @@ -0,0 +1,44 @@ +module Elasticsearch + module Rails + module Lograge + + # Rails initializer class to require Elasticsearch::Rails::Instrumentation files, + # set up Elasticsearch::Model and add Lograge configuration to display Elasticsearch-related duration + # + # Require the component in your `application.rb` file and enable Lograge: + # + # require 'elasticsearch/rails/lograge' + # + # You should see the full duration of the request to Elasticsearch as part of each log event: + # + # method=GET path=/search ... status=200 duration=380.89 view=99.64 db=0.00 es=279.37 + # + # @see https://github.com/roidrage/lograge + # + class Railtie < ::Rails::Railtie + initializer "elasticsearch.lograge" do |app| + require 'elasticsearch/rails/instrumentation/publishers' + require 'elasticsearch/rails/instrumentation/log_subscriber' + require 'elasticsearch/rails/instrumentation/controller_runtime' + + Elasticsearch::Model::Searching::SearchRequest.class_eval do + include Elasticsearch::Rails::Instrumentation::Publishers::SearchRequest + end if defined?(Elasticsearch::Model::Searching::SearchRequest) + + Elasticsearch::Persistence::Model::Find::SearchRequest.class_eval do + include Elasticsearch::Rails::Instrumentation::Publishers::SearchRequest + end if defined?(Elasticsearch::Persistence::Model::Find::SearchRequest) + + ActiveSupport.on_load(:action_controller) do + include Elasticsearch::Rails::Instrumentation::ControllerRuntime + end + + config.lograge.custom_options = lambda do |event| + { es: event.payload[:elasticsearch_runtime].to_f.round(2) } + end + end + end + + end + end +end diff --git a/elasticsearch-rails/lib/elasticsearch/rails/tasks/import.rb b/elasticsearch-rails/lib/elasticsearch/rails/tasks/import.rb new file mode 100644 index 0000000000..bb2f9ff3d0 --- /dev/null +++ b/elasticsearch-rails/lib/elasticsearch/rails/tasks/import.rb @@ -0,0 +1,112 @@ +# A collection of Rake tasks to facilitate importing data from yout models into Elasticsearch. +# +# Add this e.g. into the `lib/tasks/elasticsearch.rake` file in your Rails application: +# +# require 'elasticsearch/rails/tasks/import' +# +# To import the records from your `Article` model, run: +# +# $ bundle exec rake environment elasticsearch:import:model CLASS='MyModel' +# +# Run this command to display usage instructions: +# +# $ bundle exec rake -D elasticsearch +# +STDOUT.sync = true +STDERR.sync = true + +begin; require 'ansi/progressbar'; rescue LoadError; end + +namespace :elasticsearch do + + task :import => 'import:model' + + namespace :import do + import_model_desc = <<-DESC.gsub(/ /, '') + Import data from your model (pass name as CLASS environment variable). + + $ rake environment elasticsearch:import:model CLASS='MyModel' + + Force rebuilding the index (delete and create): + $ rake environment elasticsearch:import:model CLASS='Article' FORCE=y + + Customize the batch size: + $ rake environment elasticsearch:import:model CLASS='Article' BATCH=100 + + Set target index name: + $ rake environment elasticsearch:import:model CLASS='Article' INDEX='articles-new' + + Pass an ActiveRecord scope to limit the imported records: + $ rake environment elasticsearch:import:model CLASS='Article' SCOPE='published' + DESC + desc import_model_desc + task :model do + if ENV['CLASS'].to_s == '' + puts '='*90, 'USAGE', '='*90, import_model_desc, "" + exit(1) + end + + klass = eval(ENV['CLASS'].to_s) + total = klass.count rescue nil + pbar = ANSI::Progressbar.new(klass.to_s, total) rescue nil + pbar.__send__ :show if pbar + + unless ENV['DEBUG'] + begin + klass.__elasticsearch__.client.transport.logger.level = Logger::WARN + rescue NoMethodError; end + begin + klass.__elasticsearch__.client.transport.tracer.level = Logger::WARN + rescue NoMethodError; end + end + + total_errors = klass.__elasticsearch__.import force: ENV.fetch('FORCE', false), + batch_size: ENV.fetch('BATCH', 1000).to_i, + index: ENV.fetch('INDEX', nil), + type: ENV.fetch('TYPE', nil), + scope: ENV.fetch('SCOPE', nil) do |response| + pbar.inc response['items'].size if pbar + STDERR.flush + STDOUT.flush + end + pbar.finish if pbar + + puts "[IMPORT] #{total_errors} errors occurred" unless total_errors.zero? + puts '[IMPORT] Done' + end + + desc <<-DESC.gsub(/ /, '') + Import all indices from `app/models` (or use DIR environment variable). + + $ rake environment elasticsearch:import:all DIR=app/models + DESC + task :all do + dir = ENV['DIR'].to_s != '' ? ENV['DIR'] : Rails.root.join("app/models") + + puts "[IMPORT] Loading models from: #{dir}" + Dir.glob(File.join("#{dir}/**/*.rb")).each do |path| + model_filename = path[/#{Regexp.escape(dir.to_s)}\/([^\.]+).rb/, 1] + + next if model_filename.match(/^concerns\//i) # Skip concerns/ folder + + begin + klass = model_filename.camelize.constantize + rescue NameError + require(path) ? retry : raise(RuntimeError, "Cannot load class '#{klass}'") + end + + # Skip if the class doesn't have Elasticsearch integration + next unless klass.respond_to?(:__elasticsearch__) + + puts "[IMPORT] Processing model: #{klass}..." + + ENV['CLASS'] = klass.to_s + Rake::Task["elasticsearch:import:model"].invoke + Rake::Task["elasticsearch:import:model"].reenable + puts + end + end + + end + +end diff --git a/elasticsearch-rails/lib/elasticsearch/rails/version.rb b/elasticsearch-rails/lib/elasticsearch/rails/version.rb new file mode 100644 index 0000000000..88b2dd7589 --- /dev/null +++ b/elasticsearch-rails/lib/elasticsearch/rails/version.rb @@ -0,0 +1,5 @@ +module Elasticsearch + module Rails + VERSION = "0.1.9" + end +end diff --git a/elasticsearch-rails/lib/rails/templates/01-basic.rb b/elasticsearch-rails/lib/rails/templates/01-basic.rb new file mode 100644 index 0000000000..129d5ffd75 --- /dev/null +++ b/elasticsearch-rails/lib/rails/templates/01-basic.rb @@ -0,0 +1,335 @@ +# ===================================================================================================== +# Template for generating a no-frills Rails application with support for Elasticsearch full-text search +# ===================================================================================================== +# +# This file creates a basic, fully working Rails application with support for Elasticsearch full-text +# search via the `elasticsearch-rails` gem; https://github.com/elasticsearch/elasticsearch-rails. +# +# Requirements: +# ------------- +# +# * Git +# * Ruby >= 1.9.3 +# * Rails >= 4 +# * Java >= 7 (for Elasticsearch) +# +# Usage: +# ------ +# +# $ rails new searchapp --skip --skip-bundle --template https://raw.github.com/elasticsearch/elasticsearch-rails/master/elasticsearch-rails/lib/rails/templates/01-basic.rb +# +# ===================================================================================================== + +require 'uri' +require 'net/http' + +at_exit do + pid = File.read("#{destination_root}/tmp/pids/elasticsearch.pid") rescue nil + if pid + say_status "Stop", "Elasticsearch", :yellow + run "kill #{pid}" + end +end + +run "touch tmp/.gitignore" + +append_to_file ".gitignore", "vendor/elasticsearch-1.0.1/\n" + +git :init +git add: "." +git commit: "-m 'Initial commit: Clean application'" + +# ----- Download Elasticsearch -------------------------------------------------------------------- + +unless (Net::HTTP.get(URI.parse('http://localhost:9200')) rescue false) + COMMAND = <<-COMMAND.gsub(/^ /, '') + curl -# -O "http://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.0.1.tar.gz" + tar -zxf elasticsearch-1.0.1.tar.gz + rm -f elasticsearch-1.0.1.tar.gz + ./elasticsearch-1.0.1/bin/elasticsearch -d -p #{destination_root}/tmp/pids/elasticsearch.pid + COMMAND + + puts "\n" + say_status "ERROR", "Elasticsearch not running!\n", :red + puts '-'*80 + say_status '', "It appears that Elasticsearch is not running on this machine." + say_status '', "Is it installed? Do you want me to install it for you with this command?\n\n" + COMMAND.each_line { |l| say_status '', "$ #{l}" } + puts + say_status '', "(To uninstall, just remove the generated application directory.)" + puts '-'*80, '' + + if yes?("Install Elasticsearch?", :bold) + puts + say_status "Install", "Elasticsearch", :yellow + + commands = COMMAND.split("\n") + exec = commands.pop + inside("vendor") do + commands.each { |command| run command } + run "(#{exec})" # Launch Elasticsearch in subshell + end + end +end unless ENV['RAILS_NO_ES_INSTALL'] + +# ----- Add README -------------------------------------------------------------------------------- + +puts +say_status "README", "Adding Readme...\n", :yellow +puts '-'*80, ''; sleep 0.25 + +remove_file 'README.rdoc' + +create_file 'README.rdoc', <<-README += Ruby on Rails and Elasticsearch: Example application + +This application is an example of integrating the {Elasticsearch}[http://www.elasticsearch.org] +search engine with the {Ruby On Rails}[http://rubyonrails.org] web framework. + +It has been generated by application templates available at +https://github.com/elasticsearch/elasticsearch-rails/tree/master/elasticsearch-rails/lib/rails/templates. + +== [1] Basic + +The `basic` version provides a simple integration for a simple Rails model, `Article`, showing how +to include the search engine support in your model, automatically index changes to records, +and use a form to perform simple search require 'requests.' + +README + + +git add: "." +git commit: "-m '[01] Added README for the application'" + +# ----- Use Thin ---------------------------------------------------------------------------------- + +begin + require 'thin' + puts + say_status "Rubygems", "Adding Thin into Gemfile...\n", :yellow + puts '-'*80, ''; + + gem 'thin' +rescue LoadError +end + +# ----- Auxiliary gems ---------------------------------------------------------------------------- + +gem 'mocha', group: 'test', require: 'mocha/api' + +# ----- Remove CoffeeScript, Sass and "all that jazz" --------------------------------------------- + +comment_lines 'Gemfile', /gem 'coffee/ +comment_lines 'Gemfile', /gem 'sass/ +comment_lines 'Gemfile', /gem 'uglifier/ +uncomment_lines 'Gemfile', /gem 'therubyracer/ + +# ----- Add gems into Gemfile --------------------------------------------------------------------- + +puts +say_status "Rubygems", "Adding Elasticsearch libraries into Gemfile...\n", :yellow +puts '-'*80, ''; sleep 0.75 + +gem 'elasticsearch', git: 'git://github.com/elasticsearch/elasticsearch-ruby.git' +gem 'elasticsearch-model', git: 'git://github.com/elasticsearch/elasticsearch-rails.git' +gem 'elasticsearch-rails', git: 'git://github.com/elasticsearch/elasticsearch-rails.git' + + +git add: "Gemfile*" +git commit: "-m 'Added libraries into Gemfile'" + +# ----- Disable asset logging in development ------------------------------------------------------ + +puts +say_status "Application", "Disabling asset logging in development...\n", :yellow +puts '-'*80, ''; sleep 0.25 + +environment 'config.assets.logger = false', env: 'development' +gem 'quiet_assets', group: "development" + +git add: "Gemfile*" +git add: "config/" +git commit: "-m 'Disabled asset logging in development'" + +# ----- Install gems ------------------------------------------------------------------------------ + +puts +say_status "Rubygems", "Installing Rubygems...", :yellow +puts '-'*80, '' + +run "bundle install" + +# ----- Generate Article resource ----------------------------------------------------------------- + +puts +say_status "Model", "Generating the Article resource...", :yellow +puts '-'*80, ''; sleep 0.75 + +generate :scaffold, "Article title:string content:text published_on:date" +route "root to: 'articles#index'" +rake "db:migrate" + +git add: "." +git commit: "-m 'Added the generated Article resource'" + +# ----- Add Elasticsearch integration into the model ---------------------------------------------- + +puts +say_status "Model", "Adding search support into the Article model...", :yellow +puts '-'*80, ''; sleep 0.25 + +run "rm -f app/models/article.rb" +file 'app/models/article.rb', <<-CODE +class Article < ActiveRecord::Base + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks + #{'attr_accessible :title, :content, :published_on' if Rails::VERSION::STRING < '4'} +end +CODE + +git commit: "-a -m 'Added Elasticsearch support into the Article model'" + +# ----- Add Elasticsearch integration into the interface ------------------------------------------ + +puts +say_status "Controller", "Adding controller action, route, and HTML for searching...", :yellow +puts '-'*80, ''; sleep 0.25 + +inject_into_file 'app/controllers/articles_controller.rb', before: %r|^\s*# GET /articles/1$| do + <<-CODE + + # GET /articles/search + def search + @articles = Article.search(params[:q]).records + + render action: "index" + end + + CODE +end + +inject_into_file 'app/views/articles/index.html.erb', after: %r{

Listing articles

}i do + <<-CODE + + +
+ + <%= form_tag search_articles_path, method: 'get' do %> + <%= label_tag :query %> + <%= text_field_tag :q, params[:q] %> + <%= submit_tag :search %> + <% end %> + +
+ CODE +end + +inject_into_file 'app/views/articles/index.html.erb', after: %r{<%= link_to 'New Article', new_article_path %>} do + <<-CODE + <%= link_to 'All Articles', articles_path if params[:q] %> + CODE +end + +gsub_file 'config/routes.rb', %r{resources :articles$}, <<-CODE +resources :articles do + collection { get :search } + end +CODE + +gsub_file "#{Rails::VERSION::STRING > '4' ? 'test/controllers' : 'test/functional'}/articles_controller_test.rb", %r{setup do.*?end}m, <<-CODE +setup do + @article = articles(:one) + + Article.__elasticsearch__.import + Article.__elasticsearch__.refresh_index! + end +CODE + +inject_into_file "#{Rails::VERSION::STRING > '4' ? 'test/controllers' : 'test/functional'}/articles_controller_test.rb", after: %r{test "should get index" do.*?end}m do + <<-CODE + + + test "should get search results" do + get :search, q: 'mystring' + assert_response :success + assert_not_nil assigns(:articles) + assert_equal 2, assigns(:articles).size + end + CODE +end + +git commit: "-a -m 'Added search form and controller action'" + +# ----- Seed the database ------------------------------------------------------------------------- + +puts +say_status "Database", "Seeding the database with data...", :yellow +puts '-'*80, ''; sleep 0.25 + +remove_file "db/seeds.rb" +create_file 'db/seeds.rb', %q{ +contents = [ +'Lorem ipsum dolor sit amet.', +'Consectetur adipisicing elit, sed do eiusmod tempor incididunt.', +'Labore et dolore magna aliqua.', +'Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris.', +'Excepteur sint occaecat cupidatat non proident.' +] + +puts "Deleting all articles..." +Article.delete_all + +unless ENV['COUNT'] + + puts "Creating articles..." + %w[ One Two Three Four Five ].each_with_index do |title, i| + Article.create title: title, content: contents[i], published_on: i.days.ago.utc + end + +else + + print "Generating articles..." + (1..ENV['COUNT'].to_i).each_with_index do |title, i| + Article.create title: "Title #{title}", content: 'Lorem ipsum dolor', published_on: i.days.ago.utc + print '.' if i % ENV['COUNT'].to_i/10 == 0 + end + puts "\n" + +end +} + +run "rails runner 'Article.__elasticsearch__.create_index! force: true'" +rake "db:seed" + +git add: "db/seeds.rb" +git commit: "-m 'Added the database seeding script'" + +# ----- Print Git log ----------------------------------------------------------------------------- + +puts +say_status "Git", "Details about the application:", :yellow +puts '-'*80, '' + +git tag: "basic" +git log: "--reverse --oneline" + +# ----- Start the application --------------------------------------------------------------------- + +unless ENV['RAILS_NO_SERVER_START'] + require 'net/http' + if (begin; Net::HTTP.get(URI('http://localhost:3000')); rescue Errno::ECONNREFUSED; false; rescue Exception; true; end) + puts "\n" + say_status "ERROR", "Some other application is running on port 3000!\n", :red + puts '-'*80 + + port = ask("Please provide free port:", :bold) + else + port = '3000' + end + + puts "", "="*80 + say_status "DONE", "\e[1mStarting the application.\e[0m", :yellow + puts "="*80, "" + + run "rails server --port=#{port}" +end diff --git a/elasticsearch-rails/lib/rails/templates/02-pretty.rb b/elasticsearch-rails/lib/rails/templates/02-pretty.rb new file mode 100644 index 0000000000..7fd6e5048f --- /dev/null +++ b/elasticsearch-rails/lib/rails/templates/02-pretty.rb @@ -0,0 +1,311 @@ +# $ rails new searchapp --skip --skip-bundle --template https://raw.github.com/elasticsearch/elasticsearch-rails/master/elasticsearch-rails/lib/rails/templates/02-pretty.rb + +unless File.read('README.rdoc').include? '== [1] Basic' + say_status "ERROR", "You have to run the 01-basic.rb template first.", :red + exit(1) +end + +puts +say_status "README", "Updating Readme...\n", :yellow +puts '-'*80, ''; sleep 0.25 + +append_to_file 'README.rdoc', <<-README + +== [2] Pretty + +The `pretty` template builds on the `basic` version and brings couple of improvements: + +* Using the [Bootstrap](http://getbootstrap.com) framework to enhance the visual style of the application +* Using an `Article.search` class method to customize the default search definition +* Highlighting matching phrases in search results +* Paginating results with Kaminari + +README + +git add: "README.rdoc" +git commit: "-m '[02] Updated the application README'" + +# ----- Update application.rb --------------------------------------------------------------------- + +puts +say_status "Rubygems", "Adding Rails logger integration...\n", :yellow +puts '-'*80, ''; sleep 0.25 + +insert_into_file 'config/application.rb', + "\n\nrequire 'elasticsearch/rails/instrumentation'", + after: /Bundler\.require.+$/ + +git add: "config/application.rb" +git commit: "-m 'Added the Rails logger integration to application.rb'" + +# ----- Add gems into Gemfile --------------------------------------------------------------------- + +puts +say_status "Rubygems", "Adding Rubygems into Gemfile...\n", :yellow +puts '-'*80, ''; sleep 0.25 + +# NOTE: Kaminari has to be loaded before Elasticsearch::Model so the callbacks are executed +# +insert_into_file 'Gemfile', <<-CODE, before: /gem ["']elasticsearch["'].+$/ + +# NOTE: Kaminari has to be loaded before Elasticsearch::Model so the callbacks are executed +gem 'kaminari' + +CODE + +run "bundle install" + +git add: "Gemfile*" +git commit: "-m 'Added the Kaminari gem'" + +# ----- Add `Article.search` class method --------------------------------------------------------- + +puts +say_status "Model", "Adding a `Article.search` class method...\n", :yellow +puts '-'*80, ''; sleep 0.5 + +insert_into_file 'app/models/article.rb', <<-CODE, after: 'include Elasticsearch::Model::Callbacks' + + + def self.search(query) + __elasticsearch__.search( + { + query: { + multi_match: { + query: query, + fields: ['title^10', 'content'] + } + }, + highlight: { + pre_tags: [''], + post_tags: [''], + fields: { + title: { number_of_fragments: 0 }, + content: { fragment_size: 25 } + } + } + } + ) + end +CODE + +insert_into_file "#{Rails::VERSION::STRING > '4' ? 'test/models' : 'test/unit' }/article_test.rb", <<-CODE, after: /class ArticleTest < ActiveSupport::TestCase$/ + + teardown do + Article.__elasticsearch__.unstub(:search) + end + +CODE + +gsub_file "#{Rails::VERSION::STRING > '4' ? 'test/models' : 'test/unit' }/article_test.rb", %r{# test "the truth" do.*?# end}m, <<-CODE + + test "has a search method delegating to __elasticsearch__" do + Article.__elasticsearch__.expects(:search).with do |definition| + assert_equal 'foo', definition[:query][:multi_match][:query] + true + end + + Article.search 'foo' + end +CODE + +git add: "app/models/article.rb" +git add: "test/**/article_test.rb" +git commit: "-m 'Added an `Article.search` method'" + +# ----- Add loading Bootstrap assets -------------------------------------------------------------- + +puts +say_status "Bootstrap", "Adding Bootstrap asset links into the 'application' layout...\n", :yellow +puts '-'*80, ''; sleep 0.5 + +gsub_file 'app/views/layouts/application.html.erb', %r{<%= yield %>}, <<-CODE unless File.read('app/views/layouts/application.html.erb').include?('class="container"') +
+<%= yield %> +
+CODE + +insert_into_file 'app/views/layouts/application.html.erb', <<-CODE, before: '' + + +CODE + +git commit: "-a -m 'Added loading Bootstrap assets in the application layout'" + +# ----- Customize the search form ----------------------------------------------------------------- + +puts +say_status "Bootstrap", "Customizing the index page...\n", :yellow +puts '-'*80, ''; sleep 0.5 + +gsub_file 'app/views/articles/index.html.erb', %r{<%= label_tag .* :search %>}m do |match| +<<-CODE +
+ <%= text_field_tag :q, params[:q], class: 'form-control', placeholder: 'Search...' %> + + + + +
+CODE +end + +# ----- Customize the header ----------------------------------------------------------------- + +gsub_file 'app/views/articles/index.html.erb', %r{

Listing articles

} do |match| + "

<%= controller.action_name == 'search' ? 'Searching articles' : 'Listing articles' %>

" +end + +# ----- Customize the results listing ------------------------------------------------------------- + +gsub_file 'app/views/articles/index.html.erb', %r{} do |match| + '
' +end + +gsub_file 'app/views/articles/index.html.erb', %r{$} do |match| + "" +end + +gsub_file 'app/views/articles/index.html.erb', %r{$} do |match| + "" +end + +git commit: "-a -m 'Added highlighting for matches'" + +# ----- Paginate the results ---------------------------------------------------------------------- + +gsub_file 'app/controllers/articles_controller.rb', %r{@articles = Article.all} do |match| + "@articles = Article.page(params[:page])" +end + +gsub_file 'app/controllers/articles_controller.rb', %r{@articles = Article.search\(params\[\:q\]\).records} do |match| + "@articles = Article.search(params[:q]).page(params[:page]).records" +end + +insert_into_file 'app/views/articles/index.html.erb', after: '
<%= link_to [^%]+} do |match| + match.gsub!('', '') + match.include?("btn") ? match : (match + ", class: 'btn btn-default btn-xs'") +end + +gsub_file 'app/views/articles/index.html.erb', %r{
\s*(<\%= link_to 'New Article'.*)}m do |content| + replace = content.match(%r{
\s*(<\%= link_to 'New Article'.*)}m)[1] + <<-END.gsub(/^ /, '') +
+ +

+ #{replace} +

+ END +end + +gsub_file 'app/views/articles/index.html.erb', %r{<%= link_to 'New Article',\s*new_article_path} do |match| + return match if match.include?('btn') + match + ", class: 'btn btn-primary btn-xs', style: 'color: #fff'" +end + +gsub_file 'app/views/articles/index.html.erb', %r{<%= link_to 'All Articles',\s*articles_path} do |match| + return match if match.include?('btn') + "\n " + match + ", class: 'btn btn-primary btn-xs', style: 'color: #fff'" +end + +git add: "app/views" +git commit: "-m 'Refactored the articles listing to use Bootstrap components'" + +# ----- Use highlighted excerpts in the listing --------------------------------------------------- + +gsub_file 'app/views/articles/index.html.erb', %r{<% @articles.each do \|article\| %>$} do |match| + "<% @articles.__send__ controller.action_name == 'search' ? :each_with_hit : :each do |article, hit| %>" +end + +gsub_file 'app/views/articles/index.html.erb', %r{		<%= article.title %><%= hit.try(:highlight).try(:title) ? hit.highlight.title.join.html_safe : article.title %><%= article.content %><%= hit.try(:highlight).try(:content) ? hit.highlight.content.join('…').html_safe : article.content %>
' do + <<-CODE.gsub(/^ /, '') + + +
+ <%= paginate @articles %> +
+ CODE +end + +generate "kaminari:views", "bootstrap2", "--force" + +gsub_file 'app/views/kaminari/_paginator.html.erb', %r{