From 350f9ff36217975840e302679b45083c6ae660d2 Mon Sep 17 00:00:00 2001 From: Jason Amster Date: Thu, 27 Jan 2011 23:32:14 -0500 Subject: [PATCH 1/2] Added ability to map Hits to AR objects in one query rather than multiple --- README.markdown | 11 ++++++++++- lib/escargot/activerecord_ex.rb | 4 +++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/README.markdown b/README.markdown index cb76bb7..11fb405 100644 --- a/README.markdown +++ b/README.markdown @@ -91,11 +91,20 @@ Basic Searching ---------------- Calling `Model.search` obtains from ElasticSearch the ids of the results matching -your query and then queries your database to get the full ActiveRecord objects. +your query and then queries your database to get the full ActiveRecord objects by +mapping each result to it's corresponding ActiveRecord object. results = Post.search "dreams OR nightmares" results.each {|r| puts r.title} +The mapping by default queries the database once for each result, but you can pass +the :quick\_search option in the options and that will do one database query. + + results = Post.search "dreams OR nightmares", :quick_search => true + + + + The query is parsed using lucene's [QueryParser syntax](http://lucene.apache.org/java/2_4_0/queryparsersyntax.html). You can use boolean operators, restrict your search to a field, etc. diff --git a/lib/escargot/activerecord_ex.rb b/lib/escargot/activerecord_ex.rb index dcbe700..d4647c5 100644 --- a/lib/escargot/activerecord_ex.rb +++ b/lib/escargot/activerecord_ex.rb @@ -67,8 +67,10 @@ def search_hits(query, options = {}) # note that the collection may include nils if ElasticSearch returns a result hit for a # record that has been deleted on the database def search(query, options = {}) + quick_search = options.delete(:quick_search) + options = options.merge(:ids_only => true) if quick_search hits = search_hits(query, options) - hits_ar = hits.map{|hit| hit.to_activerecord} + hits_ar = quick_search ? self.find(hits.compact.uniq) : hits.map{|hit| hit.to_activerecord} results = WillPaginate::Collection.new(hits.current_page, hits.per_page, hits.total_entries) results.replace(hits_ar) results From 1e1f34803c0538f46ed86cd2602a37b4b0a0701b Mon Sep 17 00:00:00 2001 From: Jason Amster Date: Tue, 2 Oct 2012 12:40:22 -0400 Subject: [PATCH 2/2] adding pre-alias indexing --- .../pre_alias_distributed_indexing.rb | 48 +++++++++++++++++++ lib/tasks/escargot.rake | 12 +++++ 2 files changed, 60 insertions(+) create mode 100644 lib/escargot/pre_alias_distributed_indexing.rb diff --git a/lib/escargot/pre_alias_distributed_indexing.rb b/lib/escargot/pre_alias_distributed_indexing.rb new file mode 100644 index 0000000..1082e6a --- /dev/null +++ b/lib/escargot/pre_alias_distributed_indexing.rb @@ -0,0 +1,48 @@ + +module Escargot + + module PreAliasDistributedIndexing + + def PreAliasDistributedIndexing.load_dependencies + require 'resque' + end + + def PreAliasDistributedIndexing.create_index_for_model(model) + load_dependencies + + model.find_in_batches(:select => model.primary_key) do |batch| + Escargot.queue_backend.enqueue(IndexDocuments, model.to_s, batch.map(&:id)) + end + end + + class IndexDocuments + @queue = :indexing + + def self.perform(model_name, ids, index_version) + model = model_name.constantize + model.find(:all, :conditions => {model.primary_key => ids}).each do |record| + record.local_index_in_elastic_search + end + end + end + + class ReIndexDocuments + @queue = :nrt + + def self.perform(model_name, ids) + model = model_name.constantize + ids_found = [] + model.find(:all, :conditions => {:id => ids}).each do |record| + record.local_index_in_elastic_search + ids_found << record.id + end + + (ids - ids_found).each do |id| + model.delete_id_from_index(id) + end + end + end + + end + +end diff --git a/lib/tasks/escargot.rake b/lib/tasks/escargot.rake index a165454..a6a9f1a 100644 --- a/lib/tasks/escargot.rake +++ b/lib/tasks/escargot.rake @@ -20,6 +20,18 @@ namespace :escargot do end end + desc "indexes the models LIVE LIKE BOSS" + task :pre_alias_distributed_index, :models, :needs => [:environment, :load_all_models] do |t, args| + each_indexed_model(args) do |model| + puts "Indexing #{model}" + index_version = model.create_index_version + $elastic_search_client.deploy_index_version(index, index_version) + Escargot::PreAliasDistributedIndexing.create_index_for_model(model) + end + end + + + desc "prunes old index versions for this models" task :prune_versions, :models, :needs => [:environment, :load_all_models] do |t, args| each_indexed_model(args) do |model|