Permalink
Browse files

add/update/remove items

  • Loading branch information...
1 parent c2fa6fd commit 5daedd2c6975fb98595112b857fcc4c23174197c @erwaller erwaller committed Oct 12, 2011
Showing with 90 additions and 19 deletions.
  1. +32 −15 lib/soulmate/loader.rb
  2. +10 −4 lib/soulmate/matcher.rb
  3. +48 −0 test/test_soulmate.rb
View
@@ -21,25 +21,42 @@ def load(items)
items_loaded = 0
items.each_with_index do |item, i|
- id = item["id"]
- term = item["term"]
- score = item["score"]
-
- if id and term
- # store the raw data in a separate key to reduce memory usage
- Soulmate.redis.hset(database, id, JSON.dump(item))
-
- phrase = ([term] + (item["aliases"] || [])).join(' ')
- prefixes_for_phrase(phrase).uniq.each do |p|
- Soulmate.redis.sadd(base, p) # remember this prefix in a master set
- Soulmate.redis.zadd("#{base}:#{p}", score, id) # store the id of this term in the index
- end
- items_loaded += 1
- end
+ add(item)
+ items_loaded += 1
puts "added #{i} entries" if i % 100 == 0 and i != 0
end
items_loaded
end
+
+ # "id", "term", "score", "aliases", "data"
+ def add(item = {})
+ raise ArgumentError unless item["id"] && item["term"]
+
+ # kill any old items with this id
+ remove(item["id"])
+
+ # store the raw data in a separate key to reduce memory usage
+ Soulmate.redis.hset(database, item["id"], JSON.dump(item))
+ phrase = ([item["term"]] + (item["aliases"] || [])).join(' ')
+ prefixes_for_phrase(phrase).uniq.each do |p|
+ Soulmate.redis.sadd(base, p) # remember this prefix in a master set
+ Soulmate.redis.zadd("#{base}:#{p}", item["score"], item["id"]) # store the id of this term in the index
+ end
+ end
+
+ def remove(id)
+ prev_item = Soulmate.redis.hget(database, id)
+ if prev_item
+ prev_item = JSON.load(prev_item)
+ # undo the operations done in add
+ Soulmate.redis.hdel(database, prev_item["id"])
+ phrase = ([prev_item["term"]] + (prev_item["aliases"] || [])).join(' ')
+ prefixes_for_phrase(phrase).uniq.each do |p|
+ Soulmate.redis.srem(base, p)
+ Soulmate.redis.zrem("#{base}:#{p}", prev_item["id"])
+ end
+ end
+ end
end
end
View
@@ -3,22 +3,28 @@ module Soulmate
class Matcher < Base
def matches_for_term(term, options = {})
+ options = { :limit => 5, :cache => true }.merge(options)
+
words = normalize(term).split(' ').reject do |w|
w.size < MIN_COMPLETE or STOP_WORDS.include?(w)
end.sort
- options[:limit] ||= 5
-
cachekey = "#{cachebase}:" + words.join('|')
- if !Soulmate.redis.exists(cachekey)
+ if !options[:cache] || !Soulmate.redis.exists(cachekey)
interkeys = words.map { |w| "#{base}:#{w}" }
Soulmate.redis.zinterstore(cachekey, interkeys)
Soulmate.redis.expire(cachekey, 10 * 60) # expire after 10 minutes
end
ids = Soulmate.redis.zrevrange(cachekey, 0, options[:limit] - 1)
- ids.size > 0 ? Soulmate.redis.hmget(database, *ids).map { |r| JSON.parse(r) } : []
+ if ids.size > 0
+ Soulmate.redis.hmget(database, *ids)
+ .reject{ |r| r.nil? } # handle cached results for ids which have since been deleted
+ .map { |r| JSON.parse(r) }
+ else
+ []
+ end
end
end
end
View
@@ -42,4 +42,52 @@ def test_integration_can_load_values_and_query_via_aliases
results = matcher.matches_for_term('stadium', :limit => 5)
assert_equal 5, results.size
end
+
+ def test_can_remove_items
+
+ loader = Soulmate::Loader.new('venues')
+ matcher = Soulmate::Matcher.new('venues')
+
+ # empty the collection
+ loader.load([])
+ results = matcher.matches_for_term("te", :cache => false)
+ assert_equal 0, results.size
+
+ loader.add("id" => 1, "term" => "Testing this", "score" => 10)
+ results = matcher.matches_for_term("te", :cache => false)
+ assert_equal 1, results.size
+
+ loader.remove(1)
+ results = matcher.matches_for_term("te", :cache => false)
+ assert_equal 0, results.size
+
+ end
+
+ def test_can_update_items
+
+ loader = Soulmate::Loader.new('venues')
+ matcher = Soulmate::Matcher.new('venues')
+
+ # empty the collection
+ loader.load([])
+
+ # initial data
+ loader.add("id" => 1, "term" => "Testing this", "score" => 10)
+ loader.add("id" => 2, "term" => "Another Term", "score" => 9)
+ loader.add("id" => 3, "term" => "Something different", "score" => 5)
+
+ results = matcher.matches_for_term("te", :cache => false)
+ assert_equal 2, results.size
+ assert_equal "Testing this", results.first["term"]
+ assert_equal 10, results.first["score"]
+
+ # update id:1
+ loader.add("id" => 1, "term" => "Updated", "score" => 5)
+
+ results = matcher.matches_for_term("te", :cache => false)
+ assert_equal 1, results.size
+ assert_equal "Another Term", results.first["term"]
+ assert_equal 9, results.first["score"]
+
+ end
end

0 comments on commit 5daedd2

Please sign in to comment.