Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

avoid storing huge data in session. let ff_daemon cache it. closes #194.

  • Loading branch information...
commit 5bf152a6e4dbceb267b964ebd97148346555a808 1 parent 029b127
@nahi authored
View
4 app/controllers/entry_controller.rb
@@ -563,12 +563,10 @@ def unpin
private
def find_opt(ctx = @ctx)
- cache = session[:cached_entries] ||= CachedEntries.new
updated_id = updated_id_in_flash()
ctx.find_opt.merge(
:allow_cache => flash[:allow_cache],
- :updated_id => updated_id,
- :cached_entries => cache
+ :updated_id => updated_id
)
end
View
3  app/models/cached_entries.rb
@@ -1,3 +1,6 @@
+require 'entry'
+
+
class CachedEntries < Array
attr_accessor :opt
end
View
8 app/models/entry.rb
@@ -1,4 +1,12 @@
require 'hash_utils'
+require 'comment'
+require 'entry_user'
+require 'geo'
+require 'like'
+require 'media'
+require 'room'
+require 'service'
+require 'via'
class Entry
View
63 app/models/entry_thread.rb
@@ -1,4 +1,6 @@
require 'task'
+require 'entry'
+require 'cached_entries'
class EntryThread
@@ -13,15 +15,13 @@ class << self
def find(opt = {})
auth = opt[:auth]
- cache = opt[:cached_entries]
return nil unless auth
unless opt.key?(:merge_entry)
opt[:merge_entry] = true
end
opt.delete(:auth)
- opt.delete(:cached_entries)
logger.info('[perf] start entries fetch')
- original = entries = fetch_entries(auth, opt, cache)
+ original = entries = fetch_entries(auth, opt)
logger.info('[perf] start internal data handling')
record_last_modified(entries)
logger.info('[perf] record_last_modified done')
@@ -98,15 +98,15 @@ def logger
ActiveRecord::Base.logger
end
- def fetch_entries(auth, opt, cache)
+ def fetch_entries(auth, opt)
if opt[:id]
- fetch_single_entry_as_array(auth, opt, cache)
+ fetch_single_entry_as_array(auth, opt)
else
- entries = fetch_list_entries(auth, opt, cache)
+ entries = fetch_list_entries(auth, opt)
if updated_id = opt[:updated_id]
entry = wrap(get_entry(auth, :id => updated_id)).first
if entry
- update_cache_entry(cache, entry)
+ update_cache_entry(auth, entry)
if entries.find { |e| e.id == updated_id }
replace_entry(entries, entry)
else
@@ -118,18 +118,20 @@ def fetch_entries(auth, opt, cache)
end
end
- def fetch_single_entry_as_array(auth, opt, cache)
- if opt[:allow_cache] and cache
- if found = cache.find { |e| e.id == opt[:id] && e.id != opt[:updated_id] }
- logger.info("[cache] entry cache found for #{opt[:id]}")
- return [found]
+ def fetch_single_entry_as_array(auth, opt)
+ if opt[:allow_cache]
+ if cache = get_cached_entries(auth)
+ if found = cache.find { |e| e.id == opt[:id] && e.id != opt[:updated_id] }
+ logger.info("[cache] entry cache found for #{opt[:id]}")
+ return [found]
+ end
end
end
wrap(Task.run { get_entry(auth, opt) }.result)
end
- def fetch_list_entries(auth, opt, cache)
- cache_entries(opt, cache) {
+ def fetch_list_entries(auth, opt)
+ cache_entries(auth, opt) {
if opt[:inbox]
start = opt[:start]
num = opt[:num]
@@ -176,7 +178,7 @@ def fetch_list_entries(auth, opt, cache)
}
end
- def cache_entries(opt, cache, &block)
+ def cache_entries(auth, opt, &block)
allow_cache = opt[:allow_cache]
opt = opt.dup
opt.delete(:allow_cache)
@@ -184,26 +186,37 @@ def cache_entries(opt, cache, &block)
opt.delete(:merge_entry)
opt.delete(:merge_service)
opt.delete(:filter_inbox_except)
- if allow_cache and cache
- if opt == cache.opt
- logger.info("[cache] entries cache found for #{opt.inspect}")
- return cache
+ if allow_cache
+ if cache = get_cached_entries(auth)
+ if opt == cache.opt
+ logger.info("[cache] entries cache found for #{opt.inspect}")
+ return cache
+ end
end
end
entries = yield
- if cache
- cache.opt = opt
- cache.replace(entries)
- end
+ cache = CachedEntries.new
+ cache.opt = opt
+ cache.replace(entries)
+ set_cached_entries(auth, cache)
entries
end
- def update_cache_entry(cache, entry)
- if cache
+ def update_cache_entry(auth, entry)
+ if cache = get_cached_entries(auth)
replace_entry(cache, entry)
+ set_cached_entries(auth, cache)
end
end
+ def get_cached_entries(auth)
+ ff_client.get_cached_entries(auth.name)
+ end
+
+ def set_cached_entries(auth, cache)
+ ff_client.set_cached_entries(auth.name, cache)
+ end
+
def record_last_modified(entries)
found = LastModified.find_all_by_eid(entries.map { |e| e.id })
found_map = found.inject({}) { |r, e|
View
33 lib/ff.rb
@@ -56,6 +56,7 @@ class BaseClient
attr_accessor :logger
attr_accessor :apikey
attr_accessor :http_proxy
+ attr_accessor :httpclient_max_keepalive
class LShiftLogger
def initialize(logger)
@@ -72,21 +73,36 @@ def method_missing(msg_id, *a, &b)
end
class UserClient
+ attr_accessor :httpclient_max_keepalive
+
def initialize(name, remote_key, logger, http_proxy)
@client = HTTPClient.new(http_proxy)
@name = name
@remote_key = remote_key
#@client.debug_dev = LShiftLogger.new(logger)
+ @logger = logger
@client.extend(MonitorMixin)
+ @last_accessed = Time.now
reset_auth
end
+ def idle?
+ if @httpclient_max_keepalive
+ elapsed = Time.now - @last_accessed
+ if elapsed > @httpclient_max_keepalive
+ @client.reset_all rescue nil
+ true
+ end
+ end
+ end
+
def client(remote_key)
@client.synchronize do
if remote_key != @remote_key
@remote_key = remote_key
reset_auth
end
+ @last_accessed = Time.now
@client
end
end
@@ -106,7 +122,9 @@ def initialize(logger = nil, apikey = nil)
@logger = logger || NullLogger.new
@apikey = apikey
@http_proxy = nil
+ @httpclient_max_keepalive = 5 * 60
@clients = {}
+ @mutex = Monitor.new
end
private
@@ -119,10 +137,23 @@ def uri(part)
end
def create_client(name, remote_key)
- UserClient.new(name, remote_key, @logger, @http_proxy)
+ client = UserClient.new(name, remote_key, @logger, @http_proxy)
+ client.httpclient_max_keepalive = @httpclient_max_keepalive
+ client
end
def client_sync(uri, name, remote_key)
+ @mutex.synchronize do
+ clients = {}
+ @clients.each do |key, value|
+ if value.idle?
+ @logger.info("removed idle HTTPClient for #{key}")
+ else
+ clients[key] = value
+ end
+ end
+ @clients = clients
+ end
user_client = @clients[name] ||= create_client(name, remote_key)
client = user_client.client(remote_key)
logger.info("#{user_client.inspect} is accessing to #{uri.to_s} for #{name}")
View
15 lib/ff_daemon.rb
@@ -70,6 +70,8 @@ class APIClientProxy
define_proxy_method :get_list_profile
define_proxy_method :get_room_status
define_proxy_method :purge_cache
+ define_proxy_method :get_cached_entries
+ define_proxy_method :set_cached_entries
def initialize
@client = DRb::DRbObject.new(nil, F2P::Config.friendfeed_api_daemon_drb_uri)
@@ -230,6 +232,19 @@ def initialize(logger = nil)
@cache = {}
end
+ def set_cached_entries(name, entries)
+ basekey = name
+ cache = ((@cache ||= {})[basekey] ||= {})
+ cache[:last_entries] = entries
+ nil
+ end
+
+ def get_cached_entries(name)
+ basekey = name
+ cache = ((@cache ||= {})[basekey] ||= {})
+ cache[:last_entries]
+ end
+
def purge_cache(key)
@cache.delete(key)
if @channel.key?(key)
View
9 test/functional/entry_controller_test.rb
@@ -8,6 +8,15 @@ def setup
@ff.stubs(:get_profile).returns(profile)
@ff.stubs(:get_user_status).returns({})
@ff.stubs(:get_room_status).returns({})
+ class << @ff
+ def set_cached_entries(auth, cache)
+ @cache = cache
+ end
+
+ def get_cached_entries(auth)
+ @cache
+ end
+ end
ApplicationController.ff_client = @ff
end
View
149 test/unit/entry_thread_test.rb
@@ -1,14 +1,19 @@
require 'test_helper'
class EntryThreadTest < ActiveSupport::TestCase
+ def setup
+ @ff = mock('ff_client')
+ @ff.stubs(:get_cached_entries)
+ @ff.stubs(:set_cached_entries)
+ end
+
test 'self.find inbox' do
user = User.find_by_name('user1')
- ff = mock('ff_client')
- ApplicationController.ff_client = ff
+ ApplicationController.ff_client = @ff
#
- ff.expects(:get_inbox_entries).with('user1', nil, nil, nil).
+ @ff.expects(:get_inbox_entries).with('user1', nil, nil, nil).
returns(read_entries('entries', 'f2ptest')).times(2)
- ff.stubs(:get_profiles)
+ @ff.stubs(:get_profiles)
2.times do
threads = EntryThread.find(:auth => user, :inbox => true, :start => nil)
assert_equal(
@@ -36,12 +41,11 @@ class EntryThreadTest < ActiveSupport::TestCase
test 'self.find inbox 2nd page' do
user = User.find_by_name('user1')
- ff = mock('ff_client')
- ApplicationController.ff_client = ff
+ ApplicationController.ff_client = @ff
#
- ff.expects(:get_inbox_entries).with('user1', nil, 20, nil).
+ @ff.expects(:get_inbox_entries).with('user1', nil, 20, nil).
returns(read_entries('entries', 'f2ptest'))
- ff.stubs(:get_profiles)
+ @ff.stubs(:get_profiles)
threads = EntryThread.find(:auth => user, :inbox => true, :start => 20)
assert_equal(
[1, 2, 1, 1, 1, 4, 2, 3, 3, 3, 1, 3, 1, 1, 1, 1, 1],
@@ -65,12 +69,11 @@ class EntryThreadTest < ActiveSupport::TestCase
pin.user = user
pin.eid = 'foobar'
pin.save
- ff = mock('ff_client')
- ApplicationController.ff_client = ff
+ ApplicationController.ff_client = @ff
#
- ff.expects(:get_inbox_entries).with('user1', nil, nil, nil).
+ @ff.expects(:get_inbox_entries).with('user1', nil, nil, nil).
returns(read_entries('entries', 'f2ptest')[2..-1]).times(2)
- ff.stubs(:get_profiles)
+ @ff.stubs(:get_profiles)
2.times do
threads = EntryThread.find(:auth => user, :inbox => true, :start => nil)
assert_equal(
@@ -82,15 +85,21 @@ class EntryThreadTest < ActiveSupport::TestCase
test 'self.find inbox cache' do
user = User.find_by_name('user1')
- ff = mock('ff_client')
- ApplicationController.ff_client = ff
+ ApplicationController.ff_client = @ff
#
- ff.expects(:get_inbox_entries).with('user1', nil, nil, nil).
+ @ff.expects(:get_inbox_entries).with('user1', nil, nil, nil).
returns(read_entries('entries', 'f2ptest')).times(1) # 1 time only
- ff.stubs(:get_profiles)
- cache = CachedEntries.new
+ @ff.stubs(:get_profiles)
+ class << @ff
+ def get_cached_entries(auth)
+ @cache
+ end
+ def set_cached_entries(auth, cache)
+ @cache = cache
+ end
+ end
2.times do
- threads = EntryThread.find(:auth => user, :inbox => true, :start => nil, :allow_cache => true, :cached_entries => cache)
+ threads = EntryThread.find(:auth => user, :inbox => true, :start => nil, :allow_cache => true)
assert_equal(
[1, 2, 1, 1, 1, 4, 2, 3, 3, 3, 1, 3, 1, 1, 1, 1, 1],
threads.map { |t| t.entries.size }
@@ -100,12 +109,11 @@ class EntryThreadTest < ActiveSupport::TestCase
test 'self.find home' do
user = User.find_by_name('user1')
- ff = mock('ff_client')
- ApplicationController.ff_client = ff
+ ApplicationController.ff_client = @ff
#
- ff.expects(:get_home_entries).with('user1', nil, {:num => nil, :start => nil, :service => nil}).
+ @ff.expects(:get_home_entries).with('user1', nil, {:num => nil, :start => nil, :service => nil}).
returns(read_entries('entries', 'f2ptest'))
- ff.stubs(:get_profiles)
+ @ff.stubs(:get_profiles)
threads = EntryThread.find(:auth => user, :start => nil)
assert_equal(
[2, 1, 1, 1, 1, 4, 2, 3, 3, 3, 1, 3, 1, 1, 1, 1, 1],
@@ -115,14 +123,13 @@ class EntryThreadTest < ActiveSupport::TestCase
test 'self.find home cache' do
user = User.find_by_name('user1')
- ff = mock('ff_client')
- ApplicationController.ff_client = ff
+ ApplicationController.ff_client = @ff
#
- ff.expects(:get_inbox_entries).with('user1', nil, nil, nil).
+ @ff.expects(:get_inbox_entries).with('user1', nil, nil, nil).
returns(read_entries('entries', 'f2ptest')).times(2) # no cache used
- ff.expects(:get_home_entries).with('user1', nil, :num => nil, :start => nil, :service => nil).
+ @ff.expects(:get_home_entries).with('user1', nil, :num => nil, :start => nil, :service => nil).
returns(read_entries('entries', 'f2ptest')).times(2) # no cache used
- ff.stubs(:get_profiles)
+ @ff.stubs(:get_profiles)
assert_equal(17, EntryThread.find(:auth => user, :inbox => true, :start => nil, :allow_cache => true).size)
assert_equal(17, EntryThread.find(:auth => user, :start => nil, :allow_cache => true).size)
assert_equal(17, EntryThread.find(:auth => user, :inbox => true, :start => nil, :allow_cache => true).size)
@@ -131,12 +138,11 @@ class EntryThreadTest < ActiveSupport::TestCase
test 'self.find query' do
user = User.find_by_name('user1')
- ff = mock('ff_client')
- ApplicationController.ff_client = ff
+ ApplicationController.ff_client = @ff
#
- ff.expects(:search_entries).with('user1', nil, 'foobar', {:from => nil, :room => nil, :friends => nil, :start => nil, :num => nil, :service => nil}).
+ @ff.expects(:search_entries).with('user1', nil, 'foobar', {:from => nil, :room => nil, :friends => nil, :start => nil, :num => nil, :service => nil}).
returns(read_entries('entries', 'f2ptest'))
- ff.stubs(:get_profiles)
+ @ff.stubs(:get_profiles)
threads = EntryThread.find(:auth => user, :query => 'foobar')
assert_equal(
[2, 1, 1, 1, 1, 4, 2, 3, 3, 3, 1, 3, 1, 1, 1, 1, 1],
@@ -146,78 +152,71 @@ class EntryThreadTest < ActiveSupport::TestCase
test 'self.find id' do
user = User.find_by_name('user1')
- ff = mock('ff_client')
- ApplicationController.ff_client = ff
+ ApplicationController.ff_client = @ff
#
- ff.expects(:get_entry).with('user1', nil, 'foobar').
+ @ff.expects(:get_entry).with('user1', nil, 'foobar').
returns(read_entries('entries', 'f2ptest'))
- ff.stubs(:get_profiles)
+ @ff.stubs(:get_profiles)
assert_equal(17, EntryThread.find(:auth => user, :id => 'foobar').size)
end
test 'self.find likes' do
user = User.find_by_name('user1')
- ff = mock('ff_client')
- ApplicationController.ff_client = ff
+ ApplicationController.ff_client = @ff
#
- ff.expects(:get_likes).with('user1', nil, 'user2', {:start => nil, :num => nil, :service => nil}).
+ @ff.expects(:get_likes).with('user1', nil, 'user2', {:start => nil, :num => nil, :service => nil}).
returns([])
- ff.stubs(:get_profiles)
+ @ff.stubs(:get_profiles)
EntryThread.find(:auth => user, :user => 'user2', :like => 'likes')
end
test 'self.find liked' do
user = User.find_by_name('user1')
- ff = mock('ff_client')
- ApplicationController.ff_client = ff
+ ApplicationController.ff_client = @ff
#
- ff.expects(:search_entries).with('user1', nil, '', {:from => nil, :start => nil, :num => nil, :likes => 1, :service => nil}).
+ @ff.expects(:search_entries).with('user1', nil, '', {:from => nil, :start => nil, :num => nil, :likes => 1, :service => nil}).
returns([])
- ff.stubs(:get_profiles)
+ @ff.stubs(:get_profiles)
EntryThread.find(:auth => user, :like => 'liked')
end
test 'self.find user' do
user = User.find_by_name('user1')
- ff = mock('ff_client')
- ApplicationController.ff_client = ff
+ ApplicationController.ff_client = @ff
#
- ff.expects(:get_user_entries).with('user1', nil, 'user2', {:start => nil, :num => nil, :service => nil}).
+ @ff.expects(:get_user_entries).with('user1', nil, 'user2', {:start => nil, :num => nil, :service => nil}).
returns([])
- ff.stubs(:get_profiles)
+ @ff.stubs(:get_profiles)
EntryThread.find(:auth => user, :user => 'user2')
end
test 'self.find list' do
user = User.find_by_name('user1')
- ff = mock('ff_client')
- ApplicationController.ff_client = ff
+ ApplicationController.ff_client = @ff
#
- ff.expects(:get_list_entries).with('user1', nil, 'list1', {:start => nil, :num => nil, :service => nil}).
+ @ff.expects(:get_list_entries).with('user1', nil, 'list1', {:start => nil, :num => nil, :service => nil}).
returns([])
- ff.stubs(:get_profiles)
+ @ff.stubs(:get_profiles)
EntryThread.find(:auth => user, :list => 'list1')
end
test 'self.find room' do
user = User.find_by_name('user1')
- ff = mock('ff_client')
- ApplicationController.ff_client = ff
+ ApplicationController.ff_client = @ff
#
- ff.expects(:get_room_entries).with('user1', nil, 'room1', {:start => nil, :num => nil, :service => nil}).
+ @ff.expects(:get_room_entries).with('user1', nil, 'room1', {:start => nil, :num => nil, :service => nil}).
returns([])
- ff.stubs(:get_profiles)
+ @ff.stubs(:get_profiles)
EntryThread.find(:auth => user, :room => 'room1')
end
test 'self.find room chunk' do
user = User.find_by_name('user1')
- ff = mock('ff_client')
- ApplicationController.ff_client = ff
+ ApplicationController.ff_client = @ff
#
- ff.expects(:get_room_entries).with('user1', nil, 'room1', {:start => nil, :num => nil, :service => nil}).
+ @ff.expects(:get_room_entries).with('user1', nil, 'room1', {:start => nil, :num => nil, :service => nil}).
returns(read_entries('entries', 'room'))
- ff.stubs(:get_profiles)
+ @ff.stubs(:get_profiles)
threads = EntryThread.find(:auth => user, :room => 'room1')
assert_equal(
[3, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1],
@@ -227,35 +226,32 @@ class EntryThreadTest < ActiveSupport::TestCase
test 'self.find friends' do
user = User.find_by_name('user1')
- ff = mock('ff_client')
- ApplicationController.ff_client = ff
+ ApplicationController.ff_client = @ff
#
- ff.expects(:get_friends_entries).with('user1', nil, 'user2', {:start => nil, :num => nil, :service => nil}).
+ @ff.expects(:get_friends_entries).with('user1', nil, 'user2', {:start => nil, :num => nil, :service => nil}).
returns([])
- ff.stubs(:get_profiles)
+ @ff.stubs(:get_profiles)
EntryThread.find(:auth => user, :friends => 'user2')
end
test 'self.find link' do
user = User.find_by_name('user1')
- ff = mock('ff_client')
- ApplicationController.ff_client = ff
+ ApplicationController.ff_client = @ff
#
- ff.expects(:get_url_entries).with('user1', nil, 'http://www.example.org/', {:start => nil, :num => nil, :service => nil}).
+ @ff.expects(:get_url_entries).with('user1', nil, 'http://www.example.org/', {:start => nil, :num => nil, :service => nil}).
returns([])
- ff.stubs(:get_profiles)
+ @ff.stubs(:get_profiles)
EntryThread.find(:auth => user, :link => 'http://www.example.org/')
end
test 'update_checked_modified' do
user = User.find_by_name('user1')
- ff = mock('ff_client')
- ApplicationController.ff_client = ff
+ ApplicationController.ff_client = @ff
list = read_entries('entries', 'f2ptest')
#
- ff.expects(:get_inbox_entries).with('user1', nil, nil, nil).
+ @ff.expects(:get_inbox_entries).with('user1', nil, nil, nil).
returns(list)
- ff.stubs(:get_profiles)
+ @ff.stubs(:get_profiles)
threads = EntryThread.find(:auth => user, :inbox => true, :start => nil)
hash = {}
threads.each do |t|
@@ -268,20 +264,19 @@ class EntryThreadTest < ActiveSupport::TestCase
EntryThread.update_checked_modified(user, hash)
#
list[0]['updated'] = Time.now.xmlschema
- ff.expects(:get_inbox_entries).with('user1', nil, nil, nil).
+ @ff.expects(:get_inbox_entries).with('user1', nil, nil, nil).
returns(list)
- ff.stubs(:get_profiles)
+ @ff.stubs(:get_profiles)
threads = EntryThread.find(:auth => user, :inbox => true, :start => nil)
assert_equal(1, threads.size)
end
test 'self.find timeout' do
user = User.find_by_name('user1')
- ff = mock('ff_client')
- ApplicationController.ff_client = ff
+ ApplicationController.ff_client = @ff
#
- ff.expects(:get_inbox_entries).with('user1', nil, nil, nil).raises(Timeout::Error.new)
- ff.stubs(:get_profiles)
+ @ff.expects(:get_inbox_entries).with('user1', nil, nil, nil).raises(Timeout::Error.new)
+ @ff.stubs(:get_profiles)
assert(EntryThread.find(:auth => user, :inbox => true, :start => nil).empty?)
end
end
Please sign in to comment.
Something went wrong with that request. Please try again.