Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

add a Disk tagstore implementation, refactor Heap implementation

  • Loading branch information...
commit bf23b6b41a589f1ae4749d5de8bcf2b4623ad262 1 parent c449b5d
@svenfuchs authored
View
14 lib/core_ext/file_utils/rmdir_p.rb
@@ -0,0 +1,14 @@
+require 'fileutils'
+
+module FileUtils
+ def self.rmdir_p(path)
+ begin
+ while(true)
+ FileUtils.rmdir(path)
+ path = File.dirname(path)
+ end
+ rescue Errno::EEXIST, Errno::ENOTEMPTY, Errno::ENOENT, Errno::EINVAL, Errno::ENOTDIR
+ # Stop trying to remove parent directories
+ end
+ end
+end
View
5 lib/rack/cache/tags.rb
@@ -8,6 +8,7 @@
require 'rack/cache/tags/storage'
require 'rack/cache/tags/tag_store'
require 'rack/cache/tags/meta_store'
+require 'rack/cache/tags/purger'
module Rack::Cache
TAGS_HEADER = 'X-Cache-Tags'
@@ -22,6 +23,10 @@ def tagstore
end
end
+ Purge::Purger.class_eval do
+ include Tags::Purger
+ end
+
module Tags
class << self
def new(backend, options={}, &b)
View
36 lib/rack/cache/tags/context.rb
@@ -10,13 +10,15 @@ def call(env)
@env = env
request = Rack::Cache::Request.new(env)
response = Rack::Cache::Response.new(*@backend.call(env))
-
+
tags = response.headers.delete(Rack::Cache::PURGE_TAGS_HEADER)
- if tags
- tags = Rack::Cache::Tags.normalize(tags)
- uris = tagged_uris(tags)
- response.headers[Rack::Cache::PURGE_HEADER] = uris.join("\n")
-
+ return response.to_a unless tags
+
+ uris = tagged_uris(tags)
+ if false && env.key?('rack-cache.purger')
+ # TODO directly purge using uris?
+ else
+ set_purge_header(response, uris)
purge_taggings(request, uris)
end
@@ -24,23 +26,27 @@ def call(env)
end
protected
-
+
def tagged_uris(tags)
- tags.inject([]) { |uris, tag| uris + tagstore.by_tag[tag] }
+ tags = Rack::Cache::Tags.normalize(tags)
+ tags.inject([]) { |uris, tag| uris + tagstore.read_tag(tag) }
end
-
+
+ def set_purge_header(response, uris)
+ response.headers[Rack::Cache::PURGE_HEADER] = uris.join("\n")
+ end
+
def purge_taggings(request, uris)
- uris.each do |uri|
- key = Rack::Cache::Utils::Key.call(request, uri)
- tagstore.purge(key)
- end
+ tagstore = self.tagstore
+ keys = uris.map { |uri| Rack::Cache::Utils::Key.call(request, uri) }
+ keys.each { |key| tagstore.purge(key) }
end
-
+
def tagstore
uri = @env['rack-cache.tagstore']
storage.resolve_tagstore_uri(uri)
end
-
+
def storage
Rack::Cache::Storage.instance
end
View
9 lib/rack/cache/tags/meta_store.rb
@@ -1,13 +1,13 @@
-Rack::Cache::MetaStore::Heap.class_eval do
+module Rack::Cache::Tags::MetaStore
def store(request, response, entity_store)
key = super
tags = response.headers[Rack::Cache::TAGS_HEADER]
tagstore(request).store(key, tags) if tags
-
+
key
end
-
+
def tagstore(request)
uri = request.env['rack-cache.tagstore']
storage.resolve_tagstore_uri(uri)
@@ -16,4 +16,7 @@ def tagstore(request)
def storage
Rack::Cache::Storage.instance
end
+
+ Rack::Cache::MetaStore::Heap.send(:include, self)
+ Rack::Cache::MetaStore::Disk.send(:include, self)
end
View
18 lib/rack/cache/tags/purger.rb
@@ -0,0 +1,18 @@
+module Rack
+ module Cache
+ module Tags
+ module Purger
+ def purge(arg)
+ keys = super
+ tagstore = self.tagstore
+ keys.each { |key| tagstore.purge(key) }
+ end
+
+ def tagstore
+ uri = context.env['rack-cache.tagstore']
+ storage.resolve_tagstore_uri(uri)
+ end
+ end
+ end
+ end
+end
View
202 lib/rack/cache/tags/tag_store.rb
@@ -1,75 +1,177 @@
-require 'fileutils'
-require 'digest/sha1'
require 'rack/utils'
+require 'digest/sha1'
+require 'fileutils'
+require 'core_ext/file_utils/rmdir_p'
module Rack::Cache::Tags
class TagStore
- def store(key, tags)
- tags = Rack::Cache::Tags.normalize(tags)
- write(key, tags)
- end
-
- def by_key(key)
- raise NotImplemented
+ def read_key(key)
+ key(key).read
end
- def by_tag(tag)
- raise NotImplemented
+ def read_tag(tag)
+ tag(tag).read
end
- protected
-
- def write(key, tags)
- raise NotImplemented
+ def store(key, tags)
+ tags = Rack::Cache::Tags.normalize(tags)
+ key(key, tags).store.each { |tag| read_tag(tag).add(key) }
end
def purge(key)
- raise NotImplemented
+ read_key(key).purge.each { |tag| read_tag(tag).remove(key) }
end
- public
+ public
- class Heap < TagStore
- def self.resolve(uri)
- new
- end
-
- def initialize
- @hash = { :by_key => {}, :by_tag => {} }
- end
-
- def by_key
- @hash[:by_key]
- end
-
- def by_tag
- @hash[:by_tag]
- end
+ class Heap < TagStore
+ def self.resolve(uri)
+ new
+ end
+
+ def initialize
+ @by_key, @by_tag = {}, {}
+ end
+
+ def key(key, tags = [])
+ Collection.new(@by_key, :key, key, tags)
+ end
+
+ def tag(tag, keys = [])
+ Collection.new(@by_tag, :tag, tag, keys)
+ end
+
+ class Collection < Array
+ attr_reader :type, :owner, :hash
+
+ def initialize(hash, type, owner, elements = [])
+ @hash, @type, @owner = hash, type, owner
+ super(elements) if elements
+ compact
+ end
+
+ def add(element)
+ return if element.nil? or include?(element)
+ push(element)
+ store
+ end
+
+ def remove(element)
+ store if delete(element)
+ self
+ end
+
+ def purge
+ hash.delete(owner)
+ self
+ end
+
+ def exist?
+ hash.key?(owner)
+ end
- def purge(key)
- if tags = by_key[key]
- tags.each do |tag|
- next unless by_tag[tag]
- by_tag[tag].delete(key)
- by_tag.delete(tag) if by_tag[tag].empty?
+ def read
+ replace(hash[owner] || [])
+ self
+ end
+
+ def store
+ return purge if empty?
+ hash[owner] = self
end
end
-
- by_key.delete(key)
- nil
end
- def write(key, tags)
- by_key[key] = tags
+ HEAP = Heap
+ MEM = HEAP
+
+ class Disk < TagStore
+ def self.resolve(uri)
+ path = File.expand_path(uri.opaque || uri.path)
+ new(path)
+ end
+
+ def initialize(root)
+ @root = root
+ FileUtils.mkdir_p root, :mode => 0755
+ end
+
+ def key(key, tags = [])
+ Collection.new(@root, :key, key, tags)
+ end
- tags.each do |tag|
- by_tag[tag] ||= []
- by_tag[tag] << key unless by_tag[tag].include?(key)
+ def tag(tag, keys = [])
+ Collection.new(@root, :tag, tag, keys)
+ end
+
+ class Collection < Array
+ attr_reader :type, :owner, :root
+
+ def initialize(root, type, owner, elements)
+ @root, @type, @owner = root, type, owner
+ super(elements) if elements
+ compact
+ end
+
+ def add(element)
+ return if element.nil? or include?(element)
+ push(element)
+ store
+ end
+
+ def remove(element)
+ store if delete(element)
+ self
+ end
+
+ def purge
+ File.unlink(path) rescue Errno::ENOENT
+ FileUtils.rmdir_p(File.dirname(path))
+ self
+ end
+
+ def exist?
+ File.exist?(path)
+ end
+
+ def read
+ replace(read_file.split(','))
+ self
+ rescue Errno::ENOENT
+ self
+ end
+
+ def store
+ return purge if empty?
+
+ FileUtils.mkdir_p(File.dirname(path), :mode => 0755)
+ File.open(path, 'wb') { |f| f.write(join(',')) }
+ self
+ end
+
+ protected
+
+ def read_file
+ File.open(path, 'rb') { |f| f.read }
+ end
+
+ def path
+ File.join(root, "by_#{type}", spread(owner))
+ end
+
+ def digest(key)
+ Digest::SHA1.hexdigest(key)
+ end
+
+ def spread(arg)
+ arg = digest(arg)
+ arg[2, 0] = '/'
+ arg
+ end
end
end
- end
- HEAP = Heap
- MEM = HEAP
+ DISK = Disk
+ FILE = Disk
end
end
View
121 test/tag_store_test.rb
@@ -1,43 +1,90 @@
-require "#{File.dirname(__FILE__)}/test_setup"
+require File.expand_path("#{File.dirname(__FILE__)}/test_setup")
describe 'Rack::Cache::Tags::Tagstore' do
- before(:each) do
- @store = Rack::Cache::Tags::TagStore::Heap.new
- end
-
- it "can be resolved from an uri" do
- tag_store = Rack::Cache::Storage.new.resolve_tagstore_uri('heap:/')
- tag_store.should.be.kind_of Rack::Cache::Tags::TagStore::Heap
- end
+ describe 'Heap' do
+ before(:each) do
+ @store = Rack::Cache::Tags::TagStore::Heap.new
+ end
+
+ it "can be resolved from an uri" do
+ tag_store = Rack::Cache::Storage.new.resolve_tagstore_uri('heap:/')
+ tag_store.should.be.kind_of Rack::Cache::Tags::TagStore::Heap
+ end
+
+ it "writes to both read_key and read_tag" do
+ @store.store('1234', 'tag-1,tag-2')
+
+ @store.read_key('1234').should == ['tag-1', 'tag-2']
+ @store.read_tag('tag-1').should == ['1234']
+ @store.read_tag('tag-2').should == ['1234']
+ @store.read_tag('tag-3').should == []
- it "writes to both by_key and by_tag" do
- @store.store('1234', 'tag-1,tag-2')
-
- @store.by_key['1234'].should == ['tag-1', 'tag-2']
- @store.by_tag['tag-1'].should == ['1234']
- @store.by_tag['tag-2'].should == ['1234']
- @store.by_tag['tag-3'].should == nil
-
- @store.store('5678', 'tag-1,tag-3')
-
- @store.by_key['1234'].should == ['tag-1', 'tag-2']
- @store.by_key['5678'].should == ['tag-1', 'tag-3']
- @store.by_tag['tag-1'].should == ['1234', '5678']
- @store.by_tag['tag-2'].should == ['1234']
- @store.by_tag['tag-3'].should == ['5678']
+ @store.store('5678', 'tag-1,tag-3')
+
+ @store.read_key('1234').should == ['tag-1', 'tag-2']
+ @store.read_key('5678').should == ['tag-1', 'tag-3']
+ @store.read_tag('tag-1').should == ['1234', '5678']
+ @store.read_tag('tag-2').should == ['1234']
+ @store.read_tag('tag-3').should == ['5678']
+ end
+
+ it "purges from both read_key and read_tag" do
+ @store.store('1234', 'tag-1,tag-2')
+ @store.store('5678', 'tag-1,tag-3')
+
+ @store.purge('1234')
+
+ @store.read_key('1234').should == []
+ @store.read_key('5678').should == ['tag-1', 'tag-3']
+ @store.read_tag('tag-1').should == ['5678']
+ @store.read_tag('tag-2').should == []
+ @store.read_tag('tag-3').should == ['5678']
+ end
end
-
- it "purges from both by_key and by_tag" do
- @store.store('1234', 'tag-1,tag-2')
- @store.store('5678', 'tag-1,tag-3')
-
- @store.purge('1234')
-
- @store.by_key['1234'].should == nil
- @store.by_key['5678'].should == ['tag-1', 'tag-3']
- @store.by_tag['tag-1'].should == ['5678']
- @store.by_tag['tag-2'].should == nil
- @store.by_tag['tag-3'].should == ['5678']
+
+ describe 'File' do
+ before(:each) do
+ uri = URI.parse("file://#{TMP_DIR}/tagstore")
+ @store = Rack::Cache::Tags::TagStore::Disk.resolve(uri)
+ end
+
+ after(:each) do
+ FileUtils.rm_r(TMP_DIR) rescue Errno::ENOENT
+ end
+
+ it "can be resolved from an uri" do
+ tag_store = Rack::Cache::Storage.new.resolve_tagstore_uri("file://#{TMP_DIR}/tagstore")
+ tag_store.should.be.kind_of Rack::Cache::Tags::TagStore::Disk
+ end
+
+ it "writes to both read_key and read_tag" do
+ @store.store('1234', 'tag-1,tag-2')
+
+ @store.read_key('1234').should == ['tag-1', 'tag-2']
+ @store.read_tag('tag-1').should == ['1234']
+ @store.read_tag('tag-2').should == ['1234']
+ @store.read_tag('tag-3').should == []
+
+ @store.store('5678', 'tag-1,tag-3')
+
+ @store.read_key('1234').should == ['tag-1', 'tag-2']
+ @store.read_key('5678').should == ['tag-1', 'tag-3']
+ @store.read_tag('tag-1').should == ['1234', '5678']
+ @store.read_tag('tag-2').should == ['1234']
+ @store.read_tag('tag-3').should == ['5678']
+ end
+
+ it "purges from both read_key and read_tag" do
+ @store.store('1234', 'tag-1,tag-2')
+ @store.store('5678', 'tag-1,tag-3')
+
+ @store.purge('1234')
+
+ @store.read_key('1234').should == []
+ @store.read_key('5678').should == ['tag-1', 'tag-3']
+ @store.read_tag('tag-1').should == ['5678']
+ @store.read_tag('tag-2').should == []
+ @store.read_tag('tag-3').should == ['5678']
+ end
end
-
end
View
107 test/tags_test.rb
@@ -1,42 +1,115 @@
-require "#{File.dirname(__FILE__)}/test_setup"
+require File.expand_path("#{File.dirname(__FILE__)}/test_setup")
+
+# When the downstream app sends an X-Cache-Tag header *and* the response is
+# cacheable store taggings for the cache entry
+#
+# When the downstream app sends an X-Cache-Purge-Tags header:
+#
+# * remove the headers
+# * lookup any tagged URIs
+# * set them as X-Cache-Purge headers
+#
+# When a cache entry is purged also purge its taggings.
describe 'Rack::Cache::Tags' do
before(:each) { setup_cache_context }
after(:each) { teardown_cache_context }
- it "writes tags for a key on :store" do
- respond_with 200, { 'X-Cache-Tags' => 'page-1,user-2', 'Cache-Control' => 'public, max-age=10000' }, 'body'
+ configs = [
+ {
+ 'metastore' => 'heap:/',
+ 'entitystore' => 'heap:/',
+ 'tagstore' => 'heap:/'
+ },
+ {
+ 'metastore' => "file://#{TMP_DIR}/metastore",
+ 'entitystore' => "file://#{TMP_DIR}/entitystore",
+ 'tagstore' => "file://#{TMP_DIR}/tagstore"
+ }
+ ]
- response = get '/'
- tagstore.by_key.should == { 'http://example.org/' => ['page-1', 'user-2'] }
- tagstore.by_tag.should == { 'page-1' => ['http://example.org/'], 'user-2' => ['http://example.org/'] }
- end
+ configs.each do |config|
+ it "writes taggings for an entry on :store (#{config[config.keys.first]})" do
+ cache_config do |cache|
+ config.each { |key, value| cache.options["rack-cache.#{key}"] = value }
+ end
+ respond_with 200, { 'X-Cache-Tags' => 'page-1,user-2', 'Cache-Control' => 'public, max-age=10000' }, 'body'
+
+ response = get '/'
+ tagstore.read_key('http://example.org/').should == ['page-1', 'user-2']
+ tagstore.read_tag('page-1').should == ['http://example.org/']
+ tagstore.read_tag('user-2').should == ['http://example.org/']
+ end
- it "deletes tags for a key on :purge" do
- respond_with 200, { 'Cache-Control' => 'public, max-age=10000' }, 'body'
- respond_with 200, { 'Cache-Control' => 'public, max-age=500' }, 'body' do |req, res|
- case req.path
- when '/'
- res.headers['X-Cache-Tags'] = ['page-1,user-2']
- when '/users/2'
- res.headers['X-Cache-Purge-Tags'] = 'user-2'
+ # TODO test all three purging methods!
+ it "deletes taggings for a key on :purge (using HTTP PURGE)" do
+ respond_with 200, { 'Cache-Control' => 'public, max-age=500' }, 'body' do |req, res|
+ case req.path
+ when '/'
+ res.headers['X-Cache-Tags'] = ['page-1,user-2']
+ when '/users/2'
+ subrequest(:purge, '/')
+ end
end
+
+ it_purges_cache_entry_including_tags
end
+ it "deletes taggings for a key on :purge (using tag headers)" do
+ respond_with 200, { 'Cache-Control' => 'public, max-age=500' }, 'body' do |req, res|
+ case req.path
+ when '/'
+ res.headers['X-Cache-Tags'] = ['page-1,user-2']
+ when '/users/2'
+ res.headers['X-Cache-Purge-Tags'] = 'user-2'
+ end
+ end
+
+ it_purges_cache_entry_including_tags
+ end
+
+ it "deletes taggings for a key on :purge (using manual purge)" do
+ respond_with 200, { 'Cache-Control' => 'public, max-age=500' }, 'body' do |req, res|
+ case req.path
+ when '/'
+ res.headers['X-Cache-Tags'] = ['page-1,user-2']
+ when '/users/2'
+ req.env['rack-cache.purger'].purge('http://example.org/')
+ end
+ end
+
+ it_purges_cache_entry_including_tags
+ end
+ end
+
+ def it_purges_cache_entry_including_tags
get '/'
- tagstore.by_key['http://example.org/'].should.include 'user-2'
+ tagstore.read_key('http://example.org/').should.include 'user-2'
cache.trace.should.include :store
get '/'
cache.trace.should.include :fresh
post '/users/2'
- tagstore.by_key['http://example.org/'].should.be.nil
+ tagstore.read_key('http://example.org/').should == []
get '/'
cache.trace.should.include :miss
end
+ def subrequest(method, path)
+ purge = Rack::Cache::Purge.new(nil, :allow_http_purge => true)
+ app = Rack::Cache::Context.new(purge, :tagstore => 'heap:/')
+ app.call(
+ "REQUEST_METHOD" => method.to_s.upcase,
+ "SERVER_NAME" => "example.org",
+ "SERVER_PORT" => "80",
+ "PATH_INFO" => path,
+ "rack.url_scheme" => "http",
+ "rack.errors" => StringIO.new
+ )
+ end
+
def tagstore
cache.send(:tagstore)
end
View
41 test/test_setup.rb
@@ -1,6 +1,7 @@
# test setup largely stolen from Ryan Tomayko's rack-cache
require 'pp'
+require 'fileutils'
begin
require 'test/spec'
@@ -10,15 +11,17 @@
end
# Setup the load path ..
-$: << File.dirname(File.dirname(__FILE__)) + '/../rack-cache/lib'
-$: << File.dirname(File.dirname(__FILE__)) + '/../rack-cache-purge/lib'
-$: << File.dirname(File.dirname(__FILE__)) + '/lib'
-$: << File.dirname(__FILE__)
+$: << File.expand_path(File.dirname(__FILE__) + '/../../rack-cache/lib')
+$: << File.expand_path(File.dirname(__FILE__) + '/../../rack-cache-purge/lib')
+$: << File.expand_path(File.dirname(__FILE__) + '/../lib')
+$: << File.expand_path(File.dirname(__FILE__))
require 'rack/cache'
require 'rack/cache/purge'
require 'rack/cache/tags'
+TMP_DIR = File.expand_path(File.dirname(__FILE__) + '/tmp')
+
# Methods for constructing downstream applications / response
# generators.
module CacheContextHelpers
@@ -26,30 +29,20 @@ module CacheContextHelpers
def setup_cache_context
# holds each Rack::Cache::Context
- @app = nil
-
- # each time a request is made, a clone of @cache_template is used
- # and appended to @caches.
- @cache_template = nil
- @cache = nil
- @caches = []
+ @app, @cache, @called, @request, @response, @cache_config = nil
@errors = StringIO.new
- @cache_config = nil
-
- @called = false
- @request = nil
- @response = nil
- @responses = []
-
@storage = Rack::Cache::Storage.instance
end
def teardown_cache_context
- @app, @cache_template, @cache, @caches, @called,
- @request, @response, @responses, @cache_config = nil
+ FileUtils.rm_r(TMP_DIR) rescue Errno::ENOENT
@storage.clear
end
+ def cache_config(&cache_config)
+ @cache_config = cache_config
+ end
+
# A basic response with 200 status code and a tiny body.
def respond_with(status=200, headers={}, body=['Hello World'])
called = false
@@ -61,14 +54,10 @@ def respond_with(status=200, headers={}, body=['Hello World'])
response.finish
end
@app.meta_def(:called?) { called }
- @app.meta_def(:reset!) { called = false }
+ @app.meta_def(:reset!) { called = false }
@app
end
- def cache_config(&block)
- @cache_config = block
- end
-
def request(method, uri = '/', env = {})
fail 'response not specified (use respond_with)' if @app.nil?
@app.reset! if @app.respond_to?(:reset!)
@@ -81,8 +70,6 @@ def request(method, uri = '/', env = {})
env = { 'rack.run_once' => true }.merge(env)
@response = @request.request(method.to_s.upcase, uri, env)
- @responses << @response
- @response
end
def get(stem, env={}, &b)
Please sign in to comment.
Something went wrong with that request. Please try again.