Skip to content
Browse files

add SQLite3 storage

  • Loading branch information...
1 parent a7895d0 commit 26e9983c94d894c7cf0c119e6f3f9b37d341665a @skojin skojin committed Mar 3, 2011
Showing with 164 additions and 3 deletions.
  1. +5 −0 lib/anemone/storage.rb
  2. +90 −0 lib/anemone/storage/sqlite3.rb
  3. +22 −1 spec/core_spec.rb
  4. +19 −1 spec/page_store_spec.rb
  5. +28 −1 spec/storage_spec.rb
View
5 lib/anemone/storage.rb
@@ -29,6 +29,11 @@ def self.Redis(opts = {})
require 'anemone/storage/redis'
self::Redis.new(opts)
end
+
+ def self.SQLite3(file = 'anemone.db')
+ require 'anemone/storage/sqlite3'
+ self::SQLite3.new(file)
+ end
end
end
View
90 lib/anemone/storage/sqlite3.rb
@@ -0,0 +1,90 @@
+begin
+ require 'sqlite3'
+rescue LoadError
+ puts "You need the sqlite3 gem to use Anemone::Storage::Sqlite"
+ exit
+end
+
+module Anemone
+ module Storage
+ class SQLite3
+
+ def initialize(file)
+ @db = ::SQLite3::Database.new(file)
+ create_schema
+ end
+
+ def [](url)
+ value = @db.get_first_value('SELECT data FROM anemone_storage WHERE key = ?', url.to_s)
+ if value
+ Marshal.load(value)
+ end
+ end
+
+ def []=(url, value)
+ data = Marshal.dump(value)
+ if has_key?(url)
+ @db.execute('UPDATE anemone_storage SET data = ? WHERE key = ?', data, url.to_s)
+ else
+ @db.execute('INSERT INTO anemone_storage (data, key) VALUES(?, ?)', data, url.to_s)
+ end
+ end
+
+ def delete(url)
+ page = self[url]
+ @db.execute('DELETE FROM anemone_storage WHERE key = ?', url.to_s)
+ page
+ end
+
+ def each
+ @db.execute("SELECT key, data FROM anemone_storage ORDER BY id") do |row|
+ value = Marshal.load(row[1])
+ yield row[0], value
+ end
+ end
+
+ def merge!(hash)
+ hash.each { |key, value| self[key] = value }
+ self
+ end
+
+ def size
+ @db.get_first_value('SELECT COUNT(id) FROM anemone_storage')
+ end
+
+ def keys
+ @db.execute("SELECT key FROM anemone_storage ORDER BY id").map{|t| t[0]}
+ end
+
+ def has_key?(url)
+ !!@db.get_first_value('SELECT id FROM anemone_storage WHERE key = ?', url.to_s)
+ end
+
+ def close
+ @db.close
+ end
+
+ private
+
+ def create_schema
+ @db.execute_batch <<SQL
+ create table if not exists anemone_storage (
+ id INTEGER PRIMARY KEY ASC,
+ key TEXT,
+ data BLOB
+ );
+ create index if not exists anemone_key_idx on anemone_storage (key);
+SQL
+ end
+
+ def load_page(hash)
+ BINARY_FIELDS.each do |field|
+ hash[field] = hash[field].to_s
+ end
+ Page.from_hash(hash)
+ end
+
+ end
+ end
+end
+
View
23 spec/core_spec.rb
@@ -1,6 +1,6 @@
$:.unshift(File.dirname(__FILE__))
require 'spec_helper'
-%w[pstore tokyo_cabinet].each { |file| require "anemone/storage/#{file}.rb" }
+%w[pstore tokyo_cabinet sqlite3].each { |file| require "anemone/storage/#{file}.rb" }
module Anemone
describe Core do
@@ -277,6 +277,27 @@ module Anemone
end
end
+ describe Storage::SQLite3 do
+ it_should_behave_like "crawl"
+
+ before(:all) do
+ @test_file = 'test.db'
+ end
+
+ before(:each) do
+ File.delete(@test_file) if File.exists?(@test_file)
+ @opts = {:storage => @store = Storage.SQLite3(@test_file)}
+ end
+
+ after(:each) do
+ @store.close
+ end
+
+ after(:each) do
+ File.delete(@test_file) if File.exists?(@test_file)
+ end
+ end
+
describe "options" do
it "should accept options for the crawl" do
core = Anemone.crawl(SPEC_DOMAIN, :verbose => false,
View
20 spec/page_store_spec.rb
@@ -1,6 +1,6 @@
$:.unshift(File.dirname(__FILE__))
require 'spec_helper'
-%w[pstore tokyo_cabinet mongodb redis].each { |file| require "anemone/storage/#{file}.rb" }
+%w[pstore tokyo_cabinet sqlite3 mongodb redis].each { |file| require "anemone/storage/#{file}.rb" }
module Anemone
describe PageStore do
@@ -125,6 +125,24 @@ module Anemone
end
end
+ describe Storage::SQLite3 do
+ it_should_behave_like "page storage"
+
+ before(:each) do
+ @test_file = 'test.db'
+ File.delete(@test_file) if File.exists?(@test_file)
+ @opts = {:storage => @store = Storage.SQLite3(@test_file)}
+ end
+
+ after(:each) do
+ @store.close
+ end
+
+ after(:each) do
+ File.delete(@test_file) if File.exists?(@test_file)
+ end
+ end
+
describe Storage::MongoDB do
it_should_behave_like "page storage"
View
29 spec/storage_spec.rb
@@ -1,7 +1,7 @@
$:.unshift(File.dirname(__FILE__))
require 'spec_helper'
-%w[pstore tokyo_cabinet mongodb redis].each { |file| require "anemone/storage/#{file}.rb" }
+%w[pstore tokyo_cabinet sqlite3 mongodb redis].each { |file| require "anemone/storage/#{file}.rb" }
module Anemone
describe Storage do
@@ -25,6 +25,14 @@ module Anemone
store.close
end
+ it "should have a class method to produce a SQLite3" do
+ test_file = 'test.db'
+ Anemone::Storage.should respond_to(:SQLite3)
+ store = Anemone::Storage.SQLite3(test_file)
+ store.should be_an_instance_of(Anemone::Storage::SQLite3)
+ store.close
+ end
+
it "should have a class method to produce a MongoDB" do
Anemone::Storage.should respond_to(:MongoDB)
store = Anemone::Storage.MongoDB
@@ -143,6 +151,25 @@ module Storage
end
end
+ describe SQLite3 do
+ it_should_behave_like "storage engine"
+
+ before(:each) do
+ @test_file = 'test.db'
+ File.delete @test_file rescue nil
+ @store = Anemone::Storage.SQLite3(@test_file)
+ end
+
+ after(:each) do
+ @store.close
+ end
+
+ after(:all) do
+ File.delete @test_file rescue nil
+ end
+
+ end
+
describe Storage::MongoDB do
it_should_behave_like "storage engine"

0 comments on commit 26e9983

Please sign in to comment.
Something went wrong with that request. Please try again.