Permalink
Browse files

Down to two tests failing. This shit is going webscale

  • Loading branch information...
Josep M. Bach
Josep M. Bach committed Oct 16, 2011
1 parent 495f7fd commit a4c9250db9b8b1dcd69647e58342fd61443f1612
Showing with 209 additions and 30 deletions.
  1. +1 −0 lib/shitdb.rb
  2. +8 −10 lib/shitdb/collection.rb
  3. +73 −0 lib/shitdb/core_ext.rb
  4. +43 −4 lib/shitdb/db.rb
  5. +20 −0 lib/shitdb/document.rb
  6. +64 −16 test/shitdb_test.rb
View
@@ -1,5 +1,6 @@
require_relative 'shitdb/db'
require_relative 'shitdb/collection'
+require_relative 'shitdb/document'
module ShitDB
end
View
@@ -1,40 +1,38 @@
module ShitDB
class Collection
attr_reader :db
-
def initialize(db, name)
@db = db
@name = name
- @db.file[@name] ||= []
+ @db.file[@name] ||= {}
@db.file["_#{@name}_last_id"] ||= 0
end
- def persist
- @db.persist
+ def persist!
+ @db.persist!
end
def get(id)
- @db.file[@name].detect do |record|
- record[:id] == id
- end
+ Document.new(id, @db.file[@name][id])
end
def put(doc)
new_id = @db.file["_#{@name}_last_id"] + 1
- @db.file[@name] << doc.update(:id => new_id)
+ @db.file[@name][new_id] = doc
@db.file["_#{@name}_last_id"] = new_id
end
def all
- @db.file[@name]
+ Document.map(@db.file[@name])
end
def where(attrs)
- @db.file[@name].select do |record|
+ docs = @db.file[@name].select do |id, record|
attrs.map do |attr|
record[attr.first] == attr.last
end.all?{|n| n == true}
end
+ Document.map(docs)
end
end
end
View
@@ -0,0 +1,73 @@
+#
+# = Hash Recursive Merge
+#
+# Merges a Ruby Hash recursively, Also known as deep merge.
+# Recursive version of Hash#merge and Hash#merge!.
+#
+# Category:: Ruby
+# Package:: Hash
+# Author:: Simone Carletti <weppos@weppos.net>
+# Copyright:: 2007-2008 The Authors
+# License:: MIT License
+# Link:: http://www.simonecarletti.com/
+# Source:: http://gist.github.com/gists/6391/
+#
+module HashRecursiveMerge
+
+ #
+ # Recursive version of Hash#merge!
+ #
+ # Adds the contents of +other_hash+ to +hsh+,
+ # merging entries in +hsh+ with duplicate keys with those from +other_hash+.
+ #
+ # Compared with Hash#merge!, this method supports nested hashes.
+ # When both +hsh+ and +other_hash+ contains an entry with the same key,
+ # it merges and returns the values from both arrays.
+ #
+ # h1 = {"a" => 100, "b" => 200, "c" => {"c1" => 12, "c2" => 14}}
+ # h2 = {"b" => 254, "c" => 300, "c" => {"c1" => 16, "c3" => 94}}
+ # h1.rmerge!(h2) #=> {"a" => 100, "b" => 254, "c" => {"c1" => 16, "c2" => 14, "c3" => 94}}
+ #
+ # Simply using Hash#merge! would return
+ #
+ # h1.merge!(h2) #=> {"a" => 100, "b" = >254, "c" => {"c1" => 16, "c3" => 94}}
+ #
+ def rmerge!(other_hash)
+ merge!(other_hash) do |key, oldval, newval|
+ oldval.class == self.class ? oldval.rmerge!(newval) : newval
+ end
+ end
+
+ #
+ # Recursive version of Hash#merge
+ #
+ # Compared with Hash#merge!, this method supports nested hashes.
+ # When both +hsh+ and +other_hash+ contains an entry with the same key,
+ # it merges and returns the values from both arrays.
+ #
+ # Compared with Hash#merge, this method provides a different approch
+ # for merging nasted hashes.
+ # If the value of a given key is an Hash and both +other_hash+ abd +hsh
+ # includes the same key, the value is merged instead replaced with
+ # +other_hash+ value.
+ #
+ # h1 = {"a" => 100, "b" => 200, "c" => {"c1" => 12, "c2" => 14}}
+ # h2 = {"b" => 254, "c" => 300, "c" => {"c1" => 16, "c3" => 94}}
+ # h1.rmerge(h2) #=> {"a" => 100, "b" => 254, "c" => {"c1" => 16, "c2" => 14, "c3" => 94}}
+ #
+ # Simply using Hash#merge would return
+ #
+ # h1.merge(h2) #=> {"a" => 100, "b" = >254, "c" => {"c1" => 16, "c3" => 94}}
+ #
+ def rmerge(other_hash)
+ r = {}
+ merge(other_hash) do |key, oldval, newval|
+ r[key] = oldval.class == self.class ? oldval.rmerge(newval) : newval
+ end
+ end
+
+end
+
+class Hash
+ include HashRecursiveMerge
+end
View
@@ -1,7 +1,10 @@
require 'yaml'
+require_relative 'core_ext'
module ShitDB
class DB
+ attr_reader :name
+
def initialize(name)
@name = File.join(File.dirname(__FILE__), name)
end
@@ -11,13 +14,49 @@ def collection(name)
end
def file
- @file ||= File.exist?(@name) ? YAML.load(File.open(@name)) : {}
+ @file ||= persisted_file
end
- def persist
- File.open(@name, 'w') do |f|
- f.write YAML.dump(@file)
+ def persisted_file
+ if File.exist?(@name)
+ to_read = ""
+ while to_read == ""
+ f = File.open(@name, 'r+')
+ # f.flock(File::LOCK_EX | File::LOCK_NB)
+ to_read = f.read
+ # to_read = f.read_nonblock(10)
+ # puts to_read.inspect
+ # f.flock(File::LOCK_UN)
+ f.close
+ end
+
+ YAML.load(to_read)
+ else
+ {}
+ end
+ end
+
+ def persist!
+ new_file = nil
+ new_file = persisted_file.rmerge(@file)
+
+ to_write = YAML.dump(new_file)
+
+ result = nil
+ begin
+ f = File.open(@name, 'w')
+ # f.flock(File::LOCK_EX | File::LOCK_NB)
+ puts "writing #{to_write}"
+ f.write to_write
+ # result = f.write_nonblock(to_write)
+ # rescue IO::WaitWritable, Errno::EINTR
+ # IO.select(nil, [f])
+ # retry
+ # ensure
+ # f.flock(File::LOCK_UN)
+ f.close
end
+ result
end
end
end
View
@@ -0,0 +1,20 @@
+module ShitDB
+ class Document
+ attr_reader :id, :doc
+
+ def self.map(hash)
+ hash.map do |k, v|
+ Document.new(k, v)
+ end
+ end
+
+ def initialize(id, doc)
+ @id = id
+ @doc = doc
+ end
+
+ def method_missing(m, *a, &b)
+ @doc[m.to_sym] || super
+ end
+ end
+end
View
@@ -4,20 +4,17 @@
module ShitDB
describe 'acceptance tests' do
before do
- FileUtils.rm('my_db') if File.exist?('my_db')
- @users = DB.new('my_db').collection('users')
- end
-
- after do
- FileUtils.rm('my_db') if File.exist?('my_db')
+ @db = DB.new('my_db')
+ @users = @db.collection(:users)
+ FileUtils.rm(@db.name) if File.exist?(@db.name)
end
describe 'storage' do
it 'saves records in memory' do
@users.put(:name => 'James')
@users.put(:name => 'John', :age => 30)
- users = @users.all.map { |record| record[:name] }
+ users = @users.all.map(&:name)
assert_equal 2, users.length
assert_includes users, 'James'
assert_includes users, 'John'
@@ -27,21 +24,51 @@ module ShitDB
@users.put(:name => 'James')
@users.put(:name => 'John', :age => 30)
- persisted = DB.new('my_db').collection('users').all
+ persisted = DB.new('my_db').collection(:users).all
assert_equal 0, persisted.length
end
it 'persists them when told so' do
@users.put(:name => 'James')
@users.put(:name => 'John', :age => 30)
- @users.persist
+ @users.persist!
- users = DB.new('my_db').collection('users').all.map { |record| record[:name] }
+ users = DB.new('my_db').collection(:users).all.map(&:name)
assert_equal 2, users.length
assert_includes users, 'James'
assert_includes users, 'John'
end
+
+ it 'merges records when persisting' do
+ @users.put(:name => 'James')
+ @users.put(:name => 'John', :age => 30)
+
+ file = {}
+
+ file[:users] ||= {}
+ file[:users][1] ||= {}
+ file[:users][1][:name] = 'Charlie'
+ file[:users][1][:age] = 91
+
+ File.open(@users.db.name, 'w') do |f|
+ f.write YAML.dump(file)
+ end
+
+ @users.persist!
+
+ users = DB.new('my_db').collection(:users).all
+ assert_equal 2, users.length
+
+ names = users.map(&:name)
+ ages = users.map(&:age)
+
+ assert_includes names, 'James'
+ assert_includes names, 'John'
+
+ assert_includes ages, 91
+ assert_includes ages, 30
+ end
end
describe 'querying' do
@@ -52,26 +79,27 @@ module ShitDB
end
it 'performs simple queries with one condition inefficiently' do
- result = @users.where(:age => 30).map { |record| record[:name] }
+ result = @users.where(:age => 30).map(&:name)
assert_equal 2, result.length
assert_includes result, 'John'
assert_includes result, 'Charles'
end
it 'performs simple queries with multiple conditions inefficiently' do
- result = @users.where(:name => 'Charles', :age => 30).map { |record| record[:name]}
+ result = @users.where(:name => 'Charles', :age => 30).map(&:name)
assert_equal 1, result.length
assert_equal 'Charles', result.first
end
it 'retrieves records by id' do
+ p @users.all
james = @users.get(1)
john = @users.get(2)
charles = @users.get(3)
- assert_equal 'James', james[:name]
- assert_equal 'John', john[:name]
- assert_equal 'Charles', charles[:name]
+ assert_equal 'James', james.name
+ assert_equal 'John', john.name
+ assert_equal 'Charles', charles.name
end
end
@@ -80,11 +108,31 @@ module ShitDB
@users.put(:name => 'James')
@users.put(:name => 'John')
- ids = @users.all.map { |record| record[:id] }.compact
+ ids = @users.all.map(&:id).compact
assert_equal 2, ids.length
refute_equal ids.first, ids.last
end
end
+
+ describe 'concurrency' do
+ it 'locks the shared file on read/write' do
+ (1..1).to_a.map do |t|
+ if t % 2 == 0
+ Thread.new do
+ DB.new('my_db').persisted_file
+ end
+ else
+ Thread.new do
+ users = DB.new('my_db').collection('users')
+ users.put(:some => 'doc')
+ users.persist!
+ end
+ end
+ end.map(&:join)
+
+ assert_equal 500, DB.new('my_db').collection('users').all.length
+ end
+ end
end
end

0 comments on commit a4c9250

Please sign in to comment.