Permalink
Browse files

Merge branch 'pr/15'

  • Loading branch information...
nelhage committed Mar 18, 2013
2 parents 5182ad2 + 921f6c8 commit 5945b627f05ca478e815cf48a719562c551b45c8
Showing with 178 additions and 45 deletions.
  1. +27 −5 README.md
  2. +5 −4 lib/mosql/cli.rb
  3. +41 −13 lib/mosql/schema.rb
  4. +9 −8 lib/mosql/sql.rb
  5. +37 −0 test/functional/cli.rb
  6. +1 −1 test/functional/schema.rb
  7. +4 −4 test/functional/sql.rb
  8. +54 −10 test/unit/lib/mongo-sql/schema.rb
View
@@ -53,8 +53,12 @@ types. An example collection map might be:
mongodb:
blog_posts:
:columns:
- - _id: TEXT
- - author: TEXT
+ - id:
+ :source: _id
+ :type: TEXT
+ - author:
+ :source: author
+ :type: TEXT
- title: TEXT
- created: DOUBLE PRECISION
:meta:
@@ -67,9 +71,27 @@ mapping
<Mongo DB name> -> { <Mongo Collection Name> -> <Collection Definition> }
Where a `<Collection Definition>` is a hash with `:columns` and
-`:meta` fields. `:columns` is a list of one-element hashes, mapping
-field-name to SQL type. It is required to include at least an `_id`
-mapping. `:meta` contains metadata about this collection/table. It is
+`:meta` fields.
+
+`:columns` is a list of hashes mapping SQL column names to an hash
+describing that column. This hash may contain the following fields:
+
+ * `:source`: The name of the attribute inside of MongoDB.
+ * `:type`: (Mandatory) The SQL type.
+
+This syntax allows to rename MongoDB's attributes during the
+import. For example the MonogDB `_id` attribute will be transferred to
+a SQL column named `id`.
+
+As a shorthand, you can specify a one-elment hash of the form `name:
+TYPE`, in which case `name` will be used for both the source attribute
+and the name of the destination column. You can see this shorthand for
+the `title` and `created` attributes, above.
+
+Every defined collection must include a mapping for the `_id`
+attribute.
+
+`:meta` contains metadata about this collection/table. It is
required to include at least `:table`, naming the SQL table this
collection will be mapped to. `extra_props` determines the handling of
unknown fields in MongoDB objects -- more about that later.
View
@@ -164,7 +164,7 @@ def bulk_upsert(table, ns, items)
items.each do |it|
h = {}
cols.zip(it).each { |k,v| h[k] = v }
- @sql.upsert(table, h)
+ @sql.upsert(table, primary_sql_key_for_ns(ns), h)
end
end
end
@@ -258,12 +258,13 @@ def optail
end
def sync_object(ns, _id)
- sqlid = @sql.transform_one_ns(ns, { '_id' => _id })['_id']
- obj = collection_for_ns(ns).find_one({:_id => _id})
+ primary_sql_key = @schemamap.primary_sql_key_for_ns(ns)
+ sqlid = @sql.transform_one_ns(ns, { '_id' => _id })[primary_sql_key]
+ obj = collection_for_ns(ns).find_one({:_id => _id})
if obj
@sql.upsert_ns(ns, obj)
else
- @sql.table_for_ns(ns).where(:_id => sqlid).delete()
+ @sql.table_for_ns(ns).where(primary_sql_key.to_sym => sqlid).delete()
end
end
View
@@ -4,19 +4,33 @@ class SchemaError < StandardError; end;
class Schema
include MoSQL::Logging
- def to_ordered_hash(lst)
- hash = BSON::OrderedHash.new
+ def to_array(lst)
+ array = []
lst.each do |ent|
- raise "Invalid ordered hash entry #{ent.inspect}" unless ent.is_a?(Hash) && ent.keys.length == 1
- field, type = ent.first
- hash[field] = type
+ if ent.is_a?(Hash) && ent[:source].is_a?(String) && ent[:type].is_a?(String)
+ # new configuration format
+ array << {
+ :source => ent.delete(:source),
+ :type => ent.delete(:type),
+ :name => ent.first.first,
+ }
+ elsif ent.is_a?(Hash) && ent.keys.length == 1
+ array << {
+ :source => ent.first.first,
+ :name => ent.first.first,
+ :type => ent.first.last
+ }
+ else
+ raise "Invalid ordered hash entry #{ent.inspect}"
+ end
+
end
- hash
+ array
end
def parse_spec(spec)
out = spec.dup
- out[:columns] = to_ordered_hash(spec[:columns])
+ out[:columns] = to_array(spec[:columns])
out
end
@@ -35,13 +49,16 @@ def create_schema(db, clobber=false)
meta = collection[:meta]
log.info("Creating table '#{meta[:table]}'...")
db.send(clobber ? :create_table! : :create_table?, meta[:table]) do
- collection[:columns].each do |field, type|
- column field, type
+ collection[:columns].each do |col|
+ column col[:name], col[:type]
+
+ if col[:source].to_sym == :_id
+ primary_key [col[:name].to_sym]
+ end
end
if meta[:extra_props]
column '_extra_props', 'TEXT'
end
- primary_key [:_id]
end
end
end
@@ -67,8 +84,12 @@ def transform(ns, obj, schema=nil)
obj = obj.dup
row = []
- schema[:columns].each do |name, type|
- v = obj.delete(name)
+ schema[:columns].each do |col|
+
+ source = col[:source]
+ type = col[:type]
+
+ v = obj.delete(source)
case v
when BSON::Binary, BSON::ObjectId
v = v.to_s
@@ -91,7 +112,10 @@ def transform(ns, obj, schema=nil)
end
def all_columns(schema)
- cols = schema[:columns].keys
+ cols = []
+ schema[:columns].each do |col|
+ cols << col[:name]
+ end
if schema[:meta][:extra_props]
cols << "_extra_props"
end
@@ -144,5 +168,9 @@ def all_mongo_dbs
def collections_for_mongo_db(db)
(@map[db]||{}).keys
end
+
+ def primary_sql_key_for_ns(ns)
+ find_ns!(ns)[:columns].find {|c| c[:source] == '_id'}[:name]
+ end
end
end
View
@@ -35,35 +35,36 @@ def transform_one_ns(ns, obj)
def upsert_ns(ns, obj)
h = transform_one_ns(ns, obj)
- upsert(table_for_ns(ns), h)
+ upsert(table_for_ns(ns), @schema.primary_sql_key_for_ns(ns), h)
end
# obj must contain an _id field. All other fields will be ignored.
def delete_ns(ns, obj)
+ primary_sql_key = @schema.primary_sql_key_for_ns(ns)
h = transform_one_ns(ns, obj)
- raise "No _id found in transform of #{obj.inspect}" if h['_id'].nil?
- table_for_ns(ns).where(:_id => h['_id']).delete
+ raise "No #{primary_sql_key} found in transform of #{obj.inspect}" if h[primary_sql_key].nil?
+ table_for_ns(ns).where(primary_sql_key.to_sym => h[primary_sql_key]).delete
end
- def upsert(table, item)
+ def upsert(table, table_primary_key, item)
begin
- upsert!(table, item)
+ upsert!(table, table_primary_key, item)
rescue Sequel::DatabaseError => e
wrapped = e.wrapped_exception
if wrapped.result
- log.warn("Ignoring row (_id=#{item['_id']}): #{e}")
+ log.warn("Ignoring row (#{table_primary_key}=#{item[table_primary_key]}): #{e}")
else
raise e
end
end
end
- def upsert!(table, item)
+ def upsert!(table, table_primary_key, item)
begin
table.insert(item)
rescue Sequel::DatabaseError => e
raise e unless e.message =~ /duplicate key value violates unique constraint/
- table.where(:_id => item['_id']).update(item)
+ table.where(table_primary_key.to_sym => item[table_primary_key]).update(item)
end
end
end
View
@@ -11,6 +11,14 @@ class MoSQL::Test::Functional::CLITest < MoSQL::Test::Functional
:columns:
- _id: TEXT
- var: INTEGER
+ renameid:
+ :meta:
+ :table: sqltable2
+ :columns:
+ - id:
+ :source: _id
+ :type: TEXT
+ - goats: INTEGER
EOF
def fake_cli
@@ -29,6 +37,7 @@ def fake_cli
@adapter = MoSQL::SQLAdapter.new(@map, sql_test_uri)
@sequel.drop_table?(:sqltable)
+ @sequel.drop_table?(:sqltable2)
@map.create_schema(@sequel)
@cli = fake_cli
@@ -73,4 +82,32 @@ def fake_cli
})
assert_equal(100, sequel[:sqltable].where(:_id => o['_id'].to_s).select.first[:var])
end
+
+ it 'handle "u" ops with $set and a renamed _id' do
+ o = { '_id' => BSON::ObjectId.new, 'goats' => 96 }
+ @adapter.upsert_ns('mosql_test.renameid', o)
+
+ # $set's are currently a bit of a hack where we read the object
+ # from the db, so make sure the new object exists in mongo
+ connect_mongo['mosql_test']['renameid'].insert(o.merge('goats' => 0),
+ :w => 1)
+
+ @cli.handle_op({ 'ns' => 'mosql_test.renameid',
+ 'op' => 'u',
+ 'o2' => { '_id' => o['_id'] },
+ 'o' => { '$set' => { 'goats' => 0 } },
+ })
+ assert_equal(0, sequel[:sqltable2].where(:id => o['_id'].to_s).select.first[:goats])
+ end
+
+ it 'handles "d" ops with a renamed id' do
+ o = { '_id' => BSON::ObjectId.new, 'goats' => 1 }
+ @adapter.upsert_ns('mosql_test.renameid', o)
+
+ @cli.handle_op({ 'ns' => 'mosql_test.renameid',
+ 'op' => 'd',
+ 'o' => { '_id' => o['_id'] },
+ })
+ assert_equal(0, sequel[:sqltable2].where(:id => o['_id'].to_s).count)
+ end
end
@@ -15,7 +15,7 @@ class MoSQL::Test::Functional::SchemaTest < MoSQL::Test::Functional
:table: sqltable2
:extra_props: true
:columns:
- - _id: INTEGER
+ - _id: TEXT
EOF
before do
View
@@ -16,8 +16,8 @@ class MoSQL::Test::Functional::SQLTest < MoSQL::Test::Functional
describe 'upsert' do
it 'inserts new items' do
- @adapter.upsert!(@table, {'_id' => 0, 'color' => 'red', 'quantity' => 10})
- @adapter.upsert!(@table, {'_id' => 1, 'color' => 'blue', 'quantity' => 5})
+ @adapter.upsert!(@table, '_id', {'_id' => 0, 'color' => 'red', 'quantity' => 10})
+ @adapter.upsert!(@table, '_id', {'_id' => 1, 'color' => 'blue', 'quantity' => 5})
assert_equal(2, @table.count)
assert_equal('red', @table[:_id => 0][:color])
assert_equal(10, @table[:_id => 0][:quantity])
@@ -26,11 +26,11 @@ class MoSQL::Test::Functional::SQLTest < MoSQL::Test::Functional
end
it 'updates items' do
- @adapter.upsert!(@table, {'_id' => 0, 'color' => 'red', 'quantity' => 10})
+ @adapter.upsert!(@table, '_id', {'_id' => 0, 'color' => 'red', 'quantity' => 10})
assert_equal(1, @table.count)
assert_equal('red', @table[:_id => 0][:color])
- @adapter.upsert!(@table, {'_id' => 0, 'color' => 'blue', 'quantity' => 5})
+ @adapter.upsert!(@table, '_id', {'_id' => 0, 'color' => 'blue', 'quantity' => 5})
assert_equal(1, @table.count)
assert_equal('blue', @table[:_id => 0][:color])
end
Oops, something went wrong.

0 comments on commit 5945b62

Please sign in to comment.