Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Merge branch 'pr/15'

  • Loading branch information...
commit 5945b627f05ca478e815cf48a719562c551b45c8 2 parents 5182ad2 + 921f6c8
@nelhage nelhage authored
View
32 README.md
@@ -53,8 +53,12 @@ types. An example collection map might be:
mongodb:
blog_posts:
:columns:
- - _id: TEXT
- - author: TEXT
+ - id:
+ :source: _id
+ :type: TEXT
+ - author:
+ :source: author
+ :type: TEXT
- title: TEXT
- created: DOUBLE PRECISION
:meta:
@@ -67,9 +71,27 @@ mapping
<Mongo DB name> -> { <Mongo Collection Name> -> <Collection Definition> }
Where a `<Collection Definition>` is a hash with `:columns` and
-`:meta` fields. `:columns` is a list of one-element hashes, mapping
-field-name to SQL type. It is required to include at least an `_id`
-mapping. `:meta` contains metadata about this collection/table. It is
+`:meta` fields.
+
+`:columns` is a list of hashes mapping SQL column names to an hash
+describing that column. This hash may contain the following fields:
+
+ * `:source`: The name of the attribute inside of MongoDB.
+ * `:type`: (Mandatory) The SQL type.
+
+This syntax allows to rename MongoDB's attributes during the
+import. For example the MonogDB `_id` attribute will be transferred to
+a SQL column named `id`.
+
+As a shorthand, you can specify a one-elment hash of the form `name:
+TYPE`, in which case `name` will be used for both the source attribute
+and the name of the destination column. You can see this shorthand for
+the `title` and `created` attributes, above.
+
+Every defined collection must include a mapping for the `_id`
+attribute.
+
+`:meta` contains metadata about this collection/table. It is
required to include at least `:table`, naming the SQL table this
collection will be mapped to. `extra_props` determines the handling of
unknown fields in MongoDB objects -- more about that later.
View
9 lib/mosql/cli.rb
@@ -164,7 +164,7 @@ def bulk_upsert(table, ns, items)
items.each do |it|
h = {}
cols.zip(it).each { |k,v| h[k] = v }
- @sql.upsert(table, h)
+ @sql.upsert(table, primary_sql_key_for_ns(ns), h)
end
end
end
@@ -258,12 +258,13 @@ def optail
end
def sync_object(ns, _id)
- sqlid = @sql.transform_one_ns(ns, { '_id' => _id })['_id']
- obj = collection_for_ns(ns).find_one({:_id => _id})
+ primary_sql_key = @schemamap.primary_sql_key_for_ns(ns)
+ sqlid = @sql.transform_one_ns(ns, { '_id' => _id })[primary_sql_key]
+ obj = collection_for_ns(ns).find_one({:_id => _id})
if obj
@sql.upsert_ns(ns, obj)
else
- @sql.table_for_ns(ns).where(:_id => sqlid).delete()
+ @sql.table_for_ns(ns).where(primary_sql_key.to_sym => sqlid).delete()
end
end
View
54 lib/mosql/schema.rb
@@ -4,19 +4,33 @@ class SchemaError < StandardError; end;
class Schema
include MoSQL::Logging
- def to_ordered_hash(lst)
- hash = BSON::OrderedHash.new
+ def to_array(lst)
+ array = []
lst.each do |ent|
- raise "Invalid ordered hash entry #{ent.inspect}" unless ent.is_a?(Hash) && ent.keys.length == 1
- field, type = ent.first
- hash[field] = type
+ if ent.is_a?(Hash) && ent[:source].is_a?(String) && ent[:type].is_a?(String)
+ # new configuration format
+ array << {
+ :source => ent.delete(:source),
+ :type => ent.delete(:type),
+ :name => ent.first.first,
+ }
+ elsif ent.is_a?(Hash) && ent.keys.length == 1
+ array << {
+ :source => ent.first.first,
+ :name => ent.first.first,
+ :type => ent.first.last
+ }
+ else
+ raise "Invalid ordered hash entry #{ent.inspect}"
+ end
+
end
- hash
+ array
end
def parse_spec(spec)
out = spec.dup
- out[:columns] = to_ordered_hash(spec[:columns])
+ out[:columns] = to_array(spec[:columns])
out
end
@@ -35,13 +49,16 @@ def create_schema(db, clobber=false)
meta = collection[:meta]
log.info("Creating table '#{meta[:table]}'...")
db.send(clobber ? :create_table! : :create_table?, meta[:table]) do
- collection[:columns].each do |field, type|
- column field, type
+ collection[:columns].each do |col|
+ column col[:name], col[:type]
+
+ if col[:source].to_sym == :_id
+ primary_key [col[:name].to_sym]
+ end
end
if meta[:extra_props]
column '_extra_props', 'TEXT'
end
- primary_key [:_id]
end
end
end
@@ -67,8 +84,12 @@ def transform(ns, obj, schema=nil)
obj = obj.dup
row = []
- schema[:columns].each do |name, type|
- v = obj.delete(name)
+ schema[:columns].each do |col|
+
+ source = col[:source]
+ type = col[:type]
+
+ v = obj.delete(source)
case v
when BSON::Binary, BSON::ObjectId
v = v.to_s
@@ -91,7 +112,10 @@ def transform(ns, obj, schema=nil)
end
def all_columns(schema)
- cols = schema[:columns].keys
+ cols = []
+ schema[:columns].each do |col|
+ cols << col[:name]
+ end
if schema[:meta][:extra_props]
cols << "_extra_props"
end
@@ -144,5 +168,9 @@ def all_mongo_dbs
def collections_for_mongo_db(db)
(@map[db]||{}).keys
end
+
+ def primary_sql_key_for_ns(ns)
+ find_ns!(ns)[:columns].find {|c| c[:source] == '_id'}[:name]
+ end
end
end
View
17 lib/mosql/sql.rb
@@ -35,35 +35,36 @@ def transform_one_ns(ns, obj)
def upsert_ns(ns, obj)
h = transform_one_ns(ns, obj)
- upsert(table_for_ns(ns), h)
+ upsert(table_for_ns(ns), @schema.primary_sql_key_for_ns(ns), h)
end
# obj must contain an _id field. All other fields will be ignored.
def delete_ns(ns, obj)
+ primary_sql_key = @schema.primary_sql_key_for_ns(ns)
h = transform_one_ns(ns, obj)
- raise "No _id found in transform of #{obj.inspect}" if h['_id'].nil?
- table_for_ns(ns).where(:_id => h['_id']).delete
+ raise "No #{primary_sql_key} found in transform of #{obj.inspect}" if h[primary_sql_key].nil?
+ table_for_ns(ns).where(primary_sql_key.to_sym => h[primary_sql_key]).delete
end
- def upsert(table, item)
+ def upsert(table, table_primary_key, item)
begin
- upsert!(table, item)
+ upsert!(table, table_primary_key, item)
rescue Sequel::DatabaseError => e
wrapped = e.wrapped_exception
if wrapped.result
- log.warn("Ignoring row (_id=#{item['_id']}): #{e}")
+ log.warn("Ignoring row (#{table_primary_key}=#{item[table_primary_key]}): #{e}")
else
raise e
end
end
end
- def upsert!(table, item)
+ def upsert!(table, table_primary_key, item)
begin
table.insert(item)
rescue Sequel::DatabaseError => e
raise e unless e.message =~ /duplicate key value violates unique constraint/
- table.where(:_id => item['_id']).update(item)
+ table.where(table_primary_key.to_sym => item[table_primary_key]).update(item)
end
end
end
View
37 test/functional/cli.rb
@@ -11,6 +11,14 @@ class MoSQL::Test::Functional::CLITest < MoSQL::Test::Functional
:columns:
- _id: TEXT
- var: INTEGER
+ renameid:
+ :meta:
+ :table: sqltable2
+ :columns:
+ - id:
+ :source: _id
+ :type: TEXT
+ - goats: INTEGER
EOF
def fake_cli
@@ -29,6 +37,7 @@ def fake_cli
@adapter = MoSQL::SQLAdapter.new(@map, sql_test_uri)
@sequel.drop_table?(:sqltable)
+ @sequel.drop_table?(:sqltable2)
@map.create_schema(@sequel)
@cli = fake_cli
@@ -73,4 +82,32 @@ def fake_cli
})
assert_equal(100, sequel[:sqltable].where(:_id => o['_id'].to_s).select.first[:var])
end
+
+ it 'handle "u" ops with $set and a renamed _id' do
+ o = { '_id' => BSON::ObjectId.new, 'goats' => 96 }
+ @adapter.upsert_ns('mosql_test.renameid', o)
+
+ # $set's are currently a bit of a hack where we read the object
+ # from the db, so make sure the new object exists in mongo
+ connect_mongo['mosql_test']['renameid'].insert(o.merge('goats' => 0),
+ :w => 1)
+
+ @cli.handle_op({ 'ns' => 'mosql_test.renameid',
+ 'op' => 'u',
+ 'o2' => { '_id' => o['_id'] },
+ 'o' => { '$set' => { 'goats' => 0 } },
+ })
+ assert_equal(0, sequel[:sqltable2].where(:id => o['_id'].to_s).select.first[:goats])
+ end
+
+ it 'handles "d" ops with a renamed id' do
+ o = { '_id' => BSON::ObjectId.new, 'goats' => 1 }
+ @adapter.upsert_ns('mosql_test.renameid', o)
+
+ @cli.handle_op({ 'ns' => 'mosql_test.renameid',
+ 'op' => 'd',
+ 'o' => { '_id' => o['_id'] },
+ })
+ assert_equal(0, sequel[:sqltable2].where(:id => o['_id'].to_s).count)
+ end
end
View
2  test/functional/schema.rb
@@ -15,7 +15,7 @@ class MoSQL::Test::Functional::SchemaTest < MoSQL::Test::Functional
:table: sqltable2
:extra_props: true
:columns:
- - _id: INTEGER
+ - _id: TEXT
EOF
before do
View
8 test/functional/sql.rb
@@ -16,8 +16,8 @@ class MoSQL::Test::Functional::SQLTest < MoSQL::Test::Functional
describe 'upsert' do
it 'inserts new items' do
- @adapter.upsert!(@table, {'_id' => 0, 'color' => 'red', 'quantity' => 10})
- @adapter.upsert!(@table, {'_id' => 1, 'color' => 'blue', 'quantity' => 5})
+ @adapter.upsert!(@table, '_id', {'_id' => 0, 'color' => 'red', 'quantity' => 10})
+ @adapter.upsert!(@table, '_id', {'_id' => 1, 'color' => 'blue', 'quantity' => 5})
assert_equal(2, @table.count)
assert_equal('red', @table[:_id => 0][:color])
assert_equal(10, @table[:_id => 0][:quantity])
@@ -26,11 +26,11 @@ class MoSQL::Test::Functional::SQLTest < MoSQL::Test::Functional
end
it 'updates items' do
- @adapter.upsert!(@table, {'_id' => 0, 'color' => 'red', 'quantity' => 10})
+ @adapter.upsert!(@table, '_id', {'_id' => 0, 'color' => 'red', 'quantity' => 10})
assert_equal(1, @table.count)
assert_equal('red', @table[:_id => 0][:color])
- @adapter.upsert!(@table, {'_id' => 0, 'color' => 'blue', 'quantity' => 5})
+ @adapter.upsert!(@table, '_id', {'_id' => 0, 'color' => 'blue', 'quantity' => 5})
assert_equal(1, @table.count)
assert_equal('blue', @table[:_id => 0][:color])
end
View
64 test/unit/lib/mongo-sql/schema.rb
@@ -8,14 +8,23 @@ class MoSQL::Test::SchemaTest < MoSQL::Test
:meta:
:table: sqltable
:columns:
- - _id: TEXT
+ - id:
+ :source: _id
+ :type: TEXT
- var: INTEGER
with_extra_props:
:meta:
:table: sqltable2
:extra_props: true
:columns:
- - _id: INTEGER
+ - id:
+ :source: _id
+ :type: TEXT
+ old_conf_syntax:
+ :columns:
+ - _id: TEXT
+ :meta:
+ :table: sqltable3
EOF
before do
@@ -34,16 +43,44 @@ class MoSQL::Test::SchemaTest < MoSQL::Test
end
end
- it 'Converts columns to an ordered hash' do
+ it 'Converts columns to an array' do
table = @map.find_ns("db.collection")
- assert(table[:columns].is_a?(Hash))
- assert_equal(['_id', 'var'], table[:columns].keys)
+ assert(table[:columns].is_a?(Array))
+
+ id_mapping = table[:columns].find{|c| c[:source] == '_id'}
+ assert id_mapping
+ assert_equal '_id', id_mapping[:source]
+ assert_equal 'id', id_mapping[:name]
+ assert_equal 'TEXT', id_mapping[:type]
+
+ var_mapping = table[:columns].find{|c| c[:source] == 'var'}
+ assert var_mapping
+ assert_equal 'var', var_mapping[:source]
+ assert_equal 'var', var_mapping[:name]
+ assert_equal 'INTEGER', var_mapping[:type]
+ end
+
+ it 'Can handle the old configuration format' do
+ table = @map.find_ns('db.old_conf_syntax')
+ assert(table[:columns].is_a?(Array))
+
+ id_mapping = table[:columns].find{|c| c[:source] == '_id'}
+ assert id_mapping
+ assert_equal '_id', id_mapping[:source]
+ assert_equal '_id', id_mapping[:name]
+ assert_equal 'TEXT', id_mapping[:type]
+ end
+
+ it 'Can find the primary key of the SQL table' do
+ assert_equal('id', @map.primary_sql_key_for_ns('db.collection'))
+ assert_equal('_id', @map.primary_sql_key_for_ns('db.old_conf_syntax'))
end
it 'can create a SQL schema' do
db = stub()
db.expects(:create_table?).with('sqltable')
db.expects(:create_table?).with('sqltable2')
+ db.expects(:create_table?).with('sqltable3')
@map.create_schema(db)
end
@@ -51,22 +88,29 @@ class MoSQL::Test::SchemaTest < MoSQL::Test
it 'creates a SQL schema with the right fields' do
db = {}
stub_1 = stub()
- stub_1.expects(:column).with('_id', 'TEXT')
+ stub_1.expects(:column).with('id', 'TEXT')
stub_1.expects(:column).with('var', 'INTEGER')
stub_1.expects(:column).with('_extra_props').never
+ stub_1.expects(:primary_key).with([:id])
stub_2 = stub()
- stub_2.expects(:column).with('_id', 'INTEGER')
+ stub_2.expects(:column).with('id', 'TEXT')
stub_2.expects(:column).with('_extra_props', 'TEXT')
+ stub_2.expects(:primary_key).with([:id])
+ stub_3 = stub()
+ stub_3.expects(:column).with('_id', 'TEXT')
+ stub_3.expects(:column).with('_extra_props').never
+ stub_3.expects(:primary_key).with([:_id])
(class << db; self; end).send(:define_method, :create_table?) do |tbl, &blk|
case tbl
when "sqltable"
o = stub_1
when "sqltable2"
o = stub_2
+ when "sqltable3"
+ o = stub_3
else
assert(false, "Tried to create an unexpeced table: #{tbl}")
end
- o.expects(:primary_key).with([:_id])
o.instance_eval(&blk)
end
@map.create_schema(db)
@@ -86,8 +130,8 @@ class MoSQL::Test::SchemaTest < MoSQL::Test
end
it 'gets all_columns right' do
- assert_equal(['_id', 'var'], @map.all_columns(@map.find_ns('db.collection')))
- assert_equal(['_id', '_extra_props'], @map.all_columns(@map.find_ns('db.with_extra_props')))
+ assert_equal(['id', 'var'], @map.all_columns(@map.find_ns('db.collection')))
+ assert_equal(['id', '_extra_props'], @map.all_columns(@map.find_ns('db.with_extra_props')))
end
end
Please sign in to comment.
Something went wrong with that request. Please try again.