Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Merge pull request #6 from stedile/master

Bug fixes
  • Loading branch information...
commit 97556ba7b17bc937502ec2df482deea05a318040 2 parents 7a28c11 + 0043ac2
@IanWhalen IanWhalen authored
View
2  mongo-connector/doc_managers/mongo_doc_manager.py
@@ -95,7 +95,7 @@ def search(self, start_ts, end_ts):
db, coll = namespace.split('.', 1)
target_coll = self.mongo[db][coll]
res.extend(list(target_coll.find({'_ts': {'$lte': end_ts,
- '$gte': start_ts}})))
+ '$gte': start_ts}})))
return res
View
8 mongo-connector/doc_managers/sample_doc_manager.py
@@ -43,11 +43,11 @@ def __init__(self, url=None, auto_commit=True, unique_key='_id'):
This method should, if necessarity, verify the url to the backend
and return None if that fails.
It should also create the connection to the backend, and start a
- periodic committer if necessary.
- The unique_key should default to '_id' and it is an obligatory
+ periodic committer if necessary.
+ The unique_key should default to '_id' and it is an obligatory
parameter.
- It requires a url parameter iff mongo_connector.py is called with
- the -b parameter. Otherwise, it doesn't require any other parameter
+ It requires a url parameter iff mongo_connector.py is called with
+ the -b parameter. Otherwise, it doesn't require any other parameter
(e.g. if the target engine doesn't need a URL)
It should raise a SystemError exception if the URL is not valid.
"""
View
33 mongo-connector/oplog_manager.py
@@ -253,32 +253,14 @@ def dump_collection(self):
namespace = str(db) + "." + str(coll)
dump_set.append(namespace)
- long_ts = None
-
+ timestamp = util.retry_until_ok(self.get_last_oplog_timestamp)
+ if timestamp is None:
+ return None
for namespace in dump_set:
db, coll = namespace.split('.', 1)
target_coll = self.main_connection[db][coll]
cursor = util.retry_until_ok(target_coll.find)
- cursor = cursor.sort('$natural', pymongo.DESCENDING)
- oplog_cursor = util.retry_until_ok(self.oplog.find)
- oplog_cursor = oplog_cursor.sort('$natural', pymongo.DESCENDING)
-
- for entry in oplog_cursor:
-
- if entry['op'] != 'i':
- continue
- #The 'o' field represents the document
- search_doc = entry['o']
- cursor.rewind()
- for doc in cursor:
- if search_doc == doc:
- long_ts = util.bson_ts_to_long(entry['ts'])
- break
-
- if long_ts:
- break
-
- cursor.rewind()
+ long_ts = util.bson_ts_to_long(timestamp)
try:
for doc in cursor:
@@ -294,12 +276,7 @@ def dump_collection(self):
self.running = False
return
- if long_ts:
- long_ts = util.long_to_bson_ts(long_ts)
- else: # Implies that we are just initiating the set
- long_ts = self.get_last_oplog_timestamp()
-
- return long_ts
+ return timestamp
def get_last_oplog_timestamp(self):
"""Return the timestamp of the latest entry in the oplog.
View
3  mongo-connector/tests/doc_managers/test_elastic.py
@@ -77,7 +77,8 @@ def setUp(self):
self.c = Connector('localhost:' + PORTS_ONE["MONGOS"],
'config.txt', 'http://localhost:9200',
['test.test'],
- '_id', None, cmd_folder + '/../../doc_managers/elastic_doc_manager.py')
+ '_id', None, cmd_folder +
+ '/../../doc_managers/elastic_doc_manager.py')
self.c.start()
while len(self.c.shard_set) == 0:
pass
View
3  mongo-connector/tests/doc_managers/test_mongo.py
@@ -75,7 +75,8 @@ def setUp(self):
self.c = Connector('localhost:' + PORTS_ONE["MONGOS"],
'config.txt', 'localhost:30000',
['test.test'],
- '_id', None, cmd_folder + '/../../doc_managers/mongo_doc_manager.py')
+ '_id', None, cmd_folder +
+ '/../../doc_managers/mongo_doc_manager.py')
self.c.start()
while len(self.c.shard_set) == 0:
pass
View
3  mongo-connector/tests/doc_managers/test_solr.py
@@ -70,7 +70,8 @@ def setUp(self):
self.c = Connector('localhost:' + PORTS_ONE["MAIN"], 'config.txt',
'http://localhost:8080/solr', ['test.test'], '_id',
- None, cmd_folder + '/../../doc_managers/solr_doc_manager.py')
+ None, cmd_folder +
+ '/../../doc_managers/solr_doc_manager.py')
self.c.start()
while len(self.c.shard_set) == 0:
time.sleep(1)
View
2  setup.py
@@ -33,7 +33,7 @@
doclines = __doc__.split("\n")
setup(name="mongo-connector",
- version="0.9.2",
+ version="1.0.0",
maintainer="10Gen",
maintainer_email="leonardo.stedile@10gen.com",
#url = "https://github.com/AayushU/mongo-connector",
Please sign in to comment.
Something went wrong with that request. Please try again.