Merge pull request #6 from stedile/master

Bug fixes
yougov · Sep 7, 2012 · 97556ba · 97556ba
2 parents 7a28c11 + 0043ac2
commit 97556ba
Show file tree

Hide file tree

Showing 7 changed files with 17 additions and 37 deletions.
diff --git a/mongo-connector/doc_managers/mongo_doc_manager.py b/mongo-connector/doc_managers/mongo_doc_manager.py
@@ -95,7 +95,7 @@ def search(self, start_ts, end_ts):
             db, coll = namespace.split('.', 1)
             target_coll = self.mongo[db][coll]
             res.extend(list(target_coll.find({'_ts': {'$lte': end_ts,
-                                          '$gte': start_ts}})))
+                                                      '$gte': start_ts}})))
 
         return res
 

diff --git a/mongo-connector/doc_managers/sample_doc_manager.py b/mongo-connector/doc_managers/sample_doc_manager.py
@@ -43,11 +43,11 @@ def __init__(self, url=None, auto_commit=True, unique_key='_id'):
         This method should, if necessarity, verify the url to the backend
         and return None if that fails.
         It should also create the connection to the backend, and start a
-        periodic committer if necessary. 
-        The unique_key should default to '_id' and it is an obligatory 
+        periodic committer if necessary.
+        The unique_key should default to '_id' and it is an obligatory
         parameter.
-        It requires a url parameter iff mongo_connector.py is called with 
-        the -b parameter. Otherwise, it doesn't require any other parameter 
+        It requires a url parameter iff mongo_connector.py is called with
+        the -b parameter. Otherwise, it doesn't require any other parameter
         (e.g. if the target engine doesn't need a URL)
         It should raise a SystemError exception if the URL is not valid.
         """

diff --git a/mongo-connector/oplog_manager.py b/mongo-connector/oplog_manager.py
@@ -253,32 +253,14 @@ def dump_collection(self):
                     namespace = str(db) + "." + str(coll)
                     dump_set.append(namespace)
 
-        long_ts = None
-
+        timestamp = util.retry_until_ok(self.get_last_oplog_timestamp)
+        if timestamp is None:
+            return None
         for namespace in dump_set:
             db, coll = namespace.split('.', 1)
             target_coll = self.main_connection[db][coll]
             cursor = util.retry_until_ok(target_coll.find)
-            cursor = cursor.sort('$natural', pymongo.DESCENDING)
-            oplog_cursor = util.retry_until_ok(self.oplog.find)
-            oplog_cursor = oplog_cursor.sort('$natural', pymongo.DESCENDING)
-
-            for entry in oplog_cursor:
-
-                if entry['op'] != 'i':
-                    continue
-                #The 'o' field represents the document
-                search_doc = entry['o']
-                cursor.rewind()
-                for doc in cursor:
-                    if search_doc == doc:
-                        long_ts = util.bson_ts_to_long(entry['ts'])
-                        break
-
-                if long_ts:
-                    break
-
-            cursor.rewind()
+            long_ts = util.bson_ts_to_long(timestamp)
 
             try:
                 for doc in cursor:
@@ -294,12 +276,7 @@ def dump_collection(self):
                 self.running = False
                 return
 
-        if long_ts:
-            long_ts = util.long_to_bson_ts(long_ts)
-        else:  # Implies that we are just initiating the set
-            long_ts = self.get_last_oplog_timestamp()
-
-        return long_ts
+        return timestamp
 
     def get_last_oplog_timestamp(self):
         """Return the timestamp of the latest entry in the oplog.

diff --git a/mongo-connector/tests/doc_managers/test_elastic.py b/mongo-connector/tests/doc_managers/test_elastic.py
@@ -77,7 +77,8 @@ def setUp(self):
         self.c = Connector('localhost:' + PORTS_ONE["MONGOS"],
                            'config.txt', 'http://localhost:9200',
                            ['test.test'],
-                           '_id', None, cmd_folder + '/../../doc_managers/elastic_doc_manager.py')
+                           '_id', None, cmd_folder +
+                           '/../../doc_managers/elastic_doc_manager.py')
         self.c.start()
         while len(self.c.shard_set) == 0:
             pass

diff --git a/mongo-connector/tests/doc_managers/test_mongo.py b/mongo-connector/tests/doc_managers/test_mongo.py
@@ -75,7 +75,8 @@ def setUp(self):
         self.c = Connector('localhost:' + PORTS_ONE["MONGOS"],
                            'config.txt', 'localhost:30000',
                            ['test.test'],
-                           '_id', None, cmd_folder + '/../../doc_managers/mongo_doc_manager.py')
+                           '_id', None, cmd_folder +
+                           '/../../doc_managers/mongo_doc_manager.py')
         self.c.start()
         while len(self.c.shard_set) == 0:
             pass

diff --git a/mongo-connector/tests/doc_managers/test_solr.py b/mongo-connector/tests/doc_managers/test_solr.py
@@ -70,7 +70,8 @@ def setUp(self):
 
         self.c = Connector('localhost:' + PORTS_ONE["MAIN"], 'config.txt',
                            'http://localhost:8080/solr', ['test.test'], '_id',
-                           None, cmd_folder + '/../../doc_managers/solr_doc_manager.py')
+                           None, cmd_folder +
+                           '/../../doc_managers/solr_doc_manager.py')
         self.c.start()
         while len(self.c.shard_set) == 0:
             time.sleep(1)

diff --git a/setup.py b/setup.py
@@ -33,7 +33,7 @@
 doclines = __doc__.split("\n")
 
 setup(name="mongo-connector",
-      version="0.9.2",
+      version="1.0.0",
       maintainer="10Gen",
       maintainer_email="leonardo.stedile@10gen.com",
       #url = "https://github.com/AayushU/mongo-connector",