speedup duplicate processor using kvfile insert generator (#28)

* speedup duplicate processor using kvfile insert generator * Update duplicate.py * Update duplicate.py
datahq · Oct 15, 2018 · ae5da8b · ae5da8b
1 parent ab2f52c
commit ae5da8b
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 4 deletions.
diff --git a/dataflows/processors/duplicate.py b/dataflows/processors/duplicate.py
@@ -3,7 +3,14 @@
 
 
 def saver(resource, db, batch_size):
-    db.insert((("{:08x}".format(idx), row) for idx, row in enumerate(resource)), batch_size=batch_size)
+    gen = db.insert_generator(
+        (("{:08x}".format(idx), row)
+         for idx, row
+         in enumerate(resource)),
+        batch_size=batch_size
+    )
+    for _, row in gen:
+        yield row
 
 
 def loader(db):
@@ -37,8 +44,7 @@ def traverse_resources(resources):
         for resource in package:
             if resource.res.name == source_:
                 db = KVFile()
-                saver(resource, db, batch_size)
-                yield loader(db)
+                yield saver(resource, db, batch_size)
                 yield loader(db)
             else:
                 yield resource

diff --git a/setup.py b/setup.py
@@ -22,7 +22,7 @@ def read(*paths):
 NAME = PACKAGE.replace('_', '-')
 INSTALL_REQUIRES = [
     'datapackage>=1.3.2',
-    'kvfile',
+    'kvfile>=0.0.4',
     'click',
     'jinja2',
     'awesome-slugify',