Fix issue of missing files in ZIP package. Test modified

zniper · May 23, 2015 · 6985f33 · 6985f33
1 parent 0bd8570
commit 6985f33
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 0 deletions.
diff --git a/scraper/models.py b/scraper/models.py
@@ -213,6 +213,13 @@ def crawl_content(self, download=True, task_id=None):
                     extractor._uuid + '.zip',
                     join(settings.SCRAPER_TEMP_DIR, self.storage_location))
                 archive.write(INDEX_JSON, crawl_json)
+                # Write result files
+                for res_id in result_paths:
+                    res_path = result_paths[res_id]
+                    for item in os.listdir(res_path):
+                        archive.write(
+                            join(res_id, item),
+                            open(join(res_path, item), 'r').read())
                 storage_path = archive.move_to_storage(
                     storage, self.storage_location)
             else:

diff --git a/scraper/tests.py b/scraper/tests.py
@@ -5,6 +5,7 @@
 import os
 import simplejson as json
 
+from zipfile import ZipFile
 from os.path import join
 from shutil import rmtree
 from zipfile import ZipFile
@@ -369,6 +370,8 @@ def test_crawl_zip(self):
         result_json = result.data
         self.assertEqual(len(result_json['content']), 3)
         self.assertGreater(result.other.pk, 0)
+        zfile = ZipFile(join(storage.base_location, path))
+        self.assertEquals(len(zfile.namelist()), 6)
         storage.delete(path)