indexwarcsjob.py: update hard-coded path to be correct (TODO: load fr…

…om env)

reqs: update to correct python-hadoop git

indexwarcsjob.py

@@ -64,7 +64,7 @@ def mapper_init(self):
             'surt_ordered': True,
             'sort': True,
             'cdxj': True,
-            'minimal': True
+            #'minimal': True
         }
 
     def mapper(self, _, line):
@@ -77,7 +77,7 @@ def mapper(self, _, line):
 
     def _conv_warc_to_cdx_path(self, warc_path):
         # set cdx path
-        cdx_path = warc_path.replace('common-crawl/crawl-data', 'cdx2')
+        cdx_path = warc_path.replace('common-crawl/crawl-data', 'common-crawl/cc-index/cdx')
         cdx_path = cdx_path.replace('.warc.gz', '.cdx.gz')
         return cdx_path
 

requirements.txt

@@ -1,5 +1,5 @@
 mrjob
 boto
 pywb
-#-e git+https://github.com/ikreymer/pywb.git@0.9.0b#egg=pywb
--e git+https://github.com/matteobertozzi/Hadoop.git#egg=hadoop&subdirectory=python-hadoop
+#-e git+https://github.com/matteobertozzi/Hadoop.git#egg=hadoop&subdirectory=python-hadoop
+-e git+https://github.com/commoncrawl/python-hadoop.git#egg=master