pipeline: Make youtube-dl work even when ~/.local/bin is not in PATH

ArchiveTeam · Jul 16, 2015 · 68474dc · 68474dc
1 parent 0c752ea
commit 68474dc
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 6 deletions.
diff --git a/pipeline/archivebot/seesaw/wpull.py b/pipeline/archivebot/seesaw/wpull.py
@@ -9,7 +9,7 @@ def add_args(args, names, item):
         if value:
             args.append(value)
 
-def make_args(item, default_user_agent, wpull_exe, phantomjs_exe, finished_warcs_dir):
+def make_args(item, default_user_agent, wpull_exe, youtube_dl_exe, phantomjs_exe, finished_warcs_dir):
     # -----------------------------------------------------------------------
     # BASE ARGUMENTS
     # -----------------------------------------------------------------------
@@ -54,6 +54,7 @@ def make_args(item, default_user_agent, wpull_exe, phantomjs_exe, finished_warcs
         '--monitor-disk', '500m',
         '--monitor-memory', '50m',
         '--max-redirect', '8',
+        '--youtube-dl-exe', youtube_dl_exe
     ]
 
     if item['url'].startswith("http://www.reddit.com/") or \
@@ -113,14 +114,15 @@ def make_args(item, default_user_agent, wpull_exe, phantomjs_exe, finished_warcs
 # ---------------------------------------------------------------------------
 
 class WpullArgs(object):
-    def __init__(self, *, default_user_agent, wpull_exe, phantomjs_exe, finished_warcs_dir):
+    def __init__(self, *, default_user_agent, wpull_exe, youtube_dl_exe, phantomjs_exe, finished_warcs_dir):
         self.default_user_agent = default_user_agent
         self.wpull_exe = wpull_exe
+        self.youtube_dl_exe = youtube_dl_exe
         self.phantomjs_exe = phantomjs_exe
         self.finished_warcs_dir = finished_warcs_dir
 
     def realize(self, item):
         return make_args(item, self.default_user_agent, self.wpull_exe,
-                         self.phantomjs_exe, self.finished_warcs_dir)
+            self.youtube_dl_exe, self.phantomjs_exe, self.finished_warcs_dir)
 
 # vim:ts=4:sw=4:et:tw=78
diff --git a/pipeline/pipeline.py b/pipeline/pipeline.py
@@ -30,7 +30,7 @@
     SetFetchDepth, PreparePaths, WriteInfo, DownloadUrlFile, \
     RelabelIfAborted, MoveFiles, StopHeartbeat, MarkItemAsDone, CheckIP
 
-VERSION = "20150701.01"
+VERSION = "20150715.01"
 PHANTOMJS_VERSION = '1.9.8'
 EXPIRE_TIME = 60 * 60 * 48  # 48 hours between archive requests
 WPULL_EXE = find_executable('Wpull', None, ['./wpull'])
@@ -111,8 +111,10 @@ def __contains__(self, item):
 _, _, _, pipeline_id = monitoring.pipeline_id()
 
 wpull_args = WpullArgs(
-    default_user_agent=DEFAULT_USER_AGENT, wpull_exe=WPULL_EXE,
-    phantomjs_exe=PHANTOMJS, 
+    default_user_agent=DEFAULT_USER_AGENT,
+    wpull_exe=WPULL_EXE,
+    youtube_dl_exe=YOUTUBE_DL,
+    phantomjs_exe=PHANTOMJS,
     finished_warcs_dir=os.environ["FINISHED_WARCS_DIR"]
 )