Merge pull request #459 from Parsely/feature/speed_up_update_virtualenv

Run a single pip install command when updating a virtualenv instead of several
pystorm · Apr 4, 2019 · 0a64bd5 · 0a64bd5
2 parents 480ee00 + 2267ce0
commit 0a64bd5
Showing 1 changed file with 15 additions and 13 deletions.
diff --git a/streamparse/cli/update_virtualenv.py b/streamparse/cli/update_virtualenv.py
@@ -68,23 +68,25 @@ def _create_or_update_virtualenv(
 
         if isinstance(requirements_paths, string_types):
             requirements_paths = [requirements_paths]
+        temp_req_paths = []
         for requirements_path in requirements_paths:
             puts("Uploading {} to temporary file.".format(requirements_path))
             temp_req = run("mktemp /tmp/streamparse_requirements-XXXXXXXXX.txt")
+            temp_req_paths.append(temp_req)
             put(requirements_path, temp_req)
 
-            puts("Updating virtualenv: {}".format(virtualenv_name))
-            cmd = "source {}".format(os.path.join(virtualenv_path, "bin/activate"))
-            with prefix(cmd):
-                # Make sure we're using latest pip so options work as expected
-                run("pip install --upgrade 'pip>=9.0,!=19.0'", pty=False)
-                run(
-                    "pip install -r {} --exists-action w --upgrade "
-                    "--upgrade-strategy only-if-needed".format(temp_req),
-                    pty=False,
-                )
-
-            run("rm {}".format(temp_req))
+        puts("Updating virtualenv: {}".format(virtualenv_name))
+        pip_path = "/".join((virtualenv_path, "bin", "pip"))
+        # Make sure we're using latest pip so options work as expected
+        run("{} install --upgrade 'pip>=9.0,!=19.0'".format(pip_path), pty=False)
+        run(
+            (
+                "{} install -r {} --exists-action w --upgrade "
+                "--upgrade-strategy only-if-needed --progress-bar off"
+            ).format(pip_path, " -r ".join(temp_req_paths)),
+            pty=False,
+        )
+        run("rm -f {}".format(" ".join(temp_req_paths)))
 
 
 def create_or_update_virtualenvs(
@@ -154,7 +156,7 @@ def create_or_update_virtualenvs(
         env.virtualenv_root,
         virtualenv_name,
         requirements_paths,
-        virtualenv_flags=options.get("virtualenv_flags"),
+        virtualenv_flags=storm_options.get("virtualenv_flags"),
         hosts=env.storm_workers,
         overwrite_virtualenv=overwrite_virtualenv,
         user=user,