Enable pip package locking (#317)

* displays pip install output on anaconda-project prepare
* displays pip packages on anaconda-project list-packages
* adds anaconda-project add-packages --pip ... to install packages and add pip packages to the yaml file
* enables full pip package locking with anaconda-project lock
* adds anaconda-project remove-packages --pip
* improved docs

anaconda_project/api.py

@@ -442,7 +442,7 @@ def export_env_spec(self, project, name, filename):
         """
         return project_ops.export_env_spec(project=project, name=name, filename=filename)
 
-    def add_packages(self, project, env_spec_name, packages, channels):
+    def add_packages(self, project, env_spec_name, packages, channels, pip=False):
         """Attempt to install packages then add them to anaconda-project.yml.
 
         If the environment spec name is None rather than an env
@@ -460,6 +460,7 @@ def add_packages(self, project, env_spec_name, packages, channels):
             env_spec_name (str): environment spec name or None for all environment specs
             packages (list of str): packages (with optional version info, as for conda install)
             channels (list of str): channels (as they should be passed to conda --channel)
+            pip (bool): Flag to request packages to be installed with pip if True else use Conda.
 
         Returns:
             ``Status`` instance
@@ -468,9 +469,10 @@ def add_packages(self, project, env_spec_name, packages, channels):
         return project_ops.add_packages(project=project,
                                         env_spec_name=env_spec_name,
                                         packages=packages,
-                                        channels=channels)
+                                        channels=channels,
+                                        pip=pip)
 
-    def remove_packages(self, project, env_spec_name, packages):
+    def remove_packages(self, project, env_spec_name, packages, pip):
         """Attempt to remove packages from an environment spec in anaconda-project.yml.
 
         If the environment spec name is None rather than an env
@@ -487,12 +489,13 @@ def remove_packages(self, project, env_spec_name, packages):
             project (Project): the project
             env_spec_name (str): environment name or None for all environments
             packages (list of str): packages
+            pip (bool): Flag to request packages to be removed with pip if True else use Conda.
 
         Returns:
             ``Status`` instance
 
         """
-        return project_ops.remove_packages(project=project, env_spec_name=env_spec_name, packages=packages)
+        return project_ops.remove_packages(project=project, env_spec_name=env_spec_name, packages=packages, pip=pip)
 
     def lock(self, project, env_spec_name):
         """Attempt to freeze dependency versions in anaconda-project-lock.yml.

anaconda_project/archiver.py

@@ -358,9 +358,11 @@ def _archive_project(project, filename, pack_envs=False):
         for env in os.listdir(envs_path):
             ext = 'zip' if filename.lower().endswith(".zip") else 'tar'
             pack = os.path.join(conda_pack_dir, '{}_envs_{}.{}'.format(current_platform(), env, ext))
+            zip_symlinks = True if ext == 'zip' else False
             fn = conda_pack.pack(prefix=os.path.join(envs_path, env),
                                  arcroot=os.path.join(project.name, 'envs', env),
                                  output=pack,
+                                 zip_symlinks=zip_symlinks,
                                  verbose=True,
                                  force=True)
             packed_envs.append(fn)

anaconda_project/conda_manager.py

@@ -159,7 +159,7 @@ def fix_environment_deviations(self, prefix, spec, deviations=None, create=True)
         pass  # pragma: no cover
 
     @abstractmethod
-    def remove_packages(self, prefix, packages):
+    def remove_packages(self, prefix, packages, pip):
         """Remove the given package name from the environment in prefix.
 
         This method ideally would not exist. The ideal approach is
@@ -174,6 +174,7 @@ def remove_packages(self, prefix, packages):
         Args:
            prefix (str): environment path
            package (list of str): package names
+           pip (bool): remove packages using pip
 
         Returns:
            None
@@ -425,6 +426,11 @@ def package_specs_for_platform(self, platform):
         per_platform = self._package_specs_by_platform.get(platform, [])
         return _combine_conda_package_lists(shared, per_platform)
 
+    @property
+    def pip_package_specs(self):
+        """Sequence of pip packages."""
+        return self._package_specs_by_platform.get('pip', [])
+
     @property
     def package_specs_for_current_platform(self):
         """Sequence of package spec strings for the current platform."""

anaconda_project/docker.py

@@ -15,7 +15,7 @@
 
 try:
     FileNotFoundError  # noqa
-except NameError:
+except NameError:  # pragma: no cover
     # python 2
     FileNotFoundError = OSError
 

anaconda_project/env_spec.py

@@ -94,13 +94,20 @@ def __init__(self,
         self.conda_constrained_packages = sorted(conda_constrained_packages)
 
         pip_specs_by_name = dict()
-        for spec in self.pip_packages:
+        for spec in self.pip_packages_for_create:
             # we quietly skip invalid specs here and let them fail
             # somewhere we can more easily report an error message.
             parsed = pip_api.parse_spec(spec)
             if parsed is not None:
                 pip_specs_by_name[parsed.name] = spec
-        self._pip_specs_by_name = pip_specs_by_name
+        self._pip_specs_for_create_by_name = pip_specs_by_name
+
+        name_set = set()
+        for spec in self.pip_packages:
+            parsed = pip_api.parse_spec(spec)
+            if parsed is not None:
+                name_set.add(parsed.name)
+        self._pip_logical_specs_name_set = name_set
 
         self._conda = conda_manager.new_conda_manager()
 
@@ -237,7 +244,12 @@ def conda_package_names_constrained_set(self):
     @property
     def pip_package_names_set(self):
         """Pip package names that we require, as a Python set."""
-        return set(self._pip_specs_by_name.keys())
+        return set(self._pip_logical_specs_name_set)
+
+    @property
+    def pip_package_names_for_create_set(self):
+        """Pip package names that we require, as a Python set."""
+        return set(self._pip_specs_for_create_by_name.keys())
 
     @property
     def lock_set(self):
@@ -252,6 +264,20 @@ def conda_packages_for_create(self):
         else:
             return self.conda_packages
 
+    @property
+    def pip_packages_for_create(self):
+        """Get pip packages (preferring the lock set list if present)."""
+        if self._lock_set is not None and self._lock_set.enabled and self._lock_set.supports_current_platform:
+            # This happens during the lock procedure because pip packages
+            # cannot be determined until they are installed. Conda is the
+            # reverse.
+            if not self._lock_set.pip_package_specs and self.pip_packages:
+                return self.pip_packages
+            else:
+                return self._lock_set.pip_package_specs
+        else:
+            return self.pip_packages
+
     def _specs_for_package_names(self, names, mapping):
         specs = []
         for name in names:
@@ -266,7 +292,7 @@ def specs_for_conda_package_names(self, names):
 
     def specs_for_pip_package_names(self, names):
         """Get the full install specs given an iterable of package names."""
-        return self._specs_for_package_names(names, self._pip_specs_by_name)
+        return self._specs_for_package_names(names, self._pip_specs_for_create_by_name)
 
     @property
     def inherit_from(self):

anaconda_project/internal/cli/environment_commands.py

@@ -51,10 +51,10 @@ def export_env_spec(project_dir, name, filename):
     return _handle_status(status)
 
 
-def add_packages(project, environment, packages, channels):
+def add_packages(project, environment, packages, channels, pip=False):
     """Add packages to the project."""
     project = load_project(project)
-    status = project_ops.add_packages(project, env_spec_name=environment, packages=packages, channels=channels)
+    status = project_ops.add_packages(project, env_spec_name=environment, packages=packages, channels=channels, pip=pip)
     package_list = ", ".join(packages)
     if environment is None:
         success_message = "Added packages to project file: %s." % (package_list)
@@ -63,10 +63,10 @@ def add_packages(project, environment, packages, channels):
     return _handle_status(status, success_message)
 
 
-def remove_packages(project, environment, packages):
+def remove_packages(project, environment, packages, pip):
     """Remove packages from the project."""
     project = load_project(project)
-    status = project_ops.remove_packages(project, env_spec_name=environment, packages=packages)
+    status = project_ops.remove_packages(project, env_spec_name=environment, packages=packages, pip=pip)
     package_list = ", ".join(packages)
     if environment is None:
         success_message = "Removed packages from project file: %s." % (package_list)
@@ -120,8 +120,13 @@ def list_packages(project_dir, environment):
     if env is None:
         print("Project doesn't have an environment called '{}'".format(environment), file=sys.stderr)
         return 1
-    print("Packages for environment '{}':\n".format(env.name))
+    print("Conda packages for environment '{}':\n".format(env.name))
     print("\n".join(sorted(env.conda_packages)), end='\n\n')
+
+    if env.pip_packages:
+        print("Pip packages for environment '{}':\n".format(env.name))
+        print("\n".join(sorted(env.pip_packages)), end='\n\n')
+
     return 0
 
 
@@ -185,12 +190,12 @@ def main_export(args):
 
 def main_add_packages(args):
     """Start the add-packages command and return exit status code."""
-    return add_packages(args.directory, args.env_spec, args.packages, args.channel)
+    return add_packages(args.directory, args.env_spec, args.packages, args.channel, args.pip)
 
 
 def main_remove_packages(args):
     """Start the remove-packages command and return exit status code."""
-    return remove_packages(args.directory, args.env_spec, args.packages)
+    return remove_packages(args.directory, args.env_spec, args.packages, args.pip)
 
 
 def main_add_platforms(args):

anaconda_project/internal/cli/main.py

@@ -281,6 +281,7 @@ def add_service_variable_name(preset):
     preset.set_defaults(main=service_commands.main_list)
 
     def add_package_args(preset):
+        preset.add_argument('--pip', action='store_true', help='Install the requested packages using pip.')
         preset.add_argument('-c',
                             '--channel',
                             metavar='CHANNEL',
@@ -333,6 +334,7 @@ def add_package_args(preset):
     preset = subparsers.add_parser('remove-packages', help="Remove packages from one or all project environments")
     add_directory_arg(preset)
     add_env_spec_arg(preset)
+    preset.add_argument('--pip', action='store_true', help='Uninstall the requested packages using pip.')
     preset.add_argument('packages', metavar='PACKAGE_NAME', default=None, nargs='+')
     preset.set_defaults(main=environment_commands.main_remove_packages)
 

anaconda_project/internal/cli/test/test_dockerize.py

@@ -24,6 +24,18 @@ def mock_dockerize(*args, **kwargs):
     return params
 
 
+def _monkeypatch_dockerize_fail(monkeypatch):
+    params = {}
+
+    def mock_dockerize(*args, **kwargs):
+        params['args'] = args
+        params['kwargs'] = kwargs
+        return SimpleStatus(success=False, description="Boo")
+
+    monkeypatch.setattr('anaconda_project.project_ops.dockerize', mock_dockerize)
+    return params
+
+
 def test_dockerize(capsys, monkeypatch):
     params = _monkeypatch_dockerize(monkeypatch)
 
@@ -43,6 +55,25 @@ def check(dirname):
     with_directory_contents_completing_project_file(dict(), check)
 
 
+def test_dockerize_fail(capsys, monkeypatch):
+    params = _monkeypatch_dockerize_fail(monkeypatch)
+
+    def check(dirname):
+        code = _parse_args_and_run_subcommand(['anaconda-project', 'dockerize'])
+        assert code == 1
+
+        out, err = capsys.readouterr()
+        assert 'Boo\n' == err
+        assert '' == out
+
+        assert params['kwargs']['tag'] is None
+        assert params['kwargs']['command'] == 'default'
+        assert params['kwargs']['builder_image'] == 'conda/s2i-anaconda-project-ubi8:latest'
+        assert params['kwargs']['build_args'] == []
+
+    with_directory_contents_completing_project_file(dict(), check)
+
+
 def test_dockerize_tag(capsys, monkeypatch):
     params = _monkeypatch_dockerize(monkeypatch)