Merge pull request #313 from AlbertDeFusco/conda-pack

.github/workflows/main.yml

@@ -105,6 +105,7 @@ jobs:
       run: |
           source ./conda/etc/profile.d/conda.sh 
           conda activate anaconda-project-dev
+          [ ${{matrix.cver}} != 4.6 ] && conda config --set restore_free_channel true
           pytest -vrfe --durations=10 \
             --cov-config=.coveragerc --cov-report=term-missing \
             --cov-fail-under=98 --cov-report=xml:./coverage.xml \

anaconda_project/api.py

@@ -714,17 +714,18 @@ def clean(self, project, prepare_result):
         """
         return project_ops.clean(project=project, prepare_result=prepare_result)
 
-    def archive(self, project, filename):
+    def archive(self, project, filename, pack_envs=False):
         """Make an archive of the non-ignored files in the project.
 
         Args:
             project (``Project``): the project
             filename (str): name of a zip, tar.gz, or tar.bz2 archive file
+            pack_envs (bool): Flag to include conda-packs of each env_spec in the archive
 
         Returns:
             a ``Status``, if failed has ``errors``
         """
-        return project_ops.archive(project=project, filename=filename)
+        return project_ops.archive(project=project, filename=filename, pack_envs=pack_envs)
 
     def unarchive(self, filename, project_dir, parent_dir=None, frontend=None):
         """Unpack an archive of the project.

anaconda_project/archiver.py

@@ -19,13 +19,16 @@
 import tempfile
 import uuid
 import zipfile
+from io import BytesIO
+from conda_pack._progress import progressbar
 
 from anaconda_project.frontend import _new_error_recorder
 from anaconda_project.internal import logged_subprocess
 from anaconda_project.internal.simple_status import SimpleStatus
 from anaconda_project.internal.directory_contains import subdirectory_relative_to_directory
 from anaconda_project.internal.rename import rename_over_existing
 from anaconda_project.internal.makedirs import makedirs_ok_if_exists
+from anaconda_project.internal.conda_api import current_platform
 
 
 class _FileInfo(object):
@@ -254,7 +257,7 @@ def _leaf_infos(infos):
     return sorted(all_by_name.values(), key=lambda x: x.relative_path)
 
 
-def _write_tar(archive_root_name, infos, filename, compression, frontend):
+def _write_tar(archive_root_name, infos, filename, compression, packed_envs, frontend):
     if compression is None:
         compression = ""
     else:
@@ -265,14 +268,49 @@ def _write_tar(archive_root_name, infos, filename, compression, frontend):
             frontend.info("  added %s" % arcname)
             tf.add(info.full_path, arcname=arcname)
 
-
-def _write_zip(archive_root_name, infos, filename, frontend):
+        for pack in packed_envs:
+            env_name = os.path.basename(pack)
+            print('Joining packed env {}'.format(env_name))
+            with tarfile.open(pack, mode='r', dereference=False) as env:
+                with progressbar(env.getmembers()) as env_p:
+                    for file in env_p:
+                        try:
+                            data = env.extractfile(file)
+                            tf.addfile(file, data)
+                        except KeyError:  # pragma: no cover
+                            tf.addfile(file)
+                    env_spec = env_name.split('.')[0].split('_')[-1]
+                    dot_packed = os.path.join(archive_root_name, 'envs', env_spec, 'conda-meta', '.packed')
+                    platform = '{}\n'.format(current_platform())
+
+                    f = BytesIO()
+                    f.write(platform.encode())
+
+                    tinfo = tarfile.TarInfo(dot_packed)
+                    tinfo.size = f.tell()
+                    f.seek(0)
+                    tf.addfile(tinfo, fileobj=f)
+
+
+def _write_zip(archive_root_name, infos, filename, packed_envs, frontend):
     with zipfile.ZipFile(filename, 'w') as zf:
         for info in _leaf_infos(infos):
             arcname = os.path.join(archive_root_name, info.relative_path)
             frontend.info("  added %s" % arcname)
             zf.write(info.full_path, arcname=arcname)
 
+        for pack in packed_envs:
+            env_name = os.path.basename(pack)
+            print('Joining packed env {}'.format(env_name))
+            with zipfile.ZipFile(pack, mode='r') as env:
+                with progressbar(env.infolist()) as infolist:
+                    for file in infolist:
+                        data = env.read(file)
+                        zf.writestr(file, data)
+                    env_spec = env_name.split('.')[0].split('_')[-1]
+                    dot_packed = os.path.join(archive_root_name, 'envs', env_spec, 'conda-meta', '.packed')
+                    zf.writestr(dot_packed, '{}\n'.format(current_platform()))
+
 
 # function exported for project.py
 def _list_relative_paths_for_unignored_project_files(project_directory, frontend, requirements):
@@ -283,7 +321,7 @@ def _list_relative_paths_for_unignored_project_files(project_directory, frontend
 
 
 # function exported for project_ops.py
-def _archive_project(project, filename):
+def _archive_project(project, filename, pack_envs=False):
     """Make an archive of the non-ignored files in the project.
 
     Args:
@@ -311,6 +349,22 @@ def _archive_project(project, filename):
         frontend.error("%s has been modified but not saved." % project.project_file.basename)
         return SimpleStatus(success=False, description="Can't create an archive.", errors=frontend.pop_errors())
 
+    envs_path = os.path.join(project.project_file.project_dir, 'envs')
+
+    packed_envs = []
+    if pack_envs and os.path.isdir(envs_path):
+        conda_pack_dir = tempfile.mkdtemp()
+        import conda_pack
+        for env in os.listdir(envs_path):
+            ext = 'zip' if filename.lower().endswith(".zip") else 'tar'
+            pack = os.path.join(conda_pack_dir, '{}_envs_{}.{}'.format(current_platform(), env, ext))
+            fn = conda_pack.pack(prefix=os.path.join(envs_path, env),
+                                 arcroot=os.path.join(project.name, 'envs', env),
+                                 output=pack,
+                                 verbose=True,
+                                 force=True)
+            packed_envs.append(fn)
+
     infos = _enumerate_archive_files(project.directory_path,
                                      frontend,
                                      requirements=project.union_of_requirements_for_all_envs)
@@ -328,13 +382,13 @@ def _archive_project(project, filename):
     tmp_filename = filename + ".tmp-" + str(uuid.uuid4())
     try:
         if filename.lower().endswith(".zip"):
-            _write_zip(project.name, infos, tmp_filename, frontend)
+            _write_zip(project.name, infos, tmp_filename, packed_envs=packed_envs, frontend=frontend)
         elif filename.lower().endswith(".tar.gz"):
-            _write_tar(project.name, infos, tmp_filename, compression="gz", frontend=frontend)
+            _write_tar(project.name, infos, tmp_filename, compression="gz", packed_envs=packed_envs, frontend=frontend)
         elif filename.lower().endswith(".tar.bz2"):
-            _write_tar(project.name, infos, tmp_filename, compression="bz2", frontend=frontend)
+            _write_tar(project.name, infos, tmp_filename, compression="bz2", packed_envs=packed_envs, frontend=frontend)
         elif filename.lower().endswith(".tar"):
-            _write_tar(project.name, infos, tmp_filename, compression=None, frontend=frontend)
+            _write_tar(project.name, infos, tmp_filename, compression=None, packed_envs=packed_envs, frontend=frontend)
         else:
             frontend.error("Unsupported archive filename %s." % (filename))
             return SimpleStatus(success=False,
@@ -349,6 +403,8 @@ def _archive_project(project, filename):
     finally:
         try:
             os.remove(tmp_filename)
+            if pack_envs:
+                os.remove(conda_pack_dir)
         except (IOError, OSError):
             pass
 
@@ -378,14 +434,21 @@ def _list_files_tar(tar_path):
         return sorted([member.name for member in tf.getmembers() if member.isreg() or member.isdir()])
 
 
+def _extractall_chmod(zf, destination):
+    for zinfo in zf.infolist():
+        out_path = zf.extract(zinfo.filename, path=destination)
+        mode = zinfo.external_attr >> 16
+        os.chmod(out_path, mode)
+
+
 def _extract_files_zip(zip_path, src_and_dest, frontend):
     # the zipfile API has no way to extract to a filename of
     # our choice, so we have to unpack to a temporary location,
     # then copy those files over.
     tmpdir = tempfile.mkdtemp()
     try:
         with zipfile.ZipFile(zip_path, mode='r') as zf:
-            zf.extractall(tmpdir)
+            _extractall_chmod(zf, tmpdir)
             for (src, dest) in src_and_dest:
                 frontend.info("Unpacking %s to %s" % (src, dest))
                 src_path = os.path.join(tmpdir, src)

anaconda_project/docker.py

@@ -13,6 +13,12 @@
 
 DEFAULT_BUILDER_IMAGE = 'conda/s2i-anaconda-project-ubi8'
 
+try:
+    FileNotFoundError  # noqa
+except NameError:
+    # python 2
+    FileNotFoundError = OSError
+
 
 def build_image(path, tag, command, builder_image=DEFAULT_BUILDER_IMAGE, build_args=None):
     """Run s2i build."""

anaconda_project/internal/cli/archive.py

@@ -11,16 +11,27 @@
 from anaconda_project.internal.cli.project_load import load_project
 from anaconda_project.internal.cli import console_utils
 import anaconda_project.project_ops as project_ops
+from anaconda_project.internal.cli.prepare import prepare_command
 
 
-def archive_command(project_dir, archive_filename):
+def archive_command(project_dir, archive_filename, pack_envs):
     """Make an archive of the project.
 
     Returns:
         exit code
     """
+    if pack_envs:
+        prepare_status = prepare_command(project_dir,
+                                         ui_mode='production_defaults',
+                                         conda_environment=None,
+                                         command_name=None,
+                                         all=True)
+        if not prepare_status:
+            return 1
+
     project = load_project(project_dir)
-    status = project_ops.archive(project, archive_filename)
+
+    status = project_ops.archive(project, archive_filename, pack_envs)
     if status:
         print(status.status_description)
         return 0
@@ -31,4 +42,4 @@ def archive_command(project_dir, archive_filename):
 
 def main(args):
     """Start the archive command and return exit status code."""
-    return archive_command(args.directory, args.filename)
+    return archive_command(args.directory, args.filename, args.pack_envs)

anaconda_project/internal/cli/main.py

@@ -129,6 +129,11 @@ def add_env_spec_name_arg(preset, required):
                                    help="Create a .zip, .tar.gz, or .tar.bz2 archive with project files in it")
     add_directory_arg(preset)
     preset.add_argument('filename', metavar='ARCHIVE_FILENAME')
+    preset.add_argument('--pack-envs',
+                        action='store_true',
+                        help='Experimental: Package env_specs into the archive'
+                        ' using conda-pack')
+
     preset.set_defaults(main=archive.main)
 
     preset = subparsers.add_parser('unarchive',

anaconda_project/internal/default_conda_manager.py

@@ -11,6 +11,8 @@
 import codecs
 import glob
 import os
+import shutil
+import subprocess
 
 from anaconda_project.conda_manager import (CondaManager, CondaEnvironmentDeviations, CondaLockSet, CondaManagerError)
 import anaconda_project.internal.conda_api as conda_api
@@ -381,7 +383,37 @@ def fix_environment_deviations(self, prefix, spec, deviations=None, create=True)
         if deviations.unfixable:
             raise CondaManagerError("Unable to update environment at %s" % prefix)
 
-        if os.path.isdir(os.path.join(prefix, 'conda-meta')):
+        conda_meta = os.path.join(prefix, 'conda-meta')
+        packed = os.path.join(conda_meta, '.packed')
+
+        if os.path.isdir(conda_meta) and os.path.exists(packed):
+            with open(packed) as f:
+                packed_arch = f.read().strip()
+
+            matched = packed_arch == conda_api.current_platform()
+            if matched:
+                if 'win' in conda_api.current_platform():
+                    unpack_script = ['python', os.path.join(prefix, 'Scripts', 'conda-unpack-script.py')]
+
+                else:
+                    unpack_script = os.path.join(prefix, 'bin', 'conda-unpack')
+
+                try:
+                    subprocess.check_call(unpack_script)
+                    os.remove(packed)
+                except (subprocess.CalledProcessError, OSError) as e:
+                    self._log_info('Warning: conda-unpack could not be run: \n{}\n'
+                                   'The environment will be recreated.'.format(str(e)))
+                    create = True
+                    shutil.rmtree(prefix)
+
+            else:
+                self._log_info('Warning: The unpacked env does not match the current architecture. '
+                               'It will be recreated.')
+                create = True
+                shutil.rmtree(prefix)
+
+        if os.path.isdir(conda_meta):
             to_update = list(set(deviations.missing_packages + deviations.wrong_version_packages))
             if len(to_update) > 0:
                 specs = spec.specs_for_conda_package_names(to_update)

anaconda_project/project_ops.py

@@ -1667,7 +1667,7 @@ def cleanup_dir(dirname):
         return SimpleStatus(success=False, description="Failed to clean everything up.", errors=errors)
 
 
-def archive(project, filename):
+def archive(project, filename, pack_envs=False):
     """Make an archive of the non-ignored files in the project.
 
     Args:
@@ -1677,7 +1677,7 @@ def archive(project, filename):
     Returns:
         a ``Status``, if failed has ``errors``
     """
-    return archiver._archive_project(project, filename)
+    return archiver._archive_project(project, filename, pack_envs)
 
 
 def unarchive(filename, project_dir, parent_dir=None, frontend=None):

anaconda_project/test/test_api.py

@@ -622,7 +622,7 @@ def mock_archive(*args, **kwargs):
     monkeypatch.setattr('anaconda_project.project_ops.archive', mock_archive)
 
     p = api.AnacondaProject()
-    kwargs = dict(project=43, filename=123)
+    kwargs = dict(project=43, filename=123, pack_envs=False)
     result = p.archive(**kwargs)
     assert 42 == result
     assert kwargs == params['kwargs']

anaconda_project/test/test_docker.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021, Anaconda, Inc. All rights reserved.
+#
+# Licensed under the terms of the BSD 3-Clause License.
+# The full license is in the file LICENSE.txt, distributed with this software.
+# -----------------------------------------------------------------------------
+from __future__ import absolute_import, print_function
+
+import subprocess
+
+from anaconda_project.docker import build_image
+
+try:
+    FileNotFoundError  # noqa
+except NameError:
+    # python 2
+    FileNotFoundError = OSError
+
+
+def test_build_image_pass(monkeypatch):
+    def mock_check_call(*args, **kwargs):
+        return
+
+    monkeypatch.setattr('subprocess.check_call', mock_check_call)
+
+    status = build_image('.', 'tag', 'default')
+    assert status
+
+
+def test_build_image_failed(monkeypatch):
+    def mock_check_call(*args, **kwargs):
+        raise subprocess.CalledProcessError(1, 's2i', 'failed to build')
+
+    monkeypatch.setattr('subprocess.check_call', mock_check_call)
+
+    status = build_image('.', 'tag', 'default')
+    assert len(status.errors) == 1
+
+
+def test_build_image_not_found(monkeypatch):
+    def mock_check_call(*args, **kwargs):
+        raise FileNotFoundError
+
+    monkeypatch.setattr('subprocess.check_call', mock_check_call)
+
+    status = build_image('.', 'tag', 'default')
+    assert len(status.errors) == 1