Merge pull request #309 from Anaconda-Platform/readonly-replace

Add replace option for readonly handling, with docs

anaconda_project/env_spec.py

@@ -10,6 +10,7 @@
 
 import codecs
 import difflib
+import shutil
 import os
 import re
 
@@ -50,6 +51,7 @@ def __init__(self,
 
         self._name = name
         self._path = None
+        self._readonly = None
         self._conda_packages = tuple(conda_packages)
         self._channels = tuple(channels)
         self._pip_packages = tuple(pip_packages)
@@ -276,7 +278,7 @@ def inherit_from_names(self):
         """Env spec names that we inherit stuff from."""
         return self._inherit_from_names
 
-    def path(self, project_dir, reset=False):
+    def path(self, project_dir, reset=False, force_writable=False):
         """The filesystem path to the default conda env containing our packages."""
 
         if reset:
@@ -301,10 +303,19 @@ def _found(env_prefix):
             existing_env = None
             for base in env_paths:
                 prefix = _prefix(base)
-                if _found(prefix):
+                if _found(prefix) and (not force_writable or self._conda._is_environment_writable(prefix)):
                     existing_env = prefix
                     break
 
+            # If we need a writable environment, find the first writable location
+            if existing_env is None and force_writable:
+                for base in env_paths:
+                    prefix = _prefix(base)
+                    if not _found(prefix) and self._conda._is_environment_writable(prefix):
+                        shutil.rmtree(prefix)
+                        existing_env = prefix
+                        break
+
             if existing_env is not None:
                 self._path = existing_env
             else:

anaconda_project/requirements_registry/providers/conda_env.py

@@ -206,23 +206,20 @@ def provide(self, requirement, context):
 
             readonly_policy = os.environ.get('ANACONDA_PROJECT_READONLY_ENVS_POLICY', 'fail').lower()
 
-            if deviations.unfixable and readonly_policy == 'clone':
+            if deviations.unfixable and readonly_policy in ('clone', 'replace'):
                 # scan for writable path
-                env_paths = os.environ.get('ANACONDA_PROJECT_ENVS_PATH', '').split(os.pathsep)
-                for base_path in env_paths:
-                    base_path = os.path.expanduser(base_path) if base_path else 'envs'
-                    destination = os.path.abspath(os.path.join(project_dir, base_path, env_spec.name))
-                    if conda._is_environment_writable(destination):
-                        # _is_environment_writable leaves behind a file
-                        # that causes the clone to fail
-                        shutil.rmtree(destination)
+                destination = env_spec.path(project_dir, reset=True, force_writable=True)
+                if destination != prefix:
+                    if readonly_policy == 'replace':
+                        print('Replacing the readonly environment {}'.format(prefix))
+                        deviations = conda.find_environment_deviations(destination, env_spec)
+                    else:
                         print('Cloning the readonly environment {}'.format(prefix))
                         conda_api.clone(destination,
                                         prefix,
                                         stdout_callback=context.frontend.partial_info,
                                         stderr_callback=context.frontend.partial_error)
-                        prefix = env_spec.path(project_dir, reset=True)
-                        break
+                    prefix = destination
 
             try:
                 conda.fix_environment_deviations(prefix, env_spec, create=(not inherited))

anaconda_project/requirements_registry/providers/test/test_conda_env.py

@@ -411,6 +411,41 @@ def clone_readonly_and_prepare(dirname):
 """}, clone_readonly_and_prepare)
 
 
+@pytest.mark.slow
+@pytest.mark.skipif(platform.system() == 'Windows', reason='Windows has a hard time with read-only directories')
+def test_replace_readonly_environment_with_deviations(monkeypatch):
+    def replace_readonly_and_prepare(dirname):
+        with _readonly_env(env_name='default', packages=('python=3.7', )) as ro_prefix:
+            readonly = conda_api.installed(ro_prefix)
+            assert 'python' in readonly
+            assert 'requests' not in readonly
+
+            ro_envs = os.path.dirname(ro_prefix)
+            environ = minimal_environ(PROJECT_DIR=dirname,
+                                      ANACONDA_PROJECT_ENVS_PATH=':{}'.format(ro_envs),
+                                      ANACONDA_PROJECT_READONLY_ENVS_POLICY='replace')
+            monkeypatch.setattr('os.environ', environ)
+
+            project = Project(dirname)
+            result = prepare_without_interaction(project)
+            assert result
+            assert result.env_prefix == os.path.join(dirname, 'envs', 'default')
+
+            replaced = conda_api.installed(result.env_prefix)
+
+            assert 'python' in replaced
+            assert 'requests' in replaced
+
+    with_directory_contents_completing_project_file(
+        {DEFAULT_PROJECT_FILENAME: """
+packages:
+  - python=3.7
+  - requests
+env_specs:
+  default: {}
+"""}, replace_readonly_and_prepare)
+
+
 @pytest.mark.slow
 @pytest.mark.skipif(platform.system() == 'Windows', reason='Windows has a hard time with read-only directories')
 def test_fail_readonly_environment_with_deviations_unset_policy(monkeypatch):

docs/source/config.rst

@@ -2,8 +2,11 @@
 Configuration
 =============
 
-Anaconda Project has two modifiable configuration settings. Currently
-these setting are controlled exclusively by environment variables.
+Environment variables
+---------------------
+
+Anaconda Project has two modifiable configuration settings, both of which
+are currently controlled exclusively by environment variables.
 
 ``ANACONDA_PROJECT_ENVS_PATH``
   This variable provides a list of directories to search for environments
@@ -44,24 +47,57 @@ these setting are controlled exclusively by environment variables.
   instead of the default location of ``$PROJECT_DIR/envs/default``.
 
 ``ANACONDA_PROJECT_READONLY_ENVS_POLICY``
-  When using ``ANACONDA_PROJECT_ENVS_PATH`` one or more of the envs directories
-  may contain read-only environments. These environments can be created on shared
-  systems to speed-up project preparation steps by providing a pre-built environment
-  matching an ``env_spec`` defined in a user's ``anaconda-project.yml`` file.
-
-  For the scenario where a user specifies an ``env_spec`` that is found in a read-only
-  directory and when the package list differs from the environment on disk the
-  ``ANACONDA_PROJECT_READONLY_ENVS_POLICY`` variable controls what action is taken.
-
-  The ``ANACONDA_PROJECT_READONLY_ENVS_POLICY`` variable accepts two values ``fail``
-  and ``clone``. The default behavior is ``fail`` if this variable is not set.
+  When an ``anaconda-project.yml`` specifies the use of an existing environment,
+  but that environment is missing one or more of the requested packages,
+  Anaconda Project attempts to remedy the deficiency by installing the missing
+  packages. If the specified environment is *read-only*, however, such an
+  installation would fail. The value of the environment variable
+  ``ANACONDA_PROJECT_READONLY_ENVS_POLICY`` governs what action should be
+  taken in such a case.
 
   ``fail``
-    If a user requests changes to be made to a read-only environment the action will
-    fail with a message that the requested changes cannot be made. 
+    The attempt will fail, and a message returned indicating that the requested
+    changes could not be made.
 
   ``clone``
-    If a user requests changes to be make to a read-only environment anaconda-project
-    will first clone the read-only environment to the first writable path in the
-    ``ANACONDA_PROJECT_ENVS_PATH`` list before making modifications. If no modifications
-    to a read-only environment are requested then the environment will not be cloned.
+    A clone of the read-only environment will be made, and additional packages
+    will be installed into this cloned environment. Note that a clone will occur
+    *only* if additional packages are required.
+
+  ``replace``
+    An entirely new environment will be created.
+
+  If this environment variable is empty or contains any other value than these,
+  the ``fail`` behavior will be assumed. Note that for ``clone`` or ``replace``
+  to succeed, a writable environment location must exist somewhere in the
+  ``ANACONDA_PROJECT_ENVS_PATH`` path.
+
+
+Read-only environments
+----------------------
+
+On some systems, it is desirable to provide Anaconda Project access to one
+or more *read-only* environments. These environments can be centrally
+managed by administrators, and will speed up environment preparation
+for users that elect to use them.
+
+On Unix, a read-only environment is quite easy to enforce with standard
+POSIX permissions settings. Unfortunately, our experience on Windows
+systems suggests it is more challenging to enforce. For this reason,
+we have adopted a simple approach that allows environments to be
+explicitly marked as read-only with a flag file:
+
+- If a file called ``.readonly`` is found in the root of an environment,
+  that environment will be considered read-only.
+- If a file called ``.readonly`` is found in the *parent* of an environment
+  directory, the environment will be considered read-only.
+- An attempt is made to write a file ``var/cache/anaconda-project/status``
+  within the environment, creating the subdirectories as needed. If
+  successful, the environment is considered read-write; otherwise, it
+  is considered read-only.
+
+This second test is particularly useful when centrally managing and entire
+directory of environments. With a single ``.readonly`` flag file, all new
+environments created within that directory will be treated as read-only.
+Of course, for the best protection, POSIX or Windows read-only permissions
+should be applied nevertheless.