Merge pull request #109 from Anaconda-Platform/feature/bootstrap_env

PR: Add bootstrap envs

anaconda_project/internal/cli/environment_commands.py

@@ -9,10 +9,14 @@
 from __future__ import absolute_import, print_function
 
 import sys
+import platform
+from os import execv
+from os.path import join, exists
 
 from anaconda_project.internal.cli.project_load import load_project
 from anaconda_project import project_ops
 from anaconda_project.internal.cli import console_utils
+from anaconda_project.internal import conda_api
 
 
 def _handle_status(status, success_message=None):
@@ -227,3 +231,30 @@ def main_update(args):
 def main_unlock(args):
     """Unlock dependency versions and return exit status code."""
     return unlock(args.directory, args.name)
+
+
+def create_bootstrap_env(project):
+    """Create a project bootstrap env, if it doesn't exist.
+
+    Input:
+        project(project.Project): project
+    """
+    if not exists(project.bootstrap_env_prefix):
+        env_spec = project.env_specs['bootstrap-env']
+        command_line_packages = list(env_spec.conda_packages + env_spec.pip_packages)
+        conda_api.create(prefix=project.bootstrap_env_prefix, pkgs=command_line_packages, channels=env_spec.channels)
+
+
+def run_on_bootstrap_env(project):
+    """Run the current command in a project bootstrap env.
+
+    Input:
+        project(project.Project): project
+    """
+    if platform.system() == 'Windows':
+        script_dir = "Scripts"
+    else:
+        script_dir = "bin"
+
+    anaconda_project_exec = join(project.bootstrap_env_prefix, script_dir, 'anaconda-project')
+    execv(anaconda_project_exec, sys.argv)

anaconda_project/internal/cli/run.py

@@ -13,6 +13,7 @@
 from anaconda_project.internal.cli.prepare_with_mode import prepare_with_ui_mode_printing_errors
 from anaconda_project.internal.cli.project_load import load_project
 from anaconda_project.project_commands import ProjectCommand
+from anaconda_project.internal.cli.environment_commands import (create_bootstrap_env, run_on_bootstrap_env)
 
 
 def _command_from_name(project, command_name):
@@ -38,29 +39,36 @@ def run_command(project_dir, ui_mode, conda_environment, command_name, extra_com
         Does not return if successful.
     """
     project = load_project(project_dir)
-    environ = None
 
-    command = _command_from_name(project, command_name)
+    if project.has_bootstrap_env_spec() and not project.is_running_in_bootstrap_env():
+        print("Project should be ran by bootstrap env... fixing.")
+        create_bootstrap_env(project)
+        run_on_bootstrap_env(project)
+    else:
+        environ = None
+        command = _command_from_name(project, command_name)
 
-    result = prepare_with_ui_mode_printing_errors(project,
-                                                  ui_mode=ui_mode,
-                                                  env_spec_name=conda_environment,
-                                                  command=command,
-                                                  extra_command_args=extra_command_args,
-                                                  environ=environ)
+        result = prepare_with_ui_mode_printing_errors(project,
+                                                      ui_mode=ui_mode,
+                                                      env_spec_name=conda_environment,
+                                                      command=command,
+                                                      extra_command_args=extra_command_args,
+                                                      environ=environ)
 
-    if result.failed:
-        # errors were printed already
-        return
-    elif result.command_exec_info is None:
-        print("No known run command for project %s; try adding a 'commands:' section to anaconda-project.yml" %
-              project_dir,
-              file=sys.stderr)
-    else:
-        try:
-            result.command_exec_info.execvpe()
-        except OSError as e:
-            print("Failed to execute '%s': %s" % (" ".join(result.command_exec_info.args), e.strerror), file=sys.stderr)
+        if result.failed:
+            # errors were printed already
+            return
+        elif result.command_exec_info is None:
+            print("No known run command for project %s; try adding a 'commands:' section to anaconda-project.yml" %
+                  project_dir,
+                  file=sys.stderr)
+        else:
+            try:
+
+                result.command_exec_info.execvpe()
+            except OSError as e:
+                print("Failed to execute '%s': %s" % (" ".join(result.command_exec_info.args), e.strerror),
+                      file=sys.stderr)
 
 
 def main(args):

anaconda_project/internal/plugins.py

@@ -36,4 +36,5 @@ def get_plugins(plugin_hook_type):
     """
     command_type = 'anaconda_project.plugins.%s' % plugin_hook_type
     entry_point_plugins = _get_entry_points_plugins(entry_point_group=command_type)
+
     return entry_point_plugins

anaconda_project/prepare.py

@@ -508,7 +508,9 @@ def get_missing_to_provide(status):
                 # the status from the CondaEnvRequirement. Possibly
                 # we sometimes do a prepare with no CondaEnvRequirement?
                 # but doing one with two wouldn't make sense afaik.
-                assert current_env_spec_name is None
+
+                # TODO: Should we just remove this? (considering the case of bootstrap env)
+                # assert current_env_spec_name is None
                 current_env_spec_name = status.env_spec_name
 
         if failed:

anaconda_project/project.py

@@ -11,12 +11,14 @@
 import contextlib
 from copy import deepcopy, copy
 import os
+from os.path import join
 
 from anaconda_project.env_spec import (EnvSpec, _anaconda_default_env_spec, _find_importable_spec,
                                        _find_out_of_sync_importable_spec)
 from anaconda_project.requirements_registry.registry import RequirementsRegistry
 from anaconda_project.requirements_registry.requirement import EnvVarRequirement
-from anaconda_project.requirements_registry.requirements.conda_env import CondaEnvRequirement
+from anaconda_project.requirements_registry.requirements.conda_env import (CondaEnvRequirement,
+                                                                           CondaBootstrapEnvRequirement)
 from anaconda_project.requirements_registry.requirements.download import DownloadRequirement
 from anaconda_project.requirements_registry.requirements.service import ServiceRequirement
 from anaconda_project.project_commands import (ProjectCommand, all_known_command_attributes)
@@ -843,6 +845,10 @@ def _update_conda_env_requirements(self, requirements, problems, project_file):
         if _fatal_problem(problems):
             return
 
+        if self.has_bootstrap_env_spec():
+            requirement = CondaBootstrapEnvRequirement(registry=self.registry, env_specs=self.env_specs)
+            self._add_requirement(requirements, self.global_base_env_spec, requirement)
+
         requirement = CondaEnvRequirement(registry=self.registry, env_specs=self.env_specs)
         self._add_requirement(requirements, self.global_base_env_spec, requirement)
 
@@ -1104,6 +1110,10 @@ def add_packages_to_env_spec(project):
                                          only_a_suggestion=True)
                 problems.append(problem)
 
+    def has_bootstrap_env_spec(self):
+        """Return True if bootstrap-env is in env_specs, False otherwise."""
+        return 'bootstrap-env' in self.env_specs
+
 
 class Project(object):
     """Represents the information we've inferred about a project.
@@ -1194,6 +1204,7 @@ def req_key(req):
             return req.env_var
 
         env_spec = self.env_specs.get(env_spec_name)
+
         if env_spec is None:
             # this happens if there was a problem parsing the project
             return []
@@ -1574,3 +1585,16 @@ def use_changes_without_saving(self):
         """
         self.project_file.use_changes_without_saving()
         self.lock_file.use_changes_without_saving()
+
+    @property
+    def bootstrap_env_prefix(self):
+        """Fullpath to bootstrap environment prefix."""
+        return join(self._directory_path, 'envs', 'bootstrap-env')
+
+    def is_running_in_bootstrap_env(self):
+        """Return True if anaconda-project is running inside a project bootstrap env False otherwise."""
+        return os.environ['CONDA_PREFIX'] == self.bootstrap_env_prefix
+
+    def has_bootstrap_env_spec(self):
+        """Return True if bootstrap-env is in env_specs, False otherwise."""
+        return self._config_cache.has_bootstrap_env_spec()

anaconda_project/requirements_registry/providers/conda_env.py

@@ -231,3 +231,185 @@ def unprovide(self, requirement, environ, local_state_file, overrides, requireme
                                 description=("Current environment is not in %s, no need to delete it." % project_dir))
 
         return _remove_env_path(env_path)
+
+
+class CondaBootstrapEnvProvider(EnvVarProvider):
+    """Provides a Conda environment."""
+
+    def __init__(self):
+        """Override to create our CondaManager."""
+        super(CondaBootstrapEnvProvider, self).__init__()
+
+    def missing_env_vars_to_configure(self, requirement, environ, local_state_file):
+        """Override superclass to not require ourselves."""
+        return ()
+
+    def missing_env_vars_to_provide(self, requirement, environ, local_state_file):
+        """Override superclass to not require ourselves."""
+        return self.missing_env_vars_to_configure(requirement, environ, local_state_file)
+
+    def read_config(self, requirement, environ, local_state_file, default_env_spec_name, overrides):
+        """Override superclass to add a choice to create a project-scoped environment."""
+        assert 'PROJECT_DIR' in environ
+        project_dir = environ['PROJECT_DIR']
+
+        if overrides.env_spec_name is not None:
+            # short-circuit this whole party
+            env = requirement.env_specs.get(overrides.env_spec_name)
+            # future: it should be possible to override the env spec without using the
+            # default-created project-scoped env.
+            config = dict(source='project', env_name=overrides.env_spec_name, value=env.path(project_dir))
+            return config
+
+        config = super(CondaBootstrapEnvProvider, self).read_config(requirement, environ, local_state_file,
+                                                                    default_env_spec_name, overrides)
+
+        assert 'source' in config
+
+        if config['source'] == 'unset':
+            # if nothing is selected, default to project mode
+            # because we don't have a radio button in the UI for
+            # "do nothing" right now.
+            config['source'] = 'project'
+
+        # if we're supposed to inherit the environment, we don't want to look at
+        # anything else. This should always get rid of 'environ' source.
+        if local_state_file.get_value('inherit_environment', default=False) and overrides.inherited_env is not None:
+            config['source'] = 'inherited'
+            config['value'] = overrides.inherited_env
+
+        # convert 'environ' to 'project' when needed... this would
+        # happen if you keep the default 'project' choice, so
+        # there's nothing in anaconda-project-local.yml
+        if config['source'] == 'environ':
+            environ_value = config['value']
+            project_dir = environ['PROJECT_DIR']
+            environ_value_is_project_specific = False
+            for env in requirement.env_specs.values():
+                if env.path(project_dir) == environ_value:
+                    environ_value_is_project_specific = True
+            assert environ_value_is_project_specific
+            config['source'] = 'project'
+
+        # we should have changed 'environ' to the specific source; since for conda envs
+        # we ignore the initial environ value, we always have to track our value in
+        assert config['source'] != 'environ'
+
+        # be sure we don't get confused by alternate ways to spell the path
+        if 'value' in config:
+            config['value'] = os.path.normpath(config['value'])
+
+        config['env_name'] = default_env_spec_name
+
+        assert config['env_name'] == 'bootstrap-env'
+
+        if 'value' in config:
+            for env in requirement.env_specs.values():
+                if config['value'] == env.path(project_dir):
+                    config['env_name'] = env.name
+                    if config['source'] == 'variables':
+                        config['source'] = 'project'
+        elif config['source'] == 'project':
+            env = requirement.env_specs.get(config['env_name'])
+            config['value'] = env.path(project_dir)
+
+        assert 'env_name' in config
+
+        return config
+
+    def set_config_values_as_strings(self, requirement, environ, local_state_file, default_env_spec_name, overrides,
+                                     values):
+        """Override superclass to support 'project' source option."""
+        super(CondaBootstrapEnvProvider, self).set_config_values_as_strings(requirement, environ, local_state_file,
+                                                                            default_env_spec_name, overrides, values)
+
+        # We have to clear out the user override or it will
+        # never stop overriding the user's new choice, if they
+        # have changed to another env.
+        overrides.env_spec_name = None
+
+        if 'source' in values:
+            if values['source'] == 'inherited':
+                local_state_file.set_value('inherit_environment', True)
+                # the superclass should have unset this so we inherit instead of using it
+                assert local_state_file.get_value(['variables', requirement.env_var]) is None
+            else:
+                # don't write this out if it wasn't in there anyway
+                if local_state_file.get_value('inherit_environment') is not None:
+                    local_state_file.set_value('inherit_environment', False)
+
+            if values['source'] == 'project':
+                project_dir = environ['PROJECT_DIR']
+                name = values['env_name']
+                for env in requirement.env_specs.values():
+                    if env.name == name:
+                        prefix = env.path(project_dir)
+                        local_state_file.set_value(['variables', requirement.env_var], prefix)
+
+    def provide(self, requirement, context):
+        """Override superclass to create or update our environment."""
+        assert 'PATH' in context.environ
+        conda = new_conda_manager(context.frontend)
+
+        # set from the inherited vale if necessary
+        if context.status.analysis.config['source'] == 'inherited':
+            context.environ[requirement.env_var] = context.status.analysis.config['value']
+
+        # set the env var (but not PATH, etc. to fully activate, that's done below)
+        super_result = super(CondaBootstrapEnvProvider, self).provide(requirement, context)
+
+        project_dir = context.environ['PROJECT_DIR']
+
+        env_name = context.status.analysis.config.get('env_name', context.default_env_spec_name)
+        env_spec = requirement.env_specs.get(env_name)
+
+        prefix = os.path.join(project_dir, 'envs', 'bootstrap-env')
+
+        # if the value has changed, choose the matching env spec
+        # (something feels wrong here; should this be in read_config?
+        # or not at all?)
+        for env in requirement.env_specs.values():
+            if env.path(project_dir) == prefix:
+                env_spec = env
+                break
+
+        if context.mode != PROVIDE_MODE_CHECK:
+            # we update the environment in both prod and dev mode
+
+            # TODO if not creating a named env, we could use the
+            # shared packages, but for now we leave it alone
+            assert env_spec is not None
+            try:
+                conda.fix_environment_deviations(prefix, env_spec, create=True)
+            except CondaManagerError as e:
+                return super_result.copy_with_additions(errors=[str(e)])
+
+        conda_api.environ_set_prefix(context.environ, prefix, varname=requirement.env_var)
+
+        path = context.environ.get("PATH", "")
+
+        context.environ["PATH"] = conda_api.set_conda_env_in_path(path, prefix)
+        # Some stuff can only be done when a shell is launched:
+        #  - we can't set PS1 because it shouldn't be exported.
+        #  - we can't run conda activate scripts because they are sourced.
+        # We can do these in the output of our activate command, but not here.
+
+        return super_result
+
+    def unprovide(self, requirement, environ, local_state_file, overrides, requirement_status=None):
+        """Override superclass to delete project-scoped envs directory."""
+        config = self.read_config(requirement,
+                                  environ,
+                                  local_state_file,
+                                  # future: pass in this default_env_spec_name
+                                  default_env_spec_name='bootstrap-env',
+                                  overrides=overrides)
+
+        env_path = config.get('value', None)
+        assert env_path is not None
+        project_dir = environ['PROJECT_DIR']
+        if not env_path.startswith(project_dir):
+            return SimpleStatus(success=True,
+                                description=("Current environment is not in %s, no need to delete it." % project_dir))
+
+        return _remove_env_path(env_path)