many: refactor snapcraft.yaml loading out of load_config (#2160)

In order to have separated logical bits and clear overlaying
communicated a logical separation is introduced between the
actual snapcraft loading in raw form and its processed/validated
form.

LP: #1775849
Signed-off-by: Sergio Schvezov <sergio.schvezov@canonical.com>

snapcraft/cli/_options.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import click
 
-from snapcraft.project import Project
+from snapcraft.project import Project, get_snapcraft_yaml
 
 
 class HiddenOption(click.Option):
@@ -52,7 +52,12 @@ def _add_build_options(func):
     return _add_build_options
 
 
-def get_project_options(**kwargs):
+def get_project(*, skip_snapcraft_yaml: bool=False, **kwargs):
+    if skip_snapcraft_yaml:
+        snapcraft_yaml_file_path = None
+    else:
+        snapcraft_yaml_file_path = get_snapcraft_yaml()
+
     ctx = click.get_current_context()
     for key, value in ctx.parent.params.items():
         if not kwargs.get(key):
@@ -62,5 +67,6 @@ def get_project_options(**kwargs):
         debug=kwargs.pop('debug'),
         use_geoip=kwargs.pop('enable_geoip'),
         parallel_builds=not kwargs.pop('no_parallel_builds'),
-        target_deb_arch=kwargs.pop('target_arch'))
+        target_deb_arch=kwargs.pop('target_arch'),
+        snapcraft_yaml_file_path=snapcraft_yaml_file_path)
     return project

snapcraft/cli/ci.py

@@ -17,6 +17,8 @@
 
 import click
 
+from ._options import get_project
+
 
 _SUPPORTED_CI_SYSTEMS = (
     'travis',
@@ -39,8 +41,9 @@ def enableci(ci_system, refresh):
     module = importlib.import_module(
         'snapcraft.integrations.{}'.format(ci_system))
 
+    project = get_project()
     if refresh:
-        module.refresh()
+        module.refresh(project)
     else:
         if click.confirm(module.__doc__):
-            module.enable()
+            module.enable(project)

snapcraft/cli/containers.py

@@ -18,7 +18,7 @@
 import os
 
 from snapcraft.internal import errors, lxd, project_loader
-from ._options import get_project_options
+from ._options import get_project
 from . import env
 
 
@@ -50,8 +50,8 @@ def refresh(debug, **kwargs):
             "You can do that with the following command:\n\n"
             "snapcraft update")
 
-    project_options = get_project_options(**kwargs, debug=debug)
-    config = project_loader.load_config(project_options)
-    lxd.Project(project_options=project_options,
+    project = get_project(**kwargs, debug=debug)
+    config = project_loader.load_config(project)
+    lxd.Project(project_options=project,
                 output=None, source=os.path.curdir,
                 metadata=config.get_metadata()).refresh()

snapcraft/cli/lifecycle.py

@@ -1,6 +1,6 @@
 # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
 #
-# Copyright (C) 2017 Canonical Ltd
+# Copyright (C) 2017-2018 Canonical Ltd
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
@@ -17,26 +17,24 @@
 import click
 import os
 
-from snapcraft.internal import (
-    deprecations,
-    lifecycle,
-    lxd,
-    project_loader,
-    steps,
-)
-from ._options import add_build_options, get_project_options
 from . import echo
 from . import env
+from ._options import add_build_options, get_project
+from snapcraft.internal import (deprecations, lifecycle, lxd, project_loader,
+                                steps)
+from snapcraft.project.errors import YamlValidationError
 
 
 def _execute(command, parts, **kwargs):
-    project_options = get_project_options(**kwargs)
+    project = get_project(**kwargs)
+    project_config = project_loader.load_config(project)
     build_environment = env.BuilderEnvironmentConfig()
+
     if build_environment.is_host:
-        lifecycle.execute(command, project_options, parts)
+        lifecycle.execute(command, project_config, parts)
     else:
-        lifecycle.containerbuild(command, project_options, parts)
-    return project_options
+        lifecycle.containerbuild(command, project_config, parts)
+    return project
 
 
 @click.group()
@@ -133,15 +131,12 @@ def snap(directory, output, **kwargs):
     """
     if directory:
         deprecations.handle_deprecation_notice('dn6')
-
-    project_options = get_project_options(**kwargs)
-    build_environment = env.BuilderEnvironmentConfig()
-    if build_environment.is_host:
-        snap_name = lifecycle.snap(
-            project_options, directory=directory, output=output)
-        echo.info('Snapped {}'.format(snap_name))
     else:
-        lifecycle.containerbuild('snap', project_options, output, directory)
+        project = _execute(steps.PRIME, parts=[], **kwargs)
+        directory = project.prime_dir
+
+    snap_name = lifecycle.pack(directory, output)
+    echo.info('Snapped {}'.format(snap_name))
 
 
 @lifecyclecli.command()
@@ -178,7 +173,11 @@ def clean(parts, step_name, **kwargs):
         snapcraft clean
         snapcraft clean my-part --step build
     """
-    project_options = get_project_options(**kwargs)
+    try:
+        project = get_project(**kwargs)
+    except YamlValidationError:
+        # We need to be able to clean invalid projects too.
+        project = get_project(skip_snapcraft_yaml=True, **kwargs)
     build_environment = env.BuilderEnvironmentConfig()
 
     step = None
@@ -190,12 +189,12 @@ def clean(parts, step_name, **kwargs):
         step = steps.get_step_by_name(step_name)
 
     if build_environment.is_host:
-        lifecycle.clean(project_options, parts, step)
+        lifecycle.clean(project, parts, step)
     else:
-        config = project_loader.load_config(project_options)
-        lxd.Project(project_options=project_options,
+        project_config = project_loader.load_config(project)
+        lxd.Project(project_options=project,
                     output=None, source=os.path.curdir,
-                    metadata=config.get_metadata()).clean(parts, step)
+                    metadata=project_config.get_metadata()).clean(parts, step)
 
 
 @lifecyclecli.command()
@@ -220,13 +219,16 @@ def cleanbuild(remote, debug, **kwargs):
     If using a remote, a prior setup is required which is described on:
     https://linuxcontainers.org/lxd/getting-started-cli/#multiple-hosts
     """
+    project = get_project(**kwargs, debug=debug)
+    project_config = project_loader.load_config(project)
     # cleanbuild is a special snow flake, while all the other commands
     # would work with the host as the build_provider it makes little
     # sense in this scenario.
     build_environment = env.BuilderEnvironmentConfig(
         default='lxd', additional_providers=['multipass'])
-    project_options = get_project_options(**kwargs, debug=debug)
-    lifecycle.cleanbuild(project=project_options,
+
+    lifecycle.cleanbuild(project=project,
+                         project_config=project_config,
                          echoer=echo,
                          remote=remote,
                          build_environment=build_environment)

snapcraft/cli/parts.py

@@ -15,8 +15,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import click
 
-from snapcraft.internal import remote_parts, lifecycle
-from ._options import get_project_options
+from snapcraft.internal import project_loader, remote_parts, lifecycle
+from ._options import get_project
 from . import env
 
 
@@ -33,8 +33,9 @@ def update(ctx, **kwargs):
     # Update in the container so that it will use the parts at build time
     build_environment = env.BuilderEnvironmentConfig()
     if not build_environment.is_host:
-        project_options = get_project_options(**kwargs)
-        lifecycle.containerbuild('update', project_options)
+        project = get_project(**kwargs)
+        project_config = project_loader.load_config(project)
+        lifecycle.containerbuild('update', project_config)
 
     # Parts can be defined and searched from any folder on the host, so
     # regardless of using containers we always update these as well

snapcraft/integrations/travis.py

@@ -182,9 +182,9 @@ def requires_travis_preconditions():
 
 
 @requires_travis_preconditions()
-def refresh():
+def refresh(project):
     series = storeapi.constants.DEFAULT_SERIES
-    project_config = project_loader.load_config()
+    project_config = project_loader.load_config(project)
     snap_name = project_config.data['name']
     logger.info(
         'Refreshing credentials to push and release "{}" snaps '
@@ -200,9 +200,9 @@ def refresh():
 
 
 @requires_travis_preconditions()
-def enable():
+def enable(project):
     series = storeapi.constants.DEFAULT_SERIES
-    project_config = project_loader.load_config()
+    project_config = project_loader.load_config(project)
     snap_name = project_config.data['name']
     logger.info(
         'Enabling Travis testbeds to push and release {!r} snaps '

snapcraft/internal/lifecycle/__init__.py

@@ -18,5 +18,4 @@
 from ._containers import containerbuild   # noqa
 from ._init import init                   # noqa
 from ._packer import pack                 # noqa
-from ._packer import snap                 # noqa
 from ._runner import execute              # noqa

snapcraft/internal/lifecycle/_clean.py

@@ -1,6 +1,6 @@
 # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
 #
-# Copyright (C) 2015-2017 Canonical Ltd
+# Copyright (C) 2015-2018 Canonical Ltd
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
@@ -179,7 +179,7 @@ def clean(project_options, parts, step=None):
         _cleanup_common_directories_for_step(step, project_options)
         return
 
-    config = project_loader.load_config()
+    config = project_loader.load_config(project_options)
 
     if not parts and step <= steps.PRIME:
         # If we've been asked to clean stage or prime without being given

snapcraft/internal/lifecycle/_containers.py

@@ -17,7 +17,7 @@
 import os
 import tarfile
 
-from snapcraft.internal import build_providers, errors, lxd, project_loader
+from snapcraft.internal import build_providers, errors, lxd
 
 
 logger = logging.getLogger(__name__)
@@ -38,19 +38,21 @@ def _tar_filter(tarinfo):
     return _tar_filter
 
 
-def containerbuild(step, project_options, output=None, args=[]):
-    config = project_loader.load_config(project_options)
+def containerbuild(command, project_config, output=None, args=None):
+    if args is None:
+        args = []
+
     lxd.Project(output=output, source=os.path.curdir,
-                project_options=project_options,
-                metadata=config.get_metadata()).execute(step, args)
+                project_options=project_config.project,
+                metadata=project_config.get_metadata()).execute(command, args)
 
 
-def cleanbuild(*, project, echoer, build_environment, remote='') -> None:
-    config = project_loader.load_config(project)
+def cleanbuild(*, project, project_config, echoer, build_environment,
+               remote='') -> None:
     tar_filename = _create_tar_file(project.info.name)
 
     if build_environment.is_lxd:
-        _deprecated_cleanbuild(project, remote, config, tar_filename)
+        _deprecated_cleanbuild(project, remote, project_config, tar_filename)
         return
 
     build_provider_class = build_providers.get_provider_for('multipass')

snapcraft/internal/lifecycle/_packer.py

@@ -1,6 +1,6 @@
 # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
 #
-# Copyright (C) 2015-2017 Canonical Ltd
+# Copyright (C) 2015-2018 Canonical Ltd
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
@@ -22,9 +22,8 @@
 from progressbar import AnimatedMarker, ProgressBar
 
 from snapcraft import file_utils
-from snapcraft.internal import common, repo, steps
+from snapcraft.internal import common, repo
 from snapcraft.internal.indicators import is_dumb_terminal
-from ._runner import execute
 
 
 logger = logging.getLogger(__name__)
@@ -40,14 +39,6 @@ def _snap_data_from_dir(directory):
             'type': snap.get('type', '')}
 
 
-def snap(project_options, directory=None, output=None):
-    if not directory:
-        directory = project_options.prime_dir
-        execute(steps.PRIME, project_options)
-
-    return pack(directory, output)
-
-
 def pack(directory, output=None):
     mksquashfs_path = file_utils.get_tool_path('mksquashfs')