build_providers: new build provider using multipass (#2100)

build_providers is a new package to handle different providers snapcraft
can use to create a snap.

In addition to the scaffolding, an implementation to handle
multipass is provided. This is the MVP for multipass integration.

setup.py

@@ -35,6 +35,8 @@
     'snapcraft.integrations',
     'snapcraft.internal',
     'snapcraft.internal.cache',
+    'snapcraft.internal.build_providers',
+    'snapcraft.internal.build_providers._multipass',
     'snapcraft.internal.deltas',
     'snapcraft.internal.lifecycle',
     'snapcraft.internal.lxd',

snapcraft/cli/env.py

@@ -15,9 +15,11 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import os
 from distutils import util
+from typing import List
 
 from . import echo
 from snapcraft.internal import errors
+from snapcraft.formatting_utils import humanize_list
 
 
 class BuilderEnvironmentConfig:
@@ -31,6 +33,7 @@ class BuilderEnvironmentConfig:
 
     - host: the host will drive the build.
     - lxd: the host will setup a container to drive the build.
+    - multipass: a vm driven by multipass will be created to drive the build.
 
     Use of the lxd value is equivalent to setting the now deprecated
     SNAPCRAFT_CONTAINER_BUILDS environment variable to a value that
@@ -39,7 +42,19 @@ class BuilderEnvironmentConfig:
     results in an error.
     """
 
-    def __init__(self) -> None:
+    def __init__(self, *, default='host',
+                 additional_providers: List[str]=None) -> None:
+        """Instantiate a BuildEnvironmentConfig.
+
+        :param str default: the default provider to use among the list of valid
+                            ones.
+        :param str additional_providers: Additional providers allowed in the
+                                         environment.
+        """
+        valid_providers = ['host', 'lxd']
+        if additional_providers is not None:
+            valid_providers.extend(additional_providers)
+
         use_lxd = None
         container_builds = os.environ.get('SNAPCRAFT_CONTAINER_BUILDS')
         if container_builds:
@@ -64,12 +79,12 @@ def __init__(self) -> None:
         if use_lxd:
             build_provider = 'lxd'
         elif not build_provider:
-            echo.warning('Using the host as the build environment.')
-            build_provider = 'host'
-        # TODO add multipass
-        elif build_provider not in ['host', 'lxd']:
+            build_provider = default
+        elif build_provider not in valid_providers:
             raise errors.SnapcraftEnvironmentError(
-                'SNAPCRAFT_BUILD_ENVIRONMENT must be one of: host or lxd.')
+                'SNAPCRAFT_BUILD_ENVIRONMENT must be one of: {}.'.format(
+                    humanize_list(items=valid_providers, conjunction='or')))
 
         self.provider = build_provider
         self.is_host = build_provider == 'host'
+        self.is_lxd = build_provider == 'lxd'

snapcraft/cli/lifecycle.py

@@ -195,12 +195,11 @@ def clean(parts, step, **kwargs):
 @click.option('--debug', is_flag=True,
               help='Shells into the environment if the build fails.')
 def cleanbuild(remote, debug, **kwargs):
-    """Create a snap using a clean environment managed by lxd.
+    """Create a snap using a clean environment managed by a build provider.
 
     \b
     Examples:
         snapcraft cleanbuild
-        snapcraft cleanbuild --output
 
     The cleanbuild command requires a properly setup lxd environment that
     can connect to external networks. Refer to the "Ubuntu Desktop and
@@ -211,8 +210,16 @@ def cleanbuild(remote, debug, **kwargs):
     If using a remote, a prior setup is required which is described on:
     https://linuxcontainers.org/lxd/getting-started-cli/#multiple-hosts
     """
+    # cleanbuild is a special snow flake, while all the other commands
+    # would work with the host as the build_provider it makes little
+    # sense in this scenario.
+    build_environment = env.BuilderEnvironmentConfig(
+        default='lxd', additional_providers=['multipass'])
     project_options = get_project_options(**kwargs, debug=debug)
-    lifecycle.cleanbuild(project_options, remote)
+    lifecycle.cleanbuild(project=project_options,
+                         echoer=echo,
+                         remote=remote,
+                         build_environment=build_environment)
 
 
 if __name__ == '__main__':

snapcraft/internal/build_providers/__init__.py

@@ -0,0 +1,19 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright (C) 2018 Canonical Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from . import errors                    # noqa: F401
+from ._factory import get_provider_for  # noqa: F401
+from ._multipass import Multipass       # noqa: F401

snapcraft/internal/build_providers/_base_provider.py

@@ -0,0 +1,94 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright (C) 2018 Canonical Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import abc
+import shlex
+from typing import List
+
+import petname
+
+
+class Provider():
+
+    def __init__(self, *, project, echoer) -> None:
+        self.project = project
+        self.echoer = echoer
+        # Once https://github.com/CanonicalLtd/multipass/issues/220 is
+        # closed we can prepend snapcraft- again.
+        self.instance_name = petname.Generate(2, '-')
+        self.project_dir = shlex.quote(project.info.name)
+
+        if project.info.version:
+            self.snap_filename = '{}_{}_{}.snap'.format(
+                project.info.name, project.info.version, project.deb_arch)
+        else:
+            self.snap_filename = '{}_{}.snap'.format(
+                project.info.name, project.deb_arch)
+
+    def __enter__(self):
+        self.create()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.destroy()
+
+    @abc.abstractmethod
+    def _run(self, command: List) -> None:
+        """Run a command on the instance."""
+
+    @abc.abstractmethod
+    def _launch(self):
+        """Launch the instance."""
+
+    @abc.abstractmethod
+    def create(self) -> None:
+        """Provider steps needed to create a fully functioning environemnt."""
+
+    @abc.abstractmethod
+    def destroy(self) -> None:
+        """Provider steps needed to ensure the instance is destroyed.
+
+        This method should be safe to call multiple times and do nothing
+        if the instance to destroy is already destroyed.
+        """
+
+    @abc.abstractmethod
+    def provision_project(self, tarball: str) -> None:
+        """Provider steps needed to copy project assests to the instance."""
+
+    @abc.abstractmethod
+    def build_project(self) -> None:
+        """Provider steps needed build the project on the instance."""
+
+    @abc.abstractmethod
+    def retrieve_snap(self) -> str:
+        """
+        Provider steps needed to retrieve the built snap from the instance.
+
+        :returns: the filename of the retrieved snap.
+        :rtype: str
+        """
+
+    def launch_instance(self) -> None:
+        self.echoer.info('Creating a build environment named {!r}'.format(
+            self.instance_name))
+        self._launch()
+
+    def setup_snapcraft(self) -> None:
+        self.echoer.info('Setting up snapcraft in {!r}'.format(
+            self.instance_name))
+        install_cmd = ['sudo', 'snap', 'install', 'snapcraft', '--classic']
+        self._run(install_cmd)

snapcraft/internal/build_providers/_factory.py

@@ -0,0 +1,33 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright (C) 2018 Canonical Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from typing import TYPE_CHECKING
+
+from . import errors
+from ._multipass import Multipass
+
+if TYPE_CHECKING:
+    from typing import Type                   # noqa: F401
+
+    from ._base_provider import Provider  # noqa: F401
+
+
+def get_provider_for(provider_name: str) -> 'Type[Provider]':
+    """Returns a Type that can build with provider_name."""
+    if provider_name == 'multipass':
+        return Multipass
+    else:
+        raise errors.ProviderNotSupportedError(provider=provider_name)

snapcraft/internal/build_providers/_multipass/__init__.py

@@ -0,0 +1,18 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright (C) 2018 Canonical Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from ._multipass import Multipass                 # noqa: F401
+from ._multipass_command import MultipassCommand  # noqa: F401

snapcraft/internal/build_providers/_multipass/_instance_info.py

@@ -0,0 +1,79 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright (C) 2018 Canonical Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+from typing import Type
+
+from snapcraft.internal.build_providers import errors
+
+
+class InstanceInfo:
+
+    @classmethod
+    def from_json(cls: Type['InstanceInfo'], *, instance_name: str,
+                  json_info: str) -> 'InstanceInfo':
+        """Create an InstanceInfo from json_info retrieved from multipass.
+
+        :param str instance_name: the name of the instance.
+        :param str json_info: a json formatted string with the structure
+                              that would follow the output of a json formatted
+                              multipass info command.
+        :returns: an InstanceInfo.
+        :rtype: InstanceInfo
+        :raises snapcraft.internal.build_providers.ProviderInfoDataKeyError:
+            if the instance name cannot be found in the given json or if a
+            required key is missing from that data structure for the instance.
+        """
+        try:
+            json_data = json.loads(json_info)
+        except json.decoder.JSONDecodeError as decode_error:
+            raise errors.ProviderBadDataError(
+                provider_name='multipass',
+                data=json_info) from decode_error
+        try:
+            instance_info = json_data['info'][instance_name]
+        except KeyError as missing_key:
+            raise errors.ProviderInfoDataKeyError(
+                provider_name='multipass',
+                missing_key=str(missing_key),
+                data=json_data) from missing_key
+        try:
+            return cls(name=instance_name,
+                       state=instance_info['state'],
+                       image_release=instance_info['image_release'])
+        except KeyError as missing_key:
+            raise errors.ProviderInfoDataKeyError(
+                provider_name='multipass',
+                missing_key=str(missing_key),
+                data=instance_info) from missing_key
+
+    def __init__(self, *, name: str, state: str, image_release: str) -> None:
+        """Initialize an InstanceInfo.
+
+        :param str name: the instance name.
+        :param str state: the state of the instance which can be any one of
+                          RUNNING, STOPPED, DELETED.
+        :param str image_release: the Operating System release string for the
+                                  image.
+        """
+        # We do not check for validity of state given that new states could
+        # be introduced.
+        self.name = name
+        self.state = state
+        self.image_release = image_release
+
+    def is_stopped(self):
+        return self.state == 'STOPPED'