state: allow parametrization of the global state file

snapcraft/internal/lifecycle/_clean.py

@@ -20,7 +20,6 @@
 
 from snapcraft import formatting_utils
 from snapcraft.internal import errors, project_loader, mountinfo, steps
-from . import constants
 
 
 logger = logging.getLogger(__name__)
@@ -77,6 +76,11 @@ def _clean_parts(part_names, step, config, staged_state, primed_state):
             )
 
 
+def _remove_directory(directory: str) -> None:
+    if os.path.isdir(directory):
+        shutil.rmtree(directory)
+
+
 def _remove_directory_if_empty(directory):
     if os.path.isdir(directory) and not os.listdir(directory):
         os.rmdir(directory)
@@ -136,7 +140,7 @@ def _cleanup_common_directories_for_step(step, project_options, parts=None):
         _cleanup_parts_dir(
             project_options.parts_dir, project_options.local_plugins_dir, parts
         )
-        _cleanup_internal_snapcraft_dir()
+        _remove_directory(project_options._internal_dir)
 
     if not being_tried:
         _remove_directory_if_empty(project_options.prime_dir)
@@ -175,11 +179,6 @@ def _cleanup_parts_dir(parts_dir, local_plugins_dir, parts):
         part.mark_cleaned(steps.PULL)
 
 
-def _cleanup_internal_snapcraft_dir():
-    if os.path.exists(constants.SNAPCRAFT_INTERNAL_DIR):
-        shutil.rmtree(constants.SNAPCRAFT_INTERNAL_DIR)
-
-
 def clean(project_options, parts, step=None):
     # step defaults to None because that's how it comes from docopt when it's
     # not set.

snapcraft/internal/lifecycle/_runner.py

@@ -20,8 +20,6 @@
 from tempfile import TemporaryDirectory
 from typing import Sequence
 
-import yaml
-
 import snapcraft
 from snapcraft import config
 from snapcraft.internal import (
@@ -35,7 +33,6 @@
     steps,
 )
 from snapcraft.internal.cache import SnapCache
-from . import constants
 from ._status_cache import StatusCache
 
 
@@ -74,12 +71,15 @@ def execute(
 
     installed_snaps = repo.snaps.install_snaps(project_config.build_snaps)
 
-    os.makedirs(constants.SNAPCRAFT_INTERNAL_DIR, exist_ok=True)
-    state_path = os.path.join(constants.SNAPCRAFT_INTERNAL_DIR, "state")
-    with open(state_path, "w") as state_file:
-        state_file.write(
-            yaml.dump(states.GlobalState(installed_packages, installed_snaps))
+    try:
+        global_state = states.GlobalState.load(
+            filepath=project_config.project._global_state_file
         )
+    except FileNotFoundError:
+        global_state = states.GlobalState()
+    global_state.append_build_packages(installed_packages)
+    global_state.append_build_snaps(installed_snaps)
+    global_state.save(filepath=project_config.project._global_state_file)
 
     if _should_get_core(project_config.data.get("confinement")):
         _setup_core(

snapcraft/internal/meta/_manifest.py

@@ -22,10 +22,10 @@
 
 import snapcraft
 from snapcraft.internal import errors, os_release, steps
-from snapcraft.internal.states import get_global_state, get_state
+from snapcraft.internal.states import GlobalState, get_state
 
 
-def annotate_snapcraft(data, parts_dir: str):
+def annotate_snapcraft(data, parts_dir: str, global_state_path: str):
     manifest = OrderedDict()  # type: Dict[str, Any]
     manifest["snapcraft-version"] = snapcraft._get_version()
 
@@ -44,8 +44,11 @@ def annotate_snapcraft(data, parts_dir: str):
         except json.decoder.JSONDecodeError as exception:
             raise errors.InvalidContainerImageInfoError(image_info) from exception
         manifest["image-info"] = image_info_dict
-    for field in ("build-packages", "build-snaps"):
-        manifest[field] = get_global_state().assets.get(field, [])
+
+    global_state = GlobalState.load(filepath=global_state_path)
+    manifest["build-packages"] = global_state.get_build_packages()
+    manifest["build-snaps"] = global_state.get_build_snaps()
+
     for part in data["parts"]:
         state_dir = os.path.join(parts_dir, part, "state")
         pull_state = get_state(state_dir, steps.PULL)

snapcraft/internal/meta/_snap_packaging.py

@@ -299,6 +299,7 @@ def __init__(self, config_data: Dict[str, Any], project: Project) -> None:
         self._prime_dir = project.prime_dir
         self._parts_dir = project.parts_dir
         self._arch_triplet = project.arch_triplet
+        self._global_state_file = project._global_state_file
         self._is_host_compatible_with_base = project.is_host_compatible_with_base
         self._meta_dir = os.path.join(self._prime_dir, "meta")
         self._config_data = config_data.copy()
@@ -352,7 +353,9 @@ def _record_manifest_and_source_snapcraft_yaml(self):
             os.makedirs(prime_snap_dir, exist_ok=True)
             shutil.copy2(self._snapcraft_yaml_path, recorded_snapcraft_yaml_path)
             annotated_snapcraft = _manifest.annotate_snapcraft(
-                copy.deepcopy(self._config_data), self._parts_dir
+                copy.deepcopy(self._config_data),
+                self._parts_dir,
+                self._global_state_file,
             )
             with open(manifest_file_path, "w") as manifest_file:
                 yaml.dump(annotated_snapcraft, manifest_file, default_flow_style=False)

snapcraft/internal/states/__init__.py

@@ -19,6 +19,5 @@
 from snapcraft.internal.states._prime_state import PrimeState  # noqa
 from snapcraft.internal.states._pull_state import PullState  # noqa
 from snapcraft.internal.states._stage_state import StageState  # noqa
-from snapcraft.internal.states._state import get_global_state  # noqa
 from snapcraft.internal.states._state import get_state  # noqa
 from snapcraft.internal.states._state import get_step_state_file  # noqa

snapcraft/internal/states/_global_state.py

@@ -1,6 +1,6 @@
 # -*- Mode:Python; indent-tabs-buildnil; tab-width:4 -*-
 #
-# Copyright (C) 2017 Canonical Ltd
+# Copyright (C) 2017-2018 Canonical Ltd
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
@@ -14,23 +14,60 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
+from typing import Dict, List, Type
+
 import yaml
 
 from snapcraft.internal.states._state import State
 
 
-def _global_state_constructor(loader, node):
-    parameters = loader.construct_mapping(node)
-    return GlobalState(**parameters)
+class _GlobalStateLoader(yaml.Loader):
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
 
+        self.add_constructor(u"!GlobalState", type(self).construct_global_state)
 
-yaml.add_constructor(u"!GlobalState", _global_state_constructor)
+    def construct_global_state(self, node) -> "GlobalState":
+        parameters = self.construct_mapping(node)
+        return GlobalState(**parameters)
 
 
 class GlobalState(State):
 
     yaml_tag = u"!GlobalState"
 
-    def __init__(self, build_packages, build_snaps):
+    @classmethod
+    def load(cls: Type["GlobalState"], *, filepath: str) -> "GlobalState":
+        with open(filepath) as state_file:
+            return yaml.load(state_file, _GlobalStateLoader)
+
+    def save(self, *, filepath: str) -> None:
+        dirpath = os.path.dirname(filepath)
+        if dirpath:
+            os.makedirs(dirpath, exist_ok=True)
+        with open(filepath, "w") as state_file:
+            yaml.dump(self, stream=state_file)
+
+    def get_build_packages(self) -> List[str]:
+        return self.assets.get("build-packages", [])
+
+    def append_build_packages(self, build_packages: List[str]) -> None:
+        current_build_packages = self.get_build_packages()
+        new_packages = [b for b in build_packages if b not in current_build_packages]
+        self.assets["build-packages"] = current_build_packages + new_packages
+
+    def get_build_snaps(self) -> List[str]:
+        return self.assets.get("build-snaps", [])
+
+    def append_build_snaps(self, build_snaps: List[str]) -> None:
+        current_build_snaps = self.get_build_snaps()
+        new_snaps = [b for b in build_snaps if b not in current_build_snaps]
+        self.assets["build-snaps"] = current_build_snaps + new_snaps
+
+    def __init__(self, *, assets: Dict[str, List[str]] = None) -> None:
         super().__init__()
-        self.assets = {"build-packages": build_packages, "build-snaps": build_snaps}
+        if assets is None:
+            self.assets = dict()  # type: Dict[str, List[str]]
+        else:
+            self.assets = assets

snapcraft/internal/states/_state.py

@@ -88,11 +88,6 @@ def _get_differing_keys(dict1, dict2):
     return differing_keys
 
 
-def get_global_state():
-    with open(os.path.join("snap", ".snapcraft", "state"), "r") as state_file:
-        return yaml.load(state_file)
-
-
 def get_state(state_dir: str, step: steps.Step):
     state = None
     state_file = get_step_state_file(state_dir, step)

snapcraft/project/_project.py

@@ -14,6 +14,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
+
 from ._project_options import ProjectOptions
 from ._project_info import ProjectInfo  # noqa: F401
 
@@ -27,15 +29,28 @@ def __init__(
         *,
         use_geoip=False,
         parallel_builds=True,
-        target_deb_arch: str = None,
+        target_deb_arch=None,
         debug=False,
-        snapcraft_yaml_file_path=None
+        snapcraft_yaml_file_path=None,
+        project_dir: str = None
     ) -> None:
+
+        if project_dir is None:
+            self.project_dir = os.getcwd()
+        else:
+            self.project_dir = project_dir
+
         # This here check is mostly for backwards compatibility with the
         # rest of the code base.
         if snapcraft_yaml_file_path is None:
             self.info = None  # type: ProjectInfo
         else:
             self.info = ProjectInfo(snapcraft_yaml_file_path=snapcraft_yaml_file_path)
 
-        super().__init__(use_geoip, parallel_builds, target_deb_arch, debug)
+        # These paths maintain backwards compatibility.
+        self._internal_dir = os.path.join(self.project_dir, "snap", ".snapcraft")
+        self._global_state_file = os.path.join(self._internal_dir, "state")
+
+        super().__init__(
+            use_geoip, parallel_builds, target_deb_arch, debug, project_dir=project_dir
+        )

snapcraft/project/_project_options.py

@@ -222,10 +222,20 @@ def debug(self):
         return self.__debug
 
     def __init__(
-        self, use_geoip=False, parallel_builds=True, target_deb_arch=None, debug=False
-    ):
-        # TODO: allow setting a different project dir
-        self.__project_dir = os.getcwd()
+        self,
+        use_geoip=False,
+        parallel_builds=True,
+        target_deb_arch=None,
+        debug=False,
+        *,
+        project_dir: str = None
+    ) -> None:
+
+        if project_dir is None:
+            self.__project_dir = os.getcwd()
+        else:
+            self.__project_dir = project_dir
+
         self.__use_geoip = use_geoip
         self.__parallel_builds = parallel_builds
         self._set_machine(target_deb_arch)