Merge remote-tracking branch 'origin/master' into feature/1785109/tem…

…plates

.flake8

@@ -11,11 +11,19 @@ max-complexity = 10
 exclude =
     # No need to traverse our git directory
     .git,
+    .hg,
+    .mypy_cache,
+    .tox,
+    .venv,
+    .vscode,
+    _build,
+    buck-out,
     # There's no value in checking cache directories
     __pycache__,
     # This contains builds of flake8 that we don't want to check
+    build,
     dist,
     # snapcraft generated
     parts,
     stage,
-    prime
+    prime

.github/PULL_REQUEST_TEMPLATE.md

@@ -3,6 +3,6 @@
 - [ ] If this is a bugfix. Have you checked that there is a bug report open for the issue you are trying to fix on [bug reports](https://bugs.launchpad.net/snapcraft)?
 - [ ] If this is a new feature. Have you discussed the design on the [forum](https://forum.snapcraft.io)?
 - [ ] Have you successfully run `./runtests.sh static`?
-- [ ] Have you successfully run `./runtests.sh unit`?
+- [ ] Have you successfully run `./runtests.sh tests/unit`?
 
 -----

mypy.ini

@@ -3,4 +3,4 @@
 [mypy]
 python_version = 3.5
 ignore_missing_imports = True
-follow_imports=silent 
+follow_imports=silent

pyproject.toml

@@ -6,8 +6,10 @@ exclude = '''
   | \.mypy_cache
   | \.tox
   | \.venv
+  | \.vscode
   | _build
   | buck-out
+  | __pycache__
   | build
   | dist
 
@@ -16,4 +18,4 @@ exclude = '''
   | stage
   | prime
 )/
-'''
+'''

runtests.sh

@@ -72,7 +72,7 @@ run_static_tests(){
     mypy -p snapcraft
 
     echo "Running codespell"
-    codespell -S "*.tar,*.xz,*.zip,*.bz2,*.7z,*.gz,*.deb,*.rpm,*.snap,*.gpg,*.pyc,*.png,*.ico,*.jar,./.git,changelog,.mypy_cache,.vscode,parts,stage,prime" -q4
+    codespell -S "*.tar,*.xz,*.zip,*.bz2,*.7z,*.gz,*.deb,*.rpm,*.snap,*.gpg,*.pyc,*.png,*.ico,*.jar,changelog,.git,.hg,.mypy_cache,.tox,.venv,_build,buck-out,__pycache__,build,dist,.vscode,parts,stage,prime" -q4
 
     echo "Running shellcheck"
     # Need to skip 'demos/gradle/gradlew' as it wasn't written by us and has

setup.py

@@ -45,6 +45,7 @@
     "snapcraft.internal.project_loader",
     "snapcraft.internal.project_loader.grammar",
     "snapcraft.internal.project_loader.grammar_processing",
+    "snapcraft.internal.project_loader.inspection",
     "snapcraft.internal.repo",
     "snapcraft.internal.sources",
     "snapcraft.internal.states",

snapcraft/cli/_command_group.py

@@ -62,6 +62,9 @@ def list_commands(self, ctx):
         # Let's keep edit-collaborators hidden until we get the green light
         # from the store.
         commands.pop(commands.index("edit-collaborators"))
+
+        # Inspect is for internal usage: hide it
+        commands.pop(commands.index("inspect"))
         build_environment = env.BuilderEnvironmentConfig()
         if build_environment.is_host:
             commands.pop(commands.index("refresh"))

snapcraft/cli/_runner.py

@@ -29,6 +29,7 @@
 from .lifecycle import lifecyclecli
 from .store import storecli
 from .parts import partscli
+from .inspect import inspectcli
 from .help import helpcli
 from .templates import templatecli
 from .version import versioncli, SNAPCRAFT_VERSION_TEMPLATE
@@ -49,6 +50,7 @@
     partscli,
     templatecli,
     versioncli,
+    inspectcli,
 ]
 
 

snapcraft/cli/inspect.py

@@ -0,0 +1,128 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright (C) 2018 Canonical Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+import click
+from datetime import datetime
+import json
+import logging
+from tabulate import tabulate
+
+from snapcraft import formatting_utils
+from snapcraft.internal import errors, project_loader
+from snapcraft.internal.project_loader import inspection
+from ._options import get_project
+from . import echo
+
+
+@click.group(context_settings={})
+@click.pass_context
+def inspectcli(ctx):
+    pass
+
+
+@inspectcli.command()
+@click.option(
+    "--provides",
+    metavar="<path>",
+    help=(
+        "Show the part that provided <path>. <path> can be relative or absolute, and "
+        "must be pointing within the staging or priming area."
+    ),
+)
+@click.option(
+    "--latest-step", is_flag=True, help="Show the most recently-completed step"
+)
+@click.option("--json", "use_json", is_flag=True, help="Print the result in json")
+def inspect(*, provides: str, latest_step: bool, use_json: bool, **kwargs):
+    """Inspect the current state of the project.
+
+    By default, this command will print a tabulated summary of the current state of the
+    project.
+    """
+    count = 0
+    for option in (provides, latest_step):
+        if option:
+            count += 1
+
+    if count > 1:
+        raise errors.SnapcraftEnvironmentError(
+            "Only one of --provides or --latest-step may be supplied"
+        )
+
+    # Silence the plugin loader logger
+    logging.getLogger("snapcraft.internal.pluginhandler._plugin_loader").setLevel(
+        logging.ERROR
+    )
+    project = get_project(**kwargs)
+    config = project_loader.load_config(project)
+
+    if provides:
+        providing_parts = inspection.provides(
+            provides, config.project, config.all_parts
+        )
+        providing_part_names = sorted([p.name for p in providing_parts])
+        if use_json:
+            print(
+                json.dumps(
+                    {"path": provides, "parts": providing_part_names},
+                    sort_keys=True,
+                    indent=4,
+                )
+            )
+        else:
+            echo.info(
+                "This path was provided by the following {}:".format(
+                    formatting_utils.pluralize(providing_part_names, "part", "parts")
+                )
+            )
+            echo.info("\n".join(providing_part_names))
+
+    elif latest_step:
+        part, step, timestamp = inspection.latest_step(config.all_parts)
+        directory = part.working_directory_for_step(step)
+        if use_json:
+            print(
+                json.dumps(
+                    {
+                        "part": part.name,
+                        "step": step.name,
+                        "directory": directory,
+                        "timestamp": timestamp,
+                    },
+                    sort_keys=True,
+                    indent=4,
+                )
+            )
+        else:
+            echo.info(
+                "The latest step that was run is the {!r} step of the {!r} part, "
+                "which ran at {}. The working directory for this step is {!r}".format(
+                    step.name,
+                    part.name,
+                    datetime.fromtimestamp(timestamp).strftime("%c"),
+                    directory,
+                )
+            )
+
+    else:
+        summary = inspection.lifecycle_status(config)
+        if use_json:
+            summary_dict = dict()
+            for part_summary in summary:
+                part_name = part_summary.pop("part")
+                summary_dict[part_name] = part_summary
+            print(json.dumps(summary_dict, sort_keys=True, indent=4))
+        else:
+            print(tabulate(summary, headers="keys"))

snapcraft/cli/parts.py

@@ -68,3 +68,11 @@ def search(ctx, query):
 
     """
     remote_parts.search(" ".join(query))
+
+
+# Implemented as a generator since loading up the state could be heavy
+def _part_states_for_step(step, parts_config):
+    for part in parts_config.all_parts:
+        state = part.get_state(step)
+        if state:
+            yield (part.name, state)

snapcraft/internal/lifecycle/__init__.py

@@ -19,3 +19,4 @@
 from ._init import init  # noqa
 from ._packer import pack  # noqa
 from ._runner import execute  # noqa
+from ._status_cache import StatusCache  # noqa

snapcraft/internal/pluginhandler/__init__.py

@@ -24,7 +24,7 @@
 import subprocess
 import sys
 from glob import glob, iglob
-from typing import Dict, Set, Sequence  # noqa: F401
+from typing import cast, Dict, Set, Sequence  # noqa: F401
 
 import yaml
 
@@ -126,24 +126,27 @@ def __init__(
 
     def get_pull_state(self) -> states.PullState:
         if not self._pull_state:
-            self._pull_state = states.get_state(self.plugin.statedir, steps.PULL)
+            self._pull_state = cast(states.PullState, self.get_state(steps.PULL))
         return self._pull_state
 
     def get_build_state(self) -> states.BuildState:
         if not self._build_state:
-            self._build_state = states.get_state(self.plugin.statedir, steps.BUILD)
+            self._build_state = cast(states.BuildState, self.get_state(steps.BUILD))
         return self._build_state
 
     def get_stage_state(self) -> states.StageState:
         if not self._stage_state:
-            self._stage_state = states.get_state(self.plugin.statedir, steps.STAGE)
+            self._stage_state = cast(states.StageState, self.get_state(steps.STAGE))
         return self._stage_state
 
     def get_prime_state(self) -> states.PrimeState:
         if not self._prime_state:
-            self._prime_state = states.get_state(self.plugin.statedir, steps.PRIME)
+            self._prime_state = cast(states.PrimeState, self.get_state(steps.PRIME))
         return self._prime_state
 
+    def get_state(self, step) -> states.PartState:
+        return states.get_state(self.plugin.statedir, step)
+
     def _get_source_handler(self, properties):
         """Returns a source_handler for the source in properties."""
         # TODO: we cannot pop source as it is used by plugins. We also make
@@ -230,6 +233,18 @@ def _migrate_state_file(self):
                 os.makedirs(self.plugin.statedir)
                 self.mark_done(steps.get_step_by_name(step))
 
+    def working_directory_for_step(self, step: steps.Step) -> str:
+        if step == steps.PULL:
+            return self.plugin.sourcedir
+        elif step == steps.BUILD:
+            return self.plugin.builddir
+        elif step == steps.STAGE:
+            return self.stagedir
+        elif step == steps.PRIME:
+            return self.primedir
+
+        raise errors.InvalidStepError(step.name)
+
     def latest_step(self):
         for step in reversed(steps.STEPS):
             if os.path.exists(states.get_step_state_file(self.plugin.statedir, step)):

snapcraft/internal/project_loader/inspection/__init__.py

@@ -0,0 +1,19 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright (C) 2018 Canonical Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from ._latest_step import latest_step  # noqa: F401
+from ._lifecycle_status import lifecycle_status  # noqa: F401
+from ._provides import provides  # noqa: F401

snapcraft/internal/project_loader/inspection/_latest_step.py

@@ -0,0 +1,48 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright (C) 2018 Canonical Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import contextlib
+from typing import List, Tuple
+
+from snapcraft.internal import pluginhandler, steps
+import snapcraft.internal.errors
+from . import errors
+
+
+def latest_step(
+    parts: List[pluginhandler.PluginHandler]
+) -> Tuple[pluginhandler.PluginHandler, steps.Step, int]:
+    """Determine and return the latest step that was run.
+
+    :param list parts: List of all parts.
+    :returns: Tuple of (part, step, timestamp).
+    """
+    latest_part = None
+    latest_step = None
+    latest_timestamp = 0
+    for part in parts:
+        with contextlib.suppress(snapcraft.internal.errors.NoLatestStepError):
+            step = part.latest_step()
+            timestamp = part.step_timestamp(step)
+            if latest_timestamp < timestamp:
+                latest_part = part
+                latest_step = step
+                latest_timestamp = timestamp
+
+    if not latest_part:
+        raise errors.NoStepsRunError()
+
+    return (latest_part, latest_step, latest_timestamp)