recording: record stage packages installed in the snap

integration_tests/snaps/basic-without-arch/snap/snapcraft.yaml

@@ -0,0 +1,9 @@
+name: basic
+version: 0.1
+summary: Summary of the most simple snap
+description: Description of the most simple snap
+confinement: strict
+
+parts:
+  dummy-part:
+    plugin: nil

integration_tests/snaps/stage-package/snap/snapcraft.yaml → integration_tests/snaps/stage-package-missing-dependency/snap/snapcraft.yaml

@@ -1,13 +1,14 @@
-name: stage-package
+name: stage-package-missing-dependency
 version: '0.1'
 summary: install a stage package
 description: |
   Install a stage package.
+  This package has one dependency: gcc-6-base.
 
 grade: stable
 confinement: strict
 
 parts:
-  hello:
+  part-with-stage-package:
     plugin: nil
     stage-packages: ['hello']

integration_tests/snaps/stage-packages-without-dependencies/snap/snapcraft.yaml

@@ -0,0 +1,14 @@
+name: stage-packages-without-dependencies
+version: '0.1'
+summary: install stage packages
+description: |
+  Install stage packages.
+  These packages don't require any additional dependency.
+
+grade: stable
+confinement: strict
+
+parts:
+  part-with-stage-packages:
+    plugin: nil
+    stage-packages: [gcc-6-base, hello]

integration_tests/test_snapcraft_recording.py

@@ -18,30 +18,190 @@
 import yaml
 
 import fixtures
-from testtools.matchers import FileExists
+from testtools.matchers import (
+    FileExists,
+    MatchesRegex
+)
 
+import snapcraft
 import integration_tests
 
 
-class SnapcraftRecordingTestCase(integration_tests.TestCase):
+class SnapcraftRecordingBaseTestCase(integration_tests.TestCase):
+    """Test that the prime step records an annotated snapcraft.yaml
 
-    def test_prime_records_snapcraft_yaml(self):
+    The annotated file will be in prime/snap/snapcraft.yaml.
+
+    """
+
+    def setUp(self):
+        super().setUp()
         self.useFixture(fixtures.EnvironmentVariable(
             'SNAPCRAFT_BUILD_INFO', '1'))
 
+    def test_prime_records_snapcraft_yaml(self):
+        """Test the recorded snapcraft.yaml for a basic snap
+
+        This snap doesn't have stage or build packages and is declared that it
+        works on all architectures.
+        """
         self.run_snapcraft('prime', project_dir='basic')
         recorded_yaml_path = os.path.join(
             self.prime_dir, 'snap', 'snapcraft.yaml')
         self.assertThat(recorded_yaml_path, FileExists())
 
+        # Annotate the source snapcraft.yaml with the expected values.
         with open(os.path.join('snap', 'snapcraft.yaml')) as source_yaml_file:
             source_yaml = yaml.load(source_yaml_file)
-        # Append the default values.
-        for key in ('prime', 'stage'):
+        for key in ('prime', 'stage', 'stage-packages'):
+            # prime and stage come from when the yaml is loaded.
+            # stage-packages comes from the annotation from state.
             source_yaml['parts']['dummy-part'].update({key: []})
         source_yaml.update(grade='stable')
 
         with open(recorded_yaml_path) as recorded_yaml_file:
             recorded_yaml = yaml.load(recorded_yaml_file)
 
         self.assertEqual(recorded_yaml, source_yaml)
+
+    def test_prime_without_arch_records_current_arch(self):
+        """Test the recorded snapcraft.yaml for a basic snap
+
+        This snap doesn't have stage or build packages and it is not declared
+        that it works on all architectures, which makes it specific to the
+        current architecture.
+        """
+        self.run_snapcraft('prime', project_dir='basic-without-arch')
+        recorded_yaml_path = os.path.join(
+            self.prime_dir, 'snap', 'snapcraft.yaml')
+        self.assertThat(recorded_yaml_path, FileExists())
+
+        # Annotate the source snapcraft.yaml with the expected values.
+        with open(os.path.join('snap', 'snapcraft.yaml')) as source_yaml_file:
+            source_yaml = yaml.load(source_yaml_file)
+        for key in ('prime', 'stage', 'stage-packages'):
+            # prime and stage come from when the yaml is loaded.
+            # stage-packages comes from the annotation from state.
+            source_yaml['parts']['dummy-part'].update({key: []})
+        source_yaml.update(grade='stable')
+        source_yaml.update(
+            architectures=[snapcraft.ProjectOptions().deb_arch])
+
+        with open(recorded_yaml_path) as recorded_yaml_file:
+            recorded_yaml = yaml.load(recorded_yaml_file)
+
+        self.assertEqual(recorded_yaml, source_yaml)
+
+    def test_prime_records_stage_packages_version(self):
+        """Test the recorded snapcraft.yaml for a snap with stage packages
+
+        This snap declares all the stage packages that it requires, there are
+        no additional dependencies. The stage packages specify their version.
+        """
+        self.copy_project_to_cwd('stage-packages-without-dependencies')
+        part_name = 'part-with-stage-packages'
+        self.set_stage_package_version(
+            os.path.join('snap', 'snapcraft.yaml'),
+            part_name, package='hello')
+        self.set_stage_package_version(
+            os.path.join('snap', 'snapcraft.yaml'),
+            part_name, package='gcc-6-base')
+
+        self.run_snapcraft('prime')
+
+        with open(os.path.join('snap', 'snapcraft.yaml')) as source_yaml_file:
+            source_yaml = yaml.load(source_yaml_file)
+
+        # Safeguard assertion for the version in the source.
+        self.assertThat(
+            source_yaml['parts'][part_name]['stage-packages'][0],
+            MatchesRegex('gcc-6-base=.+'))
+        self.assertThat(
+            source_yaml['parts'][part_name]['stage-packages'][1],
+            MatchesRegex('hello=.+'))
+
+        # Annotate the source snapcraft.yaml with the expected values.
+        for key in ('prime', 'stage'):
+            source_yaml['parts'][part_name].update({key: []})
+        # stage packages end up at the end.
+        source_yaml['parts'][part_name].move_to_end('stage-packages')
+
+        source_yaml.update(
+            architectures=[snapcraft.ProjectOptions().deb_arch])
+
+        recorded_yaml_path = os.path.join(
+            self.prime_dir, 'snap', 'snapcraft.yaml')
+        with open(recorded_yaml_path) as recorded_yaml_file:
+            recorded_yaml = yaml.load(recorded_yaml_file)
+
+        self.assertEqual(recorded_yaml, source_yaml)
+
+    def test_prime_without_stage_packages_version(self):
+        """Test the recorded snapcraft.yaml for a snap with stage packages
+
+        This snap declares all the stage packages that it requires, there are
+        no additional dependencies. The stage packages don't specify their
+        version.
+        """
+        self.run_snapcraft(
+            'prime', project_dir='stage-packages-without-dependencies')
+
+        # Annotate the source snapcraft.yaml with the expected values.
+        with open(os.path.join('snap', 'snapcraft.yaml')) as source_yaml_file:
+            source_yaml = yaml.load(source_yaml_file)
+        part_name = 'part-with-stage-packages'
+        for key in ('prime', 'stage'):
+            source_yaml['parts'][part_name].update({key: []})
+        # stage packages end up at the end.
+        source_yaml['parts'][part_name].move_to_end('stage-packages')
+        # annotate the yaml with the staged versions.
+        source_yaml['parts'][part_name]['stage-packages'] = [
+            '{}={}'.format(
+                package, integration_tests.get_package_version(
+                    package, self.distro_series, self.deb_arch)) for
+            package in source_yaml['parts'][part_name]['stage-packages']
+        ]
+        source_yaml.update(
+            architectures=[snapcraft.ProjectOptions().deb_arch])
+
+        recorded_yaml_path = os.path.join(
+            self.prime_dir, 'snap', 'snapcraft.yaml')
+        with open(recorded_yaml_path) as recorded_yaml_file:
+            recorded_yaml = yaml.load(recorded_yaml_file)
+
+        self.assertEqual(recorded_yaml, source_yaml)
+
+    def test_prime_with_stage_package_missing_dependecy(self):
+        """Test the recorded snapcraft.yaml for a snap with stage packages
+
+        This snap declares one stage packages that has one undeclared
+        dependency.
+        """
+        self.copy_project_to_cwd('stage-package-missing-dependency')
+        part_name = 'part-with-stage-package'
+        self.set_stage_package_version(
+            os.path.join('snap', 'snapcraft.yaml'), part_name, package='hello')
+        self.run_snapcraft('prime')
+
+        # Annotate the source snapcraft.yaml with the expected values.
+        with open(os.path.join('snap', 'snapcraft.yaml')) as source_yaml_file:
+            source_yaml = yaml.load(source_yaml_file)
+        part_name = 'part-with-stage-package'
+        # Append the default values.
+        for key in ('prime', 'stage'):
+            source_yaml['parts'][part_name].update({key: []})
+        # stage packages end up at the end.
+        source_yaml['parts'][part_name].move_to_end('stage-packages')
+        # annotate the yaml with the installed version.
+        source_yaml['parts'][part_name]['stage-packages'].insert(
+            0, 'gcc-6-base={}'.format(integration_tests.get_package_version(
+                'gcc-6-base', self.distro_series, self.deb_arch)))
+        source_yaml.update(
+            architectures=[snapcraft.ProjectOptions().deb_arch])
+
+        recorded_yaml_path = os.path.join(
+            self.prime_dir, 'snap', 'snapcraft.yaml')
+        with open(recorded_yaml_path) as recorded_yaml_file:
+            recorded_yaml = yaml.load(recorded_yaml_file)
+
+        self.assertEqual(recorded_yaml, source_yaml)

integration_tests/test_stage_package_version.py

@@ -23,15 +23,17 @@
 class StagePackageVersionTestCase(integration_tests.TestCase):
 
     def test_stage_package_with_invalid_version_must_fail(self):
-        self.copy_project_to_cwd('stage-package')
+        self.copy_project_to_cwd('stage-package-missing-dependency')
         self.set_stage_package_version(
             os.path.join('snap', 'snapcraft.yaml'),
-            part='hello', package='hello', version='invalid')
+            part='part-with-stage-package',
+            package='hello', version='invalid')
         error = self.assertRaises(
             subprocess.CalledProcessError,
             self.run_snapcraft, 'pull')
         self.assertIn(
-            "Error downloading stage packages for part 'hello': "
+            "Error downloading stage packages for part "
+            "'part-with-stage-package': "
             "The package 'hello=invalid' was not found.",
             str(error.output)
         )

snapcraft/internal/meta.py

@@ -14,10 +14,11 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import copy
 import contextlib
-import os
 import configparser
 import logging
+import os
 import re
 import shlex
 import shutil
@@ -32,6 +33,7 @@
 from snapcraft.internal.errors import MissingGadgetError
 from snapcraft.internal.deprecations import handle_deprecation_notice
 from snapcraft.internal.sources import get_source_handler_from_type
+from snapcraft.internal.states import get_state
 
 
 logger = logging.getLogger(__name__)
@@ -137,7 +139,17 @@ def _record_snapcraft(self):
         if os.environ.get('SNAPCRAFT_BUILD_INFO'):
             os.makedirs(record_dir, exist_ok=True)
             with open(record_file_path, 'w') as record_file:
-                yaml.dump(self._config_data, record_file)
+                annotated_snapcraft = self._annotate_snapcraft(
+                    copy.deepcopy(self._config_data))
+                yaml.dump(annotated_snapcraft, record_file)
+
+    def _annotate_snapcraft(self, data):
+        for part in data['parts']:
+            pull_state = get_state(
+                os.path.join(self._parts_dir, part, 'state'), 'pull')
+            data['parts'][part]['stage-packages'] = (
+                pull_state.assets.get('stage-packages', []))
+        return data
 
     def write_snap_directory(self):
         # First migrate the snap directory. It will overwrite any conflicting

snapcraft/tests/fixture_setup.py

@@ -1,6 +1,6 @@
 # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
 #
-# Copyright (C) 2015-2016 Canonical Ltd
+# Copyright (C) 2015-2017 Canonical Ltd
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
@@ -14,14 +14,15 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import collections
 import contextlib
-from functools import partial
 import io
 import os
 import sys
 import threading
-from types import ModuleType
 import urllib.parse
+from functools import partial
+from types import ModuleType
 from unittest import mock
 from subprocess import CalledProcessError
 
@@ -553,3 +554,50 @@ def setUp(self):
             revno = call_with_output(['hg', 'id']).split()[0]
 
             self.commit = revno
+
+
+class FakeAptCache(fixtures.Fixture):
+
+    def __init__(self, packages):
+        super().__init__()
+        self.packages = packages
+
+    def setUp(self):
+        super().setUp()
+        temp_dir = fixtures.TempDir()
+        self.useFixture(temp_dir)
+        patcher = mock.patch('snapcraft.repo._deb.apt.Cache')
+        mock_apt_cache = patcher.start()
+        self.addCleanup(patcher.stop)
+
+        cache = collections.OrderedDict()
+        for package, version in self.packages:
+            cache[package] = FakeAptCachePackage(
+                temp_dir.path, package, version)
+
+        mock_apt_cache().__getitem__.side_effect = (
+            lambda item: cache[item])
+
+        mock_apt_cache().get_changes.return_value = cache.values()
+
+
+class FakeAptCachePackage():
+
+    def __init__(self, temp_dir, name, version):
+        super().__init__()
+        self.temp_dir = temp_dir
+        self.name = name
+        self.version = version
+        self.versions = {version: self}
+        self.candidate = self
+
+    def __str__(self):
+        return '{}={}'.format(self.name, self.version)
+
+    def mark_install(self):
+        pass
+
+    def fetch_binary(self, dir_, progress):
+        path = os.path.join(self.temp_dir, self.name)
+        open(path, 'w').close()
+        return path