From 9e00ffc151b46026b414c8b4b8e44a3a9d4c0b75 Mon Sep 17 00:00:00 2001 From: Kyle Fazzari Date: Wed, 8 Feb 2017 10:49:24 -0800 Subject: [PATCH] pluginhandler: support more complex stage-packages. (#1059) Currently snapcraft only supports a flat list of stage packages to be staged regardless of target architecture. This commit introduces a more complex grammar that allows one to filter stage packages depending on various selectors (target arch for right now), as well as specify optional packages. The grammar is made up of two statements: `on` and `try`. - on [,...] - ... - else[ fail]: - ... The body of the `on` clause is taken into account if every (AND, not OR) selector is true for the target build environment. Currently the only selectors supported are target architectures (e.g. amd64). If the `on` clause doesn't match and it's immediately followed by an `else` clause, the `else` clause must be satisfied. An `on` clause without an `else` clause is considered satisfied even if no selector matched. The `else fail` form allows erroring out if an `on` clause was not matched. - try: - ... - else: - ... The body of the `try` clause is taken into account only when all packages contained within it are valid. If not, if it's immediately followed by `else` clauses they are tried in order, and one of them must be satisfied. A `try` clause with no `else` clause is considered satisfied even if it contains invalid packages. LP: #1637282 Signed-off-by: Kyle Fazzari --- docs/snapcraft-syntax.md | 39 ++- .../stage-package-grammar/snapcraft.yaml | 53 ++++ .../test_stage_package_grammar.py | 93 +++++++ schema/snapcraft.yaml | 36 ++- setup.py | 1 + snapcraft/__init__.py | 81 +++++- snapcraft/_schema.py | 6 + snapcraft/internal/errors.py | 43 ++- snapcraft/internal/parts.py | 3 +- snapcraft/internal/pluginhandler/__init__.py | 72 ++--- .../pluginhandler/_stage_package_handler.py | 99 +++++++ .../stage_package_grammar/__init__.py | 17 ++ .../stage_package_grammar/_on.py | 139 ++++++++++ .../stage_package_grammar/_processor.py | 157 +++++++++++ .../stage_package_grammar/_try.py | 120 ++++++++ .../stage_package_grammar/errors.py | 42 +++ snapcraft/internal/repo.py | 4 + snapcraft/tests/commands/test_clean.py | 6 +- snapcraft/tests/commands/test_pull.py | 2 +- snapcraft/tests/pluginhandler/mocks.py | 7 +- .../stage_package_grammar/__init__.py | 38 +++ .../test_on_statement.py | 256 ++++++++++++++++++ .../stage_package_grammar/test_processor.py | 237 ++++++++++++++++ .../test_try_statement.py | 140 ++++++++++ .../test_stage_package_handler.py | 90 ++++++ snapcraft/tests/test_project_loader.py | 10 +- 26 files changed, 1732 insertions(+), 59 deletions(-) create mode 100644 integration_tests/snaps/stage-package-grammar/snapcraft.yaml create mode 100644 integration_tests/test_stage_package_grammar.py create mode 100644 snapcraft/internal/pluginhandler/_stage_package_handler.py create mode 100644 snapcraft/internal/pluginhandler/stage_package_grammar/__init__.py create mode 100644 snapcraft/internal/pluginhandler/stage_package_grammar/_on.py create mode 100644 snapcraft/internal/pluginhandler/stage_package_grammar/_processor.py create mode 100644 snapcraft/internal/pluginhandler/stage_package_grammar/_try.py create mode 100644 snapcraft/internal/pluginhandler/stage_package_grammar/errors.py create mode 100644 snapcraft/tests/pluginhandler/stage_package_grammar/__init__.py create mode 100644 snapcraft/tests/pluginhandler/stage_package_grammar/test_on_statement.py create mode 100644 snapcraft/tests/pluginhandler/stage_package_grammar/test_processor.py create mode 100644 snapcraft/tests/pluginhandler/stage_package_grammar/test_try_statement.py create mode 100644 snapcraft/tests/pluginhandler/test_stage_package_handler.py diff --git a/docs/snapcraft-syntax.md b/docs/snapcraft-syntax.md index 592e56b84b..81b8920bfd 100644 --- a/docs/snapcraft-syntax.md +++ b/docs/snapcraft-syntax.md @@ -82,8 +82,43 @@ contain. be searched for in [the wiki](https://wiki.ubuntu.com/Snappy/Parts). *If a part is supposed to run after another, the prerequisite part will be staged before the dependent part starts its lifecycle.* - * `stage-packages` (list of strings) - A list of Ubuntu packages to use that would support the part creation. + * `stage-packages` (list of strings and/or sublists) + A set of Ubuntu packages to be downloaded and unpacked to join the part + before it's built. Note that these packages are not installed on the host. + Like the rest of the part, all files from these packages will make it into + the final snap unless filtered out via the `snap` keyword. + + One may simply specify packages in a flat list, in which case the packages + will be fetched and unpacked regardless of build environment. In addition, + a specific grammar made up of sub-lists is supported here that allows one + to filter stage packages depending on various selectors (e.g. the target + arch), as well as specify optional packages. The grammar is made up of two + nestable statements: 'on' and 'try'. + + - on [,...]: + - ... + - else[ fail]: + - ... + + The body of the 'on' clause is taken into account if every (AND, not OR) + selector is true for the target build environment. Currently the only + selectors supported are target architectures (e.g. amd64). + + If the 'on' clause doesn't match and it's immediately followed by an + 'else' clause, the 'else' clause must be satisfied. An 'on' clause without + an 'else' clause is considered satisfied even if no selector matched. The + 'else fail' form allows erroring out if an 'on' clause was not matched. + + - try: + - ... + - else: + - ... + + The body of the 'try' clause is taken into account only when all packages + contained within it are valid. If not, if it's immediately followed by + 'else' clauses they are tried in order, and one of them must be satisfied. + A 'try' clause with no 'else' clause is considered satisfied even if it + contains invalid packages. * `build-packages` (list of strings) A list of Ubuntu packages to be installed on the host to aid in building the part. These packages will not go into the final snap. diff --git a/integration_tests/snaps/stage-package-grammar/snapcraft.yaml b/integration_tests/snaps/stage-package-grammar/snapcraft.yaml new file mode 100644 index 0000000000..f7f144999d --- /dev/null +++ b/integration_tests/snaps/stage-package-grammar/snapcraft.yaml @@ -0,0 +1,53 @@ +name: stage-package-grammar +version: '0.1' +summary: Test the stage package grammar +description: A few different parts that exercise the grammar differently +grade: devel +confinement: strict + +parts: + simple: + plugin: nil + stage-packages: + - hello + + try: + plugin: nil + stage-packages: + - try: + - hello + + try-skipped: + plugin: nil + stage-packages: + - try: + - invalid-package + + try-else: + plugin: nil + stage-packages: + - try: + - invalid-package + - else: + - hello + + on-other-arch: + plugin: nil + stage-packages: + - on other-arch: + - foo + + on-other-arch-else: + plugin: nil + stage-packages: + - on other-arch: + - foo + - else: + - hello + + on-other-arch-else-fail: + plugin: nil + stage-packages: + - on other-arch: + - foo + - else fail diff --git a/integration_tests/test_stage_package_grammar.py b/integration_tests/test_stage_package_grammar.py new file mode 100644 index 0000000000..5d2ffdcf97 --- /dev/null +++ b/integration_tests/test_stage_package_grammar.py @@ -0,0 +1,93 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# Copyright (C) 2017 Canonical Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import subprocess + +import integration_tests +from testtools.matchers import ( + Contains, + FileExists, + Not +) + + +class StagePackageGrammarTestCase(integration_tests.TestCase): + + def test_simple(self): + """Test that 'simple' fetches stage package.""" + + self.run_snapcraft(['prime', 'simple'], 'stage-package-grammar') + + self.assertThat( + os.path.join('prime', 'usr', 'bin', 'hello'), + FileExists()) + + def test_try(self): + """Test that 'try' fetches stage package.""" + + self.run_snapcraft(['prime', 'try'], 'stage-package-grammar') + + self.assertThat( + os.path.join('prime', 'usr', 'bin', 'hello'), + FileExists()) + + def test_try_skipped(self): + """Test that 'try-skipped' fetches nothing.""" + + self.run_snapcraft(['prime', 'try-skipped'], 'stage-package-grammar') + + self.assertThat( + os.path.join('prime', 'usr', 'bin', 'hello'), + Not(FileExists())) + + def test_try_else(self): + """Test that 'try-else' fetches stage package.""" + + self.run_snapcraft(['prime', 'try-else'], 'stage-package-grammar') + + self.assertThat( + os.path.join('prime', 'usr', 'bin', 'hello'), + FileExists()) + + def test_on_other_arch(self): + """Test that 'on-other-arch' fetches nothing.""" + + self.run_snapcraft(['prime', 'on-other-arch'], 'stage-package-grammar') + + self.assertThat( + os.path.join('prime', 'usr', 'bin', 'hello'), + Not(FileExists())) + + def test_on_other_arch_else(self): + """Test that 'on-other-arch-else' fetches stage package.""" + + self.run_snapcraft( + ['prime', 'on-other-arch-else'], 'stage-package-grammar') + + self.assertThat( + os.path.join('prime', 'usr', 'bin', 'hello'), + FileExists()) + + def test_on_other_arch_else_fail(self): + """Test that 'on-other-arch-else-fail' fails with an error.""" + + exception = self.assertRaises( + subprocess.CalledProcessError, self.run_snapcraft, + ['prime', 'on-other-arch-else-fail'], 'stage-package-grammar') + + self.assertThat(exception.output, Contains( + "Unable to satisfy 'on other-arch', failure forced")) diff --git a/schema/snapcraft.yaml b/schema/snapcraft.yaml index cb9f8fd639..305ae9b491 100644 --- a/schema/snapcraft.yaml +++ b/schema/snapcraft.yaml @@ -1,5 +1,33 @@ $schema: http://json-schema.org/draft-04/schema# +definitions: + stage-packages: + type: array + minitems: 1 + uniqueItems: true + items: + anyOf: + - type: string + usage: "" + - type: object + usage: "on [,...]:" + additionalProperties: false + patternProperties: + ^on\s+.+$: + $ref: "#/definitions/stage-packages" + - type: object + usage: "try:" + additionalProperties: false + patternProperties: + ^try$: + $ref: "#/definitions/stage-packages" + - type: object + usage: "else:" + additionalProperties: false + patternProperties: + ^else$: + $ref: "#/definitions/stage-packages" + title: snapcraft schema type: object properties: @@ -220,12 +248,8 @@ properties: type: string default: [] stage-packages: - type: array - minitems: 1 - uniqueItems: true - items: - type: string - default: [] + $ref: "#/definitions/stage-packages" + default: [] # For some reason this doesn't work if in the ref build-packages: type: array minitems: 1 diff --git a/setup.py b/setup.py index 56437b13b9..e89db73085 100755 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ 'snapcraft.internal.cache', 'snapcraft.internal.deltas', 'snapcraft.internal.pluginhandler', + 'snapcraft.internal.pluginhandler.stage_package_grammar', 'snapcraft.internal.sources', 'snapcraft.internal.states', 'snapcraft.plugins', diff --git a/snapcraft/__init__.py b/snapcraft/__init__.py index d9f4a84998..8671043266 100644 --- a/snapcraft/__init__.py +++ b/snapcraft/__init__.py @@ -144,13 +144,90 @@ binaries within the snap (in which case they'll be discovered via `ldd`), or they are explicitly described in stage-packages. - - stage-packages: [deb, deb, deb...] + - stage-packages: YAML list - A list of Ubuntu packages to be downloaded and unpacked to join the part + A set of Ubuntu packages to be downloaded and unpacked to join the part before it's built. Note that these packages are not installed on the host. Like the rest of the part, all files from these packages will make it into the final snap unless filtered out via the `snap` keyword. + One may simply specify packages in a flat list, in which case the packages + will be fetched and unpacked regardless of build environment. In addition, + a specific grammar made up of sub-lists is supported here that allows one + to filter stage packages depending on various selectors (e.g. the target + arch), as well as specify optional packages. The grammar is made up of two + nestable statements: 'on' and 'try'. + + Let's discuss `on`. + + - on [,...]: + - ... + - else[ fail]: + - ... + + The body of the 'on' clause is taken into account if every (AND, not OR) + selector is true for the target build environment. Currently the only + selectors supported are target architectures (e.g. amd64). + + If the 'on' clause doesn't match and it's immediately followed by an 'else' + clause, the 'else' clause must be satisfied. An 'on' clause without an + 'else' clause is considered satisfied even if no selector matched. The + 'else fail' form allows erroring out if an 'on' clause was not matched. + + For example, say you only wanted to stage `foo` if building for amd64 (and + not stage `foo` if otherwise): + + - on amd64: [foo] + + Building on that, say you wanted to stage `bar` if building on an arch + other than amd64: + + - on amd64: [foo] + - else: [bar] + + You can nest these for more complex behaviors: + + - on amd64: [foo] + - else: + - on i386: [bar] + - on armhf: [baz] + + If your project requires a package that is only available on amd64, you can + fail if you're not building for amd64: + + - on amd64: [foo] + - else fail + + Now let's discuss `try`: + + - try: + - ... + - else: + - ... + + The body of the 'try' clause is taken into account only when all packages + contained within it are valid. If not, if it's immediately followed by + 'else' clauses they are tried in order, and one of them must be satisfied. + A 'try' clause with no 'else' clause is considered satisfied even if it + contains invalid packages. + + For example, say you wanted to stage `foo`, but it wasn't available for all + architectures. Assuming your project builds without it, you can make it an + optional stage package: + + - try: [foo] + + You can also add alternatives: + + - try: [foo] + - else: [bar] + + Again, you can nest these for more complex behaviors: + + - on amd64: [foo] + - else: + - try: [bar] + - organize: YAML Snapcraft will rename files according to this YAML sub-section. The diff --git a/snapcraft/_schema.py b/snapcraft/_schema.py index b43fb1bfdd..cbce98005b 100644 --- a/snapcraft/_schema.py +++ b/snapcraft/_schema.py @@ -47,6 +47,12 @@ def part_schema(self): properties = sub['^(?!plugins$)[a-z0-9][a-z0-9+-\/]*$']['properties'] return properties + @property + def definitions_schema(self): + """Return sub-schema that describes definitions used within schema.""" + + return self._schema['definitions'].copy() + def _load_schema(self): schema_file = os.path.abspath(os.path.join( common.get_schemadir(), 'snapcraft.yaml')) diff --git a/snapcraft/internal/errors.py b/snapcraft/internal/errors.py index fc938e3698..0a13e96499 100644 --- a/snapcraft/internal/errors.py +++ b/snapcraft/internal/errors.py @@ -16,6 +16,8 @@ import contextlib +from snapcraft import formatting_utils + # dict of jsonschema validator -> cause pairs. Wish jsonschema just gave us # better messages. _VALIDATION_ERROR_CAUSES = { @@ -158,13 +160,16 @@ def from_validation_error(cls, error): class tries to make them a bit more understandable. """ - messages = [error.message] + messages = [] # error.validator_value may contain a custom validation error message. # If so, use it instead of the garbage message jsonschema gives us. with contextlib.suppress(TypeError, KeyError): - messages = [error.validator_value['validation-failure'].format( - error)] + messages.append( + error.validator_value['validation-failure'].format(error)) + + if not messages: + messages.append(error.message) path = [] while error.absolute_path: @@ -190,10 +195,34 @@ def __init__(self, message): def _determine_cause(error): """Attempt to determine a cause from validation error. - Returns: - A string representing the cause of the error (it may be empty if no - cause can be determined). + :return: A string representing the cause of the error (it may be empty if + no cause can be determined). + :rtype: str """ - return _VALIDATION_ERROR_CAUSES.get(error.validator, '').format( + message = _VALIDATION_ERROR_CAUSES.get(error.validator, '').format( validator_value=error.validator_value) + + if not message and error.validator == 'anyOf': + message = _interpret_anyOf(error) + + return message + + +def _interpret_anyOf(error): + """Interpret a validation error caused by the anyOf validator. + + Returns: + A string containing a (hopefully) helpful validation error message. It + may be empty. + """ + + usages = [] + try: + for validator in error.validator_value: + usages.append(validator['usage']) + except (TypeError, KeyError): + return '' + + return 'must be one of {}'.format(formatting_utils.humanize_list( + usages, 'or')) diff --git a/snapcraft/internal/parts.py b/snapcraft/internal/parts.py index 565ced11c0..6e9dd7ebb0 100644 --- a/snapcraft/internal/parts.py +++ b/snapcraft/internal/parts.py @@ -270,7 +270,8 @@ def load_plugin(self, part_name, plugin_name, part_properties): plugin_name=plugin_name, part_properties=part_properties, project_options=self._project_options, - part_schema=self._validator.part_schema) + part_schema=self._validator.part_schema, + definitions_schema=self._validator.definitions_schema) self.build_tools += part.code.build_packages if part.source_handler and part.source_handler.command: diff --git a/snapcraft/internal/pluginhandler/__init__.py b/snapcraft/internal/pluginhandler/__init__.py index b5aab195a9..1a33d9ef85 100644 --- a/snapcraft/internal/pluginhandler/__init__.py +++ b/snapcraft/internal/pluginhandler/__init__.py @@ -46,6 +46,7 @@ ) from ._scriptlets import ScriptRunner from ._build_attributes import BuildAttributes +from ._stage_package_handler import StagePackageHandler logger = logging.getLogger(__name__) @@ -66,17 +67,9 @@ def name(self): def installdir(self): return self.code.installdir - @property - def ubuntu(self): - if not self._ubuntu: - self._ubuntu = repo.Ubuntu( - self.ubuntudir, sources=self.code.PLUGIN_STAGE_SOURCES, - project_options=self._project_options) - - return self._ubuntu - def __init__(self, *, plugin_name, part_name, - part_properties, project_options, part_schema): + part_properties, project_options, part_schema, + definitions_schema): self.valid = False self.code = None self.config = {} @@ -89,7 +82,6 @@ def __init__(self, *, plugin_name, part_name, # the layout of parts inside the parts directory causing collisions # between the main project part and its subparts. part_name = part_name.replace('/', '\N{BIG SOLIDUS}') - self._ubuntu = None self._project_options = project_options self.deps = [] @@ -109,13 +101,22 @@ def __init__(self, *, plugin_name, part_name, self._migrate_state_file() try: - self._load_code(plugin_name, self._part_properties, part_schema) + self._load_code( + plugin_name, self._part_properties, part_schema, + definitions_schema) except jsonschema.ValidationError as e: error = SnapcraftSchemaError.from_validation_error(e) raise PluginError('properties failed to load for {}: {}'.format( part_name, error.message)) - def _load_code(self, plugin_name, properties, part_schema): + stage_packages = getattr(self.code, 'stage_packages', []) + sources = getattr(self.code, 'PLUGIN_STAGE_SOURCES', None) + self._stage_package_handler = StagePackageHandler( + stage_packages, self.ubuntudir, + sources=sources, project_options=self._project_options) + + def _load_code(self, plugin_name, properties, part_schema, + definitions_schema): module_name = plugin_name.replace('-', '_') module = None @@ -138,8 +139,10 @@ def _load_code(self, plugin_name, properties, part_schema): raise PluginError('unknown plugin: {}'.format(plugin_name)) plugin = _get_plugin(module) - _validate_pull_and_build_properties(plugin_name, plugin, part_schema) - options = _make_options(part_schema, properties, plugin.schema()) + _validate_pull_and_build_properties( + plugin_name, plugin, part_schema, definitions_schema) + options = _make_options( + part_schema, definitions_schema, properties, plugin.schema()) # For backwards compatibility we add the project to the plugin try: self.code = plugin(self.name, options, self._project_options) @@ -301,23 +304,14 @@ def _step_state_file(self, step): return os.path.join(self.statedir, step) def _fetch_stage_packages(self): - if not self.code.stage_packages: - return - - logger.debug('Fetching stage-packages {!r} for part {!r}'.format( - self.code.stage_packages, self.name)) - try: - self.ubuntu.get(self.code.stage_packages) + self._stage_package_handler.fetch() except repo.PackageNotFoundError as e: raise RuntimeError("Error downloading stage packages for part " "{!r}: {}".format(self.name, e.message)) def _unpack_stage_packages(self): - if self.code.stage_packages: - logger.debug('Unpacking stage-packages for part {!r} to ' - '{!r}'.format(self.name, self.installdir)) - self.ubuntu.unpack(self.installdir) + self._stage_package_handler.unpack(self.installdir) def prepare_pull(self, force=False): self.makedirs() @@ -711,19 +705,26 @@ def _expand_part_properties(part_properties, part_schema): return properties -def _merged_part_and_plugin_schemas(part_schema, plugin_schema): +def _merged_part_and_plugin_schemas(part_schema, definitions_schema, + plugin_schema): plugin_schema = plugin_schema.copy() if 'properties' not in plugin_schema: plugin_schema['properties'] = {} + if 'definitions' not in plugin_schema: + plugin_schema['definitions'] = {} + # The part schema takes precedence over the plugin's schema. plugin_schema['properties'].update(part_schema) + plugin_schema['definitions'].update(definitions_schema) + return plugin_schema -def _validate_pull_and_build_properties(plugin_name, plugin, part_schema): +def _validate_pull_and_build_properties(plugin_name, plugin, part_schema, + definitions_schema): merged_schema = _merged_part_and_plugin_schemas( - part_schema, plugin.schema()) + part_schema, definitions_schema, plugin.schema()) merged_properties = merged_schema['properties'] # First, validate pull properties @@ -754,12 +755,13 @@ def _validate_step_properties(step_properties, schema_properties): return invalid_properties -def _make_options(part_schema, properties, plugin_schema): +def _make_options(part_schema, definitions_schema, properties, plugin_schema): # Make copies as these dictionaries are tampered with part_schema = part_schema.copy() properties = properties.copy() - plugin_schema = _merged_part_and_plugin_schemas(part_schema, plugin_schema) + plugin_schema = _merged_part_and_plugin_schemas( + part_schema, definitions_schema, plugin_schema) # This is for backwards compatibility for when most of the # schema was overridable by the plugins. @@ -817,11 +819,14 @@ def _load_local(module_name, local_plugin_dir): def load_plugin(part_name, *, plugin_name, part_properties=None, - project_options=None, part_schema=None): + project_options=None, part_schema=None, + definitions_schema=None): if part_properties is None: part_properties = {} if part_schema is None: part_schema = {} + if definitions_schema is None: + definitions_schema = {} if project_options is None: project_options = snapcraft.ProjectOptions() logger.debug('Setting up part {!r} with plugin {!r} and ' @@ -832,7 +837,8 @@ def load_plugin(part_name, *, plugin_name, part_properties=None, part_name=part_name, part_properties=part_properties, project_options=project_options, - part_schema=part_schema) + part_schema=part_schema, + definitions_schema=definitions_schema) def _migratable_filesets(fileset, srcdir): diff --git a/snapcraft/internal/pluginhandler/_stage_package_handler.py b/snapcraft/internal/pluginhandler/_stage_package_handler.py new file mode 100644 index 0000000000..f56f956dad --- /dev/null +++ b/snapcraft/internal/pluginhandler/_stage_package_handler.py @@ -0,0 +1,99 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# Copyright (C) 2017 Canonical Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging + +from snapcraft.internal import repo +from snapcraft.internal.pluginhandler import stage_package_grammar + +logger = logging.getLogger(__name__) + + +class StagePackageHandler: + """Interpret stage-packages grammar, and fetch/unpack stage-packages. + + Basic example: + >>> import os + >>> import tempfile + >>> with tempfile.TemporaryDirectory() as tmp: + ... cache_dir = os.path.join(tmp, 'cache') + ... unpack_dir = os.path.join(tmp, 'unpack') + ... handler = StagePackageHandler(['foo'], cache_dir) + ... handler.fetch() + ... handler.unpack(unpack_dir) + """ + + def __init__(self, stage_packages_grammar, cache_dir, *, sources=None, + project_options=None): + """Create new StagePackageHandler. + + :param list stage_packages: Unprocessed stage-packages grammar. + :param str cache_dir: Path to working directory. + :param str sources: Alternative sources list (host's sources are + default). + :param project_options: Instance of ProjectOptions to use for this + operation. + :type project_options: snapcraft.ProjectOptions + """ + + self._grammar = stage_packages_grammar + self._cache_dir = cache_dir + self._sources = sources + self._project_options = project_options + self.__stage_packages = None + self.__ubuntu = None + + @property + def _ubuntu(self): + if not self.__ubuntu: + self.__ubuntu = repo.Ubuntu( + self._cache_dir, sources=self._sources, + project_options=self._project_options) + + return self.__ubuntu + + @property + def _stage_packages(self): + # Comparing to None here since after calculation it may be an empty set + if self.__stage_packages is None: + self.__stage_packages = stage_package_grammar.process_grammar( + self._grammar, self._project_options, self._ubuntu) + + return self.__stage_packages + + def fetch(self): + """Fetch stage packages into cache. + + The stage packages will not be fetched if they're already present in + the cache. + """ + + if self._stage_packages: + logger.debug('Fetching stage-packages {!r}'.format( + self._stage_packages)) + self._ubuntu.get(self._stage_packages) + + def unpack(self, unpack_dir): + """Unpack fetched stage packages into directory. + + :param str unpack_dir: Path to directory in which stage packages will + be unpacked. + """ + + if self._stage_packages: + logger.debug('Unpacking stage-packages to {!r}'.format( + unpack_dir)) + self._ubuntu.unpack(unpack_dir) diff --git a/snapcraft/internal/pluginhandler/stage_package_grammar/__init__.py b/snapcraft/internal/pluginhandler/stage_package_grammar/__init__.py new file mode 100644 index 0000000000..89d66c2d9d --- /dev/null +++ b/snapcraft/internal/pluginhandler/stage_package_grammar/__init__.py @@ -0,0 +1,17 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# Copyright (C) 2017 Canonical Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from ._processor import process_grammar # noqa diff --git a/snapcraft/internal/pluginhandler/stage_package_grammar/_on.py b/snapcraft/internal/pluginhandler/stage_package_grammar/_on.py new file mode 100644 index 0000000000..e1d19f78ae --- /dev/null +++ b/snapcraft/internal/pluginhandler/stage_package_grammar/_on.py @@ -0,0 +1,139 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# Copyright (C) 2017 Canonical Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import re + +from . import process_grammar +from .errors import ( + OnStatementSyntaxError, + UnsatisfiedStatementError, +) + +_SELECTOR_PATTERN = re.compile(r'\Aon\s+([^,\s](?:,?[^,]+)*)\Z') +_WHITESPACE_PATTERN = re.compile(r'\A.*\s.*\Z') + + +class OnStatement: + """Process an 'on' statement in the stage packages grammar. + + For example: + >>> import tempfile + >>> from snapcraft import repo, ProjectOptions + >>> with tempfile.TemporaryDirectory() as cache_dir: + ... repo_instance = repo.Ubuntu(cache_dir) + ... options = ProjectOptions(target_deb_arch='i386') + ... clause = OnStatement(on='on amd64', body=['foo'], + ... project_options=options, + ... repo_instance=repo_instance) + ... clause.add_else(['bar']) + ... clause.process() + {'bar'} + """ + + def __init__(self, *, on, body, project_options, repo_instance): + """Create an _OnStatement instance. + + :param str on: The 'on ' part of the clause. + :param list body: The body of the 'on' clause. + :param project_options: Instance of ProjectOptions to use to process + clause. + :type project_options: snapcraft.ProjectOptions + :param repo_instance: repo.Ubuntu instance used for checking package + validity. + :type repo_instance: repo.Ubuntu + """ + + self.selectors = _extract_on_clause_selectors(on) + self._body = body + self._project_options = project_options + self._repo_instance = repo_instance + self._else_bodies = [] + + def add_else(self, else_body): + """Add an 'else' clause to the statement. + + :param list else_body: The body of an 'else' clause. + + The 'else' clauses will be processed in the order they are added. + """ + + self._else_bodies.append(else_body) + + def process(self): + """Process the clause. + + :return: Stage packages as determined by evaluating the statement. + :rtype: list + """ + + packages = set() + target_arch = self._project_options.deb_arch + + # The only selector currently supported is the target arch. Since + # selectors are matched with an AND, not OR, there should only be one + # selector. + if (len(self.selectors) == 1) and (target_arch in self.selectors): + packages = process_grammar( + self._body, self._project_options, self._repo_instance) + else: + for else_body in self._else_bodies: + if not else_body: + # Handle the 'else fail' case. + raise UnsatisfiedStatementError(self) + + packages = process_grammar( + else_body, self._project_options, self._repo_instance) + if packages: + break + + return packages + + def __eq__(self, other): + return self.selectors == other.selectors + + def __repr__(self): + return "'on {}'".format(','.join(sorted(self.selectors))) + + +def _extract_on_clause_selectors(on): + """Extract the list of selectors within an on clause. + + :param str on: The 'on >> _extract_on_clause_selectors('on amd64,i386') == {'amd64', 'i386'} + True + """ + + match = _SELECTOR_PATTERN.match(on) + + try: + selector_group = match.group(1) + except AttributeError: + raise OnStatementSyntaxError(on, message='selectors are missing') + except IndexError: + raise OnStatementSyntaxError(on) + + # This could be part of the _SELECTOR_PATTERN, but that would require us + # to provide a very generic error when we can try to be more helpful. + if _WHITESPACE_PATTERN.match(selector_group): + raise OnStatementSyntaxError( + on, message='spaces are not allowed in the selectors') + + return {selector.strip() for selector in selector_group.split(',')} diff --git a/snapcraft/internal/pluginhandler/stage_package_grammar/_processor.py b/snapcraft/internal/pluginhandler/stage_package_grammar/_processor.py new file mode 100644 index 0000000000..05a00c5631 --- /dev/null +++ b/snapcraft/internal/pluginhandler/stage_package_grammar/_processor.py @@ -0,0 +1,157 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# Copyright (C) 2017 Canonical Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import re + +from .errors import StagePackageSyntaxError + +_ON_CLAUSE_PATTERN = re.compile(r'\Aon\s+') +_TRY_CLAUSE_PATTERN = re.compile(r'\Atry\Z') +_ELSE_CLAUSE_PATTERN = re.compile(r'\Aelse\Z') +_ELSE_FAIL_PATTERN = re.compile(r'\Aelse\s+fail\Z') + + +def process_grammar(grammar, project_options, repo_instance): + """Process stage packages grammar and extract packages to actually stage. + + :param list grammar: Unprocessed stage-packages grammar. + :param project_options: Instance of ProjectOptions to use to determine + stage packages. + :type project_options: snapcraft.ProjectOptions + :param repo_instance: repo.Ubuntu instance used for checking package + validity. + :type repo_instance: repo.Ubuntu + + :return: Packages to stage + :rtype: set + """ + + packages = set() + statements = _StatementCollection() + statement = None + + for section in grammar: + if isinstance(section, str): + # If the secion is just a string, it's either "else fail" or a + # package name. + if _ELSE_FAIL_PATTERN.match(section): + _handle_else(statement, None) + else: + packages.add(section) + elif isinstance(section, dict): + statement = _parse_dict( + section, statement, statements, project_options, repo_instance) + else: + # jsonschema should never let us get here. + raise StagePackageSyntaxError( + "expected grammar section to be either of type 'str' or " + "type 'dict', but got {!r}".format(type(section))) + + # We've parsed the entire grammar, time to process it. + statements.add(statement) + packages |= statements.process_all() + + return packages + + +def _parse_dict(section, statement, statements, project_options, + repo_instance): + from ._on import OnStatement + from ._try import TryStatement + + for key, value in section.items(): + if _ON_CLAUSE_PATTERN.match(key): + # We've come across the begining of an 'on' statement. + # That means any previous statement we found is complete. + # The first time through this may be None, but the + # collection will ignore it. + statements.add(statement) + + statement = OnStatement( + on=key, body=value, project_options=project_options, + repo_instance=repo_instance) + + if _TRY_CLAUSE_PATTERN.match(key): + # We've come across the begining of a 'try' statement. + # That means any previous statement we found is complete. + # The first time through this may be None, but the + # collection will ignore it. + statements.add(statement) + + statement = TryStatement( + body=value, project_options=project_options, + repo_instance=repo_instance) + + if _ELSE_CLAUSE_PATTERN.match(key): + _handle_else(statement, value) + + return statement + + +def _handle_else(statement, else_body): + """Add else body to current statement. + + :param statement: The currently-active statement. If None it will be + ignored. + :param else_body: The body of the else clause to add. + + :raises StagePackageSyntaxError: If there isn't a currently-active + statement. + """ + + try: + statement.add_else(else_body) + except AttributeError: + raise StagePackageSyntaxError( + "'else' doesn't seem to correspond to an 'on' or " + "'try'") + + +class _StatementCollection: + """Unique collection of statements to run at a later time.""" + + def __init__(self): + self._statements = [] + + def add(self, statement): + """Add new statement to collection. + + :param statement: New statement. + + :raises StagePackageSyntaxError: If statement is already in collection. + """ + + if not statement: + return + + if statement in self._statements: + raise StagePackageSyntaxError( + "found duplicate {!r} statements. These should be " + 'merged.'.format(statement)) + + self._statements.append(statement) + + def process_all(self): + """Process all statements in collection. + + :return: Packages to stage as judged by all statements in collection. + :rtype: set + """ + packages = set() + for statement in self._statements: + packages |= statement.process() + + return packages diff --git a/snapcraft/internal/pluginhandler/stage_package_grammar/_try.py b/snapcraft/internal/pluginhandler/stage_package_grammar/_try.py new file mode 100644 index 0000000000..5016760a1c --- /dev/null +++ b/snapcraft/internal/pluginhandler/stage_package_grammar/_try.py @@ -0,0 +1,120 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# Copyright (C) 2017 Canonical Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from . import process_grammar + + +class TryStatement: + """Process a 'try' statement in the stage packages grammar. + + For example: + >>> import tempfile + >>> from snapcraft import repo, ProjectOptions + >>> with tempfile.TemporaryDirectory() as cache_dir: + ... repo_instance = repo.Ubuntu(cache_dir) + ... options = ProjectOptions(target_deb_arch='i386') + ... clause = TryStatement(body=['invalid'], project_options=options, + ... repo_instance=repo_instance) + ... clause.add_else(['valid']) + ... clause.process() + {'valid'} + """ + + def __init__(self, *, body, project_options, repo_instance): + """Create an _OnStatement instance. + + :param list body: The body of the 'try' clause. + :param project_options: Instance of ProjectOptions to use to process + clause. + :type project_options: snapcraft.ProjectOptions + :param repo_instance: repo.Ubuntu instance used for checking package + validity. + :type repo_instance: repo.Ubuntu + """ + + self._body = body + self._project_options = project_options + self._repo_instance = repo_instance + self._else_bodies = [] + + def add_else(self, else_body): + """Add an 'else' clause to the statement. + + :param list else_body: The body of an 'else' clause. + + The 'else' clauses will be processed in the order they are added. + """ + + self._else_bodies.append(else_body) + + def process(self): + """Process the clause. + + :return: Stage packages as determined by evaluating the statement. + :rtype: list + """ + + packages = process_grammar( + self._body, self._project_options, self._repo_instance) + + # If some of the packages in the 'try' were invalid, then we need to + # process the 'else' clauses. + if not _all_packages_valid(packages, self._repo_instance): + if not self._else_bodies: + # If there are no 'else' statements, the 'try' was considered + # optional and it failed, which means it doesn't resolve to + # any packages. + return set() + + for else_body in self._else_bodies: + if not else_body: + continue + + packages = process_grammar( + else_body, self._project_options, self._repo_instance) + + # Stop once an 'else' clause gives us valid packages + if _all_packages_valid(packages, self._repo_instance): + break + + return packages + + def __repr__(self): + return "'try'" + + +def _all_packages_valid(packages, repo_instance): + """Ensure that all packages are valid. + + :param packages: Iterable container of package names. + :param repo_instance: repo.Ubuntu instance to use for validity check. + :type repo_instance: repo.Ubuntu + + For example: + >>> import tempfile + >>> from snapcraft import repo, ProjectOptions + >>> with tempfile.TemporaryDirectory() as cache_dir: + ... ubuntu = repo.Ubuntu(cache_dir) + ... _all_packages_valid(['valid'], ubuntu) + ... _all_packages_valid(['valid', 'invalid'], ubuntu) + True + False + """ + + for package in packages: + if not repo_instance.is_valid(package): + return False + return True diff --git a/snapcraft/internal/pluginhandler/stage_package_grammar/errors.py b/snapcraft/internal/pluginhandler/stage_package_grammar/errors.py new file mode 100644 index 0000000000..676e21a4fa --- /dev/null +++ b/snapcraft/internal/pluginhandler/stage_package_grammar/errors.py @@ -0,0 +1,42 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# Copyright (C) 2017 Canonical Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from snapcraft.internal import errors + + +class StagePackageSyntaxError(errors.SnapcraftError): + + fmt = 'Invalid syntax for stage packages: {message}' + + def __init__(self, message): + super().__init__(message=message) + + +class OnStatementSyntaxError(StagePackageSyntaxError): + + def __init__(self, on_statement, *, message=None): + components = ["{!r} is not a valid 'on' clause".format(on_statement)] + if message: + components.append(message) + super().__init__(message=': '.join(components)) + + +class UnsatisfiedStatementError(errors.SnapcraftError): + + fmt = 'Unable to satisfy {statement!r}, failure forced' + + def __init__(self, statement): + super().__init__(statement=statement) diff --git a/snapcraft/internal/repo.py b/snapcraft/internal/repo.py index cab6f374f2..2ac1862b68 100644 --- a/snapcraft/internal/repo.py +++ b/snapcraft/internal/repo.py @@ -296,6 +296,10 @@ def __init__(self, rootdir, recommends=False, sources_list=sources, use_geoip=project_options.use_geoip) + def is_valid(self, package_name): + with self.apt.archive(self.rootdir, self.downloaddir) as apt_cache: + return package_name in apt_cache + def get(self, package_names): with self.apt.archive(self.rootdir, self.downloaddir) as apt_cache: self._get(apt_cache, package_names) diff --git a/snapcraft/tests/commands/test_clean.py b/snapcraft/tests/commands/test_clean.py index 296ba96655..0ef02f05f6 100644 --- a/snapcraft/tests/commands/test_clean.py +++ b/snapcraft/tests/commands/test_clean.py @@ -1,6 +1,6 @@ # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- # -# Copyright (C) 2015-2016 Canonical Ltd +# Copyright (C) 2015-2017 Canonical Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3 as @@ -54,12 +54,14 @@ def make_snapcraft_yaml(self, n=1, create=True): open('icon.png', 'w').close() parts = [] + validator = project_loader.Validator() for i in range(n): part_name = 'clean{}'.format(i) handler = pluginhandler.load_plugin( part_name, plugin_name='nil', part_properties={'plugin': 'nil'}, - part_schema=project_loader.Validator().part_schema) + part_schema=validator.part_schema, + definitions_schema=validator.definitions_schema) parts.append({ 'part_dir': handler.code.partdir, }) diff --git a/snapcraft/tests/commands/test_pull.py b/snapcraft/tests/commands/test_pull.py index f1ad4be05b..c53aff5cea 100644 --- a/snapcraft/tests/commands/test_pull.py +++ b/snapcraft/tests/commands/test_pull.py @@ -144,4 +144,4 @@ def test_pull_multiarch_stage_package(self, mock_unpack, mock_get): main(['pull', 'pull1']) - mock_get.assert_called_once_with(['mir:arch']) + mock_get.assert_called_once_with({'mir:arch'}) diff --git a/snapcraft/tests/pluginhandler/mocks.py b/snapcraft/tests/pluginhandler/mocks.py index 9e92361d37..9c6ca3d02f 100644 --- a/snapcraft/tests/pluginhandler/mocks.py +++ b/snapcraft/tests/pluginhandler/mocks.py @@ -52,9 +52,12 @@ def loadplugin(part_name, plugin_name=None, part_properties=None, if not project_options: project_options = snapcraft.ProjectOptions() - schema = project_loader.Validator().part_schema + validator = project_loader.Validator() + schema = validator.part_schema + definitions_schema = validator.definitions_schema return pluginhandler.load_plugin(part_name=part_name, plugin_name=plugin_name, part_properties=properties, project_options=project_options, - part_schema=schema) + part_schema=schema, + definitions_schema=definitions_schema) diff --git a/snapcraft/tests/pluginhandler/stage_package_grammar/__init__.py b/snapcraft/tests/pluginhandler/stage_package_grammar/__init__.py new file mode 100644 index 0000000000..aea3b6287a --- /dev/null +++ b/snapcraft/tests/pluginhandler/stage_package_grammar/__init__.py @@ -0,0 +1,38 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# Copyright (C) 2017 Canonical Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from unittest import mock + +from snapcraft import tests + + +class GrammarTestCase(tests.TestCase): + + def setUp(self): + super().setUp() + + patcher = mock.patch('snapcraft.repo.Ubuntu') + self.ubuntu_mock = patcher.start() + self.addCleanup(patcher.stop) + + self.get_mock = self.ubuntu_mock.return_value.get + self.unpack_mock = self.ubuntu_mock.return_value.unpack + self.is_valid_mock = self.ubuntu_mock.return_value.is_valid + + def _is_valid(package_name): + return 'invalid' not in package_name + + self.is_valid_mock.side_effect = _is_valid diff --git a/snapcraft/tests/pluginhandler/stage_package_grammar/test_on_statement.py b/snapcraft/tests/pluginhandler/stage_package_grammar/test_on_statement.py new file mode 100644 index 0000000000..b0bfe55936 --- /dev/null +++ b/snapcraft/tests/pluginhandler/stage_package_grammar/test_on_statement.py @@ -0,0 +1,256 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# Copyright (C) 2017 Canonical Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import doctest +from unittest import mock +import testtools +from testtools.matchers import Equals + +import snapcraft +import snapcraft.internal.pluginhandler.stage_package_grammar as grammar +import snapcraft.internal.pluginhandler.stage_package_grammar._on as on + +from . import GrammarTestCase + + +def load_tests(loader, tests, ignore): + patcher = mock.patch('snapcraft.repo.Ubuntu') + + def _setup(test): + patcher.start() + + def _teardown(test): + patcher.stop() + + tests.addTests(doctest.DocTestSuite( + on, setUp=_setup, tearDown=_teardown)) + return tests + + +class OnStatementGrammarTestCase(GrammarTestCase): + + scenarios = [ + ('on amd64', { + 'on': 'on amd64', + 'body': ['foo'], + 'else_bodies': [], + 'target_arch': 'amd64', + 'expected_packages': {'foo'} + }), + ('on i386', { + 'on': 'on amd64', + 'body': ['foo'], + 'else_bodies': [], + 'target_arch': 'i386', + 'expected_packages': set() + }), + ('ignored else', { + 'on': 'on amd64', + 'body': ['foo'], + 'else_bodies': [ + ['bar'] + ], + 'target_arch': 'amd64', + 'expected_packages': {'foo'} + }), + ('used else', { + 'on': 'on amd64', + 'body': ['foo'], + 'else_bodies': [ + ['bar'] + ], + 'target_arch': 'i386', + 'expected_packages': {'bar'} + }), + ('third else ignored', { + 'on': 'on amd64', + 'body': ['foo'], + 'else_bodies': [ + ['bar'], + ['baz'] + ], + 'target_arch': 'i386', + 'expected_packages': {'bar'} + }), + ('third else followed', { + 'on': 'on amd64', + 'body': ['foo'], + 'else_bodies': [ + [{'on armhf': ['bar']}], + ['baz'] + ], + 'target_arch': 'i386', + 'expected_packages': {'baz'} + }), + ('nested amd64', { + 'on': 'on amd64', + 'body': [ + {'on amd64': ['foo']}, + {'on i386': ['bar']}, + ], + 'else_bodies': [], + 'target_arch': 'amd64', + 'expected_packages': {'foo'} + }), + ('nested i386', { + 'on': 'on i386', + 'body': [ + {'on amd64': ['foo']}, + {'on i386': ['bar']}, + ], + 'else_bodies': [], + 'target_arch': 'i386', + 'expected_packages': {'bar'} + }), + ('nested body ignored else', { + 'on': 'on amd64', + 'body': [ + {'on amd64': ['foo']}, + {'else': ['bar']}, + ], + 'else_bodies': [], + 'target_arch': 'amd64', + 'expected_packages': {'foo'} + }), + ('nested body used else', { + 'on': 'on i386', + 'body': [ + {'on amd64': ['foo']}, + {'else': ['bar']}, + ], + 'else_bodies': [], + 'target_arch': 'i386', + 'expected_packages': {'bar'} + }), + ('nested else ignored else', { + 'on': 'on armhf', + 'body': ['foo'], + 'else_bodies': [ + [ + {'on amd64': ['bar']}, + {'else': ['baz']}, + ], + ], + 'target_arch': 'amd64', + 'expected_packages': {'bar'} + }), + ('nested else used else', { + 'on': 'on armhf', + 'body': ['foo'], + 'else_bodies': [ + [ + {'on amd64': ['bar']}, + {'else': ['baz']}, + ], + ], + 'target_arch': 'i386', + 'expected_packages': {'baz'} + }), + ] + + def test_on_statement_grammar(self): + options = snapcraft.ProjectOptions(target_deb_arch=self.target_arch) + statement = on.OnStatement( + on=self.on, body=self.body, project_options=options, + repo_instance=snapcraft.repo.Ubuntu()) + + for else_body in self.else_bodies: + statement.add_else(else_body) + + self.assertThat(statement.process(), Equals(self.expected_packages)) + + +class OnStatementInvalidGrammarTestCase(GrammarTestCase): + + scenarios = [ + ('spaces in selectors', { + 'on': 'on amd64, ubuntu', + 'body': ['foo'], + 'else_bodies': [], + 'target_arch': 'amd64', + 'expected_exception': + ".*not a valid 'on' clause.*spaces are not allowed in the " + 'selectors.*', + }), + ('beginning with comma', { + 'on': 'on ,amd64', + 'body': ['foo'], + 'else_bodies': [], + 'target_arch': 'amd64', + 'expected_exception': ".*not a valid 'on' clause", + }), + ('ending with comma', { + 'on': 'on amd64,', + 'body': ['foo'], + 'else_bodies': [], + 'target_arch': 'amd64', + 'expected_exception': ".*not a valid 'on' clause", + }), + ('multiple commas', { + 'on': 'on amd64,,ubuntu', + 'body': ['foo'], + 'else_bodies': [], + 'target_arch': 'amd64', + 'expected_exception': ".*not a valid 'on' clause", + }), + ('invalid selector format', { + 'on': 'on', + 'body': ['foo'], + 'else_bodies': [], + 'target_arch': 'amd64', + 'expected_exception': + ".*not a valid 'on' clause.*selectors are missing", + }), + ('not even close', { + 'on': 'im-invalid', + 'body': ['foo'], + 'else_bodies': [], + 'target_arch': 'amd64', + 'expected_exception': ".*not a valid 'on' clause", + }), + ] + + def test_on_statement_invalid_grammar(self): + with testtools.ExpectedException( + grammar.errors.OnStatementSyntaxError, + self.expected_exception): + options = snapcraft.ProjectOptions( + target_deb_arch=self.target_arch) + statement = on.OnStatement( + on=self.on, body=self.body, project_options=options, + repo_instance=snapcraft.repo.Ubuntu()) + + for else_body in self.else_bodies: + statement.add_else(else_body) + + statement.process() + + +class OnStatementElseFail(GrammarTestCase): + + def test_else_fail(self): + options = snapcraft.ProjectOptions( + target_deb_arch='amd64') + statement = on.OnStatement( + on='on i386', body=['foo'], project_options=options, + repo_instance=snapcraft.repo.Ubuntu()) + + statement.add_else(None) + + with testtools.ExpectedException( + grammar.errors.UnsatisfiedStatementError, + "Unable to satisfy 'on i386', failure forced"): + statement.process() diff --git a/snapcraft/tests/pluginhandler/stage_package_grammar/test_processor.py b/snapcraft/tests/pluginhandler/stage_package_grammar/test_processor.py new file mode 100644 index 0000000000..12f8197062 --- /dev/null +++ b/snapcraft/tests/pluginhandler/stage_package_grammar/test_processor.py @@ -0,0 +1,237 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# Copyright (C) 2017 Canonical Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import testtools +from testtools.matchers import Equals + +import snapcraft +import snapcraft.internal.pluginhandler.stage_package_grammar as grammar + +from . import GrammarTestCase + + +class GrammarOnDuplicatesTestCase(GrammarTestCase): + + scenarios = [ + ('same order', { + 'grammar': [ + {'on amd64,i386': ['foo']}, + {'on amd64,i386': ['bar']}, + ]}), + ('different order', { + 'grammar': [ + {'on amd64,i386': ['foo']}, + {'on i386,amd64': ['bar']}, + ]}), + ] + + def test_on_duplicates_raises(self): + """Test that multiple identical selector sets is an error.""" + + with testtools.ExpectedException( + grammar.errors.StagePackageSyntaxError, + "Invalid syntax for stage packages: found duplicate 'on " + "amd64,i386' statements. These should be merged."): + grammar.process_grammar( + self.grammar, snapcraft.ProjectOptions(), + snapcraft.repo.Ubuntu()) + + +class BasicGrammarTestCase(GrammarTestCase): + + scenarios = [ + ('unconditional', { + 'grammar': [ + 'foo', + 'bar', + ], + 'target_arch': 'amd64', + 'expected_packages': {'foo', 'bar'} + }), + ('mixed including', { + 'grammar': [ + 'foo', + {'on i386': ['bar']} + ], + 'target_arch': 'i386', + 'expected_packages': {'foo', 'bar'} + }), + ('mixed excluding', { + 'grammar': [ + 'foo', + {'on i386': ['bar']} + ], + 'target_arch': 'amd64', + 'expected_packages': {'foo'} + }), + ('on amd64', { + 'grammar': [ + {'on amd64': ['foo']}, + {'on i386': ['bar']}, + ], + 'target_arch': 'amd64', + 'expected_packages': {'foo'} + }), + ('on i386', { + 'grammar': [ + {'on amd64': ['foo']}, + {'on i386': ['bar']}, + ], + 'target_arch': 'i386', + 'expected_packages': {'bar'} + }), + ('ignored else', { + 'grammar': [ + {'on amd64': ['foo']}, + {'else': ['bar']}, + ], + 'target_arch': 'amd64', + 'expected_packages': {'foo'} + }), + ('used else', { + 'grammar': [ + {'on amd64': ['foo']}, + {'else': ['bar']}, + ], + 'target_arch': 'i386', + 'expected_packages': {'bar'} + }), + ('nested amd64', { + 'grammar': [ + {'on amd64': [ + {'on amd64': ['foo']}, + {'on i386': ['bar']}, + ]}, + ], + 'target_arch': 'amd64', + 'expected_packages': {'foo'} + }), + ('nested i386', { + 'grammar': [ + {'on i386': [ + {'on amd64': ['foo']}, + {'on i386': ['bar']}, + ]}, + ], + 'target_arch': 'i386', + 'expected_packages': {'bar'} + }), + ('nested ignored else', { + 'grammar': [ + {'on amd64': [ + {'on amd64': ['foo']}, + {'else': ['bar']}, + ]}, + ], + 'target_arch': 'amd64', + 'expected_packages': {'foo'} + }), + ('nested used else', { + 'grammar': [ + {'on i386': [ + {'on amd64': ['foo']}, + {'else': ['bar']}, + ]}, + ], + 'target_arch': 'i386', + 'expected_packages': {'bar'} + }), + ('try', { + 'grammar': [ + {'try': ['valid']}, + ], + 'target_arch': 'amd64', + 'expected_packages': {'valid'} + }), + ('try else', { + 'grammar': [ + {'try': ['invalid']}, + {'else': ['valid']}, + ], + 'target_arch': 'amd64', + 'expected_packages': {'valid'} + }), + ('nested try', { + 'grammar': [ + {'on amd64': [ + {'try': ['foo']}, + {'else': ['bar']}, + ]}, + ], + 'target_arch': 'amd64', + 'expected_packages': {'foo'} + }), + ('nested try else', { + 'grammar': [ + {'on i386': [ + {'try': ['invalid']}, + {'else': ['bar']}, + ]}, + ], + 'target_arch': 'i386', + 'expected_packages': {'bar'} + }), + ('optional', { + 'grammar': [ + 'foo', + {'try': ['invalid']}, + ], + 'target_arch': 'amd64', + 'expected_packages': {'foo'} + }), + ] + + def test_basic_grammar(self): + options = snapcraft.ProjectOptions(target_deb_arch=self.target_arch) + self.assertThat( + grammar.process_grammar( + self.grammar, options, snapcraft.repo.Ubuntu()), + Equals(self.expected_packages)) + + +class InvalidGrammarTestCase(GrammarTestCase): + + scenarios = [ + ('unmatched else', { + 'grammar': [ + {'else': ['foo']} + ], + 'target_arch': 'amd64', + 'expected_exception': ".*'else' doesn't seem to correspond.*", + }), + ('unmatched else fail', { + 'grammar': [ + 'else fail' + ], + 'target_arch': 'amd64', + 'expected_exception': ".*'else' doesn't seem to correspond.*", + }), + ('unexpected type', { + 'grammar': [ + 5, + ], + 'target_arch': 'amd64', + 'expected_exception': ".*expected grammar section.*but got.*", + }), + ] + + def test_invalid_grammar(self): + with testtools.ExpectedException( + grammar.errors.StagePackageSyntaxError, + self.expected_exception): + grammar.process_grammar( + self.grammar, snapcraft.ProjectOptions(), + snapcraft.repo.Ubuntu()) diff --git a/snapcraft/tests/pluginhandler/stage_package_grammar/test_try_statement.py b/snapcraft/tests/pluginhandler/stage_package_grammar/test_try_statement.py new file mode 100644 index 0000000000..2e42f77809 --- /dev/null +++ b/snapcraft/tests/pluginhandler/stage_package_grammar/test_try_statement.py @@ -0,0 +1,140 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# Copyright (C) 2017 Canonical Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import doctest +from unittest import mock +from testtools.matchers import Equals + +import snapcraft +import snapcraft.internal.pluginhandler.stage_package_grammar._try as _try + +from . import GrammarTestCase + + +def load_tests(loader, tests, ignore): + patcher = mock.patch('snapcraft.repo.Ubuntu') + + def _setup(test): + ubuntu_mock = patcher.start() + + def _is_valid(package_name): + return 'invalid' not in package_name + + ubuntu_mock.return_value.is_valid.side_effect = _is_valid + + def _teardown(test): + patcher.stop() + + tests.addTests(doctest.DocTestSuite( + _try, setUp=_setup, tearDown=_teardown)) + return tests + + +class TryStatementGrammarTestCase(GrammarTestCase): + + scenarios = [ + ('followed body', { + 'body': ['foo', 'bar'], + 'else_bodies': [], + 'expected_packages': {'foo', 'bar'} + }), + ('followed else', { + 'body': ['invalid'], + 'else_bodies': [ + ['valid'] + ], + 'expected_packages': {'valid'} + }), + ('optional without else', { + 'body': ['invalid'], + 'else_bodies': [], + 'expected_packages': set(), + }), + ('followed chained else', { + 'body': ['invalid1'], + 'else_bodies': [ + ['invalid2'], + ['finally-valid'] + ], + 'expected_packages': {'finally-valid'} + }), + ('nested body followed body', { + 'body': [ + {'try': ['foo']}, + {'else': ['bar']}, + ], + 'else_bodies': [], + 'expected_packages': {'foo'} + }), + ('nested body followed else', { + 'body': [ + {'try': ['invalid']}, + {'else': ['bar']}, + ], + 'else_bodies': [], + 'expected_packages': {'bar'} + }), + ('nested else followed body', { + 'body': ['invalid'], + 'else_bodies': [ + [{'try': ['foo']}, {'else': ['bar']}], + ], + 'expected_packages': {'foo'} + }), + ('nested else followed else', { + 'body': ['invalid'], + 'else_bodies': [ + [{'try': ['invalid']}, {'else': ['bar']}], + ], + 'expected_packages': {'bar'} + }), + ('multiple elses', { + 'body': ['invalid1'], + 'else_bodies': [ + ['invalid2'], + ['valid'] + ], + 'expected_packages': {'valid'} + }), + ('multiple elses all invalid', { + 'body': ['invalid1'], + 'else_bodies': [ + ['invalid2'], + ['invalid3'] + ], + 'expected_packages': {'invalid3'} + }), + ('empty else', { + 'body': ['invalid'], + 'else_bodies': [[]], + 'expected_packages': {'invalid'} + }), + ('empty else followed by else', { + 'body': ['invalid'], + 'else_bodies': [[], ['valid']], + 'expected_packages': {'valid'} + }), + ] + + def test_try_statement_grammar(self): + statement = _try.TryStatement( + body=self.body, project_options=snapcraft.ProjectOptions(), + repo_instance=snapcraft.repo.Ubuntu()) + + for else_body in self.else_bodies: + statement.add_else(else_body) + + self.assertThat(statement.process(), Equals(self.expected_packages)) diff --git a/snapcraft/tests/pluginhandler/test_stage_package_handler.py b/snapcraft/tests/pluginhandler/test_stage_package_handler.py new file mode 100644 index 0000000000..f7aa20a202 --- /dev/null +++ b/snapcraft/tests/pluginhandler/test_stage_package_handler.py @@ -0,0 +1,90 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# Copyright (C) 2017 Canonical Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import doctest +from unittest import mock + +import snapcraft + +from snapcraft.internal.pluginhandler.stage_package_grammar import ( + process_grammar +) + +from snapcraft.internal.pluginhandler._stage_package_handler import ( + StagePackageHandler +) + +from snapcraft import tests + + +def load_tests(loader, tests, ignore): + patcher = mock.patch('snapcraft.repo.Ubuntu') + + def _setup(test): + patcher.start() + + def _teardown(test): + patcher.stop() + + tests.addTests(doctest.DocTestSuite( + snapcraft.internal.pluginhandler._stage_package_handler, + setUp=_setup, tearDown=_teardown)) + return tests + + +class StagePackageHandlerTestCase(tests.TestCase): + + def setUp(self): + super().setUp() + + patcher = mock.patch( + 'snapcraft.internal.pluginhandler.stage_package_grammar.' + 'process_grammar') + self.process_grammar_mock = patcher.start() + self.process_grammar_mock.side_effect = process_grammar + self.addCleanup(patcher.stop) + + patcher = mock.patch('snapcraft.repo.Ubuntu') + self.ubuntu_mock = patcher.start() + self.addCleanup(patcher.stop) + + self.get_mock = self.ubuntu_mock.return_value.get + self.unpack_mock = self.ubuntu_mock.return_value.unpack + self.is_valid_mock = self.ubuntu_mock.return_value.is_valid + + self.cache_dir = os.path.join(os.getcwd(), 'cache') + self.unpack_dir = os.path.join(os.getcwd(), 'unpack') + + def test_fetch(self): + handler = StagePackageHandler(['foo'], self.cache_dir) + + handler.fetch() + + self.process_grammar_mock.assert_called_once_with( + ['foo'], mock.ANY, mock.ANY) + self.get_mock.assert_called_once_with({'foo'}) + self.unpack_mock.assert_not_called() + + def test_unpack(self): + handler = StagePackageHandler(['foo'], self.cache_dir) + + handler.unpack(self.unpack_dir) + + self.process_grammar_mock.assert_called_once_with( + ['foo'], mock.ANY, mock.ANY) + self.get_mock.assert_not_called() + self.unpack_mock.assert_called_with(self.unpack_dir) diff --git a/snapcraft/tests/test_project_loader.py b/snapcraft/tests/test_project_loader.py index 3b7ac05af0..4ff6bf0c48 100644 --- a/snapcraft/tests/test_project_loader.py +++ b/snapcraft/tests/test_project_loader.py @@ -45,7 +45,9 @@ def setUp(self): self.mock_get_yaml.return_value = os.path.join( 'snap', 'snapcraft.yaml') self.addCleanup(patcher.stop) - self.part_schema = project_loader.Validator().part_schema + validator = project_loader.Validator() + self.part_schema = validator.part_schema + self.definitions_schema = validator.definitions_schema self.deb_arch = snapcraft.ProjectOptions().deb_arch @@ -319,7 +321,8 @@ def load_effect(*args, **kwargs): 'plugin': 'autotools', 'stage': [], 'prime': [], 'snap': [], 'source': 'http://curl.org'}, project_options=project_options, - part_schema=self.part_schema) + part_schema=self.part_schema, + definitions_schema=self.definitions_schema) call2 = unittest.mock.call( 'part1', plugin_name='go', @@ -327,7 +330,8 @@ def load_effect(*args, **kwargs): 'plugin': 'go', 'stage': [], 'prime': [], 'snap': [], 'stage-packages': ['fswebcam']}, project_options=project_options, - part_schema=self.part_schema) + part_schema=self.part_schema, + definitions_schema=self.definitions_schema) mock_load.assert_has_calls([call1, call2], any_order=True)