diff --git a/.gitignore b/.gitignore index d60cb3e..342e783 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,6 @@ docs/_build/ # PyBuilder target/ + +# Downloaded by test.sh +get-pip.py diff --git a/dockerfile_parse/parser.py b/dockerfile_parse/parser.py index 86334b6..212e1fb 100644 --- a/dockerfile_parse/parser.py +++ b/dockerfile_parse/parser.py @@ -15,59 +15,71 @@ import re from contextlib import contextmanager from six import string_types +from six.moves import shlex_quote as quote from .constants import DOCKERFILE_FILENAME, COMMENT_INSTRUCTION -from .util import (b2u, extract_labels_or_envs, get_key_val_dictionary, +from .util import (b2u, extract_key_values, get_key_val_dictionary, u2b, Context, WordSplitter) -try: - # py3 - from shlex import quote -except ImportError: - from pipes import quote - logger = logging.getLogger(__name__) -class Labels(dict): +class KeyValues(dict): """ - A class for allowing direct write access to Dockerfile labels, e.g.: + Abstract base class for allowing direct write access to Dockerfile + instructions which result in a set of key value pairs. - parser.labels['label'] = 'value' + Subclasses must override the `parser_attr` value. """ + parser_attr = None - def __init__(self, labels, parser): - super(Labels, self).__init__(labels) + def __init__(self, key_values, parser): + super(KeyValues, self).__init__(key_values) self.parser = parser def __delitem__(self, key): - super(Labels, self).__delitem__(key) - self.parser.labels = dict(self) + super(KeyValues, self).__delitem__(key) + setattr(self.parser, self.parser_attr, dict(self)) def __setitem__(self, key, value): - super(Labels, self).__setitem__(key, value) - self.parser.labels = dict(self) + super(KeyValues, self).__setitem__(key, value) + setattr(self.parser, self.parser_attr, dict(self)) + + def __eq__(self, other): + if not isinstance(other, dict): + return False + return dict(self) == other + + def __hash__(self): + return hash(json.dumps(self, separators=(',', ':'), sort_keys=True)) -class Envs(dict): +class Labels(KeyValues): + """ + A class for allowing direct write access to Dockerfile labels, e.g.: + + parser.labels['label'] = 'value' + """ + parser_attr = 'labels' + + +class Envs(KeyValues): """ A class for allowing direct write access to Dockerfile env. vars., e.g.: parser.envs['variable_name'] = 'value' """ + parser_attr = 'envs' - def __init__(self, envs, parser): - super(Envs, self).__init__(envs) - self.parser = parser - def __delitem__(self, key): - super(Envs, self).__delitem__(key) - self.parser.envs = dict(self) +class Args(KeyValues): + """ + A class for allowing direct write access to Dockerfile build args, e.g.: - def __setitem__(self, key, value): - super(Envs, self).__setitem__(key, value) - self.parser.envs = dict(self) + parser.args['variable_name'] = 'value' + """ + parser_attr = 'args' class DockerfileParser(object): @@ -75,14 +87,17 @@ def __init__(self, path=None, cache_content=False, env_replace=True, parent_env=None, - fileobj=None): + fileobj=None, + build_args=None): """ Initialize source of Dockerfile :param path: path to (directory with) Dockerfile :param cache_content: cache Dockerfile content inside DockerfileParser + :param env_replace: return content with variables replaced :param parent_env: python dict of inherited env vars from parent image :param fileobj: seekable file-like object containing Dockerfile content as bytes (will be truncated on write) + :param build_args: python dict of build args used when building image """ self.fileobj = fileobj @@ -119,6 +134,13 @@ def __init__(self, path=None, logger.debug("Setting inherited parent image ENV vars: %s", parent_env) self.parent_env = parent_env + if build_args is None: + self.build_args = {} + else: + assert isinstance(build_args, dict) + logger.debug("Setting build args: %s", build_args) + self.build_args = build_args + @contextmanager def _open_dockerfile(self, mode): if self.fileobj is not None: @@ -298,13 +320,26 @@ def parent_images(self): """ :return: list of parent images -- one image per each stage's FROM instruction """ + in_stage = False + top_args = {} parents = [] for instr in self.structure: - if instr['instruction'] != 'FROM': - continue - image, _ = image_from(instr['value']) - if image is not None: - parents.append(image) + if instr['instruction'] == 'ARG': + if not in_stage: + key_val_list = extract_key_values( + env_replace=False, + args={}, envs={}, + instruction_value=instr['value']) + for key, value in key_val_list: + if key in self.build_args: + value = self.build_args[key] + top_args[key] = value + elif instr['instruction'] == 'FROM': + in_stage = True + image, _ = image_from(instr['value']) + if image is not None: + image = WordSplitter(image, args=top_args).dequote() + parents.append(image) return parents @parent_images.setter @@ -324,7 +359,7 @@ def parent_images(self, parents): continue old_image, stage = image_from(instr['value']) - if not old_image: + if old_image is None: continue # broken FROM, fixing would just confuse things if not parents: raise RuntimeError("not enough parents to match build stages") @@ -360,7 +395,14 @@ def baseimage(self, new_image): """ change image of final stage FROM instruction """ - images = self.parent_images or [None] + images = [] + for instr in self.structure: + if instr['instruction'] == 'FROM': + image, _ = image_from(instr['value']) + if image is not None: + images.append(image) + if not images: + raise RuntimeError('No stage defined to set base image on') images[-1] = new_image self.parent_images = images @@ -414,30 +456,55 @@ def envs(self): """ return self._instruction_getter('ENV', env_replace=self.env_replace) + @property + def args(self): + """ + ARGs from Dockerfile + :return: dictionary of arg_var_name:value (value might be '') + """ + return self._instruction_getter('ARG', env_replace=self.env_replace) + def _instruction_getter(self, name, env_replace): """ - Get LABEL or ENV instructions with environment replacement + Get LABEL or ENV or ARG instructions with environment replacement - :param name: e.g. 'LABEL' or 'ENV' + :param name: e.g. 'LABEL' or 'ENV' or 'ARG' :param env_replace: bool, whether to perform ENV substitution :return: Labels instance or Envs instance """ - if name != 'LABEL' and name != 'ENV': - raise ValueError("Unsupported instruction '%s'" % name) + if name not in ('LABEL', 'ENV', 'ARG'): + raise ValueError("Unsupported instruction '{0}'".format(name)) + in_stage = False + top_args = {} instructions = {} + args = {} envs = {} for instruction_desc in self.structure: this_instruction = instruction_desc['instruction'] if this_instruction == 'FROM': + in_stage = True instructions.clear() + args = {} envs = self.parent_env.copy() - elif this_instruction in (name, 'ENV'): + elif this_instruction in (name, 'ENV', 'ARG'): logger.debug("%s value: %r", name.lower(), instruction_desc['value']) - key_val_list = extract_labels_or_envs(env_replace=env_replace, - envs=envs, - instruction_value=instruction_desc['value']) + key_val_list = extract_key_values( + env_replace=this_instruction != 'ARG' and env_replace, + args=args, envs=envs, + instruction_value=instruction_desc['value']) for key, value in key_val_list: + if this_instruction == 'ARG': + if in_stage: + if key in top_args: + value = top_args[key] + elif key in self.build_args: + value = self.build_args[key] + args[key] = value + else: + if key in self.build_args: + value = self.build_args[key] + top_args[key] = value if this_instruction == name: instructions[key] = value logger.debug("new %s %r=%r", name.lower(), key, value) @@ -445,7 +512,12 @@ def _instruction_getter(self, name, env_replace): envs[key] = value logger.debug("instructions: %r", instructions) - return Labels(instructions, self) if name == 'LABEL' else Envs(instructions, self) + if name == 'LABEL': + return Labels(instructions, self) + elif name == 'ENV': + return Envs(instructions, self) + else: + return Args(instructions, self) @labels.setter def labels(self, labels): @@ -463,6 +535,14 @@ def envs(self, envs): """ self._instructions_setter('ENV', envs) + @args.setter + def args(self, args): + """ + Setter for ARG instruction, i.e. sets ARGs per input param. + :param args: dictionary of arg names & values to be set + """ + self._instructions_setter('ARG', args) + def _instructions_setter(self, name, instructions): if not isinstance(instructions, dict): raise TypeError('instructions needs to be a dictionary {name: value}') @@ -471,6 +551,10 @@ def _instructions_setter(self, name, instructions): existing = self.labels elif name == 'ENV': existing = self.envs + elif name == 'ARG': + existing = self.args + else: + raise ValueError("Unexpected instruction '%s'" % name) logger.debug("setting %s instructions: %r", name, instructions) @@ -496,6 +580,9 @@ def _modify_instruction_label(self, label_key, instr_value): def _modify_instruction_env(self, env_var_key, env_var_value): self._modify_instruction_label_env('ENV', env_var_key, env_var_value) + def _modify_instruction_arg(self, arg_key, arg_value): + self._modify_instruction_label_env('ARG', arg_key, arg_value) + def _modify_instruction_label_env(self, instruction, instr_key, instr_value): """ set instr_key to instr_value @@ -507,6 +594,8 @@ def _modify_instruction_label_env(self, instruction, instr_key, instr_value): instructions = self.labels elif instruction == 'ENV': instructions = self.envs + elif instruction == 'ARG': + instructions = self.args else: raise ValueError("Unknown instruction '%s'" % instruction) @@ -526,8 +615,8 @@ def _modify_instruction_label_env(self, instruction, instr_key, instr_value): for candidate in candidates: words = list(WordSplitter(candidate['value']).split(dequote=False)) - # LABEL/ENV syntax is one of two types: - if '=' not in words[0]: # LABEL/ENV name value + # LABEL/ENV/ARG syntax is one of two types: + if '=' not in words[0]: # LABEL/ENV/ARG name value # Remove quotes from key name and see if it's the one # we're looking for. if WordSplitter(words[0]).dequote() == instr_key: @@ -544,7 +633,7 @@ def _modify_instruction_label_env(self, instruction, instr_key, instr_value): startline = candidate['startline'] endline = candidate['endline'] break - else: # LABEL/ENV "name"="value" + else: # LABEL/ENV/ARG "name"="value" for index, token in enumerate(words): key, _ = token.split("=", 1) if WordSplitter(key).dequote() == instr_key: @@ -589,6 +678,9 @@ def _delete_instructions(self, instruction, value=None): if instruction == 'ENV' and value: self._modify_instruction_env(value, None) return + if instruction == 'ARG' and value: + self._modify_instruction_arg(value, None) + return lines = self.lines deleted = False @@ -606,7 +698,7 @@ def _add_instruction(self, instruction, value): :param instruction: instruction name to be added :param value: instruction value """ - if (instruction == 'LABEL' or instruction == 'ENV') and len(value) == 2: + if instruction in ('LABEL', 'ENV', 'ARG') and len(value) == 2: new_line = instruction + ' ' + '='.join(map(quote, value)) + '\n' else: new_line = '{0} {1}\n'.format(instruction, value) @@ -708,23 +800,42 @@ def add_lines_at(self, anchor, *lines, **kwargs): def context_structure(self): """ :return: list of Context objects - (Contains info about labels and environment variables for each line.) + (Contains info about build arguments, labels, and environment variables for each line.) """ + in_stage = False + top_args = {} instructions = [] last_context = Context() for instr in self.structure: instruction_type = instr['instruction'] if instruction_type == "FROM": # reset per stage - last_context = Context() + in_stage = True + last_context = Context(envs=dict(self.parent_env)) - context = Context(envs=dict(last_context.envs), + context = Context(args=dict(last_context.args), + envs=dict(last_context.envs), labels=dict(last_context.labels)) - if instruction_type in ["ENV", "LABEL"]: - val = get_key_val_dictionary(instruction_value=instr['value'], - env_replace=self.env_replace, - envs=last_context.envs) - context.set_line_value(context_type=instruction_type, value=val) + if instruction_type in ('ARG', 'ENV', 'LABEL'): + values = get_key_val_dictionary( + instruction_value=instr['value'], + env_replace=instruction_type != 'ARG' and self.env_replace, + args=last_context.args, + envs=last_context.envs) + if instruction_type == 'ARG' and self.env_replace: + if in_stage: + for key in list(values.keys()): + if key in top_args: + values[key] = top_args[key] + elif key in self.build_args: + values[key] = self.build_args[key] + else: + for key, value in list(values.items()): + if key in self.build_args: + value = self.build_args[key] + top_args[key] = value + values[key] = value + context.set_line_value(context_type=instruction_type, value=values) instructions.append(context) last_context = context diff --git a/dockerfile_parse/util.py b/dockerfile_parse/util.py index fcdff41..0992f9c 100644 --- a/dockerfile_parse/util.py +++ b/dockerfile_parse/util.py @@ -46,13 +46,16 @@ class WordSplitter(object): SQUOTE = "'" DQUOTE = '"' - def __init__(self, s, envs=None): + def __init__(self, s, args=None, envs=None): """ :param s: str, string to process + :param args: dict, build arguments to use; if None, do not + attempt substitution :param envs: dict, environment variables to use; if None, do not attempt substitution """ self.stream = StringIO(s) + self.args = args self.envs = envs # Initial state @@ -143,7 +146,7 @@ def append(self, s): return if (not self.escaped and - self.envs is not None and + (self.envs is not None or self.args is not None) and ch == '$' and self.quotes != self.SQUOTE): while True: @@ -168,10 +171,10 @@ def append(self, s): varname += ch - try: + if self.envs is not None and varname in self.envs: word.append(self.envs[varname]) - except KeyError: - pass + elif self.args is not None and varname in self.args: + word.append(self.args[varname]) # Check whether there is another envvar if ch != '$': @@ -210,13 +213,14 @@ def append(self, s): word.append(ch) -def extract_labels_or_envs(env_replace, envs, instruction_value): +def extract_key_values(env_replace, args, envs, instruction_value): words = list(WordSplitter(instruction_value).split(dequote=False)) key_val_list = [] def substitute_vars(val): kwargs = {} if env_replace: + kwargs['args'] = args kwargs['envs'] = envs return WordSplitter(val, **kwargs).dequote() @@ -248,65 +252,83 @@ def substitute_vars(val): return key_val_list -def get_key_val_dictionary(instruction_value, env_replace=False, envs=None): - envs = envs or [] - return dict(extract_labels_or_envs(instruction_value=instruction_value, - env_replace=env_replace, - envs=envs)) +def get_key_val_dictionary(instruction_value, env_replace=False, args=None, envs=None): + args = args or {} + envs = envs or {} + return dict(extract_key_values(instruction_value=instruction_value, + env_replace=env_replace, + args=args, envs=envs)) class Context(object): - def __init__(self, envs=None, labels=None, line_envs=None, line_labels=None): + def __init__(self, args=None, envs=None, labels=None, + line_args=None, line_envs=None, line_labels=None): """ - Class representing current state of environment variables and labels. + Class representing current state of build arguments, environment variables and labels. + :param args: dict with arguments valid for this line + (all variables defined to this line) :param envs: dict with variables valid for this line (all variables defined to this line) :param labels: dict with labels valid for this line (all labels defined to this line) + :param line_args: dict with arguments defined on this line :param line_envs: dict with variables defined on this line :param line_labels: dict with labels defined on this line """ + self.args = args or {} self.envs = envs or {} self.labels = labels or {} + self.line_args = line_args or {} self.line_envs = line_envs or {} self.line_labels = line_labels or {} def set_line_value(self, context_type, value): """ - Set value defined on this line ('line_envs'/'line_labels') - and update 'envs'/'labels'. + Set value defined on this line ('line_args'/'line_envs'/'line_labels') + and update 'args'/'envs'/'labels'. - :param context_type: "ENV" or "LABEL" + :param context_type: "ARG" or "ENV" or "LABEL" :param value: new value for this line """ - if context_type.upper() == "ENV": + if context_type.upper() == "ARG": + self.line_args = value + self.args.update(value) + elif context_type.upper() == "ENV": self.line_envs = value self.envs.update(value) elif context_type.upper() == "LABEL": self.line_labels = value self.labels.update(value) + else: + raise ValueError("Unexpected context type: " + context_type) def get_line_value(self, context_type): """ Get the values defined on this line. - :param context_type: "ENV" or "LABEL" + :param context_type: "ARG" or "ENV" or "LABEL" :return: values of given type defined on this line """ + if context_type.upper() == "ARG": + return self.line_args if context_type.upper() == "ENV": return self.line_envs - elif context_type.upper() == "LABEL": + if context_type.upper() == "LABEL": return self.line_labels + raise ValueError("Unexpected context type: " + context_type) def get_values(self, context_type): """ Get the values valid on this line. - :param context_type: "ENV" or "LABEL" + :param context_type: "ARG" or "ENV" or "LABEL" :return: values of given type valid on this line """ + if context_type.upper() == "ARG": + return self.args if context_type.upper() == "ENV": return self.envs - elif context_type.upper() == "LABEL": + if context_type.upper() == "LABEL": return self.labels + raise ValueError("Unexpected context type: " + context_type) diff --git a/python-dockerfile-parse.spec b/python-dockerfile-parse.spec index ae80d1a..e2f7e64 100644 --- a/python-dockerfile-parse.spec +++ b/python-dockerfile-parse.spec @@ -121,7 +121,7 @@ py.test-%{python3_version} -v tests * Tue Jun 02 2020 Robert Cerven 0.0.18-1 - new upstream release: 0.0.18 -* Fri Apr 24 2020 Martin Bašti 0.0.17-1 +* Fri Apr 24 2020 Martin Basti 0.0.17-1 - new upstream release: 0.0.17 * Tue Jan 21 2020 Robert Cerven - 0.0.16-1 diff --git a/test-test.sh b/test-test.sh new file mode 100755 index 0000000..3ca8edc --- /dev/null +++ b/test-test.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -eux + +# This script tests the test.sh script using many permutations +# of operating systems, python versions, and actions. Recreating +# the container for each permutation for every action ensures +# that a prior action did not already install a required package. + +export ENGINE=${ENGINE:="docker"} +export RECREATE_CONTAINER=${RECREATE_CONTAINER:="true"} + +for os in fedora:29 fedora:30 fedora:31 centos:7 centos:8 +do + os_version=$(echo $os | cut -d : -f 2) + os=$(echo $os | cut -d : -f 1) + for python_version in 2 3 + do + for action in test bandit pylint markdownlint + do + ACTION=$action PYTHON_VERSION=$python_version OS=$os OS_VERSION=$os_version ./test.sh + done + done +done diff --git a/test.sh b/test.sh index e2e3da7..a2938f6 100755 --- a/test.sh +++ b/test.sh @@ -9,6 +9,7 @@ PYTHON_VERSION=${PYTHON_VERSION:="2"} ACTION=${ACTION:="test"} IMAGE="$OS:$OS_VERSION" CONTAINER_NAME="dockerfile-parse-$OS-$OS_VERSION-py$PYTHON_VERSION" +RECREATE_CONTAINER=${RECREATE_CONTAINER:="false"} if [[ $ACTION == "markdownlint" ]]; then IMAGE="ruby" @@ -23,6 +24,16 @@ for dir in ${EXTRA_MOUNT:-}; do engine_mounts=("${engine_mounts[@]}" -v "$dir":"$dir":z) done +# Force recreation of the container +if $RECREATE_CONTAINER; then + if [[ $($ENGINE ps -q -f name="$CONTAINER_NAME" | wc -l) -gt 0 ]]; then + $ENGINE kill $CONTAINER_NAME + fi + if [[ $($ENGINE ps -qa -f name="$CONTAINER_NAME" | wc -l) -gt 0 ]]; then + $ENGINE rm $CONTAINER_NAME + fi +fi + # Create or resurrect container if needed if [[ $($ENGINE ps -qa -f name="$CONTAINER_NAME" | wc -l) -eq 0 ]]; then $ENGINE run --name "$CONTAINER_NAME" -d "${engine_mounts[@]}" -w "$PWD" -ti "$IMAGE" sleep infinity @@ -37,32 +48,24 @@ function setup_dfp() { IMAGE="registry.fedoraproject.org/$IMAGE" fi - if [[ $OS == "fedora" ]]; then - PIP_PKG="python$PYTHON_VERSION-pip" - PIP="pip$PYTHON_VERSION" + if [[ $OS == "fedora" || ( $OS == "centos" && $OS_VERSION -gt 7 ) ]]; then PKG="dnf" PKG_EXTRA="dnf-plugins-core" BUILDDEP="dnf builddep" - PYTHON="python$PYTHON_VERSION" else - PIP_PKG="python-pip" - PIP="pip" PKG="yum" PKG_EXTRA="yum-utils epel-release" BUILDDEP="yum-builddep" - PYTHON="python" fi + PIP="pip$PYTHON_VERSION" + PYTHON="python$PYTHON_VERSION" # Install dependencies $RUN $PKG install -y $PKG_EXTRA $RUN $BUILDDEP -y python-dockerfile-parse.spec - if [[ $OS != "fedora" ]]; then - # Install dependecies for test, as check is disabled for rhel - $RUN yum install -y python-six - fi # Install package - $RUN $PKG install -y $PIP_PKG + $RUN $PKG install -y $PYTHON-pip if [[ $PYTHON_VERSION == 3 ]]; then # https://fedoraproject.org/wiki/Changes/Making_sudo_pip_safe $RUN mkdir -p /usr/local/lib/python3.6/site-packages/ @@ -71,7 +74,7 @@ function setup_dfp() { # CentOS needs to have setuptools updates to make pytest-cov work if [[ $OS != "fedora" ]]; then - $RUN $PIP install -U setuptools + $RUN $PIP install -U "setuptools$([[ $PYTHON_VERSION == 3 ]] || echo -n '<45')" # Watch out for https://github.com/pypa/setuptools/issues/937 $RUN curl -O https://bootstrap.pypa.io/2.6/get-pip.py @@ -96,7 +99,7 @@ case ${ACTION} in ;; "pylint") setup_dfp - $RUN $PKG install -y "${PYTHON}-pylint" + $RUN $PIP install pylint PACKAGES='dockerfile_parse tests' TEST_CMD="${PYTHON} -m pylint ${PACKAGES}" ;; diff --git a/tests/fixtures.py b/tests/fixtures.py index a7b963c..56f517f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -36,7 +36,7 @@ def dfparser(tmpdir, request): return DockerfileParser(path=tmpdir_path, cache_content=cache_content) -@pytest.fixture(params=['LABEL', 'ENV']) +@pytest.fixture(params=['LABEL', 'ENV', 'ARG']) def instruction(request): """ Parametrized fixture which enables to run a test once for each instruction in params diff --git a/tests/test_parser.py b/tests/test_parser.py index 5c3164f..f1f3e8f 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -21,6 +21,7 @@ from dockerfile_parse import DockerfileParser from dockerfile_parse.parser import image_from from dockerfile_parse.constants import COMMENT_INSTRUCTION +from dockerfile_parse.util import b2u, u2b, Context from tests.fixtures import dfparser, instruction NON_ASCII = "žluťoučký" @@ -51,6 +52,24 @@ def read_version(fp, regex): assert spec_version == module_version assert setup_py_version == module_version + def test_util_b2u(self): + assert isinstance(b2u(u'string'), six.text_type) + assert isinstance(b2u(b'byte'), six.text_type) + + def test_util_u2b(self): + assert isinstance(u2b(u'string'), six.binary_type) + assert isinstance(u2b(b'byte'), six.binary_type) + + def test_util_context_exceptions(self): + context = Context() + with pytest.raises(ValueError): + context.get_values('FOO') + with pytest.raises(ValueError): + context.get_line_value('FOO') + with pytest.raises(ValueError): + context.set_line_value('FOO', {}) + + def test_dockerfileparser(self, dfparser, tmpdir): df_content = dedent("""\ FROM fedora @@ -69,13 +88,39 @@ def test_dockerfileparser(self, dfparser, tmpdir): assert dfparser.lines == df_lines assert [isinstance(line, six.text_type) for line in dfparser.lines] - with open(os.path.join(str(tmpdir), 'Dockerfile'), 'wb') as fp: + dockerfile = os.path.join(str(tmpdir), 'Dockerfile') + with open(dockerfile, 'wb') as fp: fp.write(df_content.encode('utf-8')) - dfparser = DockerfileParser(str(tmpdir)) + dfparser = DockerfileParser(dockerfile) assert dfparser.content == df_content assert dfparser.lines == df_lines assert [isinstance(line, six.text_type) for line in dfparser.lines] + def test_dockerfileparser_exceptions(self, tmpdir): + df_content = dedent("""\ + FROM fedora + LABEL label={0}""".format(NON_ASCII)) + df_lines = ["FROM fedora\n", "LABEL label={0}".format(NON_ASCII)] + + dfp = DockerfileParser(os.path.join(str(tmpdir), 'no-directory')) + with pytest.raises(IOError): + assert dfp.content + with pytest.raises(IOError): + dfp.content = df_content + with pytest.raises(IOError): + assert dfp.lines + with pytest.raises(IOError): + dfp.lines = df_lines + + def test_internal_exceptions(self, tmpdir): + dfp = DockerfileParser(str(tmpdir)) + with pytest.raises(ValueError): + dfp._instruction_getter('FOO', env_replace=True) + with pytest.raises(ValueError): + dfp._instructions_setter('FOO', {}) + with pytest.raises(ValueError): + dfp._modify_instruction_label_env('FOO', 'key', 'value') + def test_constructor_cache(self, tmpdir): tmpdir_path = str(tmpdir.realpath()) df1 = DockerfileParser(tmpdir_path) @@ -266,8 +311,63 @@ def test_multistage_dockerfile_labels(self, dfparser): def test_get_baseimg_from_df(self, dfparser): dfparser.lines = ["From fedora:latest\n", "LABEL a b\n"] - base_img = dfparser.baseimage - assert base_img.startswith('fedora') + assert dfparser.baseimage == 'fedora:latest' + + def test_get_baseimg_from_arg(self, dfparser): + dfparser.lines = ["ARG BASE=fedora:latest\n", + "FROM $BASE\n", + "LABEL a b\n"] + assert dfparser.baseimage == 'fedora:latest' + + def test_get_baseimg_from_build_arg(self, tmpdir): + tmpdir_path = str(tmpdir.realpath()) + b_args = {"BASE": "fedora:latest"} + dfp = DockerfileParser(tmpdir_path, env_replace=True, build_args=b_args) + dfp.lines = ["ARG BASE=centos:latest\n", + "FROM $BASE\n", + "LABEL a b\n"] + assert dfp.baseimage == 'fedora:latest' + assert not dfp.args + + def test_set_no_baseimage(self, dfparser): + dfparser.lines = [] + with pytest.raises(RuntimeError): + dfparser.baseimage = 'fedora:latest' + assert not dfparser.baseimage + + def test_get_build_args(self, tmpdir): + tmpdir_path = str(tmpdir.realpath()) + b_args = {"bar": "baz❤"} + df1 = DockerfileParser(tmpdir_path, env_replace=True, build_args=b_args) + df1.lines = [ + "ARG foo=\"baz❤\"\n", + "ARG not=\"used\"\n", + "FROM parent\n", + "ARG foo\n", + "ARG bar\n", + "LABEL label=\"$foo $bar\"\n" + ] + + # Even though we inherit an ARG, this .args count should only be for the + # ARGs defined in *this* Dockerfile as we're parsing the Dockerfile and + # the build_args is only to satisfy use of this build. + assert len(df1.args) == 2 + assert df1.args.get('foo') == 'baz❤' + assert df1.args.get('bar') == 'baz❤' + assert len(df1.labels) == 1 + assert df1.labels.get('label') == 'baz❤ baz❤' + + def test_get_build_args_from_scratch(self, tmpdir): + tmpdir_path = str(tmpdir.realpath()) + b_args = {"bar": "baz"} + df1 = DockerfileParser(tmpdir_path, env_replace=True, build_args=b_args) + df1.lines = [ + "FROM scratch\n", + ] + + assert not df1.args + assert not (df1.args == ['bar', 'baz']) + assert hash(df1.args) def test_get_parent_env(self, tmpdir): tmpdir_path = str(tmpdir.realpath()) @@ -296,6 +396,8 @@ def test_get_parent_env_from_scratch(self, tmpdir): ] assert not df1.envs + assert not (df1.envs == ['bar', 'baz']) + assert hash(df1.envs) @pytest.mark.parametrize(('instr_value', 'expected'), [ # pylint: disable=anomalous-backslash-in-string @@ -310,8 +412,8 @@ def test_get_parent_env_from_scratch(self, tmpdir): ('"name9"="asd \\ \\n qwe"', {'name9': 'asd \\ \\n qwe'}), ('"name10"="{0}"'.format(NON_ASCII), {'name10': NON_ASCII}), ('"name1 1"=1', {'name1 1': '1'}), - ('"name12"=12 \ \n "name13"=13', {'name12': '12', 'name13': '13'}), - ('name14=1\ 4', {'name14': '1 4'}), + ('"name12"=12 \\ \n "name13"=13', {'name12': '12', 'name13': '13'}), + ('name14=1\\ 4', {'name14': '1 4'}), ('name15="with = in value"', {'name15': 'with = in value'}), ('name16=❤', {'name16': '❤'}), ('name❤=❤', {'name❤': '❤'}), @@ -322,7 +424,7 @@ def test_get_parent_env_from_scratch(self, tmpdir): ('name104 "1" 04', {'name104': '1 04'}), ('name105 1 \'05\'', {'name105': '1 05'}), ('name106 1 \'0\' 6', {'name106': '1 0 6'}), - ('name107 1 0\ 7', {'name107': '1 0 7'}), + ('name107 1 0\\ 7', {'name107': '1 0 7'}), ('name108 "with = in value"', {'name108': 'with = in value'}), ('name109 "\\"quoted\\""', {'name109': '"quoted"'}), ('name110 ❤', {'name110': '❤'}), @@ -335,6 +437,10 @@ def test_get_instructions_from_df(self, dfparser, instruction, instr_value, instructions = dfparser.labels elif instruction == 'ENV': instructions = dfparser.envs + elif instruction == 'ARG': + instructions = dfparser.args + else: + assert False, 'Unexpected instruction: {0}'.format(instruction) assert instructions == expected @@ -419,6 +525,7 @@ def test_modify_instruction(self, dfparser): assert dfparser.cmd == CMD[1] def test_modify_from_multistage(self, dfparser): + CODE_VERSION = 'latest.❤' BASE_FROM = 'base:${CODE_VERSION}' BUILDER_FROM = 'builder:${CODE_VERSION}' UPDATED_BASE_FROM = 'bass:${CODE_VERSION}' @@ -428,12 +535,12 @@ def test_modify_from_multistage(self, dfparser): UPDATED_BASE_CMD = '/code/run-main-actors' df_content = dedent("""\ - ARG CODE_VERSION=latest.❤ - FROM {0} - CMD {1} + ARG CODE_VERSION={0} + FROM {1} + CMD {2} - FROM {2} - """).format(BUILDER_FROM, BUILDER_CMD, BASE_FROM) + FROM {3} + """).format(CODE_VERSION, BUILDER_FROM, BUILDER_CMD, BASE_FROM) INDEX_FIRST_FROM = 1 INDEX_SECOND_FROM = 4 @@ -443,12 +550,12 @@ def test_modify_from_multistage(self, dfparser): dfparser.content = df_content - assert dfparser.baseimage == BASE_FROM + assert dfparser.baseimage == 'base:{0}'.format(CODE_VERSION) assert dfparser.lines[INDEX_FIRST_FROM].strip() == 'FROM {0}'.format(BUILDER_FROM) assert dfparser.lines[INDEX_SECOND_FROM].strip() == 'FROM {0}'.format(BASE_FROM) dfparser.baseimage = UPDATED_BASE_FROM # should update only last FROM - assert dfparser.baseimage == UPDATED_BASE_FROM + assert dfparser.baseimage == 'bass:{0}'.format(CODE_VERSION) assert dfparser.lines[INDEX_FIRST_FROM].strip() == 'FROM {0}'.format(BUILDER_FROM) assert dfparser.lines[INDEX_SECOND_FROM].strip() == 'FROM {0}'.format(UPDATED_BASE_FROM) @@ -469,6 +576,8 @@ def test_add_del_instruction(self, dfparser): LABEL x=\"y z\" ENV h i ENV j='k' l=m + ARG a b + ARG c='d' e=f """) dfparser.content = df_content @@ -479,6 +588,8 @@ def test_add_del_instruction(self, dfparser): dfparser._add_instruction('FROM', 'fedora') assert dfparser.baseimage == 'fedora' + dfparser._delete_instructions('FROM', 'centos') + assert dfparser.baseimage == 'fedora' dfparser._delete_instructions('FROM', 'fedora') assert dfparser.baseimage is None @@ -494,6 +605,12 @@ def test_add_del_instruction(self, dfparser): dfparser._delete_instructions('ENV') assert dfparser.envs == {} + dfparser._add_instruction('ARG', ('Name', 'self')) + assert len(dfparser.args) == 4 + assert dfparser.args.get('Name') == 'self' + dfparser._delete_instructions('ARG') + assert dfparser.envs == {} + assert dfparser.cmd == 'xyz' @pytest.mark.parametrize(('existing', @@ -621,6 +738,9 @@ def test_setter(self, dfparser, instruction, existing, new, expected): elif instruction == 'ENV': dfparser.envs = new assert dfparser.envs == new + elif instruction == 'ARG': + dfparser.args = new + assert dfparser.args == new assert set(dfparser.lines[1:]) == set(expected) @pytest.mark.parametrize(('old_instructions', 'key', 'new_value', 'expected'), [ @@ -667,6 +787,7 @@ def test_setter_direct(self, dfparser, instruction, old_instructions, key, new_v del dfparser.envs[key] assert not dfparser.labels.get(key) + @pytest.mark.parametrize('instruction', ('ARG', 'ENV')) @pytest.mark.parametrize('separator', [' ', '=']) @pytest.mark.parametrize(('label', 'expected'), [ # Expected substitutions @@ -693,13 +814,16 @@ def test_setter_direct(self, dfparser, instruction, old_instructions, key, new_v ('${}', ''), ("'\\'$V'\\'", "\\v\\"), ]) - def test_env_replace(self, dfparser, label, expected, separator): + def test_arg_env_replace(self, dfparser, instruction, separator, label, expected): dfparser.lines = ["FROM fedora\n", - "ENV V=v\n", - "ENV VS='spam maps'\n", + "{0} V=v\n".format(instruction), + "{0} VS='spam maps'\n".format(instruction), "LABEL TEST{0}{1}\n".format(separator, label)] assert dfparser.labels['TEST'] == expected + with pytest.raises(TypeError): + dfparser.labels = ['foo', 'bar'] + @pytest.mark.parametrize('instruction', ('ARG', 'ENV')) @pytest.mark.parametrize('separator', [' ', '=']) @pytest.mark.parametrize(('label', 'expected'), [ # These would have been substituted with env_replace=True @@ -710,68 +834,74 @@ def test_env_replace(self, dfparser, label, expected, separator): ('"$V"-foo', '$V-foo'), ('"$V"-❤', '$V-❤'), ]) - def test_env_noreplace(self, dfparser, label, expected, separator): + def test_arg_env_noreplace(self, dfparser, instruction, separator, label, expected): """ Make sure environment replacement can be disabled. """ dfparser.env_replace = False dfparser.lines = ["FROM fedora\n", - "ENV V=v\n", + "{0} V=v\n".format(instruction), "LABEL TEST{0}{1}\n".format(separator, label)] assert dfparser.labels['TEST'] == expected + @pytest.mark.parametrize('instruction', ('ARG', 'ENV')) @pytest.mark.parametrize('label', [ '${V', '"${V"', '${{{{V}', ]) - def test_env_invalid(self, dfparser, label): + def test_arg_env_invalid(self, dfparser, instruction, label): """ These tests are invalid, but the parser should at least terminate even if it raises an exception. """ dfparser.lines = ["FROM fedora\n", - "ENV v=v\n", + "{0} v=v\n".format(instruction), "LABEL TEST={0}\n".format(label)] try: dfparser.labels['TEST'] except KeyError: pass - def test_env_multistage(self, dfparser): + @pytest.mark.parametrize(('instruction', 'attribute'), ( + ('ARG', 'args'), + ('ENV', 'envs'), + )) + def test_arg_env_multistage(self, dfparser, instruction, attribute): dfparser.content = dedent("""\ FROM stuff - ENV a=keep❤ b=keep❤ + {instruction} a=keep❤ b=keep❤ FROM base - ENV a=delete❤ + {instruction} a=delete❤ RUN something - """) + """.format(instruction=instruction)) - dfparser.envs['a'] = "changed❤" - del dfparser.envs['a'] - dfparser.envs['b'] = "new❤" + getattr(dfparser, attribute)['a'] = "changed❤" + del getattr(dfparser, attribute)['a'] + getattr(dfparser, attribute)['b'] = "new❤" lines = dfparser.lines - assert "ENV" in lines[1] + assert instruction in lines[1] assert "a=keep❤" in lines[1] assert "b=new❤" not in lines[1] assert "a=delete❤" not in dfparser.content assert "b='new❤'" in lines[-1] # unicode quoted @pytest.mark.xfail + @pytest.mark.parametrize('instruction', ('ARG', 'ENV')) @pytest.mark.parametrize(('label', 'expected'), [ ('${V:-foo}', 'foo'), ('${V:+foo}', 'v'), ('${UNDEF:+foo}', 'foo'), ('${UNDEF:+${V}}', 'v'), ]) - def test_env_replace_notimplemented(self, dfparser, label, expected): + def test_arg_env_replace_notimplemented(self, dfparser, instruction, label, expected): """ Test for syntax we don't support yet but should. """ dfparser.lines = ["FROM fedora\n", - "ENV V=v\n", + "{0} V=v\n".format(instruction), "LABEL TEST={0}\n".format(label)] assert dfparser.labels['TEST'] == expected @@ -925,27 +1055,49 @@ def test_context_structure_mixed(self, dfparser, instruction): assert c[3].get_values(context_type=instruction) == {"key": "value❤", "key2": "value2❤"} - def test_context_structure_mixed_env_label(self, dfparser): + @pytest.mark.parametrize('instruction', ('ARG', 'ENV')) + def test_context_structure_mixed_arg_env_label(self, dfparser, instruction): dfparser.content = dedent("""\ FROM fedora:25 - ENV key=value❤ + {0} key=value❤ RUN touch /tmp/a - LABEL key2=value2❤""") + LABEL key2=value2❤""".format(instruction)) c = dfparser.context_structure - assert c[0].get_values(context_type="ENV") == {} + assert c[0].get_values(context_type=instruction) == {} assert c[0].get_values(context_type="LABEL") == {} - assert c[1].get_values(context_type="ENV") == {"key": "value❤"} + assert c[1].get_values(context_type=instruction) == {"key": "value❤"} assert c[1].get_values(context_type="LABEL") == {} - assert c[2].get_values(context_type="ENV") == {"key": "value❤"} + assert c[2].get_values(context_type=instruction) == {"key": "value❤"} assert c[2].get_values(context_type="LABEL") == {} - assert c[3].get_values(context_type="ENV") == {"key": "value❤"} + assert c[3].get_values(context_type=instruction) == {"key": "value❤"} assert c[3].get_values(context_type="LABEL") == {"key2": "value2❤"} + def test_context_structure_mixed_top_arg(self, tmpdir): + dfp = DockerfileParser( + str(tmpdir.realpath()), + build_args={"version": "8", "key": "value❤"}, + env_replace=True) + dfp.content = dedent("""\ + ARG image=centos + ARG version=latest + FROM $image:$version + ARG image + ARG key + """) + c = dfp.context_structure + + assert len(c) == 5 + assert c[0].get_values(context_type='ARG') == {"image": "centos"} + assert c[1].get_values(context_type='ARG') == {"image": "centos", "version": "8"} + assert c[2].get_values(context_type='ARG') == {} + assert c[3].get_values(context_type='ARG') == {"image": "centos"} + assert c[4].get_values(context_type='ARG') == {"image": "centos", "key": "value❤"} + def test_expand_concatenated_variables(self, dfparser): dfparser.content = dedent("""\ FROM scratch @@ -954,7 +1106,8 @@ def test_expand_concatenated_variables(self, dfparser): """) assert dfparser.labels['component'] == 'name1❤' - def test_label_env_key(self, dfparser): + @pytest.mark.parametrize('instruction', ('ARG', 'ENV')) + def test_label_arg_env_key(self, dfparser, instruction): """ Verify keys may be substituted with values containing space. @@ -964,9 +1117,9 @@ def test_label_env_key(self, dfparser): """ dfparser.content = dedent("""\ FROM scratch - ENV FOOBAR="foo bar" + {0} FOOBAR="foo bar" LABEL "$FOOBAR"="baz" - """) + """.format(instruction)) assert dfparser.labels['foo bar'] == 'baz' @pytest.mark.parametrize('label_value, bad_keyval, envs', [ @@ -993,10 +1146,7 @@ def test_label_invalid(self, dfparser, label_value, bad_keyval, envs, action): dfparser.labels # pylint: disable=pointless-statement elif action == 'set': dfparser.labels = {} - if six.PY2: - msg = exc_info.value.message - else: - msg = str(exc_info.value) + msg = exc_info.value.args[0] assert msg == ('Syntax error - can\'t find = in "{word}". ' 'Must be of the form: name=value' .format(word=bad_keyval))