Merge 025a92b into ae404e8

.gitignore

@@ -55,3 +55,6 @@ docs/_build/
 
 # PyBuilder
 target/
+
+# Downloaded by test.sh
+get-pip.py

dockerfile_parse/util.py

@@ -46,13 +46,16 @@ class WordSplitter(object):
     SQUOTE = "'"
     DQUOTE = '"'
 
-    def __init__(self, s, envs=None):
+    def __init__(self, s, args=None, envs=None):
         """
         :param s: str, string to process
+        :param args: dict, build arguments to use; if None, do not
+            attempt substitution
         :param envs: dict, environment variables to use; if None, do not
             attempt substitution
         """
         self.stream = StringIO(s)
+        self.args = args
         self.envs = envs
 
         # Initial state
@@ -143,7 +146,7 @@ def append(self, s):
                 return
 
             if (not self.escaped and
-                    self.envs is not None and
+                    (self.envs is not None or self.args is not None) and
                     ch == '$' and
                     self.quotes != self.SQUOTE):
                 while True:
@@ -168,10 +171,10 @@ def append(self, s):
 
                         varname += ch
 
-                    try:
+                    if self.envs is not None and varname in self.envs:
                         word.append(self.envs[varname])
-                    except KeyError:
-                        pass
+                    elif self.args is not None and varname in self.args:
+                        word.append(self.args[varname])
 
                     # Check whether there is another envvar
                     if ch != '$':
@@ -210,13 +213,14 @@ def append(self, s):
                 word.append(ch)
 
 
-def extract_labels_or_envs(env_replace, envs, instruction_value):
+def extract_key_values(env_replace, args, envs, instruction_value):
     words = list(WordSplitter(instruction_value).split(dequote=False))
     key_val_list = []
 
     def substitute_vars(val):
         kwargs = {}
         if env_replace:
+            kwargs['args'] = args
             kwargs['envs'] = envs
 
         return WordSplitter(val, **kwargs).dequote()
@@ -248,65 +252,83 @@ def substitute_vars(val):
     return key_val_list
 
 
-def get_key_val_dictionary(instruction_value, env_replace=False, envs=None):
-    envs = envs or []
-    return dict(extract_labels_or_envs(instruction_value=instruction_value,
-                                       env_replace=env_replace,
-                                       envs=envs))
+def get_key_val_dictionary(instruction_value, env_replace=False, args=None, envs=None):
+    args = args or {}
+    envs = envs or {}
+    return dict(extract_key_values(instruction_value=instruction_value,
+                                   env_replace=env_replace,
+                                   args=args, envs=envs))
 
 
 class Context(object):
-    def __init__(self, envs=None, labels=None, line_envs=None, line_labels=None):
+    def __init__(self, args=None, envs=None, labels=None,
+                 line_args=None, line_envs=None, line_labels=None):
         """
-        Class representing current state of environment variables and labels.
+        Class representing current state of build arguments, environment variables and labels.
 
+        :param args: dict with arguments valid for this line
+            (all variables defined to this line)
         :param envs: dict with variables valid for this line
             (all variables defined to this line)
         :param labels: dict with labels valid for this line
             (all labels defined to this line)
+        :param line_args: dict with arguments defined on this line
         :param line_envs: dict with variables defined on this line
         :param line_labels: dict with labels defined on this line
         """
+        self.args = args or {}
         self.envs = envs or {}
         self.labels = labels or {}
+        self.line_args = line_args or {}
         self.line_envs = line_envs or {}
         self.line_labels = line_labels or {}
 
     def set_line_value(self, context_type, value):
         """
-        Set value defined on this line ('line_envs'/'line_labels')
-        and update 'envs'/'labels'.
+        Set value defined on this line ('line_args'/'line_envs'/'line_labels')
+        and update 'args'/'envs'/'labels'.
 
-        :param context_type: "ENV" or "LABEL"
+        :param context_type: "ARG" or "ENV" or "LABEL"
         :param value: new value for this line
         """
-        if context_type.upper() == "ENV":
+        if context_type.upper() == "ARG":
+            self.line_args = value
+            self.args.update(value)
+        elif context_type.upper() == "ENV":
             self.line_envs = value
             self.envs.update(value)
         elif context_type.upper() == "LABEL":
             self.line_labels = value
             self.labels.update(value)
+        else:
+            raise ValueError("Unexpected context type: " + context_type)
 
     def get_line_value(self, context_type):
         """
         Get the values defined on this line.
 
-        :param context_type: "ENV" or "LABEL"
+        :param context_type: "ARG" or "ENV" or "LABEL"
         :return: values of given type defined on this line
         """
+        if context_type.upper() == "ARG":
+            return self.line_args
         if context_type.upper() == "ENV":
             return self.line_envs
-        elif context_type.upper() == "LABEL":
+        if context_type.upper() == "LABEL":
             return self.line_labels
+        raise ValueError("Unexpected context type: " + context_type)
 
     def get_values(self, context_type):
         """
         Get the values valid on this line.
 
-        :param context_type: "ENV" or "LABEL"
+        :param context_type: "ARG" or "ENV" or "LABEL"
         :return: values of given type valid on this line
         """
+        if context_type.upper() == "ARG":
+            return self.args
         if context_type.upper() == "ENV":
             return self.envs
-        elif context_type.upper() == "LABEL":
+        if context_type.upper() == "LABEL":
             return self.labels
+        raise ValueError("Unexpected context type: " + context_type)

python-dockerfile-parse.spec

@@ -121,7 +121,7 @@ py.test-%{python3_version} -v tests
 * Tue Jun 02 2020 Robert Cerven <rcerven@redhat.com> 0.0.18-1
 - new upstream release: 0.0.18
 
-* Fri Apr 24 2020 Martin Bašti <mbasti@redhat.com> 0.0.17-1
+* Fri Apr 24 2020 Martin Basti <mbasti@redhat.com> 0.0.17-1
 - new upstream release: 0.0.17
 
 * Tue Jan 21 2020 Robert Cerven <rcerven@redhat.com> - 0.0.16-1

test-all.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -eux
+
+# This script tests the test.sh script using many permutations
+# of operating systems, python versions, and actions. Recreating
+# the container for each permutation for every action ensures
+# that a prior action did not already install a required package.
+
+export ENGINE=${ENGINE:="docker"}
+export RECREATE_CONTAINER=${RECREATE_CONTAINER:="true"}
+
+for os in fedora:29 fedora:30 fedora:31 centos:7 centos:8
+do
+    os_version=$(echo $os | cut -d : -f 2)
+    os=$(echo $os | cut -d : -f 1)
+    for python_version in 2 3
+    do
+        for action in test bandit pylint markdownlint
+        do
+            ACTION=$action PYTHON_VERSION=$python_version OS=$os OS_VERSION=$os_version ./test.sh
+        done
+    done
+done

test.sh

@@ -9,6 +9,7 @@ PYTHON_VERSION=${PYTHON_VERSION:="2"}
 ACTION=${ACTION:="test"}
 IMAGE="$OS:$OS_VERSION"
 CONTAINER_NAME="dockerfile-parse-$OS-$OS_VERSION-py$PYTHON_VERSION"
+RECREATE_CONTAINER=${RECREATE_CONTAINER:="false"}
 
 if [[ $ACTION == "markdownlint" ]]; then
   IMAGE="ruby"
@@ -23,6 +24,16 @@ for dir in ${EXTRA_MOUNT:-}; do
   engine_mounts=("${engine_mounts[@]}" -v "$dir":"$dir":z)
 done
 
+# Force recreation of the container
+if $RECREATE_CONTAINER; then
+  if [[ $($ENGINE ps -qa -f name="$CONTAINER_NAME" | wc -l) -gt 0 ]]; then
+    $ENGINE kill $CONTAINER_NAME
+  fi
+  if [[ $($ENGINE ps -qa -f name="$CONTAINER_NAME" | wc -l) -gt 0 ]]; then
+    $ENGINE rm $CONTAINER_NAME
+  fi
+fi
+
 # Create or resurrect container if needed
 if [[ $($ENGINE ps -qa -f name="$CONTAINER_NAME" | wc -l) -eq 0 ]]; then
   $ENGINE run --name "$CONTAINER_NAME" -d "${engine_mounts[@]}" -w "$PWD" -ti "$IMAGE" sleep infinity
@@ -37,32 +48,24 @@ function setup_dfp() {
     IMAGE="registry.fedoraproject.org/$IMAGE"
   fi
 
-  if [[ $OS == "fedora" ]]; then
-    PIP_PKG="python$PYTHON_VERSION-pip"
-    PIP="pip$PYTHON_VERSION"
+  if [[ $OS == "fedora" || ( $OS == "centos" && $OS_VERSION -gt 7 ) ]]; then
     PKG="dnf"
     PKG_EXTRA="dnf-plugins-core"
     BUILDDEP="dnf builddep"
-    PYTHON="python$PYTHON_VERSION"
   else
-    PIP_PKG="python-pip"
-    PIP="pip"
     PKG="yum"
     PKG_EXTRA="yum-utils epel-release"
     BUILDDEP="yum-builddep"
-    PYTHON="python"
   fi
+  PIP="pip$PYTHON_VERSION"
+  PYTHON="python$PYTHON_VERSION"
 
   # Install dependencies
   $RUN $PKG install -y $PKG_EXTRA
   $RUN $BUILDDEP -y python-dockerfile-parse.spec
-  if [[ $OS != "fedora" ]]; then
-    # Install dependecies for test, as check is disabled for rhel
-    $RUN yum install -y python-six
-  fi
 
   # Install package
-  $RUN $PKG install -y $PIP_PKG
+  $RUN $PKG install -y $PYTHON-pip
   if [[ $PYTHON_VERSION == 3 ]]; then
     # https://fedoraproject.org/wiki/Changes/Making_sudo_pip_safe
     $RUN mkdir -p /usr/local/lib/python3.6/site-packages/
@@ -71,7 +74,7 @@ function setup_dfp() {
 
   # CentOS needs to have setuptools updates to make pytest-cov work
   if [[ $OS != "fedora" ]]; then
-    $RUN $PIP install -U setuptools
+    $RUN $PIP install -U setuptools$([[ $PYTHON_VERSION == 3 ]] || echo -n '<45')
 
     # Watch out for https://github.com/pypa/setuptools/issues/937
     $RUN curl -O https://bootstrap.pypa.io/2.6/get-pip.py
@@ -96,7 +99,7 @@ case ${ACTION} in
   ;;
 "pylint")
   setup_dfp
-  $RUN $PKG install -y "${PYTHON}-pylint"
+  $RUN $PIP install pylint
   PACKAGES='dockerfile_parse tests'
   TEST_CMD="${PYTHON} -m pylint ${PACKAGES}"
   ;;

tests/fixtures.py

@@ -36,7 +36,7 @@ def dfparser(tmpdir, request):
         return DockerfileParser(path=tmpdir_path, cache_content=cache_content)
 
 
-@pytest.fixture(params=['LABEL', 'ENV'])
+@pytest.fixture(params=['LABEL', 'ENV', 'ARG'])
 def instruction(request):
     """
     Parametrized fixture which enables to run a test once for each instruction in params