Build cache fix (#678)

* Include in the hash any vars passed to the role

* Include service level variables in the role hash

* Updates ubuntu apache settings

AUTHORS

@@ -18,8 +18,8 @@ Dylan Silva <thaumos@users.noreply.github.com>
 Tomas Tomecek <ttomecek@redhat.com>
 Fabian von Feilitzsch <fabianvf@users.noreply.github.com>
 Charlie Drage <charlie@charliedrage.com>
-Roderick Randolph <roderickrandolph@users.noreply.github.com>
 Marcus Levine <marcusianl@gmail.com>
+Roderick Randolph <roderickrandolph@users.noreply.github.com>
 Will Refvem <wbrefvem@gmail.com>
 Sebastien Plisson <sebastien.plisson@gmail.com>
 Mike Hume <mhumeSF@gmail.com>
@@ -52,12 +52,3 @@ szinck1 <szinck@gmail.com>
 Erik Nelson <erik@nsk.io>
 Chrrrles Paul <chrrrles@users.noreply.github.com>
 Andrea De Pirro <andrea.depirro@yameveo.com>
-Aaron Cowdin <aaronthebaron@users.noreply.github.com>
-fignew <thomas@vorget.com>
-Gabor Lekeny <gabor.lekeny@gmail.com>
-John R Barker <john@johnrbarker.com>
-Sidharth Surana <ssurana@vmware.com>
-grimmjow <ryanbrown.dev@gmail.com>
-Shea Stewart <shea.stewart@arctiq.ca>
-koralsky <karol.ossowski@gmail.com>
-Michael Carden <mike.carden@gmail.com>

container/core.py

@@ -11,6 +11,7 @@
 import gzip
 import hashlib
 import io
+import json
 import os
 import re
 import ruamel
@@ -646,8 +647,7 @@ def apply_role_to_container(role, container_id, service_name, engine, vars={},
 def conductorcmd_build(engine_name, project_name, services, cache=True, local_python=False,
                        ansible_options='', debug=False, config_vars=None, **kwargs):
     engine = load_engine(['BUILD'], engine_name, project_name, services, **kwargs)
-    logger.info(u'%s integration engine loaded. Build starting.',
-        engine.display_name, project=project_name)
+    logger.info(u'%s integration engine loaded. Build starting.', engine.display_name, project=project_name)
     services_to_build = kwargs.get('services_to_build') or services.keys()
     for service_name, service in services.items():
         if service_name not in services_to_build:
@@ -662,7 +662,8 @@ def conductorcmd_build(engine_name, project_name, services, cache=True, local_py
                     "Failed to find image {}. Try `docker image pull {}`".format(service['from'])
                 )
         # the fingerprint hash tracks cacheability
-        fingerprint_hash = hashlib.sha256('%s::' % cur_image_id)
+        service_defaults = json.dumps(service['defaults']) if service.get('defaults') else ''
+        fingerprint_hash = hashlib.sha256('%s::%s' % (cur_image_id, service_defaults))
         logger.debug(u'Base fingerprint hash = %s', fingerprint_hash.hexdigest(),
                      service=service_name, hash=fingerprint_hash.hexdigest())
         cache_busted = not cache

container/docker/templates/conductor-dockerfile.j2

@@ -61,6 +61,13 @@ RUN yum install -y yum-utils && \
     yum clean all
 {% endif %}
 
+{% if distro in ["centos"] %}
+# Removing python-chardet removed yum-utils, and now that we're done pip-installing things,
+# we'll put it back.
+RUN yum install -y yum-utils && \
+    yum clean all
+{% endif %}
+
 VOLUME /usr
 {% if distro in ["ubuntu", "debian", "alpine"] %}
 VOLUME /lib

container/utils/__init__.py

@@ -7,13 +7,14 @@
 import os
 import hashlib
 import importlib
+import json
 
 from datetime import datetime
 from distutils import dir_util
-
 from jinja2 import Environment, FileSystemLoader
+from ruamel.yaml.comments import CommentedMap
 from ruamel import yaml
-from six import iteritems
+from six import iteritems, string_types
 
 
 from ..exceptions import AnsibleContainerException, \
@@ -251,6 +252,9 @@ def get_dependencies_for_role(role_path):
             yield None
 
     hash_obj = hashlib.sha256()
+    # Account for variables passed to the role by including the invocation string
+    hash_obj.update((json.dumps(role_name) if not isinstance(role_name, string_types) else role_name) + '::')
+    # Add each of the role's files and directories
     hash_role(hash_obj, resolve_role_to_path(role_name))
     return hash_obj.hexdigest()
 

test/roles/validate-build-and-import/defaults/main.yml

@@ -10,8 +10,11 @@ distros:
     httpd_pageroot: "/var/www/html/"
     httpd_logdir: "/var/log/httpd"
     httpd_rundir: "/run/httpd"
+    httpd_pid_file: "/run/httpd/httpd.pid"
     httpd_bin: "/usr/sbin/httpd"
     httpd_user: "apache"
+    httpd_run_group: "apache"
+    httpd_lock_dir: "/run/httpd"
 
   - name: "ubuntu"
     base_image: "ubuntu:trusty"
@@ -22,8 +25,11 @@ distros:
     httpd_pageroot: "/var/www/html/"
     httpd_logdir: "/var/log/apache2"
     httpd_rundir: "/run/apache2"
+    httpd_pid_file: "/var/run/apache2/apache2.pid"
     httpd_bin: "/usr/sbin/apache2"
     httpd_user: "www-data"
+    httpd_run_group: "www-data"
+    httpd_lock_dir: "/var/run/apache2"
 
 #  - name: "alpine"
 #    base_image: "alpine:3.5"

test/roles/validate-build-and-import/files/trusty/ports.conf

@@ -2,6 +2,7 @@
 # have to change the VirtualHost statement in
 # /etc/apache2/sites-enabled/000-default.conf
 
+ServerName localhost
 Listen ${APACHE_PORT}
 
 <IfModule ssl_module>

test/roles/validate-build-and-import/templates/Dockerfile.j2

@@ -24,7 +24,17 @@ RUN ["chmod", "755", "/usr/sbin/dumb-init"]
 
 # I am defaults
 ARG apache_port=8000
+ARG apache_pid_file={{ item.0.httpd_pid_file }}
+ARG apache_run_user={{ item.0.httpd_user }}
+ARG apache_run_group={{ item.0.httpd_run_group }}
+ARG apache_log_dir={{ item.0.httpd_logdir }}
+ARG apache_lock_dir={{ item.0.httpd_lock_dir }}
 ENV APACHE_PORT=${apache_port}
+ENV APACHE_PID_FILE=${apache_pid_file}
+ENV APACHE_RUN_USER=${apache_run_user}
+ENV APACHE_RUN_GROUP=${apache_run_group}
+ENV APACHE_LOG_DIR=${apache_log_dir}
+ENV APACHE_LOCK_DIR=${apache_lock_dir}
 
 # Single label
 LABEL a=1

test/roles/validate-start-stop-restart/tasks/main.yml

@@ -26,12 +26,38 @@
 
 - include: includes/show-output.yml output_file=./task.output registered_output="{{ output }}"
 
+- name: Wait for containers to start
+  wait_for:
+    timeout: 10
+
+- name: Get docker_ps
+  command: docker ps -a
+  register: docker_ps
+
+- name: Show docker_ps
+  debug: var=docker_ps.stdout_lines
+
+- name: Get docker_logs
+  command: docker logs "test{{ item.name }}_{{ item.name }}_1"
+  with_items: "{{ distros }}"
+  register: docker_logs
+
+- name: Show docker_logs
+  debug: var=docker_logs.results
+
 - name: Wait until built containers are running
-  docker_container:
-    name: "test{{ item.name }}_{{ item.name }}_1"
+  command: "docker inspect test{{ item.name }}_{{ item.name }}_1 --format '{% raw %}{{ .State.Status }}{% endraw %}'"
   with_items: "{{ distros }}"
   register: container_run_status
-  until: container_run_status.ansible_facts.docker_container.State.Status == 'running'
+  until: container_run_status.stdout == 'running'
+  tags:
+    - restart
+    - run
+
+- name: Get container start times
+  command: "docker inspect test{{ item.name }}_{{ item.name }}_1 --format '{% raw %}{{ .State.StartedAt }}{% endraw %}'"
+  with_items: "{{ distros }}"
+  register: container_run_startedat
   tags:
     - restart
     - run
@@ -51,22 +77,28 @@
 
 - include: includes/show-output.yml output_file=./task.output registered_output="{{ output }}"
 
-- name: Get new container start times
-  docker_container:
-    name: "test{{ item.name }}_{{ item.name }}_1"
+- name: Wait until restarted containers are running
+  command: "docker inspect test{{ item.name }}_{{ item.name }}_1 --format '{% raw %}{{ .State.Status }}{% endraw %}'"
   with_items: "{{ distros }}"
   register: container_restart_status
-  until: container_restart_status.ansible_facts.docker_container.State.Status == 'running'
+  until: container_restart_status.stdout == 'running'
+  tags:
+    - restart
+
+- name: Get container restart times
+  command: "docker inspect test{{ item.name }}_{{ item.name }}_1 --format '{% raw %}{{ .State.StartedAt }}{% endraw %}'"
+  with_items: "{{ distros }}"
+  register: container_restart_startedat
   tags:
     - restart
 
 - name: Ensure containers were restarted
   assert:
     that:
-      - item.0.ansible_facts.docker_container.State.StartedAt  != item.1.ansible_facts.docker_container.State.StartedAt
+      - item.0.stdout != item.1.stdout
   with_together:
-    - "{{ container_run_status.results }}"
-    - "{{ container_restart_status.results }}"
+    - "{{ container_run_startedat.results }}"
+    - "{{ container_restart_startedat.results }}"
   tags:
     - restart
 
@@ -89,13 +121,18 @@
 
 - include: includes/show-output.yml output_file=./task.output registered_output="{{ output }}"
 
-- name: Ensure containers were stopped
-  docker_container:
-    name: 'test{{ item.name }}_{{ item.name }}_1'
-    state: 'stopped'
+- name: Get container status
+  command: "docker inspect test{{ item.name }}_{{ item.name }}_1 --format '{% raw %}{{ .State.Status }}{% endraw %}'"
   with_items: "{{ distros }}"
-  register: output
-  failed_when: output.changed
+  register: container_status
   tags:
     - stop
 
+- name: Ensure containers stop
+  assert:
+    that:
+      - item.0.stdout == 'exited'
+  with_together:
+    - "{{ container_status.results }}"
+  tags:
+    - stop