Make venv build process idempotent

The venv build process currently executes on
every build, destroying all existing venvs for
the distribution/architecture of the repo server
it executes on, then rebuilds them.

It is also terribly difficult to troubleshoot
due to the fact that it is executed in parallel
through a single bash script using backgrounded
processes.

This patch breaks the build process up into two
parts - the per-venv options files, and the
script that uses the options to build the venvs.

With this breakdown we're able to do the following:
- Only execute a venv rebuild if the venv
  options (indexes, requirements) have changed.
- Use the Ansible asynchonous execution to
  execute parallel venv builds.

As a very welcome side-effect, this also means
that the venv build execution provides individual
output for success or failures, making it much
easier to see what went wrong when failing.

As part of the patch, the removal of the *.in,
*.txt and *.html files on each wheel build is
taken out. This is to protect the venv options
files. The removal of those files was unnecessary
anyway as they're templated and therefore replaced
by Ansible if they need to be changed.

Conflicts:
        tasks/repo_venv_build.yml

Change-Id: I063c3addb6fbabb01d620be33aac2cab29a02750
(cherry picked from commit 27ec4a2)

defaults/main.yml

@@ -54,6 +54,12 @@ repo_build_wheel_selective: "{{ true if repo_build_venv_selective | bool else fa
 # Toggle whether venvs should be built selectively or not
 repo_build_venv_selective: yes
 
+# Toggle whether a venv rebuild should be forced
+repo_build_venv_rebuild: no
+
+# Timeout (in minutes) for a venv build
+repo_build_venv_timeout: 30
+
 # Optionally set this to change the default index from pypi to an alternative
 #repo_build_pip_default_index: "https://pypi.python.org/simple"
 
@@ -75,11 +81,11 @@ repo_build_timeout: 120
 repo_build_concurrency: "{{ (((ansible_processor_vcpus|default(1)) | int) > 4) | ternary(8, 4) }}"
 repo_build_venv_build_dir: "/tmp/openstack-venv-builder"
 repo_build_venv_dir: "{{ repo_build_base_path }}/venvs/{{ repo_build_release_tag }}/{{ repo_build_os_distro_version }}"
-repo_build_venv_pip_install_options: >
+repo_build_venv_pip_install_options: >-
   --timeout 120
   --find-links {{ repo_build_release_path }}
   --log /var/log/repo/repo_venv_builder.log
-repo_build_venv_command_options: >
+repo_build_venv_command_options: >-
   {{ virtualenv_bin }}
   --always-copy
   --extra-search-dir {{ repo_build_release_path }}

tasks/main.yml

@@ -31,7 +31,7 @@
 - include: repo_post_build.yml
 
 # Venv building
-- include: repo_venv.yml
+- include: repo_venv_build.yml
   tags:
     - repo-build-venvs
 

tasks/repo_pre_build.yml

@@ -30,9 +30,6 @@
     patterns:
       - "*{{ ansible_architecture | lower }}.whl"
       - "*none-any.whl"
-      - "*.in"
-      - "*.html"
-      - "*.txt"
   register: os_release_files
   tags:
     - repo-clean-workspace

tasks/repo_venv_build.yml

@@ -13,8 +13,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+- name: Create venv directories
+  file:
+    path: "{{ item }}"
+    state: "directory"
+    owner: "{{ repo_build_service_user_name }}"
+    mode: 02755
+  with_items:
+    - "{{ repo_build_venv_build_dir }}/venvs"
+    - "{{ repo_build_venv_dir }}"
+
 - name: Get venv command path
   command: which virtualenv
+  changed_when: false
   register: virtualenv_path
   tags:
     - always
@@ -25,20 +36,6 @@
   tags:
     - always
 
-- name: Check for created venvs
-  command: >
-    ls -1 "{{ repo_build_venv_dir }}/"
-  register: created_venvs
-  tags:
-    - repo-create-venv
-
-- name: Set existing venv fact
-  set_fact:
-    existing_venvs: "{{ created_venvs.stdout_lines }}"
-  tags:
-    - repo-command-bin
-    - repo-create-venv
-
 # This is removed so that virtual env is not installing unknown
 #  packages as assumed wheels, IE pip8
 - name: Ensure virtualenv_support is absent
@@ -48,27 +45,43 @@
   with_items:
     - "/usr/local/lib/python2.7/dist-packages/virtualenv_support"
     - "/usr/local/lib/python2.7/site-packages/virtualenv_support"
-  tags:
-    - repo-create-venv
 
-- name: Create venv process script
+- name: Create venv build script
   template:
-    src: "op-venv-script.sh.j2"
-    dest: "/opt/op-venv-script.sh"
-  tags:
-    - repo-create-venv
+    src: "venv-build-script.sh.j2"
+    dest: "/opt/venv-build-script.sh"
+    mode: "0755"
 
-- name: Run venv process script
-  shell: "bash /opt/op-venv-script.sh"
-  tags:
-    - repo-create-venv
-    - repo-venv-compress-archive
-    - repo-create-venv-archive
-    - repo-create-venv-checksum
+- name: Create venv build options files
+  template:
+    src: "venv-build-options.txt.j2"
+    dest: "{{ repo_build_release_path }}/venv-build-options-{{ item['role_name'] }}.txt"
+  with_items: "{{ filtered_venv_role_python_requirements }}"
+  register: _create_venv_options_files
 
-- name: Remove venv process script
-  file:
-    path: "/opt/op-venv-script.sh"
-    state: absent
+- name: Execute the venv build scripts asynchonously
+  shell: "/opt/venv-build-script.sh {{ repo_build_release_path }}/venv-build-options-{{ item['item']['role_name'] }}.txt"
+  args:
+    executable: "/bin/bash"
+  when: (item | changed) or (repo_build_venv_rebuild | bool)
+  with_items:
+    - "{{ _create_venv_options_files.results }}"
+  register: _build_venv
+  async: "{{ repo_build_venv_timeout * 60 }}"
+  poll: 0
+  # This task requires the use of the shell module, so we skip lint
+  # to avoid:
+  # ANSIBLE0013 Use shell only when shell functionality is required
   tags:
-    - repo-create-venv
+    - skip_ansible_lint
+
+- name: Wait for the venvs builds to complete
+  async_status:
+    jid: "{{ item['ansible_job_id'] }}"
+  register: _venv_jobs
+  until: _venv_jobs['finished'] | bool
+  delay: 10
+  retries: "{{ repo_build_venv_timeout * 6 }}"
+  with_items: "{{ _build_venv['results'] }}"
+  when:
+    - item['ansible_job_id'] is defined