Make wheel build process idempotent

This patch implements the following:

1. The ability to use a boolean variable to always force
   a git clone or the wheel build to happen.
2. The use of local facts to mark whether a wheel build
   or venv build is required. This ensures that if there
   is a failure, the tasks will still be done when the
   repo build is executed again.
3. The git clone or wheel build tasks are only actioned
   if there is a change to the requirements or constraints.
   This ensures that when the repo build is executed
   without any changes, those processes will be skipped.
4. Re-arranges the tasks to ensure idempotency and to
   make the process easier to follow. A smaller set of
   tags are implemented which are intended to provide
   a clear code path for each of them.
5. Log output is added to the venv build process to make
   troubleshooting easier.
6. The stdout output for the wheel and venv build
   processes is made minimal to reduce confusion and
   make it easier to spot which item failed to build.
   The log output in /var/log/repo contains the verbose
   output from pip.

Conflicts:
>------tasks/repo_build_install.yml
>------tasks/repo_build_wheels.yml
>------tasks/repo_post_build.yml
>------tasks/repo_pre_build.yml

Change-Id: I2008926b43653edf50c284f5068160e27915c90a
(cherry picked from commit 8c3933c)

defaults/main.yml

@@ -48,9 +48,15 @@ repo_build_pool_dir: "{{ repo_build_base_path }}/pools/{{ repo_build_os_distro_v
 # Toggle whether git repositories should be cloned selectively or not
 repo_build_git_selective: "{{ true if (repo_build_wheel_selective | bool and repo_build_venv_selective | bool) else false }}"
 
+# Toggle whether a git clone should be forced
+repo_build_git_reclone: no
+
 # Toggle whether wheels should be built selectively or not
 repo_build_wheel_selective: "{{ true if repo_build_venv_selective | bool else false }}"
 
+# Toggle whether a wheel rebuild should be forced
+repo_build_wheel_rebuild: no
+
 # Toggle whether venvs should be built selectively or not
 repo_build_venv_selective: yes
 

releasenotes/notes/idempotent-wheel-build-4c527045bec09fd5.yaml

@@ -0,0 +1,22 @@
+---
+upgrade:
+  - The entire repo build process is now idempotent. From now on when
+    the repo build is re-run, it will only fetch updated git
+    repositories and rebuild the wheels/venvs if the requirements
+    have changed, or a new release is being deployed.
+  - The git clone part of the repo build process now only happens when
+    the requirements change. A git reclone can be forced by using the
+    boolean variable ``repo_build_git_reclone``.
+  - The python wheel build process now only happens when requirements
+    change. A wheel rebuild may be forced by using the boolean variable
+    ``repo_build_wheel_rebuild``.
+  - The python venv build process now only happens when requirements
+    change. A venv rebuild may be forced by using the boolean variable
+    ``repo_build_venv_rebuild``.
+  - The repo build process now only has the following tags, providing
+    a clear path for each deliverable. The tag ``repo-build-install``
+    completes the installation of required packages. The tag
+    ``repo-build-wheels`` completes the wheel build process. The tag
+    ``repo-build-venvs`` completes the venv build process. Finally, the
+    tag ``repo-build-index`` completes the manifest preparation and
+    indexing of the os-releases and links folders.

tasks/main.yml

@@ -24,20 +24,35 @@
   tags:
     - always
 
-# Wheel building
-- include: repo_build_install.yml
-- include: repo_pre_build.yml
-- include: repo_build.yml
-- include: repo_post_build.yml
-
-# Venv building
-- include: repo_venv_build.yml
+- include: "repo_build_install-{{ ansible_pkg_mgr }}.yml"
+  tags:
+    - repo-build-install
+
+- include: repo_build_prepare.yml
+  tags:
+    - always
+
+- name: refresh local facts
+  setup:
+    filter: ansible_local
+    gather_subset: "!all"
+  tags:
+    - always
+
+- include: repo_build_wheels.yml
+  when:
+    - ansible_local['openstack_ansible']['repo_build']['need_wheel_build'] | bool
+  tags:
+    - repo-build-wheels
+
+- include: repo_build_venvs.yml
   tags:
     - repo-build-venvs
 
-- include: repo_index.yml
+- include: repo_build_index.yml
   tags:
     - repo-build-index
 
+# Synchronize all built packages back to the repo master
 - include: repo_package_sync.yml
   when: inventory_hostname != groups['repo_all'][0]

tasks/install-apt.yml → tasks/repo_build_install-apt.yml

@@ -20,8 +20,6 @@
     owner: root
     group: root
     mode: 0644
-  tags:
-    - repo-build-apt-packages
 
 #TODO(evrardjp): Replace the next 2 tasks by a standard apt with cache
 #when https://github.com/ansible/ansible-modules-core/pull/1517 is merged
@@ -30,16 +28,12 @@
   stat:
     path: /var/cache/apt
   register: apt_cache_stat
-  tags:
-    - repo-build-apt-packages
 
 - name: Update apt if needed
   apt:
     update_cache: yes
   when: ("ansible_date_time.epoch|float - apt_cache_stat.stat.mtime > {{cache_timeout}}" or
         add_uca_keys | changed or add_uca_repo | changed)
-  tags:
-    - repo-build-apt-packages
 
 - name: Install apt packages
   apt:
@@ -50,5 +44,3 @@
   retries: 5
   delay: 2
   with_items: "{{ repo_build_distro_packages }}"
-  tags:
-    - repo-build-apt-packages

tasks/install-yum.yml → tasks/repo_build_install-yum.yml

@@ -22,5 +22,3 @@
   retries: 5
   delay: 2
   with_items: "{{ repo_build_distro_packages }}"
-  tags:
-    - repo-build-yum-packages

tasks/repo_build_prepare.yml

@@ -0,0 +1,114 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Initialize the wheel build local fact
+  ini_file:
+    dest: "/etc/ansible/facts.d/openstack_ansible.fact"
+    section: repo_build
+    option: need_wheel_build
+    value: False
+  when:
+    - "('openstack_ansible' not in ansible_local) or
+       ('repo_build' not in ansible_local['openstack_ansible']) or
+       ('need_wheel_build' not in ansible_local['openstack_ansible']['repo_build'])"
+
+- name: Initialize the venv build local fact
+  ini_file:
+    dest: "/etc/ansible/facts.d/openstack_ansible.fact"
+    section: repo_build
+    option: need_venv_build
+    value: False
+  when:
+    - "('openstack_ansible' not in ansible_local) or
+       ('repo_build' not in ansible_local['openstack_ansible']) or
+       ('need_venv_build' not in ansible_local['openstack_ansible']['repo_build'])"
+
+# TODO(odyssey4me):
+# This is to clean up files created in early Ocata. It may be removed in Pike.
+- name: Ensure old pool index file is cleaned up
+  file:
+    path: "{{ repo_build_pool_dir }}/index.html"
+    state: "absent"
+
+- name: Create package directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ repo_build_service_user_name }}"
+  with_items:
+    - "{{ repo_build_release_path }}"
+    - "{{ repo_build_global_links_path }}"
+
+- name: Build package requirements file
+  template:
+    src: "requirements.txt.j2"
+    dest: "{{ repo_build_release_path }}/requirements.txt"
+  register: _wheel_build_requirements
+
+- include: repo_clone_git.yml
+  when:
+    - (repo_build_git_reclone | bool) or
+      (_wheel_build_requirements | changed)
+  tags:
+    - repo-clone-repos
+
+- name: Retrieve upper constraints content
+  slurp:
+    src: "{{ repo_build_git_dir }}/requirements/upper-constraints.txt"
+  register: slurp_upper_constraints
+
+- name: Decode the upper constraints content
+  set_fact:
+    upper_constraints: "{{ slurp_upper_constraints.content | b64decode | splitlines }}"
+
+- name: Build package constraints file
+  template:
+    src: "requirements_constraints.txt.j2"
+    dest: "{{ repo_build_release_path }}/requirements_constraints.txt"
+  register: _wheel_build_constraints
+
+- name: Record whether a wheel build is required
+  ini_file:
+    dest: "/etc/ansible/facts.d/openstack_ansible.fact"
+    section: repo_build
+    option: need_wheel_build
+    value: True
+  when:
+    - (_wheel_build_requirements | changed) or
+      (_wheel_build_constraints | changed) or
+      (repo_build_wheel_rebuild | bool)
+
+- name: Record whether a venv build is required
+  ini_file:
+    dest: "/etc/ansible/facts.d/openstack_ansible.fact"
+    section: repo_build
+    option: need_venv_build
+    value: True
+  when:
+    - (_wheel_build_requirements | changed) or
+      (_wheel_build_constraints | changed) or
+      (repo_build_venv_rebuild | bool)
+
+- name: Install pip packages
+  pip:
+    name: "{{ repo_pip_packages | join(' ') }}"
+    state: latest
+    extra_args: "--constraint {{ repo_build_release_path }}/requirements_constraints.txt {{ pip_install_options }}"
+  register: install_packages
+  until: install_packages|success
+  retries: 5
+  delay: 5
+  tags:
+    - repo-build-install

tasks/repo_venv_build.yml → tasks/repo_build_venvs.yml

@@ -27,14 +27,10 @@
   command: which virtualenv
   changed_when: false
   register: virtualenv_path
-  tags:
-    - always
 
 - name: Set virtualenv command path
   set_fact:
     virtualenv_bin: "{{ virtualenv_path.stdout }}"
-  tags:
-    - always
 
 # This is removed so that virtual env is not installing unknown
 #  packages as assumed wheels, IE pip8
@@ -63,7 +59,9 @@
   shell: "/opt/venv-build-script.sh {{ repo_build_release_path }}/venv-build-options-{{ item['item']['role_name'] }}.txt"
   args:
     executable: "/bin/bash"
-  when: (item | changed) or (repo_build_venv_rebuild | bool)
+  when:
+    - (item | changed) or
+      (ansible_local['openstack_ansible']['repo_build']['need_venv_build'] | bool)
   with_items:
     - "{{ _create_venv_options_files.results }}"
   register: _build_venv
@@ -85,3 +83,10 @@
   with_items: "{{ _build_venv['results'] }}"
   when:
     - item['ansible_job_id'] is defined
+
+- name: Disable the venv build requirement now that it is complete
+  ini_file:
+    dest: "/etc/ansible/facts.d/openstack_ansible.fact"
+    section: repo_build
+    option: need_venv_build
+    value: False