From 5c307f9b20df6959a8953951f3da0f474e3288c3 Mon Sep 17 00:00:00 2001 From: Jesse Pretorius Date: Wed, 18 Oct 2017 23:31:13 +0100 Subject: [PATCH] Use the deployment host to download images for tempest Currently there are two sets of image downloads happening, which is a common cause of failure in jobs due to the fact that they're downloaded from a URL outside of OpenStack-CI. This patch consolidates them into a single single download, but also uses the deploy host to do the download. The download gets done to the same location as where the image is cached in OpenStack-CI, so as long as the SHA256SUM matches, it will skip the download - making our tests faster and more reliable. As part of this, some legacy and unnecessary variables have been removed. Also, we were uploading the same image twice into Glance which is unnecessary. Instead we re-use the same image ID twice in the tempest configuration. Change-Id: I8d0afd08f6c80594d69b8d711261ae6365fad917 --- defaults/main.yml | 26 +++++--- ...-duplicated-download-99a9ec5bfe4ba749.yaml | 37 +++++++++++ tasks/tempest_post_install.yml | 12 ---- tasks/tempest_resources.yml | 65 +++++++++++-------- templates/tempest.conf.j2 | 8 +-- 5 files changed, 96 insertions(+), 52 deletions(-) create mode 100644 releasenotes/notes/remove-duplicated-download-99a9ec5bfe4ba749.yaml diff --git a/defaults/main.yml b/defaults/main.yml index bb2a2c87..ed6f13c8 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -174,16 +174,24 @@ tempest_pip_packages: - tempest - testscenarios -# Please update SHA in tempest_images below when changing the cirros version. -cirros_version: 0.3.5 -tempest_img_url: "http://download.cirros-cloud.net/{{ cirros_version }}/cirros-{{ cirros_version }}-x86_64-disk.img" -tempest_image_dir: "/opt/images" +# The list of images for tempest to download. +# url: where to download from (required) +# checksum: the checksum of the downloaded file to validate against, format: : (optional) +# format: format to use when uploading to glance (required) +# name: name to use when uploading to glance - default is to use the downloaded file's name (optional) tempest_images: - - url: "{{ tempest_img_url }}" - sha256: "e137062a4dfbb4c225971b67781bc52183d14517170e16a3841d16f962ae7470" -tempest_image_file: "cirros-{{ cirros_version }}-x86_64-disk.img" -tempest_img_disk_format: qcow2 -tempest_img_name: 'cirros' + - url: "http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img" + checksum: "sha256:e137062a4dfbb4c225971b67781bc52183d14517170e16a3841d16f962ae7470" + format: "qcow2" + name: "cirros" + +# The location where images are downloaded to +tempest_image_dir: "/opt/cache/files" + +# Where the download is executed from. +# Options are ['deployment-host', 'target-host'] +tempest_image_downloader: "deployment-host" + tempest_enable_instance_password: True tempest_flavors: diff --git a/releasenotes/notes/remove-duplicated-download-99a9ec5bfe4ba749.yaml b/releasenotes/notes/remove-duplicated-download-99a9ec5bfe4ba749.yaml new file mode 100644 index 00000000..c02b3535 --- /dev/null +++ b/releasenotes/notes/remove-duplicated-download-99a9ec5bfe4ba749.yaml @@ -0,0 +1,37 @@ +--- +features: + - | + The ``tempest_images`` data structure for the ``os_tempest`` role + now expects the values for each image to include ``name`` (optionally) + and ``format`` (the disk format). Also, the optional variable ``checksum`` + may be used to set the checksum expected for the file in the format + ``:``. + - | + The default location for the image downloads in the ``os_tempest`` + role set by the ``tempest_image_dir`` variable has now been changed + to be ``/opt/cache/files`` in order to match the default location + in nodepool. This improves the reliability of CI testing in + OpenStack CI as it will find the file already cached there. + - | + A new variable has been introduced into the ``os_tempest`` role + named ``tempest_image_downloader``. When set to ``deployment-host`` + (which is the default) it uses the deployment host to handle the + download of images to be used for tempest testing. The images are + then uploaded to the target host for uploading into Glance. +deprecations: + - | + The following variables have been removed from the ``os_tempest`` + role to simplify it. They have been replaced through the use of + the data structure ``tempest_images`` which now has equivalent + variables per image. + - cirros_version + - tempest_img_url + - tempest_image_file + - tempest_img_disk_format + - tempest_img_name + - tempest_images.sha256 (replaced by checksum) +fixes: + - | + The ``os_tempest`` tempest role was downloading images twice - once + arbitrarily, and once to use for testing. This has been consolidated + into a single download to a consistent location. diff --git a/tasks/tempest_post_install.yml b/tasks/tempest_post_install.yml index 2e416891..f035f02a 100644 --- a/tasks/tempest_post_install.yml +++ b/tasks/tempest_post_install.yml @@ -50,18 +50,6 @@ - path: "{{ tempest_image_dir }}" when: "{{ item.when | default(True) }}" -- name: Image(s) download - get_url: - url: "{{ item.url }}" - dest: "{{ tempest_image_dir }}" - sha256sum: "{{ item.sha256 }}" - with_items: "{{ tempest_images }}" - when: tempest_service_available_glance | bool - register: fetch_url - until: fetch_url | success - retries: 6 - delay: 5 - - name: Copy tempest config config_template: src: "tempest.conf.j2" diff --git a/tasks/tempest_resources.yml b/tasks/tempest_resources.yml index 756e13fb..707ae9c3 100644 --- a/tasks/tempest_resources.yml +++ b/tasks/tempest_resources.yml @@ -13,22 +13,52 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Download tempest image +- name: Create deployment-host tempest_image_dir + file: + path: "{{ tempest_image_dir }}" + state: directory + delegate_to: localhost + when: + - tempest_service_available_glance | bool + - tempest_image_downloader == "deployment-host" + +- name: Image(s) download get_url: - url: "{{ tempest_img_url }}" - dest: "/var/tmp/{{ tempest_img_url | basename }}" + url: "{{ item.url }}" + dest: "{{ tempest_image_dir }}/" + checksum: "{{ item.checksum | default(omit) }}" + with_items: "{{ tempest_images }}" when: tempest_service_available_glance | bool + register: fetch_url + until: fetch_url | success + retries: 6 + delay: 5 + delegate_to: "{{ (tempest_image_downloader == 'deployment-host') | ternary('localhost', omit) }}" + +- name: Copy download images from deployment-host to target-host + copy: + src: "{{ tempest_image_dir }}/{{ item.url | basename }}" + dest: "{{ tempest_image_dir }}/" + with_items: "{{ tempest_images }}" + when: + - tempest_service_available_glance | bool + - tempest_image_downloader == "deployment-host" + register: fetch_url + until: fetch_url | success + retries: 6 + delay: 5 -- name: Ensure tempest image +- name: Upload tempest images to glance os_image: cloud: default endpoint_type: internal validate_certs: "{{ keystone_service_internaluri_insecure | ternary(false, true) }}" - name: "{{ tempest_img_name }}" - filename: "/var/tmp/{{ tempest_img_url | basename }}" + name: "{{ item.name | default(item.url | basename) }}" + filename: "{{ tempest_image_dir }}/{{ item.url | basename }}" container_format: bare - disk_format: "{{ tempest_img_disk_format }}" + disk_format: "{{ item.format }}" is_public: True + with_items: "{{ tempest_images }}" register: tempest_image_create until: tempest_image_create | success retries: 5 @@ -39,26 +69,7 @@ # set above to ensure the template will parse correctly. - name: Store tempest image id set_fact: - tempest_glance_image_id: "{{ tempest_service_available_glance | ternary(tempest_image_create.id, '') }}" - -- name: Ensure alt tempest image - os_image: - cloud: default - endpoint_type: internal - validate_certs: "{{ keystone_service_internaluri_insecure | ternary(false, true) }}" - name: "{{ tempest_img_name }}_alt" - filename: "/var/tmp/{{ tempest_img_url | basename }}" - container_format: bare - disk_format: "{{ tempest_img_disk_format }}" - is_public: True - register: tempest_image_alt_create - when: tempest_service_available_glance | bool - -# This fact is used in tempest.conf.j2; we set an empty string if it doesn't get -# set above to ensure the template will parse correctly. -- name: Store alt tempest image id - set_fact: - tempest_glance_image_alt_id: "{{ tempest_service_available_glance | ternary(tempest_image_alt_create.id, '') }}" + tempest_glance_image_id: "{{ tempest_service_available_glance | ternary(tempest_image_create['results'][0]['id'], '') }}" - name: Ensure tempest tenants keystone: diff --git a/templates/tempest.conf.j2 b/templates/tempest.conf.j2 index 0210dcdd..28daf388 100644 --- a/templates/tempest.conf.j2 +++ b/templates/tempest.conf.j2 @@ -24,7 +24,7 @@ endpoint_type = internalURL [compute] image_ref = {{ tempest_glance_image_id }} -image_ref_alt = {{ tempest_glance_image_alt_id }} +image_ref_alt = {{ tempest_glance_image_id }} flavor_ref = 201 flavor_ref_alt = 202 fixed_network_name = private @@ -71,7 +71,7 @@ trust = false [image] endpoint_type = internalURL -http_image = {{ tempest_img_url }} +http_image = {{ tempest_images[0]['url'] }} [image-feature-enabled] @@ -124,9 +124,9 @@ lock_path = {{ tempest_venv_bin | dirname }}/locks [scenario] img_dir = {{ tempest_image_dir }} -img_file = {{ tempest_image_file }} +img_file = {{ tempest_images[0]['url'] | basename }} img_container_format = bare -img_disk_format = {{ tempest_img_disk_format }} +img_disk_format = {{ tempest_images[0]['format'] }} [service_available]