Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #779 from ksatchit/master
Optimizations to reduce the CI run duration (Prepull images)
- Loading branch information
Showing
10 changed files
with
211 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
--- | ||
- hosts: kubernetes-kubeminions | ||
|
||
tasks: | ||
- name: Check for presence of tarfiles | ||
shell: ls *.tar | ||
args: | ||
chdir: "{{ playbook_dir }}/roles/load-test-images/files" | ||
register: tar | ||
delegate_to: 127.0.0.1 | ||
|
||
- name: Execute load-test-images role if tarfile exists | ||
include_role: | ||
name: load-test-images | ||
when: tar.stdout != "" | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
--- | ||
|
||
tarfile_path: "{{ ansible_env.HOME}}" | ||
|
||
test_images: | ||
# keys -> label:image:tag | ||
- {label: percona, image: percona, tag: latest} | ||
- {label: jupyter, image: satyamz/docker-jupyter, tag: v0.4} | ||
- {label: postgres, image: crunchydata/crunchy-postgres, tag: centos7-9.6-1.4.0} | ||
- {label: mysqlclient, image: openebs/tests-mysql-client, tag: latest} | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
--- | ||
- block: | ||
|
||
- name: Load test images from Archive | ||
command: docker load --input "{{item.label}}_{{item.tag}}.tar" | ||
args: | ||
chdir: "/vagrant/roles/load-test-images/files" | ||
become: true | ||
with_items: "{{test_images}}" | ||
|
||
when: is_vagrant_vm | ||
|
||
- block: | ||
|
||
- name: Copy test image tarfile to kubernetes hosts | ||
copy: | ||
src: "{{item.label}}_{{item.tag}}.tar" | ||
dest: "{{ tarfile_path }}" | ||
become: true | ||
with_items: "{{test_images}}" | ||
|
||
- name: Load test images from Archive | ||
command: docker load --input "{{item.label}}_{{item.tag}}.tar" | ||
args: | ||
chdir: "{{ tarfile_path }}" | ||
become: true | ||
with_items: "{{test_images}}" | ||
|
||
when: not is_vagrant_vm | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
--- | ||
deb_local_packages: | ||
- docker.io | ||
|
||
pip_local_packages: | ||
- 'docker-py==1.9.0' | ||
|
||
test_images: | ||
# keys -> label:image:tag | ||
- {label: percona, image: percona, tag: latest} | ||
- {label: jupyter, image: satyamz/docker-jupyter, tag: v0.4} | ||
- {label: postgres, image: crunchydata/crunchy-postgres, tag: centos7-9.6-1.4.0} | ||
- {label: mysqlclient, image: openebs/tests-mysql-client, tag: latest} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,75 @@ | ||
--- | ||
- name: Install APT Packages | ||
apt: | ||
name: "{{item}}" | ||
state: present | ||
with_items: "{{deb_local_packages}}" | ||
become: true | ||
delegate_to: 127.0.0.1 | ||
|
||
- name: Install PIP Packages | ||
pip: | ||
name: "{{ item }}" | ||
state: present | ||
with_items: "{{ pip_local_packages }}" | ||
delegate_to: 127.0.0.1 | ||
become: true | ||
|
||
- name: Create files directory in load-test-images role if not present | ||
file: | ||
path: "{{ playbook_dir }}/roles/load-test-images/files" | ||
state: directory | ||
mode: 0755 | ||
|
||
- name: Check for available image tarballs | ||
shell: ls *.tar | ||
args: | ||
chdir: "{{ playbook_dir }}/roles/load-test-images/files" | ||
register: tar | ||
delegate_to: 127.0.0.1 | ||
|
||
- name: Fetch test images from dockerhub | ||
docker_image: | ||
name: "{{item.image}}:{{item.tag}}" | ||
state: present | ||
pull: true | ||
force: yes | ||
register: result | ||
until: "'Pulled image' and item.image in result.actions[0]" | ||
delay: 60 | ||
retries: 3 | ||
when: "item.label ~ '_' ~ item.tag ~ '.tar' not in tar.stdout_lines" | ||
with_items: "{{ test_images }}" | ||
become: true | ||
|
||
- name: Create TAR files for test images | ||
shell: > | ||
docker save {{item.image}}:{{item.tag}} | ||
-o {{item.label}}_{{item.tag}}.tar | ||
args: | ||
chdir: "{{playbook_dir}}/roles/load-test-images/files" | ||
creates: "{{item.label}}_{{item.tag}}.tar" | ||
become: true | ||
with_items: "{{ test_images }}" | ||
|
||
- name: Get current user details | ||
command: whoami | ||
register: user | ||
|
||
- name: Change permissions of tarfile | ||
file: | ||
path: "{{playbook_dir}}/roles/load-test-images/files" | ||
owner: "{{ user.stdout }}" | ||
group: "{{ user.stdout }}" | ||
recurse: true | ||
become: true | ||
|
||
- name: Remove test images from localhost | ||
docker_image: | ||
name: "{{item.image}}:{{item.tag}}" | ||
state: absent | ||
with_items: "{{ test_images }}" | ||
become: true | ||
|
||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
--- | ||
- hosts: localhost | ||
roles: | ||
- {role: setup-test-images} | ||
|
||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
#!/bin/bash | ||
|
||
############################################################################# | ||
# This is a script that will be run periodically as part of a cronjob on the | ||
# Jenkins master (build machine) to create docker image tarballs for the e2e | ||
# test images. The script checks for presence of the versioned image tarballs | ||
# and if the desired version is not present, fetches the image and creates the | ||
# tarball. It uses the setup-test-images ansible role to perform these tasks. | ||
# | ||
# These tarfiles are used by the load-test-images ansible role to load the | ||
# images into the test VMs created as part of the CI test run. | ||
############################################################################# | ||
|
||
source ~/.profile | ||
|
||
# file touched by the jenkins job to indicate run in progress | ||
file=ci.running | ||
|
||
# log file path | ||
log=~/openebs/tar_status.log | ||
|
||
# message definitions | ||
no_run_msg="##### CI RUN IN PROGRESS, WON'T ATTEMPT TEST IMAGE UPDATES #####" | ||
run_msg="##### RUNNING PLAYBOOK TO PEROFRM IMAGE CHECK #####" | ||
error_msg="##### ERRORS DURING PLAYBOOK RUN, CHECK ARA LOGS #####" | ||
success_msg="##### PLAYBOOK COMPLETED SUCCESSFULLY #####" | ||
|
||
# look for file indicating ongoing CI job, exit if present | ||
if [ -f "$file" ]; then | ||
echo $no_run_msg | awk '{ print strftime("%c:"),$0; fflush();}' >> $log | ||
exit | ||
fi | ||
|
||
# invoke the setup-images.yml playbook | ||
echo $run_msg | awk '{ print strftime("%c:"),$0; fflush();}' >> $log | ||
cd ~/openebs/e2e/ansible && ansible-playbook setup-images.yml 2>&1 >> $log | ||
|
||
# log failed playbook runs | ||
retcode=$? | ||
|
||
if [ $retcode -ne 0 ] | ||
then | ||
echo $error_msg | awk '{ print strftime("%c:"),$0; fflush();}' >> $log | ||
echo $retcode >> $log | ||
else | ||
echo $success_msg | awk '{ print strftime("%c:"),$0; fflush();}' >> $log | ||
fi |