diff --git a/README.md b/README.md index a11527c0..3d595237 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,12 @@ Role Variables build_nodepool: version: HEAD refspec: refs/for/master +* `nodepool_provider` -- Type of nodepool provider to use, it has three + possible values: + - openstack: Use an openstack tenant + - host: Use the host where docker-compose runs + - libvirt: Start up a pair of libvirt nodes at install and connects nodepool + to it Example Playbook diff --git a/defaults/main.yaml b/defaults/main.yaml index 5929edde..c762fd1a 100644 --- a/defaults/main.yaml +++ b/defaults/main.yaml @@ -56,3 +56,20 @@ nodepool_launcher_image: rdoci/nodepool-launcher:stable zuul_merger_image: rdoci/zuul-merger:stable # logs_image: httpd logs_image: rdoci/httpd:stable + +# To control nodepool setup +setup_nodepool: true + +# For libvirt +libvirt_volume_path: /opt/vm_images +libvirt_uri: qemu:///system +overcloud_nodes: + - name: subnode-0 + - name: subnode-1 +libvirt_nodepool: true +local_working_dir: "{{ install_path }}" +working_dir: "{{ install_path }}" +non_root_user: "{{ ansible_user }}" +non_root_group: "{{ ansible_user }}" +restore_snapshot: false +create_snapshot: false diff --git a/playbooks/tripleo-ci-reproducer/post.yaml b/playbooks/tripleo-ci-reproducer/post.yaml index 96ff1d28..9eb7fd57 100644 --- a/playbooks/tripleo-ci-reproducer/post.yaml +++ b/playbooks/tripleo-ci-reproducer/post.yaml @@ -42,7 +42,7 @@ chdir: "{{ ansible_user_dir }}/tripleo-ci-reproducer" changed_when: true - - name: Copy files from {{ ansible_user_dir }} on node to log + - name: Copy files from reproducer on node to log synchronize: src: '{{ ansible_user_dir }}/tripleo-ci-reproducer/' dest: '{{ zuul.executor.log_root }}/tripleo-ci-reproducer/' @@ -51,3 +51,4 @@ verify_host: true rsync_opts: - "--exclude=secrets.env" + - "--exclude=*.qcow2" diff --git a/playbooks/tripleo-ci-reproducer/pre.yaml b/playbooks/tripleo-ci-reproducer/pre.yaml index f06f2fa0..d59eaa98 100644 --- a/playbooks/tripleo-ci-reproducer/pre.yaml +++ b/playbooks/tripleo-ci-reproducer/pre.yaml @@ -21,7 +21,7 @@ become: true package: name: epel-release - - name: Install package dependencies + - name: Install pip become: true package: name: @@ -32,6 +32,7 @@ package: name: - "{{ docker_package | default('docker-1.13.1') }}" + - "{{ netaddr_package | default('python-netaddr') }}" - name: Install python dependencies pip: diff --git a/playbooks/tripleo-ci-reproducer/run.yaml b/playbooks/tripleo-ci-reproducer/run.yaml index f7bbbe93..536de7f9 100644 --- a/playbooks/tripleo-ci-reproducer/run.yaml +++ b/playbooks/tripleo-ci-reproducer/run.yaml @@ -1,4 +1,9 @@ --- + +# Zuul executor has some limitations that prevents from running the role +# directly there so we have to generate a playbook and run it at the +# node + - hosts: primary vars: repro_role_path: >- @@ -22,18 +27,23 @@ tasks: - name: Start reproducer using host as the nodepool provider vars: - nodepool_provider: host + nodepool_provider: "{{ nodepool_provider }}" upstream_gerrit_user: "{{ upstream_gerrit_user }}" upstream_gerrit_key: "{{ tripleo_gerrit_key_name }}" rdo_gerrit_user: "{{ rdo_gerrit_user }}" rdo_gerrit_key: "{{ tripleo_gerrit_key_name }}" + control_vcpu: 4 + control_memory: 4096 include_role: name: tripleo-ci-reproducer copy: content: "{{ reproducer_playbook | to_nice_yaml }}" dest: "{{ ansible_user_dir }}/start.yaml" - + - name: Generate run script + template: + src: run.sh.j2 + dest: "{{ ansible_user_dir }}/run.sh" + mode: 0755 - name: Run reproducer - command: | - ~/.local/bin/ansible-playbook -vv {{ ansible_user_dir }}/start.yaml + command: ~/run.sh changed_when: true diff --git a/playbooks/tripleo-ci-reproducer/templates/run.sh.j2 b/playbooks/tripleo-ci-reproducer/templates/run.sh.j2 new file mode 100644 index 00000000..edc1c6da --- /dev/null +++ b/playbooks/tripleo-ci-reproducer/templates/run.sh.j2 @@ -0,0 +1,17 @@ +#!/bin/bash -ex + +{% if 'git.openstack.org/openstack/tripleo-quickstart' in zuul.projects %} + +tq={{ ansible_user_dir }}/{{ zuul.projects['git.openstack.org/openstack/tripleo-quickstart'].src_dir }} +export ANSIBLE_LIBRARY="${tq}/library" +export ANSIBLE_ROLES_PATH="${tq}/roles" + +{% endif %} + +export ANSIBLE_STDOUT_CALLBACK=debug + +# To be able to use stuff installed with pip --user +# At zuul images we don't have any .bash_profile or .profile +export PATH=$PATH:$HOME/.local/bin + +ansible-playbook -vv ~/start.yaml diff --git a/tasks/encrypt-gerrit-key.yaml b/tasks/encrypt-gerrit-key.yaml index 43d90a87..b03d4bc8 100644 --- a/tasks/encrypt-gerrit-key.yaml +++ b/tasks/encrypt-gerrit-key.yaml @@ -40,7 +40,7 @@ shell: chdir: "{{ zuul_config.path }}" cmd: | - pwd + set -e git remote add gerrit ssh://admin@localhost:29418/zuul-config git add zuul.d/tripleo_gerrit_key_secret.yaml git commit -m "Add secrets" diff --git a/tasks/install.yaml b/tasks/install.yaml index d88f2be3..bad2e7d7 100644 --- a/tasks/install.yaml +++ b/tasks/install.yaml @@ -21,16 +21,6 @@ - etc_nodepool - etc_zuul -- name: Generate nodepool main configuration - template: - src: "nodepool-{{ nodepool_provider }}.yaml.j2" - dest: "{{ install_path }}/etc_nodepool/nodepool.yaml" - -- name: Generate nodepool logs configuration - template: - src: "launcher-logging.yaml.j2" - dest: "{{ install_path }}/etc_nodepool/launcher-logging.yaml" - - name: Generate zuul configuration template: src: "{{ item }}.j2" @@ -55,43 +45,16 @@ dest: "{{ install_path }}/secrets.env" mode: 0600 -- name: Do openstack setup - block: - - name: Open needed ports at default security group - os_security_group_rule: - cloud: "{{ os_cloud_name }}" - security_group: default - protocol: tcp - port_range_min: "{{ item }}" - port_range_max: "{{ item }}" - remote_ip_prefix: 0.0.0.0/0 - with_items: - - 19885 - - 22 +- name: Setup nodepool provider + include_tasks: "{{ nodepool_provider }}/main.yaml" + when: nodepool_provider != "host" and setup_nodepool - - name: Get installed images names - os_image_facts: - cloud: "{{ os_cloud_name }}" - - - name: Check image are uploaded - fail: - msg: | - Cannot find {{ item }} at the openstack cloud, you can upload one from - https://nb02.openstack.org/images/ and add your ssh pub key with - virt-edit and upload it to your openstack cloud. - when: | - openstack_image|selectattr( 'name', 'equalto', item )|list|length == 0 - with_items: - - "{{ os_centos7_image }}" - - "{{ os_fedora28_image }}" - - - name: Gather facts about a previously created network by name - os_networks_facts: - cloud: "{{ os_cloud_name }}" +- name: Generate nodepool main configuration + template: + src: "nodepool-{{ nodepool_provider }}.yaml.j2" + dest: "{{ install_path }}/etc_nodepool/nodepool.yaml" - - name: Check if networks present in the tenant - fail: - msg: 'Network "{{ item }} is not found!' - when: "item not in openstack_networks|map(attribute='name')|list" - with_items: "{{ cloud_networks }}" - when: nodepool_provider == "openstack" +- name: Generate nodepool logs configuration + template: + src: "launcher-logging.yaml.j2" + dest: "{{ install_path }}/etc_nodepool/launcher-logging.yaml" diff --git a/tasks/libvirt/main.yaml b/tasks/libvirt/main.yaml new file mode 100644 index 00000000..c45d24b5 --- /dev/null +++ b/tasks/libvirt/main.yaml @@ -0,0 +1,62 @@ +--- +- name: Install libvirt packages + include_role: + name: parts/libvirt + +- name: Add user to libvirt group + become: true + user: + name: "{{ ansible_user }}" + groups: + - libvirt + - wheel + append: true +- name: reset ssh connection to allow user changes to affect + meta: reset_connection +- name: Restore a libvirt snapshot + when: restore_snapshot + vars: + ansible_become: true + snapshot_restore: true + include_role: + name: snapshot-libvirt + +- name: Setup libvirt nodes + when: not restore_snapshot + vars: + libvirt_nodepool: true + local_working_dir: "{{ install_path }}" + working_dir: "{{ install_path }}" + non_root_user: "{{ ansible_user }}" + non_root_group: "{{ ansible_user }}" + + block: + - name: Start nodes + block: + - name: Tear down nodes + include_role: + name: libvirt/teardown/nodes + + - name: Set up nodes + include_role: + name: libvirt/setup/overcloud + + - name: Set up tripleo inventory + include_role: + name: tripleo-inventory + + - name: prepare nodes + # We call playbook to parallelize + shell: | + set -e + PATH=$PATH:$HOME/.local/bin + ANSIBLE_STDOUT_CALLBACK=debug + ansible-playbook -i {{ install_path }}/hosts -vv \ + {{ role_path }}/tasks/libvirt/prepare.yaml + - name: Create libvirt snapshot + when: create_snapshot + vars: + ansible_become: true + snapshot_create: true + include_role: + name: snapshot-libvirt diff --git a/tasks/libvirt/prepare.yaml b/tasks/libvirt/prepare.yaml new file mode 100644 index 00000000..9f413c0c --- /dev/null +++ b/tasks/libvirt/prepare.yaml @@ -0,0 +1,82 @@ +--- +- hosts: subnodes + vars: + update_subnodes: true + packages_list: + - git + - unbound + - tmux + - screen + - vim + - wget + - "{% if ansible_python.version.major == 3 -%}python3-virtualenv{%- + else -%}python-virtualenv{%- endif -%}" + tasks: + - name: Create /etc/nodepool directory + file: + path: /etc/nodepool + state: directory + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: 0755 + become: true + + - name: Install repos (will be removed by toci_gate_test) + package: + name: "{{ item }}" + with_items: + - centos-release-openstack-queens + - epel-release + become: true + + - name: Install packages + package: + name: "{{ packages_list }}" + state: latest + become: true + tags: + # We really want to install latest + - skip_ansible_lint + + - name: Scan SSH keys from subnodes + shell: | + for key in {{ hostvars[item].subnode_public_ip }} \ + {{ hostvars[item].subnode_private_ip }}; do + if ! grep -q $key /etc/ssh/ssh_known_hosts ; then + ssh-keyscan $key >> /etc/ssh/ssh_known_hosts + fi + done + changed_when: true + with_inventory_hostnames: + - subnodes + become: true + + - when: update_subnodes|bool + block: + + - name: Update packages + package: + name: '*' + state: latest + become: true + tags: + # We really want to update + - skip_ansible_lint + + - name: Reboot hosts + shell: sleep 2 && shutdown -r now + async: 1 + poll: 0 + ignore_errors: true + become: true + + - name: Wait for provisioned hosts to become reachable + delegate_to: localhost + wait_for: + host: "{{ subnode_public_ip }}" + port: 22 + delay: 35 + sleep: 10 + state: started + connect_timeout: 10 + timeout: 180 diff --git a/tasks/main.yaml b/tasks/main.yaml index 8fb149fd..6ab9dded 100644 --- a/tasks/main.yaml +++ b/tasks/main.yaml @@ -30,6 +30,7 @@ - name: Start it include_tasks: start.yaml + - name: Add gerrit localhost to known_hosts known_hosts: name: "[localhost]:29418" diff --git a/tasks/openstack/main.yaml b/tasks/openstack/main.yaml new file mode 100644 index 00000000..ebe1b460 --- /dev/null +++ b/tasks/openstack/main.yaml @@ -0,0 +1,38 @@ +--- +- name: Open needed ports at default security group + os_security_group_rule: + cloud: "{{ os_cloud_name }}" + security_group: default + protocol: tcp + port_range_min: "{{ item }}" + port_range_max: "{{ item }}" + remote_ip_prefix: 0.0.0.0/0 + with_items: + - 19885 + - 22 + +- name: Get installed images names + os_image_facts: + cloud: "{{ os_cloud_name }}" + +- name: Check image are uploaded + fail: + msg: | + Cannot find {{ item }} at the openstack cloud, you can upload one from + https://nb02.openstack.org/images/ and add your ssh pub key with + virt-edit and upload it to your openstack cloud. + when: | + openstack_image|selectattr( 'name', 'equalto', item )|list|length == 0 + with_items: + - "{{ os_centos7_image }}" + - "{{ os_fedora28_image }}" + +- name: Gather facts about a previously created network by name + os_networks_facts: + cloud: "{{ os_cloud_name }}" + +- name: Check if networks present in the tenant + fail: + msg: 'Network "{{ item }} is not found!' + when: "item not in openstack_networks|map(attribute='name')|list" + with_items: "{{ cloud_networks }}" diff --git a/tasks/start.yaml b/tasks/start.yaml index 7b3c26bd..39d751b0 100644 --- a/tasks/start.yaml +++ b/tasks/start.yaml @@ -4,13 +4,19 @@ name: docker state: started +# ansible module does not support removal of anon volumes at restart +# so we do a stop/start +# +- name: Stop up zuul and friends + docker_service: + project_src: "{{ install_path }}" + state: absent + remove_volumes: true - name: Start up zuul and friends docker_service: project_src: "{{ install_path }}" state: present - build: true pull: "{{ pull | default(false) | bool }}" - restarted: true - name: Wait for zuul tenant uri: diff --git a/templates/nodepool-host.yaml.j2 b/templates/nodepool-host.yaml.j2 index 934f8f67..8ff8574d 100644 --- a/templates/nodepool-host.yaml.j2 +++ b/templates/nodepool-host.yaml.j2 @@ -8,7 +8,7 @@ labels: - name: tripleo-ovb-centos-7 providers: - - name: static-host + - name: static-{{ nodepool_provider }} driver: static pools: - name: main diff --git a/templates/nodepool-libvirt.yaml.j2 b/templates/nodepool-libvirt.yaml.j2 new file mode 100644 index 00000000..1e2173e9 --- /dev/null +++ b/templates/nodepool-libvirt.yaml.j2 @@ -0,0 +1,24 @@ +--- +zookeeper-servers: + - host: zk + +labels: + - name: centos-7 + - name: fedora-28 + - name: tripleo-ovb-centos-7 + +providers: + - name: static-{{ nodepool_provider }} + driver: static + pools: + - name: main + nodes: +{% for subnode in groups['subnodes'] %} + - name: {{ hostvars[subnode].ansible_host }} + labels: + - fedora-28 + - centos-7 + - tripleo-ovb-centos-7 + host-key: "{{ lookup('pipe', 'ssh-keyscan -t ed25519 ' + hostvars[subnode].ansible_host ).split(" ", 1)[1] }}" + username: zuul +{% endfor %} diff --git a/zuul.yaml b/zuul.yaml index 4f07b5be..8cfd75e7 100644 --- a/zuul.yaml +++ b/zuul.yaml @@ -15,24 +15,55 @@ rdo_gerrit_user: os-tripleo-ci - job: - name: tripleo-ci-reproducer-fedora-28 + name: tripleo-ci-reproducer-host + abstract: true parent: tripleo-ci-reproducer - nodeset: single-fedora-28-node + vars: + nodepool_provider: host - job: - name: tripleo-ci-reproducer-centos-7 + name: tripleo-ci-reproducer-libvirt + abstract: true parent: tripleo-ci-reproducer + vars: + nodepool_provider: libvirt + required-projects: + - openstack/tripleo-quickstart + +- job: + name: tripleo-ci-reproducer-fedora-28-host + parent: tripleo-ci-reproducer-host + nodeset: single-fedora-28-node + +- job: + name: tripleo-ci-reproducer-centos-7-host + parent: tripleo-ci-reproducer-host + nodeset: single-centos-7-node + +- job: + name: tripleo-ci-reproducer-fedora-28-libvirt + parent: tripleo-ci-reproducer-libvirt + nodeset: single-fedora-28-node + +- job: + name: tripleo-ci-reproducer-centos-7-libvirt + parent: tripleo-ci-reproducer-libvirt nodeset: single-centos-7-node + - project: name: rdo-infra/ansible-role-tripleo-ci-reproducer check: jobs: - tox-linters - - tripleo-ci-reproducer-fedora-28 - - tripleo-ci-reproducer-centos-7 + - tripleo-ci-reproducer-fedora-28-host + - tripleo-ci-reproducer-centos-7-host + - tripleo-ci-reproducer-fedora-28-libvirt + - tripleo-ci-reproducer-centos-7-libvirt gate: jobs: - tox-linters - - tripleo-ci-reproducer-fedora-28 - - tripleo-ci-reproducer-centos-7 + - tripleo-ci-reproducer-fedora-28-host + - tripleo-ci-reproducer-centos-7-host + - tripleo-ci-reproducer-fedora-28-libvirt + - tripleo-ci-reproducer-centos-7-libvirt