Add libvirt nodepool_provider type

Implement it by starting up two libvirt nodes as nodepool ones  and
before start zuul repro connecting nodepool-launcher to them.

Zuul executor has some issues with the repro role that's why we are
generating a playbook to run the role inside the nodes.

Only Centos7 is tested.

Change-Id: I127306e2bed51df08301ae76d28f34828d1bfcba

README.md

@@ -40,6 +40,12 @@ Role Variables
        build_nodepool:
           version: HEAD
           refspec: refs/for/master
+* `nodepool_provider` -- Type of nodepool provider to use, it has three
+  possible values:
+  - openstack: Use an openstack tenant
+  - host: Use the host where docker-compose runs
+  - libvirt: Start up a pair of libvirt nodes at install and connects nodepool
+    to it
 
 
 Example Playbook

defaults/main.yaml

@@ -56,3 +56,20 @@ nodepool_launcher_image: rdoci/nodepool-launcher:stable
 zuul_merger_image: rdoci/zuul-merger:stable
 # logs_image: httpd
 logs_image: rdoci/httpd:stable
+
+# To control nodepool setup
+setup_nodepool: true
+
+# For libvirt
+libvirt_volume_path: /opt/vm_images
+libvirt_uri: qemu:///system
+overcloud_nodes:
+  - name: subnode-0
+  - name: subnode-1
+libvirt_nodepool: true
+local_working_dir: "{{ install_path }}"
+working_dir: "{{ install_path }}"
+non_root_user: "{{ ansible_user }}"
+non_root_group: "{{ ansible_user }}"
+restore_snapshot: false
+create_snapshot: false

playbooks/tripleo-ci-reproducer/post.yaml

@@ -42,7 +42,7 @@
         chdir: "{{ ansible_user_dir }}/tripleo-ci-reproducer"
       changed_when: true
 
-    - name: Copy files from {{ ansible_user_dir }} on node to log
+    - name: Copy files from reproducer on node to log
       synchronize:
         src: '{{ ansible_user_dir }}/tripleo-ci-reproducer/'
         dest: '{{ zuul.executor.log_root }}/tripleo-ci-reproducer/'
@@ -51,3 +51,4 @@
         verify_host: true
         rsync_opts:
           - "--exclude=secrets.env"
+          - "--exclude=*.qcow2"

playbooks/tripleo-ci-reproducer/pre.yaml

@@ -21,7 +21,7 @@
           become: true
           package:
             name: epel-release
-        - name: Install package dependencies
+        - name: Install pip
           become: true
           package:
             name:
@@ -32,6 +32,7 @@
       package:
         name:
           - "{{ docker_package | default('docker-1.13.1') }}"
+          - "{{ netaddr_package | default('python-netaddr') }}"
 
     - name: Install python dependencies
       pip:

playbooks/tripleo-ci-reproducer/run.yaml

@@ -1,4 +1,9 @@
 ---
+
+# Zuul executor has some limitations that prevents from running the role
+# directly there so we have to generate a playbook and run it at the
+# node
+
 - hosts: primary
   vars:
     repro_role_path: >-
@@ -22,18 +27,23 @@
             tasks:
               - name: Start reproducer using host as the nodepool provider
                 vars:
-                  nodepool_provider: host
+                  nodepool_provider: "{{ nodepool_provider }}"
                   upstream_gerrit_user: "{{ upstream_gerrit_user }}"
                   upstream_gerrit_key: "{{ tripleo_gerrit_key_name }}"
                   rdo_gerrit_user: "{{ rdo_gerrit_user }}"
                   rdo_gerrit_key: "{{ tripleo_gerrit_key_name }}"
+                  control_vcpu: 4
+                  control_memory: 4096
                 include_role:
                   name: tripleo-ci-reproducer
       copy:
         content: "{{ reproducer_playbook | to_nice_yaml }}"
         dest: "{{ ansible_user_dir }}/start.yaml"
-
+    - name: Generate run script
+      template:
+        src: run.sh.j2
+        dest: "{{ ansible_user_dir }}/run.sh"
+        mode: 0755
     - name: Run reproducer
-      command: |
-       ~/.local/bin/ansible-playbook -vv {{ ansible_user_dir }}/start.yaml
+      command: ~/run.sh
       changed_when: true

playbooks/tripleo-ci-reproducer/templates/run.sh.j2

@@ -0,0 +1,17 @@
+#!/bin/bash -ex
+
+{% if 'git.openstack.org/openstack/tripleo-quickstart' in zuul.projects %}
+
+tq={{ ansible_user_dir }}/{{ zuul.projects['git.openstack.org/openstack/tripleo-quickstart'].src_dir }}
+export ANSIBLE_LIBRARY="${tq}/library"
+export ANSIBLE_ROLES_PATH="${tq}/roles"
+
+{% endif %}
+
+export ANSIBLE_STDOUT_CALLBACK=debug
+
+# To be able to use stuff installed with pip --user
+# At zuul images we don't have any .bash_profile or .profile
+export PATH=$PATH:$HOME/.local/bin
+
+ansible-playbook -vv ~/start.yaml

tasks/encrypt-gerrit-key.yaml

@@ -40,7 +40,7 @@
       shell:
         chdir: "{{ zuul_config.path }}"
         cmd: |
-          pwd
+          set -e
           git remote add gerrit ssh://admin@localhost:29418/zuul-config
           git add zuul.d/tripleo_gerrit_key_secret.yaml
           git commit -m "Add secrets"

tasks/install.yaml

@@ -21,16 +21,6 @@
     - etc_nodepool
     - etc_zuul
 
-- name: Generate nodepool main configuration
-  template:
-    src: "nodepool-{{ nodepool_provider }}.yaml.j2"
-    dest: "{{ install_path }}/etc_nodepool/nodepool.yaml"
-
-- name: Generate nodepool logs configuration
-  template:
-    src: "launcher-logging.yaml.j2"
-    dest: "{{ install_path }}/etc_nodepool/launcher-logging.yaml"
-
 - name: Generate zuul configuration
   template:
     src: "{{ item }}.j2"
@@ -55,43 +45,16 @@
     dest: "{{ install_path }}/secrets.env"
     mode: 0600
 
-- name: Do openstack setup
-  block:
-    - name: Open needed ports at default security group
-      os_security_group_rule:
-        cloud: "{{ os_cloud_name }}"
-        security_group: default
-        protocol: tcp
-        port_range_min: "{{ item }}"
-        port_range_max: "{{ item }}"
-        remote_ip_prefix: 0.0.0.0/0
-      with_items:
-        - 19885
-        - 22
+- name: Setup nodepool provider
+  include_tasks: "{{ nodepool_provider }}/main.yaml"
+  when: nodepool_provider != "host" and setup_nodepool
 
-    - name: Get installed images names
-      os_image_facts:
-        cloud: "{{ os_cloud_name }}"
-
-    - name: Check image are uploaded
-      fail:
-        msg: |
-          Cannot find {{ item }} at the openstack cloud, you can upload one from
-          https://nb02.openstack.org/images/ and add your ssh pub key with
-          virt-edit and upload it to your openstack cloud.
-      when: |
-        openstack_image|selectattr( 'name', 'equalto', item )|list|length == 0
-      with_items:
-        - "{{ os_centos7_image }}"
-        - "{{ os_fedora28_image }}"
-
-    - name: Gather facts about a previously created network by name
-      os_networks_facts:
-        cloud: "{{ os_cloud_name }}"
+- name: Generate nodepool main configuration
+  template:
+    src: "nodepool-{{ nodepool_provider }}.yaml.j2"
+    dest: "{{ install_path }}/etc_nodepool/nodepool.yaml"
 
-    - name: Check if networks present in the tenant
-      fail:
-        msg: 'Network "{{ item }} is not found!'
-      when: "item not in openstack_networks|map(attribute='name')|list"
-      with_items: "{{ cloud_networks }}"
-  when: nodepool_provider == "openstack"
+- name: Generate nodepool logs configuration
+  template:
+    src: "launcher-logging.yaml.j2"
+    dest: "{{ install_path }}/etc_nodepool/launcher-logging.yaml"

tasks/libvirt/main.yaml

@@ -0,0 +1,62 @@
+---
+- name: Install libvirt packages
+  include_role:
+    name: parts/libvirt
+
+- name: Add user to libvirt group
+  become: true
+  user:
+    name: "{{ ansible_user }}"
+    groups:
+      - libvirt
+      - wheel
+    append: true
+- name: reset ssh connection to allow user changes to affect
+  meta: reset_connection
+- name: Restore a libvirt snapshot
+  when: restore_snapshot
+  vars:
+    ansible_become: true
+    snapshot_restore: true
+  include_role:
+    name: snapshot-libvirt
+
+- name: Setup libvirt nodes
+  when: not restore_snapshot
+  vars:
+    libvirt_nodepool: true
+    local_working_dir: "{{ install_path }}"
+    working_dir: "{{ install_path }}"
+    non_root_user: "{{ ansible_user }}"
+    non_root_group: "{{ ansible_user }}"
+
+  block:
+    - name: Start nodes
+      block:
+        - name: Tear down nodes
+          include_role:
+            name: libvirt/teardown/nodes
+
+        - name: Set up nodes
+          include_role:
+            name: libvirt/setup/overcloud
+
+        - name: Set up tripleo inventory
+          include_role:
+            name: tripleo-inventory
+
+    - name: prepare nodes
+      # We call playbook to parallelize
+      shell: |
+        set -e
+        PATH=$PATH:$HOME/.local/bin
+        ANSIBLE_STDOUT_CALLBACK=debug
+        ansible-playbook -i {{ install_path }}/hosts -vv \
+          {{ role_path }}/tasks/libvirt/prepare.yaml
+    - name: Create libvirt snapshot
+      when: create_snapshot
+      vars:
+        ansible_become: true
+        snapshot_create: true
+      include_role:
+        name: snapshot-libvirt

tasks/libvirt/prepare.yaml

@@ -0,0 +1,82 @@
+---
+- hosts: subnodes
+  vars:
+    update_subnodes: true
+    packages_list:
+      - git
+      - unbound
+      - tmux
+      - screen
+      - vim
+      - wget
+      - "{% if ansible_python.version.major == 3 -%}python3-virtualenv{%-
+          else -%}python-virtualenv{%- endif -%}"
+  tasks:
+    - name: Create /etc/nodepool directory
+      file:
+        path: /etc/nodepool
+        state: directory
+        owner: "{{ ansible_user }}"
+        group: "{{ ansible_user }}"
+        mode: 0755
+      become: true
+
+    - name: Install repos (will be removed by toci_gate_test)
+      package:
+        name: "{{ item }}"
+      with_items:
+        - centos-release-openstack-queens
+        - epel-release
+      become: true
+
+    - name: Install packages
+      package:
+        name: "{{ packages_list }}"
+        state: latest
+      become: true
+      tags:
+        # We really want to install latest
+        - skip_ansible_lint
+
+    - name: Scan SSH keys from subnodes
+      shell: |
+        for key in {{ hostvars[item].subnode_public_ip }} \
+          {{ hostvars[item].subnode_private_ip }}; do
+          if ! grep -q $key /etc/ssh/ssh_known_hosts ; then
+            ssh-keyscan $key >> /etc/ssh/ssh_known_hosts
+          fi
+        done
+      changed_when: true
+      with_inventory_hostnames:
+        - subnodes
+      become: true
+
+    - when: update_subnodes|bool
+      block:
+
+        - name: Update packages
+          package:
+            name: '*'
+            state: latest
+          become: true
+          tags:
+            # We really want to update
+            - skip_ansible_lint
+
+        - name: Reboot hosts
+          shell: sleep 2 && shutdown -r now
+          async: 1
+          poll: 0
+          ignore_errors: true
+          become: true
+
+        - name: Wait for provisioned hosts to become reachable
+          delegate_to: localhost
+          wait_for:
+            host: "{{ subnode_public_ip }}"
+            port: 22
+            delay: 35
+            sleep: 10
+            state: started
+            connect_timeout: 10
+            timeout: 180

tasks/main.yaml

@@ -30,6 +30,7 @@
 
     - name: Start it
       include_tasks: start.yaml
+
     - name: Add gerrit localhost to known_hosts
       known_hosts:
         name: "[localhost]:29418"

tasks/openstack/main.yaml

@@ -0,0 +1,38 @@
+---
+- name: Open needed ports at default security group
+  os_security_group_rule:
+    cloud: "{{ os_cloud_name }}"
+    security_group: default
+    protocol: tcp
+    port_range_min: "{{ item }}"
+    port_range_max: "{{ item }}"
+    remote_ip_prefix: 0.0.0.0/0
+  with_items:
+    - 19885
+    - 22
+
+- name: Get installed images names
+  os_image_facts:
+    cloud: "{{ os_cloud_name }}"
+
+- name: Check image are uploaded
+  fail:
+    msg: |
+      Cannot find {{ item }} at the openstack cloud, you can upload one from
+      https://nb02.openstack.org/images/ and add your ssh pub key with
+      virt-edit and upload it to your openstack cloud.
+  when: |
+    openstack_image|selectattr( 'name', 'equalto', item )|list|length == 0
+  with_items:
+    - "{{ os_centos7_image }}"
+    - "{{ os_fedora28_image }}"
+
+- name: Gather facts about a previously created network by name
+  os_networks_facts:
+    cloud: "{{ os_cloud_name }}"
+
+- name: Check if networks present in the tenant
+  fail:
+    msg: 'Network "{{ item }} is not found!'
+  when: "item not in openstack_networks|map(attribute='name')|list"
+  with_items: "{{ cloud_networks }}"