Fix stackviz and subunit2html report generation

This is a better way to install and run stackviz, also removing some
tasks that's no longer necessary:
* Remove stackviz static files from the repo, since it's already in the
tarball
* Remove stackviz cloning, and using tarball
* Install stackviz directly from pip module
* Removing tempest-status.yml since we don't use to anything
* Move tempest-results.yml to run before stackviz, so we can take
advantage of the testrepository.subunit file to generate stackviz data
* Use testrepository.subunit to generate subunit2html report instead
of stestr/testr stream

Closes-Bug: 1779562
Change-Id: I918e5401b4f46e947bbaca99a48c2b21bce9a4bd

roles/validate-tempest/README.md

@@ -62,6 +62,7 @@ Role Variables
 * `tempest_deployer_input_file`: <file path> The path to tripleo deployer input file which contains the pre configured
                                  configuration for the deployed cloud using TripleO.
 * `tempest_os_cloud`: <string> String name of the cloud to export as OS_CLOUD when using clouds.yaml rather than stackrc
+* `stackviz_tarball`: <string> Stackviz tarball url to be installed via pip
 
 
 Skip tests file

roles/validate-tempest/defaults/main.yml

@@ -82,3 +82,4 @@ tempest_conf_removal_queens:
   volume.max_microversion: 3.50
   network-feature-enabled.api_extensions: dvr
 tempest_os_cloud: ""
+stackviz_tarball: "https://tarballs.openstack.org/package-stackviz-element/stackviz-latest.tar.gz"

roles/validate-tempest/tasks/main.yml

@@ -24,6 +24,11 @@
   tags:
     - run-tempest
 
+- include: tempest-results.yml
+  when: run_tempest|bool
+  tags:
+    - tempest-results
+
 - include: stackviz.yml
   when: run_tempest|bool
   tags:

roles/validate-tempest/tasks/post-tempest.yml

@@ -1,3 +1,4 @@
 ---
-- include: tempest-status.yml
-- include: tempest-results.yml
+- name: Exit with tempest result code if configured
+  shell: tail -10 {{ tempest_log_file }}; exit {{ tempest_result.rc }}
+  when: tempest_result.rc != 0 and tempest_exit_on_failure|bool

roles/validate-tempest/tasks/pre-tempest.yml

@@ -52,6 +52,12 @@
   package: name=python-stestr state=present
   when: tempest_format in ['container', 'packages'] and release not in ['newton', 'ocata']
 
+- name: Install pip
+  easy_install:
+    name: pip
+    state: present
+  become: true
+
 - ignore_errors: true
   block:
     - name: Copying bugcheck files

roles/validate-tempest/tasks/stackviz.yml

@@ -1,44 +1,8 @@
 ---
-- name: Check for .stestr directory
-  stat:
-    path: "{{ working_dir }}/tempest/.stestr"
-  register: testr_dir
-
-- name: Define testr facts
-  set_fact:
-    testr_command: testr
-    testr_dir: .testrepository
-  when: not testr_dir.stat.exists
-
-- name: Define stestr facts
-  set_fact:
-    testr_command: stestr
-    testr_dir: .stestr
-  when: testr_dir.stat.exists
-
-- name: Cloning stackviz from openstack
-  git:
-    repo: git://git.openstack.org/openstack/stackviz
-    dest: "{{ working_dir }}/stackviz"
-    version: master
-
-- name: Copying stackviz static files
-  synchronize:
-      src: stackviz/
-      dest: "{{ working_dir }}/stackviz_static"
-      use_ssh_args: true
-
-- name: Install pip
-  easy_install:
-    name: pip
-    state: present
-  become: true
-
 - name: Install stackviz
   pip:
-    name: "."
-    chdir: "{{ working_dir }}/stackviz"
-  become: true
+    name: "{{ stackviz_tarball }}"
+    virtualenv: "{{ working_dir }}/tempest/.stackviz"
 
 - name: Check if dstats file exists
   stat: path=/var/log/extra/dstat-csv.log
@@ -57,14 +21,18 @@
     recurse: yes
   become: true
 
-- name: Collecting data from tempest
+- name: Copy stackviz static file to home directory
   shell: >
-    set -o pipefail &&
-    {% if tempest_format == 'venv' %}source {{ working_dir }}/tempest_git/.venv/bin/activate; {% endif %}
-    {{ testr_command }} last --subunit | subunit-1to2 | stackviz-export {{ tempest_dstat_opt | default('') }} --env --stdin {{ working_dir }}/stackviz_static/data
+    cp -r "{{ working_dir }}/tempest/.stackviz/share/stackviz-html" "{{ working_dir }}/stackviz"
+
+- name: Collecting data from tempest
+  shell: |
+    set -o pipefail
+    source "{{ working_dir }}/tempest/.stackviz/bin/activate"
+    stackviz-export {{ tempest_dstat_opt | default('') }} --env -f {{ working_dir }}/tempest/testrepository.subunit {{ working_dir }}/stackviz/data
   args:
       chdir: "{{ working_dir }}/tempest"
 
 - name: Moving stackviz to /var/log/extra
-  shell: mv {{ working_dir }}/stackviz_static/ /var/log/extra/stackviz
+  shell: mv {{ working_dir }}/stackviz/ /var/log/extra/stackviz
   become: true

roles/validate-tempest/tasks/tempest-results.yml

@@ -1,6 +1,15 @@
 ---
 - ignore_errors: true
   block:
+    - name: Check for .stestr directory
+      stat:
+        path: "{{ working_dir }}/tempest/.stestr"
+      register: testr_dir
+
+    - name: Define testr facts
+      set_fact:
+        testr_command: "{{ testr_dir.stat.exists | ternary('stestr', 'testr') }}"
+
     - name: Change permission of tempest container log directory
       shell: |
         sudo chmod -R 777 /var/log/containers/tempest
@@ -18,14 +27,14 @@
       shell: |
         set -o pipefail &&
         {% if tempest_format == 'venv' %}source {{ working_dir }}/tempest_git/.venv/bin/activate; {% endif %}
-        subunit2html $(find {{ working_dir }}/tempest/{{ testr_dir }} -name [0-9] \
-        | head -1) {{ working_dir }}/tempest/tempest.html \
-        2>&1 >> {{ tempest_log_file }}
+        subunit2html {{ working_dir }}/tempest/testrepository.subunit {{ working_dir }}/tempest/tempest.html
 
     - name: Generate XML results file
-      shell: >
+      shell: |
         set -o pipefail &&
-        {% if tempest_format == 'venv' %}source {{ working_dir }}/tempest_git/.venv/bin/activate; {% endif %}
+        {%- if tempest_format == 'venv' -%}
+        source {{ working_dir }}/tempest_git/.venv/bin/activate;
+        {%- endif -%}
         subunit2junitxml {{ working_dir }}/tempest/testrepository.subunit --output-to {{ working_dir }}/tempest/tempest.xml 2>&1 >> {{ tempest_log_file }}
 
     - name: Fetch HTML results
@@ -64,6 +73,3 @@
         chdir: "{{ working_dir }}/tempestmail"
       ignore_errors: yes
 
-- name: Exit with tempest result code if configured
-  shell: tail -10 {{ tempest_log_file }}; exit {{ tempest_result.rc }}
-  when: tempest_result.rc != 0 and tempest_exit_on_failure|bool