Various tweaks

Dockerfile

@@ -10,7 +10,7 @@ RUN apt-get update && apt-get install --yes sudo python3-dev python3-pip vim git
     echo "rally ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/00-rally-user && \
     mkdir /rally && chown -R rally:rally /rally
 
-RUN pip install git+https://github.com/openstack/rally-openstack.git  --constraint https://raw.githubusercontent.com/openstack/rally-openstack/master/upper-constraints.txt --no-cache-dir && \
+RUN pip install git+https://github.com/openstack/rally-openstack.git --no-cache-dir && \
     pip3 install pymysql psycopg2-binary --no-cache-dir
 
 COPY ./etc/motd_for_docker /etc/motd
@@ -32,6 +32,7 @@ RUN rally verify create-verifier --name default --type tempest
 
 COPY bin/rally-verify-wrapper.sh /usr/bin/rally-verify-wrapper.sh
 COPY bin/rally-extract-tests.sh /usr/bin/rally-extract-tests.sh
+COPY bin/rally-normalize.py /usr/bin/rally-normalize.py
 
 # Data generated during the image creation is copied to volume only when it's
 # attached for the first time (volume initialization)

bin/rally-normalize.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+
+# Originally from: https://opendev.org/osf/refstack-client/src/branch/master/refstack_client/list_parser.py
+
+# Copyright (c) 2015 IBM Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+
+import atexit
+import logging
+import os
+import re
+import requests
+import subprocess
+import tempfile
+
+
+class TestListParser(object):
+
+    """This class is for normalizing test lists to match the tests in the
+    current Tempest environment.
+    """
+
+    def __init__(self, insecure=False):
+        """
+        Initialize the TestListParser.
+        :param tempest_dir: Absolute path of the Tempest directory.
+        :param insecure: Whether https requests, if any, should be insecure.
+        """
+        self.logger = logging.getLogger(__name__)
+        self.insecure = insecure
+
+    def _get_tempest_test_ids(self):
+        """This does a 'testr list-tests' or 'stestr list' according to
+        Tempest version on the Tempest directory in order to get a list
+        of full test IDs for the current Tempest environment. Test ID
+        mappings are then formed for these tests.
+        """
+        cmd = ('rally', 'verify', 'list-verifier-tests')
+        process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+        (stdout, stderr) = process.communicate()
+
+        if process.returncode != 0:
+            self.logger.error(stdout)
+            self.logger.error(stderr)
+            raise subprocess.CalledProcessError(process.returncode,
+                                                ' '.join(cmd))
+        try:
+            return self._form_test_id_mappings(stdout.split('\n'))
+        except TypeError:
+            return self._form_test_id_mappings(stdout.decode().split('\n'))
+
+    def _form_test_id_mappings(self, test_list):
+        """This takes in a list of full test IDs and forms a dict containing
+        base test IDs mapped to their attributes. A full test ID also contains
+        test attributes such as '[gate,smoke]'
+        Ex:
+            'tempest.api.test1': '[gate]'
+            'tempest.api.test2': ''
+            'tempest.api.test3(some_scenario)': '[smoke,gate]'
+        :param test_list: List of full test IDs
+        """
+        test_mappings = {}
+        for testcase in test_list:
+            if testcase.startswith("tempest"):
+                # Search for any strings like '[smoke, gate]' in the test ID.
+                match = re.search('(\[.*\])', testcase)
+
+                if match:
+                    testcase = re.sub('\[.*\]', '', testcase)
+                    test_mappings[testcase] = match.group(1)
+                else:
+                    test_mappings[testcase] = ""
+        return test_mappings
+
+    def _get_base_test_ids_from_list_file(self, list_location):
+        """This takes in a test list file and finds all the base test IDs
+        for the tests listed.
+        Ex:
+            'tempest.test1[gate,id-2]' -> 'tempest.test1'
+            'tempest.test2[gate,id-3](scenario)' -> 'tempest.test2(scenario)'
+        :param list_location: file path or URL location of list file
+        """
+        try:
+            response = requests.get(list_location,
+                                    verify=not self.insecure)
+            testcase_list = response.text.split('\n')
+            test_mappings = self._form_test_id_mappings(testcase_list)
+        # If the location isn't a valid URL, we assume it is a file path.
+        except requests.exceptions.MissingSchema:
+            try:
+                with open(list_location) as data_file:
+                    testcase_list = [line.rstrip('\n') for line in data_file]
+                test_mappings = self._form_test_id_mappings(testcase_list)
+            except Exception:
+                self.logger.error("Error reading the passed in test list " +
+                                  "file.")
+                raise
+        except Exception:
+            self.logger.error("Error reading the passed in test list file.")
+            raise
+
+        return list(test_mappings.keys())
+
+    def _get_full_test_ids(self, tempest_ids, base_ids):
+        """This will remake the test ID list with the full IDs of the current
+        Tempest environment. The Tempest test ID dict should have the correct
+        mappings.
+        :param tempest_ids: dict containing test ID mappings
+        :param base_ids: list containing base test IDs
+        """
+        test_list = []
+        for test_id in base_ids:
+            try:
+                attr = tempest_ids[test_id]
+                # If the test has a scenario in the test ID, but also has some
+                # additional attributes, the attributes need to go before the
+                # scenario.
+                if '(' in test_id and attr:
+                    components = test_id.split('(', 1)
+                    test_portion = components[0]
+                    scenario = "(" + components[1]
+                    test_list.append(test_portion + attr + scenario)
+                else:
+                    test_list.append(test_id + attr)
+            except KeyError:
+                self.logger.warning("Test %s not found in Tempest list." %
+                                    test_id)
+        self.logger.debug("Number of tests: " + str(len(test_list)))
+        return test_list
+
+    def _write_normalized_test_list(self, test_ids):
+        """Create a temporary file to pass into testr containing a list of test
+        IDs that should be tested.
+        :param test_ids: list of full test IDs
+        """
+        temp = tempfile.NamedTemporaryFile(delete=False)
+        for test_id in test_ids:
+            temp.write(("%s\n" % test_id).encode('utf-8'))
+        temp.flush()
+
+        # Register the created file for cleanup.
+        atexit.register(self._remove_test_list_file, temp.name)
+        return temp.name
+
+    def _remove_test_list_file(self, file_path):
+        """Delete the given file.
+        :param file_path: string containing the location of the file
+        """
+        if os.path.isfile(file_path):
+            os.remove(file_path)
+
+    def get_normalized_test_list(self, list_location):
+        """This will take in the user's test list and will normalize it
+        so that the test cases in the list map to actual full test IDS in
+        the Tempest environment.
+        :param list_location: file path or URL of the test list
+        """
+        tempest_test_ids = self._get_tempest_test_ids()
+        #raise ValueError(tempest_test_ids)
+        if not tempest_test_ids:
+            return None
+        base_test_ids = self._get_base_test_ids_from_list_file(list_location)
+        full_capability_test_ids = self._get_full_test_ids(tempest_test_ids,
+                                                           base_test_ids)
+        list_file = self._write_normalized_test_list(full_capability_test_ids)
+        return list_file
+
+    def create_whitelist(self, list_location):
+        """This takes in a test list file, get normalized, and get whitelist
+        regexes using full qualified test names (one per line).
+        Ex:
+            'tempest.test1[id-2,gate]' -> tempest.test1\[
+            'tempest.test2[id-3,smoke](scenario)' -> tempest.test2\[
+            'tempest.test3[compute,id-4]' -> tempest.test3\[
+        :param list_location: file path or URL location of list file
+        """
+        normalized_list = open(self.get_normalized_test_list(list_location),
+                               'r').read()
+        # Keep the names
+        tests_list = [re.sub("\[", "\[", test)
+                      for test in re.findall(".*\[", normalized_list)]
+
+        return self._write_normalized_test_list(tests_list)
+
+
+if __name__ == "__main__":
+   import sys
+   a = TestListParser()
+   result = open(a.get_normalized_test_list(sys.argv[1]),
+                               'r').read()
+   with open(sys.argv[1], 'w') as f:
+      print(result, file=f)
+

bin/rally-verify-wrapper.sh

@@ -22,10 +22,26 @@ if [ ! -z ${TEMPEST_VERSION:+x} ]; then
     tempest_version="--version $TEMPEST_VERSION"
 fi
 
+if [ ! -z ${TEMPEST_LOAD_LIST:+x} ]; then
+    echo "$TEMPEST_LOAD_LIST" > ~/tempest-load-list
+fi
+
+if [ ! -z ${TEMPEST_SKIP_LIST:+x} ]; then
+    echo "$TEMPEST_SKIP_LIST" > ~/tempest-skip-list
+fi
+
+if [ ! -z ${TEMPEST_CONF_OVERRIDES:+x} ]; then
+    echo "$TEMPEST_CONF_OVERRIDES" > ~/tempest-overrides.conf
+fi
+
 load_list=""
 # You can't have a load list and a pattern, pattern takes priority
 if [ -f ~/tempest-load-list ] && [ -z ${TEMPEST_PATTERN:+x} ]; then
     load_list="--load-list /home/rally/tempest-load-list"
+    if [ $(wc -l /home/rally/tempest-load-list | cut -d ' ' -f 1) -lt 1]; then
+        echo >&2 "The load list appears to be empty, exiting..."
+        exit -1
+    fi
 fi
 
 skip_list=""
@@ -63,6 +79,10 @@ set -x
 unset OS_CACERT
 
 crudini --set ~/.rally/rally.conf DEFAULT openstack_client_http_timeout 300
+crudini --set ~/.rally/rally.conf openstack flavor_ref_ram 128
+crudini --set ~/.rally/rally.conf openstack flavor_ref_alt_ram 256
+crudini --set ~/.rally/rally.conf openstack flavor_ref_disk 1
+crudini --set ~/.rally/rally.conf openstack flavor_ref_alt_disk 1
 
 rally deployment create --fromenv --name openstack
 
@@ -74,7 +94,18 @@ if [ -f ~/tempest-overrides.conf ]; then
     rally verify configure-verifier --reconfigure --extend ~/tempest-overrides.conf
 fi
 
-rally verify start $skip_list $load_list $pattern $concurrency > >(tee -a $artifacts_dir/stdout.log) 2> >(tee -a $artifacts_dir/stderr.log >&2)
+if [ -f ~/tempest-load-list ] && [ -z ${TEMPEST_PATTERN:+x} ]; then
+    if [ ${TEMPEST_NORMALIZE_LOAD_LIST:-1} -eq 1 ]; then
+        echo normalizing load-list
+        rally-normalize.py /home/rally/tempest-load-list
+    fi
+    if [ $(wc -l /home/rally/tempest-load-list | cut -d ' ' -f 1) -lt 1 ]; then
+        echo >&2 "The load list appears to be empty, exiting..."
+        exit -1
+    fi
+fi
+
+rally verify start $skip_list $load_list $pattern $concurrency > >(tee -a $artifacts_dir/stdout.log) 2> >(tee -a $artifacts_dir/stderr.log >&2) || export failed=1
 
 rally verify report --type html --to $artifacts_dir/rally-verify-report.html
 rally verify report --type json --to $artifacts_dir/rally-verify-report.json