From 0f3e410aab31dcc49f7d3dfc805079300b65b21f Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 2 Jul 2021 19:49:29 +0100 Subject: [PATCH 1/4] Tweak rally generated default flavor sizes Was having issues with the cirros instances failing to boot. --- bin/rally-verify-wrapper.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/rally-verify-wrapper.sh b/bin/rally-verify-wrapper.sh index d7757af..b29f320 100755 --- a/bin/rally-verify-wrapper.sh +++ b/bin/rally-verify-wrapper.sh @@ -75,6 +75,10 @@ set -x unset OS_CACERT crudini --set ~/.rally/rally.conf DEFAULT openstack_client_http_timeout 300 +crudini --set ~/.rally/rally.conf openstack flavor_ref_ram 128 +crudini --set ~/.rally/rally.conf openstack flavor_ref_alt_ram 256 +crudini --set ~/.rally/rally.conf openstack flavor_ref_disk 1 +crudini --set ~/.rally/rally.conf openstack flavor_ref_alt_disk 1 rally deployment create --fromenv --name openstack From 09cb6e7f0d969c55557183bcd4eb007f21dc4e4a Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 10 Sep 2021 17:14:42 +0100 Subject: [PATCH 2/4] Stop using upper-constraints Hitting issues with dependency resolution. These can be added back when the issue is fixed. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f2e597a..35845d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ RUN apt-get update && apt-get install --yes sudo python3-dev python3-pip vim git echo "rally ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/00-rally-user && \ mkdir /rally && chown -R rally:rally /rally -RUN pip install git+https://github.com/openstack/rally-openstack.git --constraint https://raw.githubusercontent.com/openstack/rally-openstack/master/upper-constraints.txt --no-cache-dir && \ +RUN pip install git+https://github.com/openstack/rally-openstack.git --no-cache-dir && \ pip3 install pymysql psycopg2-binary --no-cache-dir COPY ./etc/motd_for_docker /etc/motd From 5c0852daccd1abc8128a262347196aa66674ce92 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 10 Sep 2021 17:18:07 +0100 Subject: [PATCH 3/4] Don't exit early on failure This seems to be a change in behaviour. We want to be able to collect the artifacts after a run. --- bin/rally-verify-wrapper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/rally-verify-wrapper.sh b/bin/rally-verify-wrapper.sh index b29f320..6f05230 100755 --- a/bin/rally-verify-wrapper.sh +++ b/bin/rally-verify-wrapper.sh @@ -90,7 +90,7 @@ if [ -f ~/tempest-overrides.conf ]; then rally verify configure-verifier --reconfigure --extend ~/tempest-overrides.conf fi -rally verify start $skip_list $load_list $pattern $concurrency > >(tee -a $artifacts_dir/stdout.log) 2> >(tee -a $artifacts_dir/stderr.log >&2) +rally verify start $skip_list $load_list $pattern $concurrency > >(tee -a $artifacts_dir/stdout.log) 2> >(tee -a $artifacts_dir/stderr.log >&2) || export failed=1 rally verify report --type html --to $artifacts_dir/rally-verify-report.html rally verify report --type json --to $artifacts_dir/rally-verify-report.json From af3cc165642d80ad27c07d75e870355d4eaff484 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 10 Sep 2021 17:16:48 +0100 Subject: [PATCH 4/4] Support normalizing tempest test list This is uses an adapted version of the tool shipped in refstack so that we can use refstack test lists unmodified. --- Dockerfile | 1 + bin/rally-normalize.py | 206 ++++++++++++++++++++++++++++++++++++ bin/rally-verify-wrapper.sh | 15 +++ 3 files changed, 222 insertions(+) create mode 100755 bin/rally-normalize.py diff --git a/Dockerfile b/Dockerfile index 35845d4..a4ed3fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,6 +32,7 @@ RUN rally verify create-verifier --name default --type tempest COPY bin/rally-verify-wrapper.sh /usr/bin/rally-verify-wrapper.sh COPY bin/rally-extract-tests.sh /usr/bin/rally-extract-tests.sh +COPY bin/rally-normalize.py /usr/bin/rally-normalize.py # Data generated during the image creation is copied to volume only when it's # attached for the first time (volume initialization) diff --git a/bin/rally-normalize.py b/bin/rally-normalize.py new file mode 100755 index 0000000..956d954 --- /dev/null +++ b/bin/rally-normalize.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 + +# Originally from: https://opendev.org/osf/refstack-client/src/branch/master/refstack_client/list_parser.py + +# Copyright (c) 2015 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + + +import atexit +import logging +import os +import re +import requests +import subprocess +import tempfile + + +class TestListParser(object): + + """This class is for normalizing test lists to match the tests in the + current Tempest environment. + """ + + def __init__(self, insecure=False): + """ + Initialize the TestListParser. + :param tempest_dir: Absolute path of the Tempest directory. + :param insecure: Whether https requests, if any, should be insecure. + """ + self.logger = logging.getLogger(__name__) + self.insecure = insecure + + def _get_tempest_test_ids(self): + """This does a 'testr list-tests' or 'stestr list' according to + Tempest version on the Tempest directory in order to get a list + of full test IDs for the current Tempest environment. Test ID + mappings are then formed for these tests. + """ + cmd = ('rally', 'verify', 'list-verifier-tests') + process = subprocess.Popen(cmd, stdout=subprocess.PIPE) + (stdout, stderr) = process.communicate() + + if process.returncode != 0: + self.logger.error(stdout) + self.logger.error(stderr) + raise subprocess.CalledProcessError(process.returncode, + ' '.join(cmd)) + try: + return self._form_test_id_mappings(stdout.split('\n')) + except TypeError: + return self._form_test_id_mappings(stdout.decode().split('\n')) + + def _form_test_id_mappings(self, test_list): + """This takes in a list of full test IDs and forms a dict containing + base test IDs mapped to their attributes. A full test ID also contains + test attributes such as '[gate,smoke]' + Ex: + 'tempest.api.test1': '[gate]' + 'tempest.api.test2': '' + 'tempest.api.test3(some_scenario)': '[smoke,gate]' + :param test_list: List of full test IDs + """ + test_mappings = {} + for testcase in test_list: + if testcase.startswith("tempest"): + # Search for any strings like '[smoke, gate]' in the test ID. + match = re.search('(\[.*\])', testcase) + + if match: + testcase = re.sub('\[.*\]', '', testcase) + test_mappings[testcase] = match.group(1) + else: + test_mappings[testcase] = "" + return test_mappings + + def _get_base_test_ids_from_list_file(self, list_location): + """This takes in a test list file and finds all the base test IDs + for the tests listed. + Ex: + 'tempest.test1[gate,id-2]' -> 'tempest.test1' + 'tempest.test2[gate,id-3](scenario)' -> 'tempest.test2(scenario)' + :param list_location: file path or URL location of list file + """ + try: + response = requests.get(list_location, + verify=not self.insecure) + testcase_list = response.text.split('\n') + test_mappings = self._form_test_id_mappings(testcase_list) + # If the location isn't a valid URL, we assume it is a file path. + except requests.exceptions.MissingSchema: + try: + with open(list_location) as data_file: + testcase_list = [line.rstrip('\n') for line in data_file] + test_mappings = self._form_test_id_mappings(testcase_list) + except Exception: + self.logger.error("Error reading the passed in test list " + + "file.") + raise + except Exception: + self.logger.error("Error reading the passed in test list file.") + raise + + return list(test_mappings.keys()) + + def _get_full_test_ids(self, tempest_ids, base_ids): + """This will remake the test ID list with the full IDs of the current + Tempest environment. The Tempest test ID dict should have the correct + mappings. + :param tempest_ids: dict containing test ID mappings + :param base_ids: list containing base test IDs + """ + test_list = [] + for test_id in base_ids: + try: + attr = tempest_ids[test_id] + # If the test has a scenario in the test ID, but also has some + # additional attributes, the attributes need to go before the + # scenario. + if '(' in test_id and attr: + components = test_id.split('(', 1) + test_portion = components[0] + scenario = "(" + components[1] + test_list.append(test_portion + attr + scenario) + else: + test_list.append(test_id + attr) + except KeyError: + self.logger.warning("Test %s not found in Tempest list." % + test_id) + self.logger.debug("Number of tests: " + str(len(test_list))) + return test_list + + def _write_normalized_test_list(self, test_ids): + """Create a temporary file to pass into testr containing a list of test + IDs that should be tested. + :param test_ids: list of full test IDs + """ + temp = tempfile.NamedTemporaryFile(delete=False) + for test_id in test_ids: + temp.write(("%s\n" % test_id).encode('utf-8')) + temp.flush() + + # Register the created file for cleanup. + atexit.register(self._remove_test_list_file, temp.name) + return temp.name + + def _remove_test_list_file(self, file_path): + """Delete the given file. + :param file_path: string containing the location of the file + """ + if os.path.isfile(file_path): + os.remove(file_path) + + def get_normalized_test_list(self, list_location): + """This will take in the user's test list and will normalize it + so that the test cases in the list map to actual full test IDS in + the Tempest environment. + :param list_location: file path or URL of the test list + """ + tempest_test_ids = self._get_tempest_test_ids() + #raise ValueError(tempest_test_ids) + if not tempest_test_ids: + return None + base_test_ids = self._get_base_test_ids_from_list_file(list_location) + full_capability_test_ids = self._get_full_test_ids(tempest_test_ids, + base_test_ids) + list_file = self._write_normalized_test_list(full_capability_test_ids) + return list_file + + def create_whitelist(self, list_location): + """This takes in a test list file, get normalized, and get whitelist + regexes using full qualified test names (one per line). + Ex: + 'tempest.test1[id-2,gate]' -> tempest.test1\[ + 'tempest.test2[id-3,smoke](scenario)' -> tempest.test2\[ + 'tempest.test3[compute,id-4]' -> tempest.test3\[ + :param list_location: file path or URL location of list file + """ + normalized_list = open(self.get_normalized_test_list(list_location), + 'r').read() + # Keep the names + tests_list = [re.sub("\[", "\[", test) + for test in re.findall(".*\[", normalized_list)] + + return self._write_normalized_test_list(tests_list) + + +if __name__ == "__main__": + import sys + a = TestListParser() + result = open(a.get_normalized_test_list(sys.argv[1]), + 'r').read() + with open(sys.argv[1], 'w') as f: + print(result, file=f) + diff --git a/bin/rally-verify-wrapper.sh b/bin/rally-verify-wrapper.sh index 6f05230..05c4359 100755 --- a/bin/rally-verify-wrapper.sh +++ b/bin/rally-verify-wrapper.sh @@ -38,6 +38,10 @@ load_list="" # You can't have a load list and a pattern, pattern takes priority if [ -f ~/tempest-load-list ] && [ -z ${TEMPEST_PATTERN:+x} ]; then load_list="--load-list /home/rally/tempest-load-list" + if [ $(wc -l /home/rally/tempest-load-list | cut -d ' ' -f 1) -lt 1]; then + echo >&2 "The load list appears to be empty, exiting..." + exit -1 + fi fi skip_list="" @@ -90,6 +94,17 @@ if [ -f ~/tempest-overrides.conf ]; then rally verify configure-verifier --reconfigure --extend ~/tempest-overrides.conf fi +if [ -f ~/tempest-load-list ] && [ -z ${TEMPEST_PATTERN:+x} ]; then + if [ ${TEMPEST_NORMALIZE_LOAD_LIST:-1} -eq 1 ]; then + echo normalizing load-list + rally-normalize.py /home/rally/tempest-load-list + fi + if [ $(wc -l /home/rally/tempest-load-list | cut -d ' ' -f 1) -lt 1 ]; then + echo >&2 "The load list appears to be empty, exiting..." + exit -1 + fi +fi + rally verify start $skip_list $load_list $pattern $concurrency > >(tee -a $artifacts_dir/stdout.log) 2> >(tee -a $artifacts_dir/stderr.log >&2) || export failed=1 rally verify report --type html --to $artifacts_dir/rally-verify-report.html