From e6f55d172d55e641279ba2f1404e00d043e8bd0a Mon Sep 17 00:00:00 2001 From: Wojciech Wypior Date: Mon, 8 Jun 2020 22:38:26 +0200 Subject: [PATCH] Added shippable as a part of CI/CD (#1762) * added helpful info * added shippable * updated gitignore --- .gitignore | 2 + .travis.yml | 2 +- .../f5networks/f5_modules/README.md | 52 +++++ .../f5networks/f5_modules/docs/.gitkeep | 0 .../f5networks/f5_modules/plugins/filter/q.py | 17 -- .../modules/bigip_device_group_member.py | 2 +- .../plugins/modules/bigip_device_info.py | 6 +- .../modules/bigip_gtm_monitor_external.py | 8 - .../plugins/modules/bigip_monitor_mysql.py | 2 +- .../f5_modules/plugins/terminal/bigip.py | 2 +- .../f5_modules/tests/unit/mock/path.py | 3 + .../f5_modules/tests/unit/mock/yaml_helper.py | 3 + .../f5_modules/tests/unit/modules/conftest.py | 2 + .../modules/network/f5/test_module_utils.py | 202 ------------------ .../f5_modules/tests/unit/modules/utils.py | 3 + shippable.yml | 34 +++ tasks/collection.py | 1 + tasks/test.py | 5 +- test/utils/shippable/check_matrix.py | 121 +++++++++++ test/utils/shippable/sanity.sh | 7 + test/utils/shippable/shippable.sh | 182 ++++++++++++++++ test/utils/shippable/timing.py | 16 ++ test/utils/shippable/timing.sh | 5 + test/utils/shippable/units.sh | 11 + 24 files changed, 452 insertions(+), 236 deletions(-) create mode 100644 ansible_collections/f5networks/f5_modules/README.md create mode 100644 ansible_collections/f5networks/f5_modules/docs/.gitkeep delete mode 100644 ansible_collections/f5networks/f5_modules/plugins/filter/q.py delete mode 100644 ansible_collections/f5networks/f5_modules/tests/unit/modules/network/f5/test_module_utils.py create mode 100644 shippable.yml create mode 100755 test/utils/shippable/check_matrix.py create mode 100755 test/utils/shippable/sanity.sh create mode 100755 test/utils/shippable/shippable.sh create mode 100755 test/utils/shippable/timing.py create mode 100755 test/utils/shippable/timing.sh create mode 100755 test/utils/shippable/units.sh diff --git a/.gitignore b/.gitignore index afc6f3317..732eccfc8 100644 --- a/.gitignore +++ b/.gitignore @@ -111,3 +111,5 @@ ansible_py3/ *f5devcentral* # other collections ansible +# ignore build artifacts +builds/* \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index f25f08bf8..de9972c91 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ jobs: name: F5 Unit Tests script: - export PYTHONPATH=$PYTHONPATH:$TRAVIS_BUILD_DIR - - inv test.install-dep + - travis_retry inv test.install-dep - travis_retry pip install pytest --upgrade - pytest -x -s $TRAVIS_BUILD_DIR/ansible_collections/f5networks/f5_modules/tests/unit/modules/network/f5/ python: "3.6" diff --git a/ansible_collections/f5networks/f5_modules/README.md b/ansible_collections/f5networks/f5_modules/README.md new file mode 100644 index 000000000..a2942cc1e --- /dev/null +++ b/ansible_collections/f5networks/f5_modules/README.md @@ -0,0 +1,52 @@ +# f5networks.f5_modules + +This collection includes the most recently released ansible modules for BIG-IP and BIG-IQ from F5Networks. This collection packages and distributes playbooks, roles, modules, and plugins. + +## Requirements + + - ansible >= 2.9 + +## Installation + +To install in ansible default or defined paths use: +```bash +ansible-galaxy collection install f5networks.f5_modules +``` + +To specify the installation location use `-p`. If specifying a folder, make sure to update the `ansible.cfg` so ansible will check this folder as well. +```bash +ansible-galaxy collection install f5networks.f5_modules -p collections/ +``` + +To specify the version of the collection to install, include it at the end of the collection with `:==1.0.0`: +```bash +ansible-galaxy collection install f5networks.f5_modules:==1.0.0 +``` + +Sematic Versioning examples below: +- Increment major (for example: x in x.y.z) version number for an incompatible API change. +- Increment minor (for example: y in x.y.z) version number for new functionality in a backwards compatible manner. +- Increment patch (for example: z in x.y.z) version number for backwards compatible bug fixes. + +## Example Usage + + +To use a module from a collection, reference the full namespace, collection, and modules name that you want to use: + +``` +--- +- name: Using Collections + hosts: f5 + connection: local + + tasks: + - f5networks.f5_modules.bigip_pool: + name: my-pool + .... + +``` + +## Author Information + +F5 Networks +[F5 Networks](http://www.f5.com) diff --git a/ansible_collections/f5networks/f5_modules/docs/.gitkeep b/ansible_collections/f5networks/f5_modules/docs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/ansible_collections/f5networks/f5_modules/plugins/filter/q.py b/ansible_collections/f5networks/f5_modules/plugins/filter/q.py deleted file mode 100644 index e17bacce3..000000000 --- a/ansible_collections/f5networks/f5_modules/plugins/filter/q.py +++ /dev/null @@ -1,17 +0,0 @@ - -from __future__ import absolute_import, division, print_function -__metaclass__ = type - -import q -import os - - -def q_debug(value): - q.q(value) - - -class FilterModule(object): - def filters(self): - return { - 'q': q_debug - } diff --git a/ansible_collections/f5networks/f5_modules/plugins/modules/bigip_device_group_member.py b/ansible_collections/f5networks/f5_modules/plugins/modules/bigip_device_group_member.py index 778c03c4f..cc72cc290 100644 --- a/ansible_collections/f5networks/f5_modules/plugins/modules/bigip_device_group_member.py +++ b/ansible_collections/f5networks/f5_modules/plugins/modules/bigip_device_group_member.py @@ -84,7 +84,7 @@ from ..module_utils.bigip import F5RestClient from ..module_utils.common import ( - F5ModuleError, AnsibleF5Parameters, transform_name, f5_argument_spec, flatten_boolean + F5ModuleError, AnsibleF5Parameters, f5_argument_spec ) diff --git a/ansible_collections/f5networks/f5_modules/plugins/modules/bigip_device_info.py b/ansible_collections/f5networks/f5_modules/plugins/modules/bigip_device_info.py index f354818ad..7d9a2e18b 100644 --- a/ansible_collections/f5networks/f5_modules/plugins/modules/bigip_device_info.py +++ b/ansible_collections/f5networks/f5_modules/plugins/modules/bigip_device_info.py @@ -7219,11 +7219,11 @@ try: from ansible_collections.ansible.netcommon.plugins.module_utils.compat.ipaddress import ( - ip_network, ip_interface, ip_address + ip_interface ) except ImportError: from ansible.module_utils.compat.ipaddress import ( - ip_network, ip_interface, ip_address + ip_interface ) from collections import namedtuple @@ -12614,9 +12614,9 @@ def address(self): @property def netmask(self): + result = None parts = self._values['address'].split('/') if is_valid_ip(parts[0]): - addr = ip_address(u'{0}'.format(parts[0])) ip = ip_interface(u'{0}'.format(self._values['address'])) result = ip.netmask return str(result) diff --git a/ansible_collections/f5networks/f5_modules/plugins/modules/bigip_gtm_monitor_external.py b/ansible_collections/f5networks/f5_modules/plugins/modules/bigip_gtm_monitor_external.py index fd538156c..8970a300d 100644 --- a/ansible_collections/f5networks/f5_modules/plugins/modules/bigip_gtm_monitor_external.py +++ b/ansible_collections/f5networks/f5_modules/plugins/modules/bigip_gtm_monitor_external.py @@ -169,14 +169,6 @@ from ansible.module_utils.basic import env_fallback from ansible.module_utils.six import iteritems -try: - from ansible_collections.ansible.netcommon.plugins.module_utils.compat.ipaddress import ( - ip_network, ip_interface - ) -except ImportError: - from ansible.module_utils.compat.ipaddress import ( - ip_network, ip_interface - ) from ..module_utils.bigip import F5RestClient from ..module_utils.common import ( diff --git a/ansible_collections/f5networks/f5_modules/plugins/modules/bigip_monitor_mysql.py b/ansible_collections/f5networks/f5_modules/plugins/modules/bigip_monitor_mysql.py index 238c07710..912661bd2 100644 --- a/ansible_collections/f5networks/f5_modules/plugins/modules/bigip_monitor_mysql.py +++ b/ansible_collections/f5networks/f5_modules/plugins/modules/bigip_monitor_mysql.py @@ -320,7 +320,7 @@ from ..module_utils.bigip import F5RestClient from ..module_utils.common import ( - F5ModuleError, AnsibleF5Parameters, transform_name, f5_argument_spec, flatten_boolean, fq_name + F5ModuleError, AnsibleF5Parameters, transform_name, f5_argument_spec, flatten_boolean ) from ..module_utils.compare import cmp_str_with_none from ..module_utils.ipaddress import is_valid_ip diff --git a/ansible_collections/f5networks/f5_modules/plugins/terminal/bigip.py b/ansible_collections/f5networks/f5_modules/plugins/terminal/bigip.py index 4367c5fce..334446310 100644 --- a/ansible_collections/f5networks/f5_modules/plugins/terminal/bigip.py +++ b/ansible_collections/f5networks/f5_modules/plugins/terminal/bigip.py @@ -58,5 +58,5 @@ def on_open_shell(self): try: self._exec_cli_command(b'tmsh modify cli preference display-threshold 0 pager disabled') self._exec_cli_command(b'stty cols 1000000 2> /dev/null') - except AnsibleConnectionFailure as ex: + except AnsibleConnectionFailure: raise AnsibleConnectionFailure('unable to set terminal parameters') diff --git a/ansible_collections/f5networks/f5_modules/tests/unit/mock/path.py b/ansible_collections/f5networks/f5_modules/tests/unit/mock/path.py index e68a5d38e..ec88f77f3 100644 --- a/ansible_collections/f5networks/f5_modules/tests/unit/mock/path.py +++ b/ansible_collections/f5networks/f5_modules/tests/unit/mock/path.py @@ -1,3 +1,6 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + from ansible_collections.f5networks.f5_modules.tests.unit.compat.mock import MagicMock from ansible.utils.path import unfrackpath diff --git a/ansible_collections/f5networks/f5_modules/tests/unit/mock/yaml_helper.py b/ansible_collections/f5networks/f5_modules/tests/unit/mock/yaml_helper.py index cc095fea1..1ef172159 100644 --- a/ansible_collections/f5networks/f5_modules/tests/unit/mock/yaml_helper.py +++ b/ansible_collections/f5networks/f5_modules/tests/unit/mock/yaml_helper.py @@ -1,3 +1,6 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + import io import yaml diff --git a/ansible_collections/f5networks/f5_modules/tests/unit/modules/conftest.py b/ansible_collections/f5networks/f5_modules/tests/unit/modules/conftest.py index 5f16f2255..e1bde0c80 100644 --- a/ansible_collections/f5networks/f5_modules/tests/unit/modules/conftest.py +++ b/ansible_collections/f5networks/f5_modules/tests/unit/modules/conftest.py @@ -1,5 +1,7 @@ # Copyright (c) 2017 Ansible Project # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type import json diff --git a/ansible_collections/f5networks/f5_modules/tests/unit/modules/network/f5/test_module_utils.py b/ansible_collections/f5networks/f5_modules/tests/unit/modules/network/f5/test_module_utils.py deleted file mode 100644 index 8b14f4ff9..000000000 --- a/ansible_collections/f5networks/f5_modules/tests/unit/modules/network/f5/test_module_utils.py +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# -# Copyright (c) 2017 F5 Networks Inc. -# GNU General Public License v3.0 (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) - -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - - -from ansible_collections.f5networks.f5_modules.tests.unit.compat import unittest -from ansible_collections.f5networks.f5_modules.plugins.module_utils.common import AnsibleF5Parameters - - -class TestRegular(unittest.TestCase): - class Foo(AnsibleF5Parameters): - @property - def dns(self): - return self._values['dns'] - - @dns.setter - def dns(self, value): - self._values['dns'] = value - - @property - def lb_method(self): - return self._values['lb_method'] - - @lb_method.setter - def lb_method(self, value): - self._values['lb_method'] = value - - def test_run(self): - args = dict( - dns='alice', - lb_method='bob' - ) - test = TestRegular.Foo(params=args) - assert test.dns == 'alice' - assert test.lb_method == 'bob' - - -class TestKeysWithPunctuation(unittest.TestCase): - class Foo(AnsibleF5Parameters): - api_map = { - 'dns.proxy.__iter__': 'bar' - } - - @property - def bar(self): - return self._values['bar'] - - @bar.setter - def bar(self, value): - self._values['bar'] = value - - @property - def baz(self): - return self._values['baz'] - - @baz.setter - def baz(self, value): - self._values['baz'] = value - - def test_run(self): - args = { - 'dns.proxy.__iter__': 'alice', - 'baz': 'bob' - } - test = TestKeysWithPunctuation.Foo(params=args) - assert test.bar == 'alice' - assert test.baz == 'bob' - assert 'dns.proxy.__iter__' not in test._values - - -class TestInheritence(unittest.TestCase): - class Baz(AnsibleF5Parameters): - @property - def baz(self): - return self._values['baz'] - - @baz.setter - def baz(self, value): - self._values['baz'] = value - - class Bar(Baz): - @property - def bar(self): - return self._values['bar'] - - @bar.setter - def bar(self, value): - self._values['bar'] = value - - class Foo(Bar): - @property - def foo(self): - return self._values['foo'] - - @foo.setter - def foo(self, value): - self._values['foo'] = value - - def test_run(self): - args = { - 'foo': 'alice', - 'bar': 'bob', - 'baz': 'carol', - 'dns.proxy.__iter__': 'dan', - } - test = TestInheritence.Foo(params=args) - assert test.foo == 'alice' - assert test.bar == 'bob' - assert test.baz == 'carol' - - -class TestNoProperties(unittest.TestCase): - class Foo(AnsibleF5Parameters): - pass - - def test_run(self): - args = dict( - dns='alice', - lb_method='bob' - ) - test = TestNoProperties.Foo(params=args) - assert test.dns == 'alice' - assert test.lb_method == 'bob' - - -class TestReferenceAnother(unittest.TestCase): - class Foo(AnsibleF5Parameters): - @property - def poolLbMode(self): - return self.lb_method - - @poolLbMode.setter - def poolLbMode(self, value): - self.lb_method = value - - @property - def lb_method(self): - return self._values['lb_method'] - - @lb_method.setter - def lb_method(self, value): - self._values['lb_method'] = value - - def test_run(self): - args = dict( - lb_method='bob' - ) - test = TestReferenceAnother.Foo(params=args) - assert test.lb_method == 'bob' - assert test.poolLbMode == 'bob' - - -class TestMissingAttrSetter(unittest.TestCase): - class Foo(AnsibleF5Parameters): - @property - def reject(self): - return self._values['reject'] - - @property - def destination(self): - return self._values['destination'] - - def test_run(self): - args = dict( - name='test-route', - password='admin', - server='localhost', - user='admin', - state='present', - destination='10.10.10.10', - reject='yes' - ) - test = TestMissingAttrSetter.Foo(params=args) - assert test.destination == '10.10.10.10' - assert test.reject == 'yes' - - -class TestAssureNoInstanceAttributes(unittest.TestCase): - class Foo(AnsibleF5Parameters): - @property - def reject(self): - return self._values['reject'] - - def test_run(self): - args = dict( - name='test-route', - password='admin', - server='localhost', - user='admin', - state='present', - destination='10.10.10.10', - reject='yes' - ) - test = TestAssureNoInstanceAttributes.Foo(params=args) - assert test.destination == '10.10.10.10' - assert test.reject == 'yes' - assert 'destination' not in dir(test) diff --git a/ansible_collections/f5networks/f5_modules/tests/unit/modules/utils.py b/ansible_collections/f5networks/f5_modules/tests/unit/modules/utils.py index 6c4e4cd8f..5df4053af 100644 --- a/ansible_collections/f5networks/f5_modules/tests/unit/modules/utils.py +++ b/ansible_collections/f5networks/f5_modules/tests/unit/modules/utils.py @@ -1,3 +1,6 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + import json from ..compat import unittest diff --git a/shippable.yml b/shippable.yml new file mode 100644 index 000000000..8930b78d9 --- /dev/null +++ b/shippable.yml @@ -0,0 +1,34 @@ +language: python +# TODO enable unit tests once root cause of permissions denied in shippable is identified +env: + matrix: + - T=none +matrix: + exclude: + - env: T=none + include: + - env: T=sanity + #- env: T=units/2.6 + #- env: T=units/2.7 + #- env: T=units/3.5 + #- env: T=units/3.6 + #- env: T=units/3.7 + #- env: T=units/3.8 + #- env: T=units/3.9 +branches: + except: + - "*-patch-*" + - "revert-*-*" + +build: + ci: + - test/utils/shippable/timing.sh test/utils/shippable/shippable.sh $T + +integrations: + notifications: + - integrationName: email + type: email + on_success: never + on_failure: never + on_start: never + on_pull_request: never \ No newline at end of file diff --git a/tasks/collection.py b/tasks/collection.py index 42cbba596..eae9c9e0b 100644 --- a/tasks/collection.py +++ b/tasks/collection.py @@ -60,6 +60,7 @@ def validate_version(version): collection="The collection name for which the galaxy.yml file to be changed, DEFAULT: 'f5_modules'.", )) def change_galaxy_version(c, version, collection='f5_modules'): + """Changes version of the collection in galaxy.yml file.""" validate_version(version) update_galaxy_file(version, collection) print("File galaxy.yml updated.") diff --git a/tasks/test.py b/tasks/test.py index 45847ca81..1c7145423 100644 --- a/tasks/test.py +++ b/tasks/test.py @@ -52,15 +52,16 @@ def install_dependency(c): @task(name='ansible-test') def ansible_test(c, python_version='3.7', requirements=False): + """Runs ansible-test sanity tests against modules.""" net_dir = '{0}/ansible_collections/ansible/netcommon/'.format(BASE_DIR) collection = '{0}/ansible_collections/f5networks/f5_modules'.format(BASE_DIR) if not os.path.exists(net_dir): install_dependency(c) with c.cd(collection): if requirements: - execute = 'ansible-test sanity plugins/ --requirements --python {0}'.format(python_version) + execute = 'ansible-test sanity --requirements --python {0}'.format(python_version) else: - execute = 'ansible-test sanity plugins/ --python {0}'.format(python_version) + execute = 'ansible-test sanity --python {0}'.format(python_version) result = c.run(execute, warn=True) if result.failed: sys.exit(1) diff --git a/test/utils/shippable/check_matrix.py b/test/utils/shippable/check_matrix.py new file mode 100755 index 000000000..50ff85aa7 --- /dev/null +++ b/test/utils/shippable/check_matrix.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python +"""Verify the currently executing Shippable test matrix matches the one defined in the "shippable.yml" file.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import datetime +import json +import os +import re +import sys +import time + +try: + from typing import NoReturn +except ImportError: + NoReturn = None + +try: + # noinspection PyCompatibility + from urllib2 import urlopen # pylint: disable=ansible-bad-import-from +except ImportError: + # noinspection PyCompatibility + from urllib.request import urlopen + + +def main(): # type: () -> None + """Main entry point.""" + repo_full_name = os.environ['REPO_FULL_NAME'] + required_repo_full_name = 'F5Networks/f5-ansible' + build_dir = os.environ['SHIPPABLE_BUILD_DIR'] + + if repo_full_name != required_repo_full_name: + sys.stderr.write('Skipping matrix check on repo "%s" which is not "%s".\n' % (repo_full_name, required_repo_full_name)) + return + + with open('{0}/shippable.yml'.format(build_dir), 'rb') as yaml_file: + yaml = yaml_file.read().decode('utf-8').splitlines() + + defined_matrix = [match.group(1) for match in [re.search(r'^ *- env: T=(.*)$', line) for line in yaml] if match and match.group(1) != 'none'] + + if not defined_matrix: + fail('No matrix entries found in the "shippable.yml" file.', + 'Did you modify the "shippable.yml" file?') + + run_id = os.environ['SHIPPABLE_BUILD_ID'] + sleep = 1 + jobs = [] + + for attempts_remaining in range(4, -1, -1): + try: + jobs = json.loads(urlopen('https://api.shippable.com/jobs?runIds=%s' % run_id).read()) + + if not isinstance(jobs, list): + raise Exception('Shippable run %s data is not a list.' % run_id) + + break + except Exception as ex: + if not attempts_remaining: + fail('Unable to retrieve Shippable run %s matrix.' % run_id, + str(ex)) + + sys.stderr.write('Unable to retrieve Shippable run %s matrix: %s\n' % (run_id, ex)) + sys.stderr.write('Trying again in %d seconds...\n' % sleep) + time.sleep(sleep) + sleep *= 2 + + if len(jobs) != len(defined_matrix): + if len(jobs) == 1: + hint = '\n\nMake sure you do not use the "Rebuild with SSH" option.' + else: + hint = '' + + fail('Shippable run %s has %d jobs instead of the expected %d jobs.' % (run_id, len(jobs), len(defined_matrix)), + 'Try re-running the entire matrix.%s' % hint) + + actual_matrix = dict((job.get('jobNumber'), dict(tuple(line.split('=', 1)) for line in job.get('env', [])).get('T', '')) for job in jobs) + errors = [(job_number, test, actual_matrix.get(job_number)) for job_number, test in enumerate(defined_matrix, 1) if actual_matrix.get(job_number) != test] + + if len(errors): + error_summary = '\n'.join('Job %s expected "%s" but found "%s" instead.' % (job_number, expected, actual) for job_number, expected, actual in errors) + + fail('Shippable run %s has a job matrix mismatch.' % run_id, + 'Try re-running the entire matrix.\n\n%s' % error_summary) + + +def fail(message, output): # type: (str, str) -> NoReturn + # Include a leading newline to improve readability on Shippable "Tests" tab. + # Without this, the first line becomes indented. + output = '\n' + output.strip() + + timestamp = datetime.datetime.utcnow().replace(microsecond=0).isoformat() + + # hack to avoid requiring junit-xml, which isn't pre-installed on Shippable outside our test containers + xml = ''' + + +\t +\t\t +\t\t\t%s +\t\t +\t + +''' % (timestamp, message, output) + + path = 'shippable/testresults/check-matrix.xml' + dir_path = os.path.dirname(path) + + if not os.path.exists(dir_path): + os.makedirs(dir_path) + + with open(path, 'w') as junit_fd: + junit_fd.write(xml.lstrip()) + + sys.stderr.write(message + '\n') + sys.stderr.write(output + '\n') + + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/test/utils/shippable/sanity.sh b/test/utils/shippable/sanity.sh new file mode 100755 index 000000000..0c54a42c8 --- /dev/null +++ b/test/utils/shippable/sanity.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -o pipefail -eux + +# shellcheck disable=SC2086 +ansible-test sanity --color -v --junit ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ + --docker \ diff --git a/test/utils/shippable/shippable.sh b/test/utils/shippable/shippable.sh new file mode 100755 index 000000000..b3fd9a630 --- /dev/null +++ b/test/utils/shippable/shippable.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash + +set -o pipefail -eux + +declare -a args +IFS='/:' read -ra args <<< "$1" + +script="${args[0]}" + +test="$1" + +docker images ansible/ansible +docker images quay.io/ansible/* +docker ps + +for container in $(docker ps --format '{{.Image}} {{.ID}}' | grep -v '^drydock/' | sed 's/^.* //'); do + docker rm -f "${container}" || true # ignore errors +done + +docker ps + +if [ -d /home/shippable/cache/ ]; then + ls -la /home/shippable/cache/ +fi + +command -v python +python -V + +function retry +{ + # shellcheck disable=SC2034 + for repetition in 1 2 3; do + set +e + "$@" + result=$? + set -e + if [ ${result} == 0 ]; then + return ${result} + fi + echo "@* -> ${result}" + done + echo "Command '@*' failed 3 times!" + exit -1 +} + +command -v pip +pip --version +pip list --disable-pip-version-check + + +if [ "${JOB_TRIGGERED_BY_NAME:-}" == "nightly-trigger" ]; then + COVERAGE=yes + COMPLETE=yes +fi + +if [ -n "${COVERAGE:-}" ]; then + # on-demand coverage reporting triggered by setting the COVERAGE environment variable to a non-empty value + export COVERAGE="--coverage" +elif [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]]; then + # on-demand coverage reporting triggered by having 'ci_coverage' in the latest commit message + export COVERAGE="--coverage" +else + # on-demand coverage reporting disabled (default behavior, always-on coverage reporting remains enabled) + export COVERAGE="--coverage-check" +fi + +if [ -n "${COMPLETE:-}" ]; then + # disable change detection triggered by setting the COMPLETE environment variable to a non-empty value + export CHANGED="" +elif [[ "${COMMIT_MESSAGE}" =~ ci_complete ]]; then + # disable change detection triggered by having 'ci_complete' in the latest commit message + export CHANGED="" +else + # enable change detection (default behavior) + export CHANGED="--changed" +fi + +if [ "${IS_PULL_REQUEST:-}" == "true" ]; then + # run unstable tests which are targeted by focused changes on PRs + export UNSTABLE="--allow-unstable-changed" +else + # do not run unstable tests outside PRs + export UNSTABLE="" +fi + + +export ANSIBLE_COLLECTIONS_PATHS="${SHIPPABLE_BUILD_DIR}" +TEST_DIR="${SHIPPABLE_BUILD_DIR}/ansible_collections/f5networks/f5_modules" +SHIPPABLE_RESULT_DIR="$(pwd)/shippable" +retry pip install ansible==2.9.9 --disable-pip-version-check +retry ansible-galaxy -vvv collection install ansible.netcommon +cd "${TEST_DIR}" + + +export PYTHONIOENCODING='utf-8' +function cleanup +{ + # for complete on-demand coverage generate a report for all files with no coverage on the "sanity/5" job so we only have one copy + if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ] && [ "${test}" == "sanity/5" ]; then + stub="--stub" + # trigger coverage reporting for stubs even if no other coverage data exists + mkdir -p tests/output/coverage/ + else + stub="" + fi + + if [ -d tests/output/coverage/ ]; then + if find tests/output/coverage/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then + process_coverage='yes' # process existing coverage files + elif [ "${stub}" ]; then + process_coverage='yes' # process coverage when stubs are enabled + else + process_coverage='' + fi + + if [ "${process_coverage}" ]; then + # use python 3.7 for coverage to avoid running out of memory during coverage xml processing + # only use it for coverage to avoid the additional overhead of setting up a virtual environment for a potential no-op job + virtualenv --python /usr/bin/python3.7 ~/ansible-venv + set +ux + . ~/ansible-venv/bin/activate + set -ux + + # shellcheck disable=SC2086 + ansible-test coverage xml --color --requirements --group-by command --group-by version ${stub:+"$stub"} + cp -a tests/output/reports/coverage=*.xml "$SHIPPABLE_RESULT_DIR/codecoverage/" + + # analyze and capture code coverage aggregated by integration test target + ansible-test coverage analyze targets generate -v "$SHIPPABLE_RESULT_DIR/testresults/coverage-analyze-targets.json" + + # upload coverage report to codecov.io only when using complete on-demand coverage + if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ]; then + for file in tests/output/reports/coverage=*.xml; do + flags="${file##*/coverage=}" + flags="${flags%-powershell.xml}" + flags="${flags%.xml}" + # remove numbered component from stub files when converting to tags + flags="${flags//stub-[0-9]*/stub}" + flags="${flags//=/,}" + flags="${flags//[^a-zA-Z0-9_,]/_}" + + bash <(curl -s https://codecov.io/bash) \ + -f "${file}" \ + -F "${flags}" \ + -n "${test}" \ + -t 20636cf5-4d6a-4b9a-8d2d-6f22ebbaa752 \ + -X coveragepy \ + -X gcov \ + -X fix \ + -X search \ + -X xcode \ + || echo "Failed to upload code coverage report to codecov.io: ${file}" + done + fi + fi + fi + + if [ -d tests/output/junit/ ]; then + cp -aT tests/output/junit/ "$SHIPPABLE_RESULT_DIR/testresults/" + fi + + if [ -d tests/output/data/ ]; then + cp -a tests/output/data/ "$SHIPPABLE_RESULT_DIR/testresults/" + fi + + if [ -d tests/output/bot/ ]; then + cp -aT tests/output/bot/ "$SHIPPABLE_RESULT_DIR/testresults/" + fi +} + +trap cleanup EXIT + +if [[ "${COVERAGE:-}" == "--coverage" ]]; then + timeout=60 +else + timeout=45 +fi + +ansible-test env --dump --show --timeout "${timeout}" --color -v + +"${SHIPPABLE_BUILD_DIR}/test/utils/shippable/check_matrix.py" +"${SHIPPABLE_BUILD_DIR}/test/utils/shippable/${script}.sh" "${test}" \ No newline at end of file diff --git a/test/utils/shippable/timing.py b/test/utils/shippable/timing.py new file mode 100755 index 000000000..825b727e5 --- /dev/null +++ b/test/utils/shippable/timing.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3.7 +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import sys +import time + +start = time.time() + +sys.stdin.reconfigure(errors='surrogateescape') +sys.stdout.reconfigure(errors='surrogateescape') + +for line in sys.stdin: + seconds = time.time() - start + sys.stdout.write('%02d:%02d %s' % (seconds // 60, seconds % 60, line)) + sys.stdout.flush() \ No newline at end of file diff --git a/test/utils/shippable/timing.sh b/test/utils/shippable/timing.sh new file mode 100755 index 000000000..31532376b --- /dev/null +++ b/test/utils/shippable/timing.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -o pipefail -eu + +"$@" 2>&1 | "$(dirname "$0")/timing.py" \ No newline at end of file diff --git a/test/utils/shippable/units.sh b/test/utils/shippable/units.sh new file mode 100755 index 000000000..4a14378eb --- /dev/null +++ b/test/utils/shippable/units.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -o pipefail -eux + +declare -a args +IFS='/:' read -ra args <<< "$1" + +version="${args[1]}" + +# shellcheck disable=SC2086 +ansible-test units --color -v --docker default --python "${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ No newline at end of file