diff --git a/.travis.yml b/.travis.yml index 160a858..8151320 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,16 @@ language: generic -sudo: required services: - docker env: - - FEDORA=28 - - FEDORA=29 - - FEDORA=rawhide + - FEDORA=28 TOXENV=integration + - FEDORA=28 TOXENV=py3 + - FEDORA=29 TOXENV=integration + - FEDORA=29 TOXENV=py3 + - FEDORA=29 TOXENV=style + - FEDORA=rawhide TOXENV=integration + - FEDORA=rawhide TOXENV=py3 install: - sed -i "s/fedora-28-x86_64/fedora-${FEDORA}-x86_64/" mock.cfg @@ -16,4 +19,4 @@ install: - docker build -t taskotron . script: - - docker run --cap-add=SYS_ADMIN -v $(pwd):$(pwd) -w $(pwd) -i -t taskotron + - docker run --cap-add=SYS_ADMIN -v $(pwd):$(pwd) -w $(pwd) -i -e "TOXENV=$TOXENV" -t taskotron diff --git a/Dockerfile b/Dockerfile index 2d4f18f..4bc2fcc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM fedora RUN dnf -y install --setopt=install_weak_deps=false --setopt=tsflags=nodocs \ - --setopt=deltarpm=false --allowerasing --best \ + --setopt=deltarpm=false --allowerasing --best --disablerepo=\*modular \ mock tox python3-rpm python3-dnf && dnf clean all ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 diff --git a/python_versions_check.py b/python_versions_check.py index c4360d0..30fd944 100644 --- a/python_versions_check.py +++ b/python_versions_check.py @@ -8,6 +8,7 @@ logging.basicConfig() import os +import pathlib import sys from libtaskotron import check @@ -31,9 +32,14 @@ def run(koji_build, workdir='.', artifactsdir='artifacts', testcase='dist.python-versions', arches=['x86_64', 'noarch', 'src']): '''The main method to run from Taskotron''' - workdir = os.path.abspath(workdir) - results_path = os.path.join(artifactsdir, 'taskotron', 'results.yml') - artifact = os.path.join(artifactsdir, 'output.log') + artifactsdir = pathlib.Path(artifactsdir) + workdir = pathlib.Path(workdir).resolve() + resultsdir = artifactsdir / 'taskotron' + resultspath = resultsdir / 'results.yml' + artifact = artifactsdir / 'output.log' + + artifactsdir.mkdir(parents=True, exist_ok=True) + resultsdir.mkdir(parents=True, exist_ok=True) # find files to run on files = sorted(os.listdir(workdir)) @@ -41,7 +47,7 @@ def run(koji_build, workdir='.', artifactsdir='artifacts', packages = [] srpm_packages = [] for file_ in files: - path = os.path.join(workdir, file_) + path = workdir / file_ if file_.endswith('.rpm'): try: package = Package(path) @@ -97,7 +103,7 @@ def run(koji_build, workdir='.', artifactsdir='artifacts', outcome=outcome, keyvals={'arch': arches}) if outcome == 'FAILED': - overall_detail.artifact = artifact + overall_detail.artifact = str(artifact) details.append(overall_detail) summary = 'python-versions {} for {} ({}).'.format( @@ -106,8 +112,7 @@ def run(koji_build, workdir='.', artifactsdir='artifacts', # generate output reportable to ResultsDB output = check.export_YAML(details) - with open(results_path, 'w') as results_file: - results_file.write(output) + resultspath.write_text(output) return 0 if overall_detail.outcome in ['PASSED', 'INFO'] else 1 diff --git a/setup.py b/setup.py index 507c08b..5383ff4 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,11 @@ #!/usr/bin/env python3 +import pathlib from setuptools import setup, find_packages description = """Taskotron checks regarding Python versions""" - -with open('README.rst') as readme: - long_description = readme.read() +long_description = pathlib.Path('README.rst').read_text() setup( name='taskotron-python-versions', diff --git a/taskotron_python_versions/common.py b/taskotron_python_versions/common.py index a81d300..22c176e 100644 --- a/taskotron_python_versions/common.py +++ b/taskotron_python_versions/common.py @@ -42,6 +42,8 @@ def packages_by_version(packages): """ pkg_by_version = collections.defaultdict(list) for package in packages: + if package.py_versions is None: # SRPMS + continue for version in package.py_versions: pkg_by_version[version].append(package) return pkg_by_version diff --git a/taskotron_python_versions/executables.py b/taskotron_python_versions/executables.py index 1e8736c..e3de6fd 100644 --- a/taskotron_python_versions/executables.py +++ b/taskotron_python_versions/executables.py @@ -96,8 +96,8 @@ def task_executables(packages, koji_build, artifact): outcome=outcome) if message: - detail.artifact = artifact write_to_artifact(artifact, MESSAGE.format(message), INFO_URL) + detail.artifact = str(artifact) log.info('subcheck executables {} for {}'.format( outcome, koji_build)) diff --git a/taskotron_python_versions/naming_scheme.py b/taskotron_python_versions/naming_scheme.py index b3429f4..5e39ac0 100644 --- a/taskotron_python_versions/naming_scheme.py +++ b/taskotron_python_versions/naming_scheme.py @@ -97,9 +97,9 @@ def task_naming_scheme(packages, koji_build, artifact): outcome=outcome) if incorrect_names: - detail.artifact = artifact names = ', '.join(incorrect_names) write_to_artifact(artifact, MESSAGE.format(names), INFO_URL) + detail.artifact = str(artifact) problems = 'Problematic RPMs:\n' + names else: problems = 'No problems found.' diff --git a/taskotron_python_versions/py3_support.py b/taskotron_python_versions/py3_support.py index de82edd..af12151 100644 --- a/taskotron_python_versions/py3_support.py +++ b/taskotron_python_versions/py3_support.py @@ -115,8 +115,8 @@ def task_py3_support(packages, koji_build, artifact): outcome=outcome) if message: - detail.artifact = artifact write_to_artifact(artifact, MESSAGE.format(message), INFO_URL) + detail.artifact = str(artifact) log.info('subcheck py3_support {} for {}'.format( outcome, koji_build)) diff --git a/taskotron_python_versions/python_usage.py b/taskotron_python_versions/python_usage.py index 8279a2b..c28c356 100644 --- a/taskotron_python_versions/python_usage.py +++ b/taskotron_python_versions/python_usage.py @@ -47,10 +47,10 @@ def task_python_usage(packages, koji_build, artifact): outcome=outcome) if problem_rpms: - detail.artifact = artifact write_to_artifact( artifact, MESSAGE.format('\n * '.join(problem_rpms)), INFO_URL) + detail.artifact = str(artifact) problems = 'Problematic RPMs:\n' + ', '.join(problem_rpms) else: problems = 'No problems found.' diff --git a/taskotron_python_versions/python_usage_obsoleted.py b/taskotron_python_versions/python_usage_obsoleted.py index 5f860fe..af35d5f 100644 --- a/taskotron_python_versions/python_usage_obsoleted.py +++ b/taskotron_python_versions/python_usage_obsoleted.py @@ -40,7 +40,7 @@ def task_python_usage_obsoleted(logs, koji_build, artifact): if file_contains(buildlog, WARNING): log.debug('{} contains our warning'.format(buildlog)) - _, _, arch = buildlog.rpartition('.') + arch = buildlog.suffix.lstrip('.') problem_arches.add(arch) outcome = 'FAILED' @@ -51,9 +51,9 @@ def task_python_usage_obsoleted(logs, koji_build, artifact): outcome=outcome) if problem_arches: - detail.artifact = artifact info = '{}: {}'.format(koji_build, ', '.join(sorted(problem_arches))) write_to_artifact(artifact, MESSAGE.format(info), INFO_URL) + detail.artifact = str(artifact) problems = 'Problematic architectures: ' + info else: problems = 'No problems found.' diff --git a/taskotron_python_versions/requires.py b/taskotron_python_versions/requires.py index bba549a..1d7052d 100644 --- a/taskotron_python_versions/requires.py +++ b/taskotron_python_versions/requires.py @@ -160,8 +160,8 @@ def task_requires_naming_scheme(packages, koji_build, artifact): outcome=outcome) if problem_rpms: - detail.artifact = artifact write_to_artifact(artifact, MESSAGE.format(message_rpms), INFO_URL) + detail.artifact = str(artifact) problems = 'Problematic RPMs:\n' + ', '.join(problem_rpms) else: problems = 'No problems found.' diff --git a/taskotron_python_versions/two_three.py b/taskotron_python_versions/two_three.py index f0eb142..7e24e48 100644 --- a/taskotron_python_versions/two_three.py +++ b/taskotron_python_versions/two_three.py @@ -145,7 +145,6 @@ def task_two_three(packages, koji_build, artifact): outcome=outcome) if bads: - detail.artifact = artifact rpms = '' for name, py_versions in bads.items(): rpms += ('{}\n' @@ -154,6 +153,7 @@ def task_two_three(packages, koji_build, artifact): py_versions[2], py_versions[3])) write_to_artifact(artifact, MESSAGE.format(rpms), INFO_URL) + detail.artifact = str(artifact) names = ', '.join(str(k) for k in bads.keys()) problems = 'Problematic RPMs:\n' + names else: diff --git a/taskotron_python_versions/unversioned_shebangs.py b/taskotron_python_versions/unversioned_shebangs.py index 350eda3..e038581 100644 --- a/taskotron_python_versions/unversioned_shebangs.py +++ b/taskotron_python_versions/unversioned_shebangs.py @@ -45,7 +45,7 @@ def get_problematic_files(archive, query): the shebang query is encoded as well. We only test for ASCII shebangs. """ problematic = set() - with libarchive.file_reader(archive) as a: + with libarchive.file_reader(str(archive)) as a: for entry in a: try: first_line = next(entry.get_blocks(), '').splitlines()[0] @@ -112,7 +112,7 @@ def check_logs(logs): for buildlog in logs: if file_contains(buildlog, WARNING): log.debug('{} contains our warning'.format(buildlog)) - _, _, arch = buildlog.rpartition('.') + arch = buildlog.suffix.lstrip('.') problem_arches.add(arch) return ', '.join(sorted(problem_arches)) @@ -148,8 +148,8 @@ def task_unversioned_shebangs(packages, logs, koji_build, artifact): outcome=outcome) if outcome == 'FAILED': - detail.artifact = artifact write_to_artifact(artifact, message, INFO_URL) + detail.artifact = str(artifact) else: problems = 'No problems found.' diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 24298f1..0b8e4ee 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -1,6 +1,14 @@ +import pytest from xdist.scheduler import LoadScopeScheduling +def parse_fixture_name(nodeid): + if '[' in nodeid: + parameters = nodeid.rsplit('[')[-1].replace(']', '') + return parameters.split('-')[0] + return None + + def pytest_addoption(parser): parser.addoption('--fake', action='store_true', default=False, help='don\'t run the code, reuse the result from ' @@ -12,11 +20,15 @@ class FixtureScheduling(LoadScopeScheduling): See https://github.com/pytest-dev/pytest-xdist/issues/18 """ def _split_scope(self, nodeid): - if '[' in nodeid: - parameters = nodeid.rsplit('[')[-1].replace(']', '') - return parameters.split('-')[0] - return None + return parse_fixture_name(nodeid) def pytest_xdist_make_scheduler(log, config): return FixtureScheduling(config, log) + + +def pytest_collection_modifyitems(items): + for item in items: + fixture = parse_fixture_name(item.nodeid) + if fixture: + item.add_marker(getattr(pytest.mark, fixture)) diff --git a/test/integration/test_integration.py b/test/integration/test_integration.py index 1df2094..ede9014 100644 --- a/test/integration/test_integration.py +++ b/test/integration/test_integration.py @@ -1,7 +1,7 @@ from collections import namedtuple import contextlib import glob -import os +import pathlib import pprint import shutil import subprocess @@ -30,7 +30,7 @@ def root(self): @property def rootdir(self): - return os.path.join(os.path.abspath('.'), 'mockroots', self.root) + return pathlib.Path('./mockroots').resolve() / self.root def _run(self, what, **kwargs): command = list(self.mock) # needs a copy not to change in place @@ -110,7 +110,8 @@ def run_task(nevr, *, mock): def fix_artifact_path(path): if path is None: return None - return path.replace('/artifacts/', '/{}/'.format(artifacts)) + newpath = path.replace('artifacts/', '{}/'.format(artifacts)) + return pathlib.Path(newpath) return {r['checkname']: Result(r.get('outcome'), fix_artifact_path(r.get('artifact')), @@ -177,6 +178,9 @@ def results(request): _pycallgraph = fixtures_factory('python-pycallgraph-0.5.1-13.fc28') pycallgraph = fixtures_factory('_pycallgraph') +_ttystatus = fixtures_factory('python-ttystatus-0.34-7.fc29') +ttystatus = fixtures_factory('_ttystatus') + # TODO: remove with Fedora 28 EOL. _jsonrpc = fixtures_factory('jsonrpc-glib-3.27.4-2.fc28') jsonrpc = fixtures_factory('_jsonrpc') @@ -190,7 +194,8 @@ def parametrize(*fixtrues): @parametrize('eric', 'six', 'admesh', 'tracer', 'copr', 'epub', 'twine', 'yum', - 'vdirsyncer', 'docutils', 'nodejs', 'pycallgraph', 'teeworlds') + 'vdirsyncer', 'docutils', 'nodejs', 'pycallgraph', 'teeworlds', + 'ttystatus') def test_number_of_results(results, request): # getting a fixture by name # https://github.com/pytest-dev/pytest/issues/349#issuecomment-112203541 @@ -232,8 +237,7 @@ def test_artifact_is_the_same(results, task, request): def test_artifact_contains_two_three_and_looks_as_expected(results, request): results = request.getfixturevalue(results) result = results['dist.python-versions.two_three'] - with open(result.artifact) as f: - artifact = f.read() + artifact = result.artifact.read_text() assert dedent(''' These RPMs require both Python 2 and Python 3: @@ -260,8 +264,7 @@ def test_artifact_contains_naming_scheme_and_looks_as_expected(results, request): results = request.getfixturevalue(results) result = results['dist.python-versions.naming_scheme'] - with open(result.artifact) as f: - artifact = f.read() + artifact = result.artifact.read_text() assert dedent(""" These RPMs' names violate the new Python package naming guidelines: @@ -288,8 +291,7 @@ def test_artifact_contains_requires_naming_scheme_and_looks_as_expected( results, request): results = request.getfixturevalue(results) result = results['dist.python-versions.requires_naming_scheme'] - with open(result.artifact) as f: - artifact = f.read() + artifact = result.artifact.read_text() print(artifact) @@ -312,8 +314,7 @@ def test_artifact_contains_requires_naming_scheme_and_looks_as_expected( def test_requires_naming_scheme_contains_python(results, request): results = request.getfixturevalue(results) result = results['dist.python-versions.requires_naming_scheme'] - with open(result.artifact) as f: - artifact = f.read() + artifact = result.artifact.read_text() print(artifact) @@ -340,8 +341,7 @@ def test_artifact_contains_executables_and_looks_as_expected( results, request): results = request.getfixturevalue(results) result = results['dist.python-versions.executables'] - with open(result.artifact) as f: - artifact = f.read() + artifact = result.artifact.read_text() print(artifact) @@ -387,8 +387,7 @@ def test_artifact_contains_unversioned_shebangs_and_looks_as_expected( results, request): results = request.getfixturevalue(results) result = results['dist.python-versions.unversioned_shebangs'] - with open(result.artifact) as f: - artifact = f.read() + artifact = result.artifact.read_text() print(artifact) @@ -415,8 +414,7 @@ def test_artifact_contains_mangled_unversioned_shebangs_and_looks_as_expected( results, request): results = request.getfixturevalue(results) result = results['dist.python-versions.unversioned_shebangs'] - with open(result.artifact) as f: - artifact = f.read() + artifact = result.artifact.read_text() print(artifact) @@ -447,26 +445,25 @@ def test_py3_support_passed(results, request): assert task_result.outcome == 'PASSED' -@parametrize('pycallgraph') +@parametrize('ttystatus') def test_py3_support_failed(results, request): results = request.getfixturevalue(results) task_result = results['dist.python-versions.py3_support'] assert task_result.outcome == 'FAILED' -@parametrize('pycallgraph') +@parametrize('ttystatus') def test_artifact_contains_py3_support_and_looks_as_expected( results, request): """Test that py3_support check fails if the package is mispackaged. - NOTE: The test will start to fail as soon as python-pycallgraph + NOTE: The test will start to fail as soon as python-ttystatus gets ported to Python 3 and its Bugzilla gets closed. - See https://bugzilla.redhat.com/show_bug.cgi?id=1309383 + See https://bugzilla.redhat.com/show_bug.cgi?id=1458531 """ results = request.getfixturevalue(results) result = results['dist.python-versions.py3_support'] - with open(result.artifact) as f: - artifact = f.read() + artifact = result.artifact.read_text() print(artifact) @@ -476,7 +473,7 @@ def test_artifact_contains_py3_support_and_looks_as_expected( Software MUST be packaged for Python 3 if upstream supports it. See the following Bugzilla: - https://bugzilla.redhat.com/show_bug.cgi?id=1309383 + https://bugzilla.redhat.com/show_bug.cgi?id=1458531 """).strip() in artifact.strip() @@ -503,8 +500,7 @@ def test_artifact_of_python_usage_obsoleted_looks_as_expected(results, request): results = request.getfixturevalue(results) result = results['dist.python-versions.python_usage_obsoleted'] - with open(result.artifact) as f: - artifact = f.read() + artifact = result.artifact.read_text() print(artifact) @@ -542,8 +538,7 @@ def test_artifact_contains_python_usage_and_looks_as_expected(results, request): results = request.getfixturevalue(results) result = results['dist.python-versions.python_usage'] - with open(result.artifact) as f: - artifact = f.read() + artifact = result.artifact.read_text() print(artifact)