From fb65762868129d389b640ac6c793b64eddd2ddf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 1 Oct 2018 15:28:26 +0200 Subject: [PATCH 1/6] Use python3 everywhere Enable https://copr.fedorainfracloud.org/coprs/kparal/taskotron-dev/ --- Dockerfile | 4 ++-- README.rst | 2 +- mock.cfg | 11 +++++++++++ setup.py | 1 - taskotron_python_versions/common.py | 10 +++------- tests.yml | 12 ++++++------ tox.ini | 2 +- 7 files changed, 24 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7b0ea39..2d4f18f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ FROM fedora RUN dnf -y install --setopt=install_weak_deps=false --setopt=tsflags=nodocs \ - --setopt=deltarpm=false python2-rpm python3-rpm tox python2-dnf \ - python3-dnf mock --allowerasing --best && dnf clean all + --setopt=deltarpm=false --allowerasing --best \ + mock tox python3-rpm python3-dnf && dnf clean all ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 diff --git a/README.rst b/README.rst index b250624..69118da 100644 --- a/README.rst +++ b/README.rst @@ -50,7 +50,7 @@ Tests This task is covered with functional and integration tests. You can run them using `tox `__, but -you will need ``mock``, ``python{2,3}-rpm`` and ``python{2,3}-dnf`` installed. +you will need ``mock``, ``python3-rpm`` and ``python3-dnf`` installed. For mock configuration see `mock setup `__ instructions. Use the following command to run the test suite:: diff --git a/mock.cfg b/mock.cfg index 6ad302d..753c0b2 100644 --- a/mock.cfg +++ b/mock.cfg @@ -6,3 +6,14 @@ config_opts['rpmbuild_networking'] = True config_opts['use_nspawn'] = False config_opts['root'] = 'taskotron-python-versions-master' config_opts['plugin_conf']['root_cache_opts']['dir'] = "%(cache_topdir)s/taskotron-python-versions/root_cache/" +config_opts['yum.conf'] += """ +[kparal-taskotron-dev] +name=kparal-taskotron-dev +baseurl=https://copr-be.cloud.fedoraproject.org/results/kparal/taskotron-dev/fedora-$releasever-$basearch/ +type=rpm-md +skip_if_unavailable=False +gpgcheck=1 +gpgkey=https://copr-be.cloud.fedoraproject.org/results/kparal/taskotron-dev/pubkey.gpg +repo_gpgcheck=0 +enabled=1 +""" diff --git a/setup.py b/setup.py index 2b91c50..6fb6e3a 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,6 @@ 'License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 3', 'Topic :: Software Development :: Testing', ] diff --git a/taskotron_python_versions/common.py b/taskotron_python_versions/common.py index 1ab64fd..bbca549 100644 --- a/taskotron_python_versions/common.py +++ b/taskotron_python_versions/common.py @@ -61,14 +61,10 @@ def file_contains(path, needle): # let's optimize prematurely because practicality beats purity # Memory-mapped file object behaving like bytearray - mmf = mmap.mmap(f.fileno(), - length=0, # = determine automatically - access=mmap.ACCESS_READ) - try: + with mmap.mmap(f.fileno(), + length=0, # = determine automatically + access=mmap.ACCESS_READ) as mmf: return mmf.find(needle) != -1 - finally: - # mmap context manager is Python 3 only - mmf.close() class PackageException(Exception): diff --git a/tests.yml b/tests.yml index 2333433..2326ea6 100644 --- a/tests.yml +++ b/tests.yml @@ -16,10 +16,10 @@ name: "{{ item }}" state: latest with_items: - - rpm-python - - python2-dnf - - python2-libarchive-c - - python-bugzilla + - python3-rpm + - python3-dnf + - python3-libarchive-c + - python3-bugzilla - libtaskotron-core - libtaskotron-fedora register: dnf_output @@ -66,13 +66,13 @@ - block: - name: Download RPMs from Koji shell: > - python2 download_rpms.py {{ taskotron_item }} {{ workdir.path }} + python3 download_rpms.py {{ taskotron_item }} {{ workdir.path }} {{ test_arches | join(',') }} &> {{ artifacts }}/test.log - name: Run task shell: > - python2 python_versions_check.py {{ taskotron_item }} {{ workdir.path }} + python3 python_versions_check.py {{ taskotron_item }} {{ workdir.path }} {{ artifacts }} {{ testcase }} {{ test_arches | join(',') }} &>> {{ artifacts }}/test.log always: diff --git a/tox.ini b/tox.ini index 8e99374..3af6d1b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py3,integration,style +envlist = py3,integration,style skipsdist = True [testenv] From ea226b3cca2522d002f67ddd82312b8e3c1b4377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 1 Oct 2018 19:33:49 +0200 Subject: [PATCH 2/6] The copr repo on rawhide doesn't work with $releasever --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3bd8959..e889fdc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ env: install: - sed -i "s/fedora-28-x86_64/fedora-${FEDORA}-x86_64/" mock.cfg + - sed -i "s/\$releasever/${FEDORA}/" mock.cfg # needed on rawhide - sed -i "s|FROM fedora|FROM registry.fedoraproject.org/fedora:${FEDORA}|" Dockerfile - docker build -t taskotron . From c5fa7d0da34c6169cecf9ab9e8142c73a5a4f91e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 1 Oct 2018 17:10:13 +0200 Subject: [PATCH 3/6] Decode early and only deal with bytes when reading arbitrary files --- taskotron_python_versions/common.py | 4 +- taskotron_python_versions/python_usage.py | 2 - taskotron_python_versions/requires.py | 2 - taskotron_python_versions/two_three.py | 68 +++++++++---------- .../unversioned_shebangs.py | 18 ++--- test/functional/test_unversioned_shebangs.py | 14 ++-- 6 files changed, 48 insertions(+), 60 deletions(-) diff --git a/taskotron_python_versions/common.py b/taskotron_python_versions/common.py index bbca549..a81d300 100644 --- a/taskotron_python_versions/common.py +++ b/taskotron_python_versions/common.py @@ -108,11 +108,11 @@ def nvr(self): @property def require_names(self): - return self.hdr[rpm.RPMTAG_REQUIRENAME] + return [r.decode() for r in self.hdr[rpm.RPMTAG_REQUIRENAME]] @property def require_nevrs(self): - return self.hdr[rpm.RPMTAG_REQUIRENEVRS] + return [r.decode() for r in self.hdr[rpm.RPMTAG_REQUIRENEVRS]] @property def files(self): diff --git a/taskotron_python_versions/python_usage.py b/taskotron_python_versions/python_usage.py index ca0da7b..8279a2b 100644 --- a/taskotron_python_versions/python_usage.py +++ b/taskotron_python_versions/python_usage.py @@ -34,8 +34,6 @@ def task_python_usage(packages, koji_build, artifact): log.debug('Checking {}'.format(package.filename)) for name in package.require_names: - name = name.decode() - if name in PYTHON_COMMAND: log.error( '{} requires {}'.format(package.filename, name)) diff --git a/taskotron_python_versions/requires.py b/taskotron_python_versions/requires.py index c698402..bba549a 100644 --- a/taskotron_python_versions/requires.py +++ b/taskotron_python_versions/requires.py @@ -107,8 +107,6 @@ def check_requires_naming_scheme(package, repoquery): misnamed_requires = set() for name in package.require_names: - name = name.decode() - if is_unversioned(name): versioned = get_versioned_name(name, repoquery) diff --git a/taskotron_python_versions/two_three.py b/taskotron_python_versions/two_three.py index a21f0f9..8e08653 100644 --- a/taskotron_python_versions/two_three.py +++ b/taskotron_python_versions/two_three.py @@ -2,52 +2,52 @@ NEVRS_STARTS = { - 2: (b'python(abi) = 2.',), - 3: (b'python(abi) = 3.',) + 2: ('python(abi) = 2.',), + 3: ('python(abi) = 3.',) } NAME_STARTS = { 2: ( - b'python-', - b'python2', - b'/usr/bin/python2', - b'libpython2', - b'pygtk2', - b'pygobject2', - b'pycairo', - b'py-', + 'python-', + 'python2', + '/usr/bin/python2', + 'libpython2', + 'pygtk2', + 'pygobject2', + 'pycairo', + 'py-', ), 3: ( - b'python3', - b'/usr/bin/python3', - b'libpython3', - b'system-python' + 'python3', + '/usr/bin/python3', + 'libpython3', + 'system-python' ) } NAME_EXACTS = { 2: ( - b'/usr/bin/python', - b'python', + '/usr/bin/python', + 'python', ) } NAME_NOTS = ( - b'python-rpm-macros', - b'python-srpm-macros', - b'python-sphinx-locale', - b'python-multilib-conf', - b'python-ldb-devel-common', - b'python-qt5-rpm-macros', - b'python-btchip-common', - b'python-matplotlib-data', - b'python-matplotlib-data-fonts', - b'python-cartopy-common', - b'python-django-bash-completion', - b'python-jupyter-filesystem', - b'python-pip-wheel', - b'python-setuptools-wheel', - b'python-wheel-wheel', + 'python-rpm-macros', + 'python-srpm-macros', + 'python-sphinx-locale', + 'python-multilib-conf', + 'python-ldb-devel-common', + 'python-qt5-rpm-macros', + 'python-btchip-common', + 'python-matplotlib-data', + 'python-matplotlib-data-fonts', + 'python-cartopy-common', + 'python-django-bash-completion', + 'python-jupyter-filesystem', + 'python-pip-wheel', + 'python-setuptools-wheel', + 'python-wheel-wheel', ) @@ -80,7 +80,7 @@ def check_two_three(package): for nevr in package.require_nevrs: for py_version, starts in NEVRS_STARTS.items(): if nevr.startswith(starts): - log.debug('Found dependency {}'.format(nevr.decode())) + log.debug('Found dependency {}'.format(nevr)) log.debug('Requires Python {}'.format(py_version)) py_versions[py_version] = nevr @@ -88,14 +88,14 @@ def check_two_three(package): for py_version, starts in NAME_STARTS.items(): if py_version not in py_versions: if name.startswith(starts) and name not in NAME_NOTS: - log.debug('Found dependency {}'.format(name.decode())) + log.debug('Found dependency {}'.format(name)) log.debug('Requires Python {}'.format(py_version)) py_versions[py_version] = name for py_version, exacts in NAME_EXACTS.items(): if py_version not in py_versions: if name in exacts: - log.debug('Found dependency {}'.format(name.decode())) + log.debug('Found dependency {}'.format(name)) log.debug('Requires Python {}'.format(py_version)) py_versions[py_version] = name diff --git a/taskotron_python_versions/unversioned_shebangs.py b/taskotron_python_versions/unversioned_shebangs.py index a738d22..350eda3 100644 --- a/taskotron_python_versions/unversioned_shebangs.py +++ b/taskotron_python_versions/unversioned_shebangs.py @@ -42,7 +42,7 @@ def get_problematic_files(archive, query): """Search for the files inside archive with the first line matching given query. Some of the files can contain data, which are not in the plain text format. Bytes are read from the file and - the shebang query has to be of the same type. + the shebang query is encoded as well. We only test for ASCII shebangs. """ problematic = set() with libarchive.file_reader(archive) as a: @@ -51,21 +51,15 @@ def get_problematic_files(archive, query): first_line = next(entry.get_blocks(), '').splitlines()[0] except IndexError: continue # file is empty - if matches(first_line, query.encode()): + if matches(first_line, query.encode('ascii')): problematic.add(entry.pathname.lstrip('.')) return problematic -def shebang_to_require(shebang, use_bytes=True): - """Convert shebang to the format of requirement. - If the use_bytes argument is set to False, executable path - is returned as a string instead of the default bytes type.""" - executable_path = shebang.split()[0][2:] - if use_bytes: - return executable_path.encode() - else: - return executable_path +def shebang_to_require(shebang): + """Convert shebang to the format of requirement.""" + return shebang.split()[0][2:] def get_scripts_summary(package): @@ -79,7 +73,7 @@ def get_scripts_summary(package): if shebang_to_require(shebang) in package.require_names: log.debug('Package {} requires {}'.format( package.filename, shebang_to_require( - shebang, use_bytes=False))) + shebang))) problematic = get_problematic_files(package.path, shebang) if problematic: log.debug('{} shebang was found in scripts: {}'.format( diff --git a/test/functional/test_unversioned_shebangs.py b/test/functional/test_unversioned_shebangs.py index 2511193..aee8f2f 100644 --- a/test/functional/test_unversioned_shebangs.py +++ b/test/functional/test_unversioned_shebangs.py @@ -40,15 +40,13 @@ def test_get_problematic_files(archive, query, expected): assert get_problematic_files(gpkg_path(archive), query) == expected -@pytest.mark.parametrize(('shebang', 'use_bytes', 'expected'), ( - ("#!/foo", True, b"/foo"), - ("#!/usr/bin/python", True, b"/usr/bin/python"), - ("#!/usr/bin/env python", True, b"/usr/bin/env"), - ("#!/usr/bin/python", False, "/usr/bin/python"), - ("#!/usr/bin/env python", False, "/usr/bin/env"), +@pytest.mark.parametrize(('shebang', 'expected'), ( + ("#!/foo", "/foo"), + ("#!/usr/bin/python", "/usr/bin/python"), + ("#!/usr/bin/env python", "/usr/bin/env"), )) -def test_shebang_to_require(shebang, use_bytes, expected): - assert shebang_to_require(shebang, use_bytes) == expected +def test_shebang_to_require(shebang, expected): + assert shebang_to_require(shebang) == expected @pytest.mark.parametrize(('glob', 'expected'), ( From b048f756cf449bfb3196d22e7d07ac30132c1ef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 18 Oct 2018 12:41:24 +0200 Subject: [PATCH 4/6] Release 0.1.dev4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6fb6e3a..507c08b 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name='taskotron-python-versions', - version='0.1.dev3', + version='0.1.dev4', description=description, long_description=long_description, keywords='taskotron fedora python rpm', From 4aad12ca301f1c29f43ffd936a69f14b88c14035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 15 Nov 2018 17:06:39 +0100 Subject: [PATCH 5/6] Flake8: Ignore W504 If W504 or W503 is not ignored, it always bugs you. Normally, one of them is ignored by default, but we redefine what's ignored. W503: line break before binary operator W504: line break after binary operator --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 3af6d1b..bd5e0e1 100644 --- a/tox.ini +++ b/tox.ini @@ -23,5 +23,5 @@ sitepackages = False [testenv:style] deps = flake8 basepython = python3 -commands = python -m flake8 . --ignore=E402 --exclude=.git,__pycache__,.tox,.eggs,dist,build,mockroots +commands = python -m flake8 . --ignore=E402,W504 --exclude=.git,__pycache__,.tox,.eggs,dist,build,mockroots sitepackages = False From 930abc353a1b0d1437a47eeecc96030cf77087e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 15 Nov 2018 17:25:25 +0100 Subject: [PATCH 6/6] Drop the Copr repo, use Fedora proper Update the list of taskotron packages we need --- mock.cfg | 11 ----------- tests.yml | 4 ++-- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/mock.cfg b/mock.cfg index 753c0b2..6ad302d 100644 --- a/mock.cfg +++ b/mock.cfg @@ -6,14 +6,3 @@ config_opts['rpmbuild_networking'] = True config_opts['use_nspawn'] = False config_opts['root'] = 'taskotron-python-versions-master' config_opts['plugin_conf']['root_cache_opts']['dir'] = "%(cache_topdir)s/taskotron-python-versions/root_cache/" -config_opts['yum.conf'] += """ -[kparal-taskotron-dev] -name=kparal-taskotron-dev -baseurl=https://copr-be.cloud.fedoraproject.org/results/kparal/taskotron-dev/fedora-$releasever-$basearch/ -type=rpm-md -skip_if_unavailable=False -gpgcheck=1 -gpgkey=https://copr-be.cloud.fedoraproject.org/results/kparal/taskotron-dev/pubkey.gpg -repo_gpgcheck=0 -enabled=1 -""" diff --git a/tests.yml b/tests.yml index 2326ea6..eadb316 100644 --- a/tests.yml +++ b/tests.yml @@ -20,8 +20,8 @@ - python3-dnf - python3-libarchive-c - python3-bugzilla - - libtaskotron-core - - libtaskotron-fedora + - python3-libtaskotron + - taskotron-runner register: dnf_output # retry for network failures retries: 3