From 284fe2547ad5a433bf8cfc5f6119f930efd5ab6b Mon Sep 17 00:00:00 2001 From: Joachim Metz Date: Sat, 23 Feb 2019 09:08:57 +0100 Subject: [PATCH] Migrated tox tests to Bionic on Docker (#313) --- .travis.yml | 76 ++++++++++++++++++++++++--------------- artifacts/reader.py | 3 +- config/travis/install.sh | 43 ++++++++++++++++++++-- config/travis/runtests.sh | 25 ++++++++++--- setup.py | 2 +- 5 files changed, 112 insertions(+), 37 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9ebb4d98..126806ef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,68 +30,88 @@ matrix: python: 3.5 virtualenv: system_site_packages: true - - name: "Ubuntu Xenial (16.04) with Python 2.7 and tox" - env: [TARGET="linux-python27-tox", TOXENV="py27"] + - name: "Fedora Core 29 (Docker) with Python 2.7" + env: FEDORA_VERSION="29" os: linux dist: xenial sudo: required group: edge language: python python: 2.7 - virtualenv: - system_site_packages: false - - name: "Ubuntu Xenial (16.04) with Python 3.4 and tox" - env: [TARGET="linux-python34-tox", TOXENV="py34"] + services: + - docker + - name: "Fedora Core 29 (Docker) with Python 3.7" + env: FEDORA_VERSION="29" os: linux dist: xenial sudo: required group: edge language: python - python: 3.4 - virtualenv: - system_site_packages: false - - name: "Ubuntu Xenial (16.04) with Python 3.5 and tox" - env: [TARGET="linux-python35-tox", TOXENV="py35"] + python: 3.7 + services: + - docker + - name: "Ubuntu Bionic (18.04) (Docker) with Python 2.7" + env: UBUNTU_VERSION="18.04" os: linux dist: xenial sudo: required group: edge language: python - python: 3.5 - virtualenv: - system_site_packages: false - - name: "Ubuntu Xenial (16.04) with Python 3.6 and tox" - env: [TARGET="linux-python36-tox", TOXENV="py36"] + python: 2.7 + services: + - docker + - name: "Ubuntu Bionic (18.04) (Docker) with Python 3.6" + env: UBUNTU_VERSION="18.04" os: linux dist: xenial sudo: required group: edge language: python python: 3.6 - virtualenv: - system_site_packages: false - - name: "Ubuntu Xenial (16.04) with Python 3.7 and tox" - env: [TARGET="linux-python37-tox", TOXENV="py37"] + services: + - docker + - name: "Ubuntu Bionic (18.04) (Docker) with Python 2.7 and tox" + env: [TOXENV="py27", UBUNTU_VERSION="18.04"] os: linux dist: xenial sudo: required group: edge language: python - python: 3.7 - virtualenv: - system_site_packages: false - - name: "Fedora Core 29 (Docker) with Python 2.7" - env: FEDORA_VERSION="29" + python: 2.7 + services: + - docker + - name: "Ubuntu Bionic (18.04) (Docker) with Python 3.4 and tox" + env: [TOXENV="py34", UBUNTU_VERSION="18.04"] os: linux dist: xenial sudo: required group: edge language: python - python: 2.7 + python: 3.4 services: - docker - - name: "Fedora Core 29 (Docker) with Python 3.7" - env: FEDORA_VERSION="29" + - name: "Ubuntu Bionic (18.04) (Docker) with Python 3.5 and tox" + env: [TOXENV="py35", UBUNTU_VERSION="18.04"] + os: linux + dist: xenial + sudo: required + group: edge + language: python + python: 3.5 + services: + - docker + - name: "Ubuntu Bionic (18.04) (Docker) with Python 3.6 and tox" + env: [TOXENV="py36", UBUNTU_VERSION="18.04"] + os: linux + dist: xenial + sudo: required + group: edge + language: python + python: 3.6 + services: + - docker + - name: "Ubuntu Bionic (18.04) (Docker) with Python 3.7 and tox" + env: [TOXENV="py37", UBUNTU_VERSION="18.04"] os: linux dist: xenial sudo: required diff --git a/artifacts/reader.py b/artifacts/reader.py index 391278d2..fabcff7f 100644 --- a/artifacts/reader.py +++ b/artifacts/reader.py @@ -5,6 +5,7 @@ import abc import glob +import io import os import json import yaml @@ -281,7 +282,7 @@ def ReadFile(self, filename): Yields: ArtifactDefinition: an artifact definition. """ - with open(filename, 'r') as file_object: + with io.open(filename, 'r', encoding='utf-8') as file_object: for artifact_definition in self.ReadFileObject(file_object): yield artifact_definition diff --git a/config/travis/install.sh b/config/travis/install.sh index 646a9f39..24d8802e 100755 --- a/config/travis/install.sh +++ b/config/travis/install.sh @@ -11,11 +11,11 @@ L2TBINARIES_TEST_DEPENDENCIES="funcsigs mock pbr six"; DPKG_PYTHON2_DEPENDENCIES="python-yaml"; -DPKG_PYTHON2_TEST_DEPENDENCIES="python-coverage python-funcsigs python-mock python-pbr python-six python-tox"; +DPKG_PYTHON2_TEST_DEPENDENCIES="python-coverage python-funcsigs python-mock python-pbr python-six"; DPKG_PYTHON3_DEPENDENCIES="python3-yaml"; -DPKG_PYTHON3_TEST_DEPENDENCIES="python3-mock python3-pbr python3-setuptools python3-six python3-tox"; +DPKG_PYTHON3_TEST_DEPENDENCIES="python3-mock python3-pbr python3-setuptools python3-six"; RPM_PYTHON2_DEPENDENCIES="python2-pyyaml"; @@ -62,13 +62,50 @@ then docker exec ${CONTAINER_NAME} dnf copr -y enable @gift/dev; - if test ${TRAVIS_PYTHON_VERSION} = "2.7"; + if test -n "${TOXENV}"; + then + docker exec ${CONTAINER_NAME} dnf install -y python3-tox; + + elif test ${TRAVIS_PYTHON_VERSION} = "2.7"; then docker exec ${CONTAINER_NAME} dnf install -y git python2 ${RPM_PYTHON2_DEPENDENCIES} ${RPM_PYTHON2_TEST_DEPENDENCIES}; else docker exec ${CONTAINER_NAME} dnf install -y git python3 ${RPM_PYTHON3_DEPENDENCIES} ${RPM_PYTHON3_TEST_DEPENDENCIES}; fi + docker cp ../artifacts ${CONTAINER_NAME}:/ + +elif test -n "${UBUNTU_VERSION}"; +then + CONTAINER_NAME="ubuntu${UBUNTU_VERSION}"; + + docker pull ubuntu:${UBUNTU_VERSION}; + + docker run --name=${CONTAINER_NAME} --detach -i ubuntu:${UBUNTU_VERSION}; + + docker exec ${CONTAINER_NAME} apt-get update -q; + docker exec ${CONTAINER_NAME} sh -c "DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common"; + + docker exec ${CONTAINER_NAME} add-apt-repository ppa:gift/dev -y; + + if test -n "${TOXENV}"; + then + docker exec ${CONTAINER_NAME} add-apt-repository universe; + docker exec ${CONTAINER_NAME} add-apt-repository ppa:deadsnakes/ppa -y; + + DPKG_PYTHON="python${TRAVIS_PYTHON_VERSION}"; + + docker exec ${CONTAINER_NAME} sh -c "DEBIAN_FRONTEND=noninteractive apt-get install -y ${DPKG_PYTHON} tox"; + + elif test ${TRAVIS_PYTHON_VERSION} = "2.7"; + then + docker exec ${CONTAINER_NAME} sh -c "DEBIAN_FRONTEND=noninteractive apt-get install -y git python ${DPKG_PYTHON2_DEPENDENCIES} ${DPKG_PYTHON2_TEST_DEPENDENCIES}"; + else + docker exec ${CONTAINER_NAME} sh -c "DEBIAN_FRONTEND=noninteractive apt-get install -y git python3 ${DPKG_PYTHON3_DEPENDENCIES} ${DPKG_PYTHON3_TEST_DEPENDENCIES}"; + fi + + docker cp ../artifacts ${CONTAINER_NAME}:/ + elif test ${TRAVIS_OS_NAME} = "linux" && test ${TARGET} != "jenkins"; then sudo rm -f /etc/apt/sources.list.d/travis_ci_zeromq3-source.list; diff --git a/config/travis/runtests.sh b/config/travis/runtests.sh index 5563145b..c4056368 100755 --- a/config/travis/runtests.sh +++ b/config/travis/runtests.sh @@ -42,13 +42,30 @@ elif test -n "${FEDORA_VERSION}"; then CONTAINER_NAME="fedora${FEDORA_VERSION}"; - docker exec "${CONTAINER_NAME}" sh -c "git clone https://github.com/ForensicArtifacts/artifacts.git"; + if test -n "${TOXENV}"; + then + docker exec ${CONTAINER_NAME} sh -c "export LANG=en_US.UTF-8; cd artifacts && tox -e ${TOXENV}"; + + elif test ${TRAVIS_PYTHON_VERSION} = "2.7"; + then + docker exec ${CONTAINER_NAME} sh -c "export LANG=en_US.UTF-8; cd artifacts && python2 run_tests.py"; + else + docker exec ${CONTAINER_NAME} sh -c "export LANG=en_US.UTF-8; cd artifacts && python3 run_tests.py"; + fi + +elif test -n "${UBUNTU_VERSION}"; +then + CONTAINER_NAME="ubuntu${UBUNTU_VERSION}"; + + if test -n "${TOXENV}"; + then + docker exec ${CONTAINER_NAME} sh -c "export LANG=en_US.UTF-8; cd artifacts && tox -e ${TOXENV}"; - if test ${TRAVIS_PYTHON_VERSION} = "2.7"; + elif test ${TRAVIS_PYTHON_VERSION} = "2.7"; then - docker exec "${CONTAINER_NAME}" sh -c "cd artifacts && python2 run_tests.py"; + docker exec ${CONTAINER_NAME} sh -c "export LANG=en_US.UTF-8; cd artifacts && python2 run_tests.py"; else - docker exec "${CONTAINER_NAME}" sh -c "cd artifacts && python3 run_tests.py"; + docker exec ${CONTAINER_NAME} sh -c "export LANG=en_US.UTF-8; cd artifacts && python3 run_tests.py"; fi elif test "${TRAVIS_OS_NAME}" = "linux"; diff --git a/setup.py b/setup.py index ba23eca7..786dafee 100755 --- a/setup.py +++ b/setup.py @@ -124,7 +124,7 @@ def _make_spec_file(self): elif line.startswith('%files'): python_spec_file.extend([ '%package -n %{name}-tools', - 'Requires: python-artifacts', + 'Requires: {0:s}-artifacts'.format(python_package), 'Summary: Tools for {0:s}'.format(summary), '', '%description -n %{name}-tools'])