From d278890414762b8e0f63bcbf72996311d3f2c5ee Mon Sep 17 00:00:00 2001 From: Trym Bremnes Date: Wed, 24 Jun 2020 06:36:53 +0200 Subject: [PATCH] Restructure pipeline workflow Prework for updating the pip packages. The latest version of pylint requires that it knows about various plugins for pytest in order to successfully lint. This demand requires the dependencies of setup, test and lint to be separated, since before this commit all of the steps were more or less done in one big blob. By restructuring the azure-pipeline worklfow into setup, build and test/lint it is much easier to reason about the pipeline, and now easier to install the correct dependency for the correct stage. --- azure-pipelines.yml | 19 ++++--- continuous-integration/linux/build.sh | 10 ++++ continuous-integration/linux/lint.sh | 4 +- continuous-integration/linux/test.sh | 8 +-- .../lint.txt} | 0 .../python-requirements/setup.txt | 1 + .../test.txt} | 0 continuous-integration/windows/build.py | 46 +---------------- continuous-integration/windows/setup.py | 51 +++++++++++++++++++ continuous-integration/windows/test.py | 48 +++++++++++++++++ 10 files changed, 126 insertions(+), 61 deletions(-) create mode 100755 continuous-integration/linux/build.sh rename continuous-integration/{requirements-lint.txt => python-requirements/lint.txt} (100%) create mode 100644 continuous-integration/python-requirements/setup.txt rename continuous-integration/{requirements-build-and-test.txt => python-requirements/test.txt} (100%) create mode 100644 continuous-integration/windows/setup.py create mode 100644 continuous-integration/windows/test.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c19481f9..e75e4b68 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,11 +22,13 @@ jobs: Python39: python.version: '3.9' steps: - - script: pip install -r continuous-integration\requirements-build-and-test.txt" - displayName: 'Install python requirements' + - script: python continuous-integration\windows\setup.py + displayName: 'Setup' - script: python continuous-integration\windows\build.py --root %CD% - displayName: 'Build and test' + displayName: 'Build' env: { 'Zivid_DIR': 'C:\Program Files\Zivid\CMake\Zivid', 'CXX': 'cl.exe', 'CC': 'cl.exe'} + - script: python continuous-integration\windows\test.py --root %CD% + displayName: 'Test' - job: Linux pool: vmImage: ubuntu-16.04 @@ -35,23 +37,28 @@ jobs: ArchLinux: os: 'archlinux/base' command: './lint.sh' + display_name: 'lint' Ubuntu2004: os: 'ubuntu:20.04' - command: './test.sh' + command: './test.sh' + display_name: 'test' Ubuntu1804: os: 'ubuntu:18.04' command: './test.sh' + display_name: 'test' Ubuntu1604: os: 'ubuntu:16.04' command: './test.sh' + display_name: 'test' Fedora30: os: 'fedora:30' command: './test.sh' + display_name: 'test' steps: - script: docker run --volume $PWD:/host --workdir /host/continuous-integration/linux --env "PYTHONDONTWRITEBYTECODE=1" $(os) - bash -c "./setup.sh && $(command)" - displayName: 'Build and test' + bash -c "./setup.sh && ./build.sh && $(command)" + displayName: 'Setup, build and $(display_name)' diff --git a/continuous-integration/linux/build.sh b/continuous-integration/linux/build.sh new file mode 100755 index 00000000..5b92a888 --- /dev/null +++ b/continuous-integration/linux/build.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOT_DIR=$(realpath "$SCRIPT_DIR/../..") + +pip install --upgrade pip || exit $? +echo "Building zivid-python..." +pip install "$ROOT_DIR" || exit $? + +echo Success! ["$0"] \ No newline at end of file diff --git a/continuous-integration/linux/lint.sh b/continuous-integration/linux/lint.sh index bc835239..89a0d584 100755 --- a/continuous-integration/linux/lint.sh +++ b/continuous-integration/linux/lint.sh @@ -10,9 +10,7 @@ nonPublicPythonFiles=$(comm -23 <(echo $pythonFiles| tr " " "\n" |sort) \ bashFiles=$(find "$ROOT_DIR" -name '*.sh') -pip install pip --upgrade || exit $? -pip install "$ROOT_DIR" || exit $? -pip install -r "$SCRIPT_DIR/../requirements-lint.txt" || exit $? +pip install --requirement "$SCRIPT_DIR/../python-requirements/lint.txt" || exit $? echo Running non public pylint on: echo "$nonPublicPythonFiles" diff --git a/continuous-integration/linux/test.sh b/continuous-integration/linux/test.sh index d1891263..a5b02f6d 100755 --- a/continuous-integration/linux/test.sh +++ b/continuous-integration/linux/test.sh @@ -2,14 +2,8 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" ROOT_DIR=$(realpath "$SCRIPT_DIR/../..") -VENV=$(mktemp --tmpdir --directory zivid-python-test-venv-XXXX) || exit $? -python -m venv $VENV || exit $? -source $VENV/bin/activate || exit $? - -pip install --upgrade pip || exit $? -pip install "$ROOT_DIR" || exit $? -pip install -r "$SCRIPT_DIR/../requirements-build-and-test.txt" || exit $? +pip install --requirement "$SCRIPT_DIR/../python-requirements/test.txt" || exit $? python -m pytest "$ROOT_DIR" -c "$ROOT_DIR/pytest.ini" --unmarked || exit $? diff --git a/continuous-integration/requirements-lint.txt b/continuous-integration/python-requirements/lint.txt similarity index 100% rename from continuous-integration/requirements-lint.txt rename to continuous-integration/python-requirements/lint.txt diff --git a/continuous-integration/python-requirements/setup.txt b/continuous-integration/python-requirements/setup.txt new file mode 100644 index 00000000..e20605c4 --- /dev/null +++ b/continuous-integration/python-requirements/setup.txt @@ -0,0 +1 @@ +requests==2.22.0 \ No newline at end of file diff --git a/continuous-integration/requirements-build-and-test.txt b/continuous-integration/python-requirements/test.txt similarity index 100% rename from continuous-integration/requirements-build-and-test.txt rename to continuous-integration/python-requirements/test.txt diff --git a/continuous-integration/windows/build.py b/continuous-integration/windows/build.py index 00b7c919..2f8a8543 100644 --- a/continuous-integration/windows/build.py +++ b/continuous-integration/windows/build.py @@ -1,24 +1,13 @@ import argparse -import tempfile import sys import subprocess from pathlib import Path -import requests def _options(): parser = argparse.ArgumentParser() parser.add_argument("--root", required=True, help="The repository root", type=Path) - parser.add_argument( - "--skip-setup", - action="store_true", - help="Skip setting up build and test dependencies", - ) - parser.add_argument("--skip-build", action="store_true", help="Skip the build step") - parser.add_argument( - "--skip-test", action="store_true", help="Skip the unit test step" - ) return parser.parse_args() @@ -36,47 +25,14 @@ def _run_process(args): sys.stdout.flush() -def _setup(): - with tempfile.TemporaryDirectory() as temp_dir: - zivid_installer_url = "https://www.zivid.com/hubfs/softwarefiles/releases/1.8.1+6967bc1b-1/windows/ZividSetup_1.8.1+6967bc1b-1.exe" - print("Downloading {}".format(zivid_installer_url), flush=True) - zivid_installer = Path(temp_dir) / "ZividSetup.exe" - response = requests.get(zivid_installer_url) - zivid_installer.write_bytes(response.content) - print("Installing {}".format(zivid_installer), flush=True) - _run_process((str(zivid_installer), "/S")) - - def _build(root): _run_process(("pip", "install", str(root))) -def _test(root): - _run_process( - ( - "echo", - "TODO: ", - "python", - "-m", - "pytest", - str(root), - "-c", - str(root / "pytest.ini"), - ) - ) - - def _main(): options = _options() - if not options.skip_setup: - _setup() - - if not options.skip_build: - _build(options.root) - - if not options.skip_test: - _test(options.root) + _build(options.root) if __name__ == "__main__": diff --git a/continuous-integration/windows/setup.py b/continuous-integration/windows/setup.py new file mode 100644 index 00000000..a8cf3221 --- /dev/null +++ b/continuous-integration/windows/setup.py @@ -0,0 +1,51 @@ +import tempfile +import sys +import subprocess +from pathlib import Path + + +def _run_process(args): + sys.stdout.flush() + try: + process = subprocess.Popen(args) + exit_code = process.wait() + if exit_code != 0: + raise RuntimeError("Wait failed with exit code {}".format(exit_code)) + except Exception as ex: + raise type(ex)("Process failed: '{}'.".format(" ".join(args))) from ex + finally: + sys.stdout.flush() + + +def _install_pip_dependencies(): + print("Installing python build requirements", flush=True) + _run_process( + ( + "pip", + "install", + "--requirement", + "continuous-integration/python-requirements/setup.txt", + ) + ) + + +def _install_zivid_sdk(): + import requests + + with tempfile.TemporaryDirectory() as temp_dir: + zivid_installer_url = "https://www.zivid.com/hubfs/softwarefiles/releases/1.8.1+6967bc1b-1/windows/ZividSetup_1.8.1+6967bc1b-1.exe" + print("Downloading {}".format(zivid_installer_url), flush=True) + zivid_installer = Path(temp_dir) / "ZividSetup.exe" + response = requests.get(zivid_installer_url) + zivid_installer.write_bytes(response.content) + print("Installing {}".format(zivid_installer), flush=True) + _run_process((str(zivid_installer), "/S")) + + +def _main(): + _install_pip_dependencies() + _install_zivid_sdk() + + +if __name__ == "__main__": + _main() diff --git a/continuous-integration/windows/test.py b/continuous-integration/windows/test.py new file mode 100644 index 00000000..3b0f8052 --- /dev/null +++ b/continuous-integration/windows/test.py @@ -0,0 +1,48 @@ +import argparse +import sys +import subprocess +from pathlib import Path + + +def _options(): + parser = argparse.ArgumentParser() + parser.add_argument("--root", required=True, help="The repository root", type=Path) + return parser.parse_args() + + +def _run_process(args): + sys.stdout.flush() + try: + process = subprocess.Popen(args) + exit_code = process.wait() + if exit_code != 0: + raise RuntimeError("Wait failed with exit code {}".format(exit_code)) + except Exception as ex: + raise type(ex)("Process failed: '{}'.".format(" ".join(args))) from ex + finally: + sys.stdout.flush() + + +def _test(root): + _run_process( + ( + "echo", + "TODO: ", + "python", + "-m", + "pytest", + str(root), + "-c", + str(root / "pytest.ini"), + ) + ) + + +def _main(): + options = _options() + + _test(options.root) + + +if __name__ == "__main__": + _main()