Restructure pipeline workflow

Prework for updating the pip packages. The latest version of pylint
requires that it knows about various plugins for pytest in order to
successfully lint. This demand requires the dependencies of setup, test
and lint to be separated, since before this commit all of the steps were
more or less done in one big blob.

By restructuring the azure-pipeline worklfow into setup, build and
test/lint it is much easier to reason about the pipeline, and now easier
to install the correct dependency for the correct stage.

azure-pipelines.yml

@@ -22,11 +22,13 @@ jobs:
       Python39:
         python.version: '3.9'
   steps:
-  - script: pip install -r continuous-integration\requirements-build-and-test.txt"
-    displayName: 'Install python requirements'
+  - script: python continuous-integration\windows\setup.py
+    displayName: 'Setup'
   - script: python continuous-integration\windows\build.py --root %CD%
-    displayName: 'Build and test'
+    displayName: 'Build'
     env: { 'Zivid_DIR': 'C:\Program Files\Zivid\CMake\Zivid', 'CXX': 'cl.exe', 'CC': 'cl.exe'}
+  - script: python continuous-integration\windows\test.py --root %CD%
+    displayName: 'Test'
 - job: Linux
   pool:
     vmImage: ubuntu-16.04
@@ -35,23 +37,28 @@ jobs:
       ArchLinux:
         os: 'archlinux/base'
         command: './lint.sh'
+        display_name: 'lint'
       Ubuntu2004:
         os: 'ubuntu:20.04'
-        command: './test.sh'  
+        command: './test.sh'
+        display_name: 'test'
       Ubuntu1804:
         os: 'ubuntu:18.04'
         command: './test.sh'
+        display_name: 'test'
       Ubuntu1604:
         os: 'ubuntu:16.04'
         command: './test.sh'
+        display_name: 'test'
       Fedora30:
         os: 'fedora:30'
         command: './test.sh'
+        display_name: 'test'
   steps:
   - script: docker run
               --volume $PWD:/host
               --workdir /host/continuous-integration/linux
               --env "PYTHONDONTWRITEBYTECODE=1"
               $(os)
-              bash -c "./setup.sh && $(command)"
-    displayName: 'Build and test'
+              bash -c "./setup.sh && ./build.sh && $(command)"
+    displayName: 'Setup, build and $(display_name)'

continuous-integration/linux/build.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+ROOT_DIR=$(realpath "$SCRIPT_DIR/../..")
+
+pip install --upgrade pip || exit $?
+echo "Building zivid-python..."
+pip install "$ROOT_DIR" || exit $?
+
+echo Success! ["$0"]

continuous-integration/linux/lint.sh

@@ -10,9 +10,7 @@ nonPublicPythonFiles=$(comm -23 <(echo $pythonFiles| tr " " "\n" |sort) \
 
 bashFiles=$(find "$ROOT_DIR" -name '*.sh')
 
-pip install pip --upgrade || exit $?
-pip install "$ROOT_DIR" || exit $?
-pip install -r "$SCRIPT_DIR/../requirements-lint.txt" || exit $?
+pip install --requirement "$SCRIPT_DIR/../python-requirements/lint.txt" || exit $?
 
 echo Running non public pylint on:
 echo "$nonPublicPythonFiles"

continuous-integration/linux/test.sh

@@ -2,14 +2,8 @@
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 ROOT_DIR=$(realpath "$SCRIPT_DIR/../..")
-VENV=$(mktemp --tmpdir --directory zivid-python-test-venv-XXXX) || exit $?
 
-python -m venv $VENV || exit $?
-source $VENV/bin/activate || exit $?
-
-pip install --upgrade pip || exit $?
-pip install "$ROOT_DIR" || exit $?
-pip install -r "$SCRIPT_DIR/../requirements-build-and-test.txt" || exit $?
+pip install --requirement "$SCRIPT_DIR/../python-requirements/test.txt" || exit $?
 
 python -m pytest "$ROOT_DIR" -c "$ROOT_DIR/pytest.ini" --unmarked || exit $?
 

continuous-integration/python-requirements/setup.txt

@@ -0,0 +1 @@
+requests==2.22.0

continuous-integration/windows/build.py

@@ -1,24 +1,13 @@
 import argparse
-import tempfile
 import sys
 import subprocess
 from pathlib import Path
-import requests
 
 
 def _options():
     parser = argparse.ArgumentParser()
 
     parser.add_argument("--root", required=True, help="The repository root", type=Path)
-    parser.add_argument(
-        "--skip-setup",
-        action="store_true",
-        help="Skip setting up build and test dependencies",
-    )
-    parser.add_argument("--skip-build", action="store_true", help="Skip the build step")
-    parser.add_argument(
-        "--skip-test", action="store_true", help="Skip the unit test step"
-    )
 
     return parser.parse_args()
 
@@ -36,47 +25,14 @@ def _run_process(args):
         sys.stdout.flush()
 
 
-def _setup():
-    with tempfile.TemporaryDirectory() as temp_dir:
-        zivid_installer_url = "https://www.zivid.com/hubfs/softwarefiles/releases/1.8.1+6967bc1b-1/windows/ZividSetup_1.8.1+6967bc1b-1.exe"
-        print("Downloading {}".format(zivid_installer_url), flush=True)
-        zivid_installer = Path(temp_dir) / "ZividSetup.exe"
-        response = requests.get(zivid_installer_url)
-        zivid_installer.write_bytes(response.content)
-        print("Installing {}".format(zivid_installer), flush=True)
-        _run_process((str(zivid_installer), "/S"))
-
-
 def _build(root):
     _run_process(("pip", "install", str(root)))
 
 
-def _test(root):
-    _run_process(
-        (
-            "echo",
-            "TODO: ",
-            "python",
-            "-m",
-            "pytest",
-            str(root),
-            "-c",
-            str(root / "pytest.ini"),
-        )
-    )
-
-
 def _main():
     options = _options()
 
-    if not options.skip_setup:
-        _setup()
-
-    if not options.skip_build:
-        _build(options.root)
-
-    if not options.skip_test:
-        _test(options.root)
+    _build(options.root)
 
 
 if __name__ == "__main__":

continuous-integration/windows/setup.py

@@ -0,0 +1,51 @@
+import tempfile
+import sys
+import subprocess
+from pathlib import Path
+
+
+def _run_process(args):
+    sys.stdout.flush()
+    try:
+        process = subprocess.Popen(args)
+        exit_code = process.wait()
+        if exit_code != 0:
+            raise RuntimeError("Wait failed with exit code {}".format(exit_code))
+    except Exception as ex:
+        raise type(ex)("Process failed: '{}'.".format(" ".join(args))) from ex
+    finally:
+        sys.stdout.flush()
+
+
+def _install_pip_dependencies():
+    print("Installing python build requirements", flush=True)
+    _run_process(
+        (
+            "pip",
+            "install",
+            "--requirement",
+            "continuous-integration/python-requirements/setup.txt",
+        )
+    )
+
+
+def _install_zivid_sdk():
+    import requests
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        zivid_installer_url = "https://www.zivid.com/hubfs/softwarefiles/releases/1.8.1+6967bc1b-1/windows/ZividSetup_1.8.1+6967bc1b-1.exe"
+        print("Downloading {}".format(zivid_installer_url), flush=True)
+        zivid_installer = Path(temp_dir) / "ZividSetup.exe"
+        response = requests.get(zivid_installer_url)
+        zivid_installer.write_bytes(response.content)
+        print("Installing {}".format(zivid_installer), flush=True)
+        _run_process((str(zivid_installer), "/S"))
+
+
+def _main():
+    _install_pip_dependencies()
+    _install_zivid_sdk()
+
+
+if __name__ == "__main__":
+    _main()

continuous-integration/windows/test.py

@@ -0,0 +1,48 @@
+import argparse
+import sys
+import subprocess
+from pathlib import Path
+
+
+def _options():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--root", required=True, help="The repository root", type=Path)
+    return parser.parse_args()
+
+
+def _run_process(args):
+    sys.stdout.flush()
+    try:
+        process = subprocess.Popen(args)
+        exit_code = process.wait()
+        if exit_code != 0:
+            raise RuntimeError("Wait failed with exit code {}".format(exit_code))
+    except Exception as ex:
+        raise type(ex)("Process failed: '{}'.".format(" ".join(args))) from ex
+    finally:
+        sys.stdout.flush()
+
+
+def _test(root):
+    _run_process(
+        (
+            "echo",
+            "TODO: ",
+            "python",
+            "-m",
+            "pytest",
+            str(root),
+            "-c",
+            str(root / "pytest.ini"),
+        )
+    )
+
+
+def _main():
+    options = _options()
+
+    _test(options.root)
+
+
+if __name__ == "__main__":
+    _main()