Merge pull request #76 from zivid/split-requirements-files

Restructure pipeline workflow

azure-pipelines.yml

@@ -22,11 +22,13 @@ jobs:
       Python39:
         python.version: '3.9'
   steps:
-  - script: pip install -r continuous-integration\requirements-build-and-test.txt"
-    displayName: 'Install python requirements'
+  - script: python continuous-integration\windows\setup.py
+    displayName: 'Setup'
   - script: python continuous-integration\windows\build.py --root %CD%
-    displayName: 'Build and test'
+    displayName: 'Build'
     env: { 'Zivid_DIR': 'C:\Program Files\Zivid\CMake\Zivid', 'CXX': 'cl.exe', 'CC': 'cl.exe'}
+  - script: python continuous-integration\windows\test.py --root %CD%
+    displayName: 'Test'
 - job: Linux
   pool:
     vmImage: ubuntu-16.04
@@ -35,23 +37,28 @@ jobs:
       ArchLinux:
         os: 'archlinux/base'
         command: './lint.sh'
+        display_name: 'lint'
       Ubuntu2004:
         os: 'ubuntu:20.04'
-        command: './test.sh'  
+        command: './test.sh'
+        display_name: 'test'
       Ubuntu1804:
         os: 'ubuntu:18.04'
         command: './test.sh'
+        display_name: 'test'
       Ubuntu1604:
         os: 'ubuntu:16.04'
         command: './test.sh'
+        display_name: 'test'
       Fedora30:
         os: 'fedora:30'
         command: './test.sh'
+        display_name: 'test'
   steps:
   - script: docker run
               --volume $PWD:/host
               --workdir /host/continuous-integration/linux
               --env "PYTHONDONTWRITEBYTECODE=1"
               $(os)
-              bash -c "./setup.sh && $(command)"
-    displayName: 'Build and test'
+              bash -c "./setup.sh && ./build.sh && $(command)"
+    displayName: 'Setup, build and $(display_name)'

continuous-integration/linux/build.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+ROOT_DIR=$(realpath "$SCRIPT_DIR/../..")
+
+pip install --upgrade pip || exit $?
+echo "Building zivid-python..."
+pip install "$ROOT_DIR" || exit $?
+
+echo Success! ["$0"]

continuous-integration/linux/lint.sh

@@ -10,9 +10,7 @@ nonPublicPythonFiles=$(comm -23 <(echo $pythonFiles| tr " " "\n" |sort) \
 
 bashFiles=$(find "$ROOT_DIR" -name '*.sh')
 
-pip install pip --upgrade || exit $?
-pip install "$ROOT_DIR" || exit $?
-pip install -r "$SCRIPT_DIR/../requirements-lint.txt" || exit $?
+pip install --requirement "$SCRIPT_DIR/../python-requirements/lint.txt" || exit $?
 
 echo Running non public pylint on:
 echo "$nonPublicPythonFiles"

continuous-integration/linux/test.sh

@@ -2,14 +2,8 @@
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 ROOT_DIR=$(realpath "$SCRIPT_DIR/../..")
-VENV=$(mktemp --tmpdir --directory zivid-python-test-venv-XXXX) || exit $?
 
-python -m venv $VENV || exit $?
-source $VENV/bin/activate || exit $?
-
-pip install --upgrade pip || exit $?
-pip install "$ROOT_DIR" || exit $?
-pip install -r "$SCRIPT_DIR/../requirements-build-and-test.txt" || exit $?
+pip install --requirement "$SCRIPT_DIR/../python-requirements/test.txt" || exit $?
 
 python -m pytest "$ROOT_DIR" -c "$ROOT_DIR/pytest.ini" --unmarked || exit $?
 

continuous-integration/python-requirements/setup.txt

@@ -0,0 +1 @@
+requests==2.22.0

continuous-integration/windows/build.py

@@ -1,24 +1,13 @@
 import argparse
-import tempfile
 import sys
 import subprocess
 from pathlib import Path
-import requests
 
 
 def _options():
     parser = argparse.ArgumentParser()
 
     parser.add_argument("--root", required=True, help="The repository root", type=Path)
-    parser.add_argument(
-        "--skip-setup",
-        action="store_true",
-        help="Skip setting up build and test dependencies",
-    )
-    parser.add_argument("--skip-build", action="store_true", help="Skip the build step")
-    parser.add_argument(
-        "--skip-test", action="store_true", help="Skip the unit test step"
-    )
 
     return parser.parse_args()
 
@@ -36,47 +25,14 @@ def _run_process(args):
         sys.stdout.flush()
 
 
-def _setup():
-    with tempfile.TemporaryDirectory() as temp_dir:
-        zivid_installer_url = "https://www.zivid.com/hubfs/softwarefiles/releases/1.8.1+6967bc1b-1/windows/ZividSetup_1.8.1+6967bc1b-1.exe"
-        print("Downloading {}".format(zivid_installer_url), flush=True)
-        zivid_installer = Path(temp_dir) / "ZividSetup.exe"
-        response = requests.get(zivid_installer_url)
-        zivid_installer.write_bytes(response.content)
-        print("Installing {}".format(zivid_installer), flush=True)
-        _run_process((str(zivid_installer), "/S"))
-
-
 def _build(root):
     _run_process(("pip", "install", str(root)))
 
 
-def _test(root):
-    _run_process(
-        (
-            "echo",
-            "TODO: ",
-            "python",
-            "-m",
-            "pytest",
-            str(root),
-            "-c",
-            str(root / "pytest.ini"),
-        )
-    )
-
-
 def _main():
     options = _options()
 
-    if not options.skip_setup:
-        _setup()
-
-    if not options.skip_build:
-        _build(options.root)
-
-    if not options.skip_test:
-        _test(options.root)
+    _build(options.root)
 
 
 if __name__ == "__main__":

continuous-integration/windows/setup.py

@@ -0,0 +1,51 @@
+import tempfile
+import sys
+import subprocess
+from pathlib import Path
+
+
+def _run_process(args):
+    sys.stdout.flush()
+    try:
+        process = subprocess.Popen(args)
+        exit_code = process.wait()
+        if exit_code != 0:
+            raise RuntimeError("Wait failed with exit code {}".format(exit_code))
+    except Exception as ex:
+        raise type(ex)("Process failed: '{}'.".format(" ".join(args))) from ex
+    finally:
+        sys.stdout.flush()
+
+
+def _install_pip_dependencies():
+    print("Installing python build requirements", flush=True)
+    _run_process(
+        (
+            "pip",
+            "install",
+            "--requirement",
+            "continuous-integration/python-requirements/setup.txt",
+        )
+    )
+
+
+def _install_zivid_sdk():
+    import requests
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        zivid_installer_url = "https://www.zivid.com/hubfs/softwarefiles/releases/1.8.1+6967bc1b-1/windows/ZividSetup_1.8.1+6967bc1b-1.exe"
+        print("Downloading {}".format(zivid_installer_url), flush=True)
+        zivid_installer = Path(temp_dir) / "ZividSetup.exe"
+        response = requests.get(zivid_installer_url)
+        zivid_installer.write_bytes(response.content)
+        print("Installing {}".format(zivid_installer), flush=True)
+        _run_process((str(zivid_installer), "/S"))
+
+
+def _main():
+    _install_pip_dependencies()
+    _install_zivid_sdk()
+
+
+if __name__ == "__main__":
+    _main()

continuous-integration/windows/test.py

@@ -0,0 +1,48 @@
+import argparse
+import sys
+import subprocess
+from pathlib import Path
+
+
+def _options():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--root", required=True, help="The repository root", type=Path)
+    return parser.parse_args()
+
+
+def _run_process(args):
+    sys.stdout.flush()
+    try:
+        process = subprocess.Popen(args)
+        exit_code = process.wait()
+        if exit_code != 0:
+            raise RuntimeError("Wait failed with exit code {}".format(exit_code))
+    except Exception as ex:
+        raise type(ex)("Process failed: '{}'.".format(" ".join(args))) from ex
+    finally:
+        sys.stdout.flush()
+
+
+def _test(root):
+    _run_process(
+        (
+            "echo",
+            "TODO: ",
+            "python",
+            "-m",
+            "pytest",
+            str(root),
+            "-c",
+            str(root / "pytest.ini"),
+        )
+    )
+
+
+def _main():
+    options = _options()
+
+    _test(options.root)
+
+
+if __name__ == "__main__":
+    _main()