tox.ini

[tox]
envlist =
    hagrid.publish
    lint
    stack.test.integration
    syft.docs
    syft.jupyter
    syft.publish
    syft.test.security
    syft.test.unit
    syft.test.notebook
    stack.test.notebook
    stack.test.integration.enclave.oblv
    stack.test.integration.k8s
    stack.test.vm
    frontend.test.unit
    frontend.test.e2e
    frontend.generate.types
    syft.build.helm
    syft.package.helm
    syft.test.helm
skipsdist = True

[testenv]
basepython = python3
install_command = pip install --find-links https://whls.blob.core.windows.net/unstable/index.html {opts} {packages}
commands =
    python --version
setenv =
    PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html

# Syft
[testenv:syft]
deps =
    -e{toxinidir}/packages/syft[dev]
changedir = {toxinidir}/packages/syft
description = Syft
setenv =
    PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html
commands =
    pip list

[testenv:hagrid]
deps =
    -e{toxinidir}/packages/hagrid[dev]
changedir = {toxinidir}/packages/hagrid
description = Syft
setenv =
    PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html
commands =
    pip list

[testenv:hagrid.publish]
changedir = {toxinidir}/packages/hagrid
description = Build and Publish Hagrid Wheel
commands =
    python -m pip install --upgrade pip
    pip install --upgrade setuptools wheel twine tox build
    python -c 'from shutil import rmtree; rmtree("build", True); rmtree("dist", True)'
    python -m build .

[testenv:lint]
description = Linting
allowlist_externals =
    bash
deps =
    black[python2]
    isort
    pre-commit
commands =
    black .
    isort .
    pre-commit run --all-files

[testenv:frontend.test.unit]
description = Frontend Unit Tests
deps =
    {[testenv:hagrid]deps}
allowlist_externals =
    docker
    bash
    pnpm
passenv=HOME, USER
changedir = {toxinidir}/packages/grid/frontend
setenv =
    DOCKER = {env:DOCKER:false}
commands =
    bash -c "echo Running with DOCKER=$DOCKER; date"

    bash -c 'if [[ "$DOCKER" == "false" ]]; then \
        bash ./scripts/check_pnpm.sh; \
        pnpm install; \
        pnpm run test:unit; \
    else \
        docker build --target grid-ui-tests -t ui-test -f frontend.dockerfile .; \
        docker run -t ui-test; \
    fi'

[testenv:frontend.test.e2e]
description = Frontend Unit Tests
deps =
    {[testenv:hagrid]deps}
allowlist_externals =
    docker
    bash
    pnpm
    sleep
passenv=HOME, USER
changedir = {toxinidir}/packages/grid/frontend
setenv =
    HAGRID_FLAGS = {env:HAGRID_FLAGS:--tag=local --test}
    ENABLE_SIGNUP=True
commands =
    bash ./scripts/check_pnpm.sh

    bash -c "echo Running with HAGRID_FLAGS=$HAGRID_FLAGS; date"

    ; install hagrid
    bash -c 'if [[ "$HAGRID_FLAGS" == *"local"* ]]; then \
        pip install -e ../../hagrid; \
    else \
        pip install --force hagrid; \
    fi'

    ; fix windows encoding
    - chcp 65001

    ; check docker versions
    bash -c "docker --version"
    bash -c "docker compose version"

    ; reset volumes and create nodes
    bash -c "echo Starting Nodes; date"
    bash -c "docker rm -f $(docker ps -a -q) || true"
    bash -c "docker volume rm test_domain_1_mongo-data --force || true"
    bash -c "docker volume rm test_domain_1_credentials-data --force || true"

    bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test_domain_1 domain to docker:9081 $HAGRID_FLAGS --enable-signup --no-health-checks --verbose'

    bash -c '(docker logs test_domain_1-frontend-1 -f &) | grep -q -E "Network:\s+https?://[a-zA-Z0-9.-]+:[0-9]+/" || true'
    bash -c '(docker logs test_domain_1-backend-1 -f &) | grep -q "Application startup complete" || true'

    pnpm install
    pnpm dlx playwright@1.33 install --with-deps
    pnpm test:e2e

    ; shutdown
    bash -c "echo Killing Nodes; date"
    bash -c 'HAGRID_ART=false hagrid land all --force'


[testenv:stack.test.integration]
description = Integration Tests for Core Stack
deps =
    {[testenv:syft]deps}
    {[testenv:hagrid]deps}
changedir = {toxinidir}
allowlist_externals =
    docker
    grep
    sleep
    bash
    chcp
passenv=HOME, USER
setenv =
    PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html
    HAGRID_FLAGS = {env:HAGRID_FLAGS:--tag=local --test}
    EMULATION = {env:EMULATION:false}
    HAGRID_ART = false
    PYTHONIOENCODING = utf-8
    PYTEST_MODULES = {env:PYTEST_MODULES:frontend network e2e security redis}
commands =
    bash -c "echo Running with HAGRID_FLAGS=$HAGRID_FLAGS EMULATION=$EMULATION PYTEST_MODULES=$PYTEST_MODULES; date"

    ; install syft and hagrid
    bash -c 'if [[ "$HAGRID_FLAGS" == *"latest"* ]]; then \
        pip install --force pytest hagrid syft; \
    elif [[ "$HAGRID_FLAGS" == *"beta"* ]]; then \
        pip install --force pytest hagrid; \
        pip install --force -U --pre syft; \
    else \
        pip install -e packages/hagrid -e packages/syft[dev]; \
    fi'

    ; fix windows encoding
    - chcp 65001

    ; check docker versions
    bash -c "docker --version"
    bash -c "docker compose version"

    ; reset volumes and create nodes
    bash -c "echo Starting Nodes; date"
    bash -c "docker rm -f $(docker ps -a -q) || true"
    bash -c "docker volume rm test_domain_1_mongo-data --force || true"
    bash -c "docker volume rm test_domain_1_credentials-data --force || true"
    bash -c "docker volume rm test_domain_2_mongo-data --force || true"
    bash -c "docker volume rm test_domain_2_credentials-data --force || true"
    bash -c "docker volume rm test_gateway_1_mongo-data --force || true"
    bash -c "docker volume rm test_gateway_1_credentials-data --force || true"
    bash -c "docker volume rm test_domain_1_seaweedfs-data --force || true"
    bash -c "docker volume rm test_domain_2_seaweedfs-data --force || true"
    bash -c "docker volume rm test_domain_1_app-redis-data --force || true"
    bash -c "docker volume rm test_domain_2_app-redis-data --force || true"
    bash -c "docker volume rm test_gateway_1_app-redis-data --force || true"
    bash -c "docker volume rm test_domain_1_tailscale-data --force || true"
    bash -c "docker volume rm test_domain_2_tailscale-data --force || true"
    bash -c "docker volume rm test_gateway_1_tailscale-data --force || true"
    bash -c "docker volume rm test_gateway_1_headscale-data --force || true"

    bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test_gateway_1 network to docker:9081 $HAGRID_FLAGS --no-health-checks --verbose'
    bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test_domain_1 domain to docker:9082 $HAGRID_FLAGS --no-health-checks --enable-signup --verbose'
    bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test_domain_2 domain to docker:9083 --headless $HAGRID_FLAGS --no-vpn --enable-signup --no-health-checks --verbose'

    ; wait for nodes to start
    docker ps
    bash -c "echo Waiting for Nodes; date"
    bash -c '(docker logs test_domain_1-frontend-1 -f &) | grep -q -E "Network:\s+https?://[a-zA-Z0-9.-]+:[0-9]+/" || true'
    bash -c '(docker logs test_domain_1-backend-1 -f &) | grep -q "Application startup complete" || true'
    bash -c '(docker logs test_domain_2-backend-1 -f &) | grep -q "Application startup complete" || true'
    bash -c '(docker logs test_gateway_1-backend-1 -f &) | grep -q "Application startup complete" || true'

    ; frontend
    bash -c 'if [[ "$PYTEST_MODULES" == *"frontend"* ]]; then \
        echo "Starting frontend"; date; \
        pytest tests/integration -m frontend -p no:randomly --co; \
        pytest tests/integration -m frontend -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \
        return=$?; \
        docker stop test_domain_1-frontend-1 || true; \
        echo "Finished frontend"; date; \
        exit $return; \
    fi'

    ; ; network
    ; bash -c 'if [[ "$PYTEST_MODULES" == *"network"* ]]; then \
    ;     echo "Starting network"; date; \
    ;     pytest tests/integration -m network -p no:randomly --co; \
    ;     pytest tests/integration -m network -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \
    ;     return=$?; \
    ;     echo "Finished network"; date; \
    ;     exit $return; \
    ; fi'

    ; shutdown
    bash -c "echo Killing Nodes; date"
    bash -c 'HAGRID_ART=false hagrid land all --force'

[testenv:syft.docs]
description = Build Docs for Syft
changedir = {toxinidir}/docs
deps =
    {[testenv:syft]deps}
    {[testenv:hagrid]deps}
    -r {toxinidir}/docs/requirements.txt
allowlist_externals =
    make
    echo
    cd
    rm
    ls
    xargs
    bash
commands =
    python --version
    bash -c "cd source/api_reference && ls | grep -v index.rst | xargs rm"
    sphinx-apidoc -f -M -d 2 -o ./source/api_reference/ ../packages/syft/src/syft
    make html
    echo "Open: {toxinidir}/docs/build/html/index.html"

[testenv:syft.jupyter]
description = Jupyter Notebook with Editable Syft
setenv =
    PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html
deps =
    {[testenv:syft]deps}
    {[testenv:hagrid]deps}
    jupyter
    jupyterlab
commands =
    pip install -e packages/hagrid
    pip install jupyter jupyterlab --upgrade
    jupyter lab --ip 0.0.0.0 --ServerApp.token={posargs}

[testenv:syft.publish]
changedir = {toxinidir}/packages/syft
description = Build and Publish Syft Wheel
commands =
    python -m pip install --upgrade pip
    pip install --upgrade setuptools wheel twine tox build
    python -c 'from shutil import rmtree; rmtree("build", True); rmtree("dist", True)'
    python -m build .


[testenv:syft.test.security]
description = Security Checks for Syft
changedir = {toxinidir}/packages/syft
deps =
    {[testenv:syft]deps}
    {[testenv:hagrid]deps}
commands =
    pip install --upgrade pip
    bandit -r src
    safety check -i 42923 -i 54229 -i 54230

[testenv:syft.test.unit]
description = Syft Unit Tests
deps =
    {[testenv:syft]deps}
    {[testenv:hagrid]deps}
changedir = {toxinidir}/packages/syft
setenv =
    ENABLE_SIGNUP=False
commands =
    pip list
    pytest -n auto

[testenv:stack.test.integration.enclave.oblv]
description = Integration Tests for Oblv Enclave
changedir = {toxinidir}
deps =
    {[testenv:syft]deps}
    {[testenv:hagrid]deps}
allowlist_externals =
    grep
    bash
passenv=HOME, USER
setenv =
    LOCAL_ENCLAVE_PORT=8010
    ENABLE_OBLV=true
    DOMAIN_CONNECTION_PORT=8010
    ENABLE_SIGNUP=True
commands =
    pip install oblv-ctl==0.3.1
    python -c "import platform; import os; os.system('pip install jaxlib==0.4.10 -f https://whls.blob.core.windows.net/unstable/index.html') if platform.system().lower() == 'windows' else ''"
    # run at start to kill any process started beforehand
    bash -c 'chmod +x scripts/kill_process_in_port.sh && ./scripts/kill_process_in_port.sh $LOCAL_ENCLAVE_PORT'

    bash -c 'rm -rf ~/.syft/syft-enclave'
    bash -c 'git clone https://github.com/OpenMined/syft-enclave.git ~/.syft/syft-enclave || true'
    bash -c 'cd ~/.syft/syft-enclave && git fetch && git checkout dev && git pull && pip install -r requirements_test.txt || true'

    # Starting FastAPI server locally
    bash -c 'cd ~/.syft/syft-enclave/src && uvicorn app:app --host 0.0.0.0 --port $LOCAL_ENCLAVE_PORT > /dev/null 2>&1 &'

    bash -c 'cd tests/integration/external/oblv && pytest -p no:randomly -vvvv'
    bash -c 'chmod +x scripts/kill_process_in_port.sh && ./scripts/kill_process_in_port.sh $LOCAL_ENCLAVE_PORT'

[testenv:syft.test.notebook]
description = Syft Notebook Tests
deps =
    {[testenv:syft]deps}
    {[testenv:hagrid]deps}
    nbmake
changedir = {toxinidir}/notebooks
allowlist_externals =
    bash
setenv =
    ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:python}
    DEV_MODE = {env:DEV_MODE:True}
    TEST_NOTEBOOK_PATHS = {env:TEST_NOTEBOOK_PATHS:api/0.8,tutorials}
    ENABLE_SIGNUP=True
commands =
    bash -c "echo Running with ORCHESTRA_DEPLOYMENT_TYPE=$ORCHESTRA_DEPLOYMENT_TYPE DEV_MODE=$DEV_MODE TEST_NOTEBOOK_PATHS=$TEST_NOTEBOOK_PATHS; date"
    bash -c "for subfolder in $(echo ${TEST_NOTEBOOK_PATHS} | tr ',' ' '); do \
    if [[ $subfolder == *tutorials* ]]; then \
        pytest --nbmake "$subfolder" -p no:randomly -n $(python -c 'import multiprocessing; print(multiprocessing.cpu_count())') -vvvv; \
    else \
        pytest --nbmake "$subfolder" -p no:randomly -vvvv; \
    fi \
    done"
    ; pytest --nbmake api/0.8 -p no:randomly -vvvv
    ; pytest --nbmake api/0.9 -p no:randomly -vvvv
    ; pytest --nbmake tutorials -p no:randomly -vvvv
    ; pytest --nbmake tutorials/pandas-cookbook -p no:randomly -vvvv

[testenv:stack.test.notebook]
description = Stack Notebook Tests
deps =
    {[testenv:syft]deps}
    {[testenv:hagrid]deps}
    nbmake
changedir = {toxinidir}/notebooks
allowlist_externals =
    bash
setenv =
    ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:single_container}
    DEV_MODE = {env:DEV_MODE:True}
    TEST_NOTEBOOK_PATHS = {env:TEST_NOTEBOOK_PATHS:api/0.8,tutorials}
    ENABLE_SIGNUP=True
commands =
    bash -c "echo Running with ORCHESTRA_DEPLOYMENT_TYPE=$ORCHESTRA_DEPLOYMENT_TYPE DEV_MODE=$DEV_MODE TEST_NOTEBOOK_PATHS=$TEST_NOTEBOOK_PATHS; date"
    bash -c "for subfolder in $(echo ${TEST_NOTEBOOK_PATHS} | tr ',' ' ');\
    do \
        pytest --nbmake "$subfolder" -p no:randomly -vvvv --nbmake-timeout=1000;\
    done"

    ; pytest --nbmake api/0.8 -p no:randomly -vvvv
    ; pytest --nbmake api/0.9 -p no:randomly -vvvv
    ; pytest --nbmake tutorials -p no:randomly -vvvv
    ; pytest --nbmake tutorials/pandas-cookbook -p no:randomly -vvvv

[testenv:stack.test.vm]
description = Stack VM Tests
deps =
    {[testenv:syft]deps}
    {[testenv:hagrid]deps}
    nbmake
allowlist_externals =
    cd
    vagrant
    bash
changedir = {toxinidir}
setenv =
    ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:vm}
    VAGRANT_DESTROY = {env:VAGRANT_DESTROY:skip}
commands =
    bash -c 'if [[ "$(uname -m)" == *"arm"* ]]; then \
        export VAGRANT_BOX="ubuntu-22-04-arm64"; \
    elif [[ "$(uname -m)" == *"x86"* ]]; then \
        export VAGRANT_BOX="ubuntu-22-04-x86"; \
    else \
        echo "Unsupported architecture."; \
    fi; \
    echo $VAGRANT_BOX; \
    cd packages/grid; \
    if [[ "$VAGRANT_DESTROY" == *"true"* ]]; then \
        vagrant destroy $VAGRANT_BOX --force || true; \
    else \
        vagrant ssh $VAGRANT_BOX -c "docker ps -aq | xargs -I {:} docker rm {:} --force"; \
        vagrant ssh $VAGRANT_BOX -c "docker volume prune --filter all=1 --force || true"; \
    fi; \
    vagrant up $VAGRANT_BOX --provision; \
    '

    pytest --nbmake notebooks/api/0.8 -p no:randomly -vvvv
    ; pytest --nbmake notebooks/api/0.9 -p no:randomly -vvvv

    bash -c 'if [[ "$(uname -m)" == *"arm"* ]]; then \
        export VAGRANT_BOX="ubuntu-22-04-arm64"; \
    elif [[ "$(uname -m)" == *"x86"* ]]; then \
        export VAGRANT_BOX="ubuntu-22-04-x86"; \
    else \
        echo "Unsupported architecture."; \
    fi; \
    echo $VAGRANT_BOX; \
    cd packages/grid; \
    if [[ "$VAGRANT_DESTROY" == *"true"* ]]; then \
        vagrant destroy $VAGRANT_BOX --force || true; \
    fi; \
    '

[testenv:frontend.generate.types]
description = Generate Types for Frontend
deps =
    {[testenv:syft]deps}
    {[testenv:hagrid]deps}
allowlist_externals =
    cd
    bash
    pnpm
changedir = {toxinidir}/packages/grid/frontend
passenv =
    PNPM_HOME
commands =
    bash -c ./scripts/check_pnpm.sh
    pnpm add -g json-schema-to-typescript

    ; clear the old ones
    bash -c 'rm -rf ./schema'
    bash -c 'rm -rf ./src/types/generated'

    ; generate new ones
    bash -c 'python3 -c "import syft as sy;sy.util.schema.generate_json_schemas()"'
    bash -c "json2ts -i './schema/**/*.json' -o ./src/types/generated"
    bash -c "python3 ./scripts/replace_imports.py ./src/types/generated"

[mypy]
python_version = 3.11
disable_error_code = attr-defined, valid-type, no-untyped-call, arg-type


[testenv:stack.test.integration.k8s]
description = Integration Tests for Core Stack
deps =
    {[testenv:syft]deps}
    {[testenv:hagrid]deps}
    nbmake
changedir = {toxinidir}
passenv=HOME, USER
allowlist_externals =
    devspace
    kubectl
    grep
    sleep
    bash
    kubectx
    k3d
    echo
setenv =
    PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html
    ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:k8s}
    NODE_PORT = {env:NODE_PORT:9082}
commands =
    k3d version

    ; bash -c "docker rm $(docker ps -aq) --force || true"
    # bash -c "k3d cluster delete test-gateway-1 || true"
    bash -c "k3d cluster delete test-domain-1 || true"
    ; bash -c "k3d cluster delete test-domain-2 || true"
    bash -c "k3d registry delete k3d-registry.localhost || true"
    # bash -c "docker volume rm k3d-test-gateway-1-images --force || true"
    bash -c "docker volume rm k3d-test-domain-1-images --force || true"
    # bash -c "docker volume rm k3d-test-domain-2-images --force || true"

    bash -c 'k3d registry create registry.localhost --port 12345  -v `pwd`/k3d-registry:/var/lib/registry || true'

    # bash -c 'NODE_NAME=test-gateway-1 NODE_PORT=9081 && \
    #     k3d cluster create $NODE_NAME -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \
    #     k3d cluster start $NODE_NAME'

    # bash -c 'NODE_NAME=test-gateway-1 NODE_PORT=9081 && \
    #     cd packages/grid && \
    #     devspace --no-warn --kube-context "k3d-$NODE_NAME" --namespace $NODE_NAME \
    #     --var DOMAIN_NAME=$NODE_NAME \
    #     --var NETWORK_CHECK_INTERVAL=5 \
    #     --var TEST_MODE=1 \
    #     --var CONTAINER_REGISTRY=k3d-registry.localhost:12345/ \
    #     build -b'

    # bash -c 'NODE_NAME=test-gateway-1 NODE_PORT=9081 && \
    #     cd packages/grid && \
    #     (r=5;while ! \
    #     devspace --no-warn --kube-context "k3d-$NODE_NAME" --namespace $NODE_NAME \
    #     --var DOMAIN_NAME=$NODE_NAME \
    #     --var NETWORK_CHECK_INTERVAL=5 \
    #     --var ASSOCIATION_TIMEOUT=100 \
    #     --var TEST_MODE=1 \
    #     --var CONTAINER_REGISTRY=k3d-registry.localhost:12345/ \
    #     deploy -b -p gateway; \
    #     do ((--r))||exit;echo "retrying" && sleep 20;done)'

    bash -c 'NODE_NAME=test-domain-1 NODE_PORT=9082 && \
        k3d cluster create $NODE_NAME -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \
        k3d cluster start $NODE_NAME'

    bash -c 'NODE_NAME=test-domain-1 NODE_PORT=9082 && \
        cd packages/grid && \
        (r=5;while ! \
        devspace --no-warn --kube-context "k3d-$NODE_NAME" --namespace $NODE_NAME \
        --var DOMAIN_NAME=$NODE_NAME \
        --var DOMAIN_CHECK_INTERVAL=5 \
        --var ASSOCIATION_TIMEOUT=100 \
        --var TEST_MODE=1 \
        --var CONTAINER_REGISTRY=k3d-registry.localhost:12345/ \
        deploy -b; \
        do ((--r))||exit;echo "retrying" && sleep 20;done)'

    # bash -c 'NODE_NAME=test-domain-2 NODE_PORT=9083 && \
    #     k3d cluster create $NODE_NAME -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \
    #     k3d cluster start $NODE_NAME'

    # bash -c 'NODE_NAME=test-domain-2 NODE_PORT=9083 && \
    #     cd packages/grid && \
    #     (r=5;while ! \
    #     devspace --no-warn --kube-context "k3d-$NODE_NAME" --namespace $NODE_NAME \
    #     --var DOMAIN_NAME=$NODE_NAME \
    #     --var DOMAIN_CHECK_INTERVAL=5 \
    #     --var ASSOCIATION_TIMEOUT=100 \
    #     --var TEST_MODE=1 \
    #     --var CONTAINER_REGISTRY=k3d-registry.localhost:12345/ \
    #     deploy -b -p domain; \
    #     do ((--r))||exit;echo "retrying" && sleep 20;done)'

    sleep 30

    ; wait for front end
    bash packages/grid/scripts/wait_for.sh service frontend --context k3d-test-domain-1 --namespace test-domain-1
    bash -c '(kubectl logs service/frontend --context k3d-test-domain-1 --namespace test-domain-1 -f &) | grep -q -E "Network:\s+https?://[a-zA-Z0-9.-]+:[0-9]+/" || true'

    ; wait for everything else to be loaded
    # bash packages/grid/scripts/wait_for.sh service proxy --context k3d-test-gateway-1 --namespace test-gateway-1
    # bash packages/grid/scripts/wait_for.sh service queue --context k3d-test-gateway-1 --namespace test-gateway-1
    # bash packages/grid/scripts/wait_for.sh service redis --context k3d-test-gateway-1 --namespace test-gateway-1
    # bash packages/grid/scripts/wait_for.sh service mongo --context k3d-test-gateway-1 --namespace test-gateway-1
    # bash packages/grid/scripts/wait_for.sh service backend --context k3d-test-gateway-1 --namespace test-gateway-1
    # bash packages/grid/scripts/wait_for.sh service backend-stream --context k3d-test-gateway-1 --namespace test-gateway-1
    # bash packages/grid/scripts/wait_for.sh service headscale --context k3d-test-gateway-1 --namespace test-gateway-1

    # bash packages/grid/scripts/wait_for.sh service frontend --context k3d-test-domain-1 --namespace test-domain-1
    # bash packages/grid/scripts/wait_for.sh service proxy --context k3d-test-domain-1 --namespace test-domain-1
    # bash packages/grid/scripts/wait_for.sh service queue --context k3d-test-domain-1 --namespace test-domain-1
    # bash packages/grid/scripts/wait_for.sh service redis --context k3d-test-domain-1 --namespace test-domain-1
    bash packages/grid/scripts/wait_for.sh service mongo --context k3d-test-domain-1 --namespace test-domain-1
    bash packages/grid/scripts/wait_for.sh service backend --context k3d-test-domain-1 --namespace test-domain-1
    bash packages/grid/scripts/wait_for.sh service proxy --context k3d-test-domain-1 --namespace test-domain-1
    # bash packages/grid/scripts/wait_for.sh service backend-stream --context k3d-test-domain-1 --namespace test-domain-1
    # bash packages/grid/scripts/wait_for.sh service seaweedfs --context k3d-test-domain-1 --namespace test-domain-1

    # bash packages/grid/scripts/wait_for.sh service frontend --context k3d-test-domain-2 --namespace test-domain-2
    # bash packages/grid/scripts/wait_for.sh service proxy --context k3d-test-domain-2 --namespace test-domain-2
    # bash packages/grid/scripts/wait_for.sh service queue --context k3d-test-domain-2 --namespace test-domain-2
    # bash packages/grid/scripts/wait_for.sh service redis --context k3d-test-domain-2 --namespace test-domain-2
    # bash packages/grid/scripts/wait_for.sh service db --context k3d-test-domain-2 --namespace test-domain-2
    # bash packages/grid/scripts/wait_for.sh service backend --context k3d-test-domain-2 --namespace test-domain-2
    # bash packages/grid/scripts/wait_for.sh service backend-stream --context k3d-test-domain-2 --namespace test-domain-2
    # bash packages/grid/scripts/wait_for.sh service seaweedfs --context k3d-test-domain-2 --namespace test-domain-2

    # pytest tests/integration -m frontend -p no:randomly --co
    # bash -c "CONTAINER_HOST=$CONTAINER_HOST pytest tests/integration -m frontend -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no"

    bash -c '(kubectl logs service/backend --context k3d-test-domain-1 --namespace test-domain-1 -f &) | grep -q "Application startup complete" || true'

    ; frontend
    bash -c 'if [[ "$PYTEST_MODULES" == *"frontend"* ]]; then \
        echo "Starting frontend"; date; \
        pytest tests/integration -m frontend -p no:randomly -k "test_serves_domain_frontend" --co; \
        pytest tests/integration -m frontend -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no -k "test_serves_domain_frontend"; \
        return=$?; \
        echo "Finished frontend"; date; \
        exit $return; \
    fi'

    ; ignore 06 because of opendp on arm64
    pytest --nbmake notebooks/api/0.8 -p no:randomly -vvvv -k 'not 06'

    ; bash -c "k3d cluster delete test-gateway-1 || true"
    bash -c "k3d cluster delete test-domain-1 || true"
    ; bash -c "k3d cluster delete test-domain-2 || true"
    bash -c "k3d registry delete k3d-registry.localhost || true"
    bash -c "docker rm $(docker ps -aq) --force || true"
    # bash -c "docker volume rm k3d-test-gateway-1-images --force || true"
    bash -c "docker volume rm k3d-test-domain-1-images --force || true"
    # bash -c "docker volume rm k3d-test-domain-2-images --force || true"


[testenv:syft.build.helm]
description = Build Helm Chart for Kubernetes
deps =
    pyyaml
changedir = {toxinidir}
passenv=HOME, USER
allowlist_externals =
    devspace
    kubectl
    grep
    sleep
    bash
    kubectx
    k3d
    echo
    rm
    helm
setenv =
    PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html
commands =
    k3d version

    ; bash -c "docker rm $(docker ps -aq) --force || true"
    bash -c "k3d cluster delete build || true"
    bash -c "docker volume rm k3d-build-images --force || true"

    bash -c 'k3d cluster create build || true \
        k3d cluster start build'

    bash -c 'rm packages/grid/helm/syft/templates/* || true'

    bash -c 'cd packages/grid && \
        devspace deploy --skip-build --no-warn --kube-context "k3d-build" --render 2>/dev/null | python3 helm/helm.py'

    bash -c 'cd packages/grid/helm && \
        helm lint syft'

    bash -c "k3d cluster delete build || true"

[testenv:syft.package.helm]
description = Package Helm Chart for Kubernetes
deps =
changedir = {toxinidir}
passenv=HOME, USER
allowlist_externals =
    devspace
    kubectl
    grep
    sleep
    bash
    kubectx
    k3d
    echo
    rm
    helm
setenv =
    PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html
commands =
    bash -c 'cd packages/grid/helm && \
        helm lint syft'

    bash -c 'cd packages/grid/helm/syft && \
        helm dependency update'

    bash -c 'cd packages/grid/helm && \
        helm package syft --destination repo'

    bash -c 'cd packages/grid/helm/repo && \
        helm repo index . --url https://openmined.github.io/PySyft/helm'

[testenv:syft.test.helm]
description = Test Helm Chart for Kubernetes
deps =
    {[testenv:syft]deps}
    {[testenv:hagrid]deps}
    nbmake
changedir = {toxinidir}
passenv=HOME, USER
allowlist_externals =
    devspace
    kubectl
    grep
    sleep
    bash
    kubectx
    k3d
    echo
    rm
    helm
setenv =
    PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html
    ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:k8s}
    NODE_PORT = {env:NODE_PORT:8080}
commands =
    k3d version

    ; bash -c "docker rm $(docker ps -aq) --force || true"
    bash -c "k3d cluster delete syft || true"
    bash -c "docker volume rm k3d-syft-images --force || true"

    bash -c 'k3d registry create registry.localhost --port 12345  -v `pwd`/k3d-registry:/var/lib/registry || true'

    bash -c 'NODE_NAME=syft NODE_PORT=8080 && \
        k3d cluster create syft -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \
        k3d cluster start syft'

    ; skopeo list-tags --tls-verify=false docker://k3d-registry.localhost:12345/openmined/grid-backend
    ; skopeo inspect --tls-verify=false docker://k3d-registry.localhost:12345/openmined/grid-backend:f1725f
    ; helm uninstall --kube-context k3d-syft --namespace syft syft
    ; helm install --kube-context k3d-syft --namespace syft syft ./syft
    ; k3d cluster create syft -p "8080:80@loadbalancer" && k3d cluster start syft

    bash -c 'cd packages/grid && \
        kubectl --context k3d-syft create namespace syft; \
        helm install --kube-context k3d-syft --namespace syft syft ./helm/syft --dry-run --debug; \
        helm install --kube-context k3d-syft --namespace syft syft ./helm/syft'

    sleep 50

    bash packages/grid/scripts/wait_for.sh service frontend --context k3d-syft --namespace syft
    bash -c '(kubectl logs service/frontend --context k3d-syft --namespace syft -f &) | grep -q -E "Network:\s+https?://[a-zA-Z0-9.-]+:[0-9]+/" || true'

    ; wait for everything else to be loaded
    bash packages/grid/scripts/wait_for.sh service mongo --context k3d-syft --namespace syft
    bash packages/grid/scripts/wait_for.sh service backend --context k3d-syft --namespace syft
    bash packages/grid/scripts/wait_for.sh service proxy --context k3d-syft --namespace syft

    bash -c '(kubectl logs service/backend --context k3d-syft --namespace syft -f &) | grep -q "Application startup complete" || true'

    ; frontend
    bash -c 'if [[ "$PYTEST_MODULES" == *"frontend"* ]]; then \
        echo "Starting frontend"; date; \
        pytest tests/integration -m frontend -p no:randomly -k "test_serves_domain_frontend" --co; \
        pytest tests/integration -m frontend -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no -k "test_serves_domain_frontend"; \
        return=$?; \
        echo "Finished frontend"; date; \
        exit $return; \
    fi'

    ; ignore 06 because of opendp on arm64
    pytest --nbmake notebooks/api/0.8 -p no:randomly -vvvv -k 'not 06'

    bash -c "k3d cluster delete syft || true"
    bash -c "docker volume rm k3d-syft-images --force || true"