Spark runtime #462
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test CI | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - "dev" | |
| pull_request: | |
| branches: | |
| - "dev" | |
| jobs: | |
| test-ray-lib: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Test data-processing-lib/ray | |
| run: | | |
| make -C data-processing-lib/ray DOCKER=docker venv test | |
| test-spark-lib: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Test data-processing-lib/spark | |
| run: | | |
| make -C data-processing-lib/spark DOCKER=docker venv test | |
| test-code: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Test Code Transforms | |
| run: | | |
| make -C transforms/code DOCKER=docker venv test-src | |
| test-universal: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Test Universal Transforms | |
| run: | | |
| make -C transforms/universal DOCKER=docker venv test-src | |
| test-tools: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Test tools | |
| run: | | |
| make -C tools DOCKER=docker venv test | |
| test-kfp-lib: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Test KFP lib | |
| run: | | |
| source kind/requirements.env | |
| export PATH=$PATH:/tmp/ | |
| curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 | |
| curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | |
| chmod 700 /tmp/get_helm.sh | |
| HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo | |
| chmod 777 /tmp/helm | |
| chmod 777 /tmp/kind | |
| curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl | |
| chmod 777 /tmp/kubectl | |
| curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc | |
| chmod +x /tmp/mc | |
| export DEPLOY_KUBEFLOW=0 | |
| make -C kind setup | |
| make -C kfp/kfp_support_lib build test | |
| test-kfp-workflow-run: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Free up space in github runner | |
| # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 | |
| run: | | |
| df -h | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup | |
| sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true | |
| df -h | |
| - name: Test KFP worflow run | |
| timeout-minutes: 120 | |
| run: | | |
| source kind/requirements.env | |
| export PATH=$PATH:/tmp/ | |
| curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 | |
| curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | |
| chmod 700 /tmp/get_helm.sh | |
| HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo | |
| chmod 777 /tmp/helm | |
| chmod 777 /tmp/kind | |
| curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl | |
| chmod 777 /tmp/kubectl | |
| curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc | |
| chmod +x /tmp/mc | |
| make -C kind setup | |
| make -C kfp/transform_workflows venv build | |
| make -C kfp/transform_workflows/universal/noop test |