e2e.yaml

# Copyright 2020-2023 Alibaba Group Holding Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

setup:
  env: kind
  kubeconfig: /tmp/e2e-k8s.config
  steps:
    - name: download the dataset 
      command: |
        worker=($(docker ps | grep kind-worker | awk -F ' ' '{print $1}')); for c in ${worker[@]}; do docker exec $c sh -c "mkdir -p /datasets; \
        cd /datasets/; curl -sOL https://raw.githubusercontent.com/GraphScope/gstest/master/vineyard-mars-showcase-dataset/{item,txn,user}.csv"; done
    - name: build examples images
      command: |
        make -C k8s/examples/vineyard-mars-pytorch REGISTRY=localhost:5001
    - name: push examples images to local registry
      command: |
        make -C k8s/examples/vineyard-mars-pytorch docker-push REGISTRY=localhost:5001
    - name: install vineyard cluster(with operator)
      command: |
        go run k8s/cmd/main.go deploy vineyard-cluster
    - name: run prepare-data job
      command: |
        kubectl create namespace vineyard-job

        yq k8s/examples/vineyard-mars-pytorch/prepare-data/resources/* | \
        sed 's/vineyardcloudnative/localhost:5001/' | \
        kubectl apply -f -

        kubectl wait job -n vineyard-job -l app=prepare-data --for condition=complete --timeout=1200s
    - name: run process-data job
      command: |
        yq k8s/examples/vineyard-mars-pytorch/process-data/resources/* | \
        sed 's/vineyardcloudnative/localhost:5001/' | \
        kubectl apply -f -

        kubectl wait job -n vineyard-job -l app=process-data --for condition=complete --timeout=2400s
    - name: run train-data job
      command: |
        yq k8s/examples/vineyard-mars-pytorch/train-data/resources/* | \
        sed 's/vineyardcloudnative/localhost:5001/' | \
        kubectl apply -f -

        kubectl wait pods -n vineyard-job -l app=train-data --for condition=Ready --timeout=1200s
  timeout: 60m

cleanup:
  # always never success failure
  on: success

verify:
  # verify with retry strategy
  retry:
    # max retry count
    count: 20
    # the interval between two attempts, e.g. 10s, 1m.
    interval: 20s
  cases:
    - query: |
        # find the master training pods' log
        for pod in $(kubectl get po -l app=train-data -oname -n vineyard-job | awk -F '/' '{print $2}')
        do
        logs=$(kubectl logs $pod -n vineyard-job | tail -n 1)
        if [[ "$logs" == "test passed" ]]
        then
          echo $logs | \
          yq e '{"result": .}' - | \
          yq e 'to_entries' -
          break
        fi
        done
      expected: ../verify/serialize.yaml