test.yml

name: Test
on:
  push:
    branches: [ master ]
  pull_request:

jobs:
  format:
    name: Format
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-node@v3
        with:
          node-version: 14.x
      - name: Set up Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: 3.9
      - run: npm ci
        working-directory: ./frontend
      - run: npm run check-ci
        working-directory: ./frontend
        env:
          CI: true
      - uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: pip-${{ hashFiles('requirements.dev.txt') }}
          restore-keys: |
            pip-
      - run: ./pre-commit.sh && git diff --exit-code

  install:
    name: Install
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python-version: [3.9]
        os: [ubuntu-20.04, macos-latest]
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      # - uses: actions/cache@v3
      #   with:
      #     path: ~/.cache/pip
      #     key: pip-${{ hashFiles('requirements.txt') }}-${{ matrix.python-version }}
      #     restore-keys: |
      #       pip-
      - run: pip install --upgrade pip
      - run: pip install -e .
      - run: cl
  
  test_frontend:
    name: Test Frontend
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-node@v3
        with:
          node-version: 14.x
      - run: npm ci
        working-directory: ./frontend
      - run: npm test
        working-directory: ./frontend
        env:
          CI: true

  build:
    name: Build
    runs-on: ubuntu-latest
    strategy:
      matrix:
        service: [rest-server, worker, frontend]
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: 3.9
      - uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: pip-${{ hashFiles('requirements.txt') }}
          restore-keys: |
            pip-
      - run: pip install -r requirements.txt
      - run: python3 codalab_service.py build --pull --version ${VERSION} -s ${SERVICE} $([ -z "${CODALAB_DOCKER_USERNAME}" ] || echo "--push")
        env:
          CODALAB_DOCKER_USERNAME: ${{ secrets.CODALAB_DOCKER_USERNAME }}
          CODALAB_DOCKER_PASSWORD: ${{ secrets.CODALAB_DOCKER_PASSWORD }}
          # Gives us the branch name of the PR if on a pull_request-triggered build,
          # otherwise, "master" if on a push-triggered build
          VERSION: ${{ github.head_ref || 'master' }}
          SERVICE: ${{ matrix.service }}

  test_backend:
    name: Test backend
    runs-on: ubuntu-latest
    needs: [build]
    strategy:
      matrix:
        test:
          - unittest gen-rest-docs gen-cli-docs gen-readthedocs basic auth status batch anonymous competition unicode rest1 upload1 upload2 upload3 upload4 download disk
          - refs binary rm make worksheet_search worksheet_tags bundle_freeze_unfreeze worksheet_freeze_unfreeze detach perm search_time groups
          - worker_manager service
          - run time
          - run2
          - search link read kill write mimic workers edit_user sharing_workers
          - resources
          - memoize
          - copy
          - netcat netcurl
          - edit
          - open wopen
          - store_add parallel
        runtime: [docker, kubernetes]
        exclude:
          # netcat / netcurl not supported for kubernetes.
          - test: netcat netcurl
            runtime: kubernetes
    steps:
      - name: Clear free space
        run: |
            sudo rm -rf /opt/ghc
            df -h
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: 3.9
      - uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: pip-${{ hashFiles('requirements.txt') }}
          restore-keys: |
            pip-
      - run: pip install -r requirements.txt
      - name: Setup tests
        run: |
          sudo service mysql stop
          python3 codalab_service.py build services --version ${VERSION} --pull
        env:
          VERSION: ${{ github.head_ref || 'master' }}
      - name: Run tests using Docker runtime
        if: matrix.runtime == 'docker'
        run: |
          sh ./tests/test-setup.sh
          python3 codalab_service.py start --services default --version ${VERSION}
          python3 test_runner.py --version ${VERSION} ${TEST}
        env:
          TEST: ${{ matrix.test }}
          VERSION: ${{ github.head_ref || 'master' }}
          CODALAB_LINK_MOUNTS: /tmp
      - uses: actions/setup-go@v3
        if: matrix.runtime == 'kubernetes'
        with:
          go-version: '1.18.1'
      - name: Run tests using Kubernetes runtime
        if: matrix.runtime == 'kubernetes'
        run: |
          sh ./tests/test-setup.sh
          sh ./scripts/local-k8s/setup-ci.sh
          python3 test_runner.py --version ${VERSION} ${TEST}
        env:
          TEST: ${{ matrix.test }}
          VERSION: ${{ github.head_ref || 'master' }}
          CODALAB_LINK_MOUNTS: /tmp
      - name: Save logs
        if: (failure() || cancelled())
        run: |
          mkdir /tmp/logs
          for c in $(docker ps -a --format="{{.Names}}"); do docker logs $c > /tmp/logs/$c.log 2> /tmp/logs/$c.err.log; done
      - name: Save kubernetes logs
        if: (failure() || cancelled()) && matrix.runtime == 'kubernetes'
        run: |
          kubectl config use-context kind-codalab
          kubectl cluster-info dump --output-directory /tmp/logs
      - name: Upload logs
        if: (failure() || cancelled())
        uses: actions/upload-artifact@v1
        with:
          name: logs-test-${{ matrix.runtime }}-${{ matrix.test }}
          path: /tmp/logs

  test_backend_on_worker_restart:
    name: Test backend - on worker restart
    runs-on: ubuntu-latest
    needs: [build]
    strategy:
      matrix:
        test: [run]
    steps:
      - name: Clear free space
        run: |
          sudo rm -rf /opt/ghc
          df -h
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: 3.9
      - uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: pip-${{ hashFiles('requirements.txt') }}
          restore-keys: |
            pip-
      - run: pip install -r requirements.txt
      - name: Setup tests
        run: |
          sudo service mysql stop
          python3 codalab_service.py build services --version ${VERSION} --pull
        env:
          VERSION: ${{ github.head_ref || 'master' }}
      - name: Run tests
        # Make sure restarting worker doesn't cause any issues (ie in serialization/deserialization)
        run: |
          python3 codalab_service.py start --services default --version ${VERSION}
          docker restart codalab_worker_1
          python3 test_runner.py --version ${VERSION} ${TEST}
        env:
          TEST: ${{ matrix.test }}
          VERSION: ${{ github.head_ref || 'master' }}
      - name: Save logs
        run: |
          mkdir /tmp/logs
          for c in $(docker ps -a --format="{{.Names}}"); do docker logs $c > /tmp/logs/$c.log 2> /tmp/logs/$c.err.log; done
      - name: Upload logs
        if: (failure() || cancelled())
        uses: actions/upload-artifact@v1
        with:
          name: logs-test-${{ matrix.test }}
          path: /tmp/logs

  test_backend_sharedfs:
    name: Test backend - shared FS
    runs-on: ubuntu-latest
    needs: [build]
    strategy:
      matrix:
        test: [run,run2,link read write kill resources]
    steps:
      - name: Clear free space
        run: |
            sudo rm -rf /opt/ghc
            df -h
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: 3.9
      - uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: pip-${{ hashFiles('requirements.txt') }}
          restore-keys: |
            pip-
      - run: pip install -r requirements.txt
      - name: Setup tests
        run: |
          sudo service mysql stop
          python3 codalab_service.py build services --version ${VERSION} --pull
        env:
          VERSION: ${{ github.head_ref || 'master' }}
      - name: Run shared filesystem tests
        run: |
          sh ./tests/test-setup.sh
          python3 codalab_service.py start --services default --version ${VERSION} --shared-file-system
          python3 test_runner.py --version ${VERSION} ${TEST}
        env:
          TEST: ${{ matrix.test }}
          VERSION: ${{ github.head_ref || 'master' }}
          CODALAB_LINK_MOUNTS: /tmp
      - name: Save logs
        if: (failure() || cancelled())
        run: |
          mkdir /tmp/logs
          for c in $(docker ps -a --format="{{.Names}}"); do docker logs $c > /tmp/logs/$c.log 2> /tmp/logs/$c.err.log; done
      - name: Upload logs
        if: (failure() || cancelled())
        uses: actions/upload-artifact@v1
        with:
          name: logs-test-sharedfs-${{ matrix.test }}
          path: /tmp/logs

  test_backend_protected_mode:
    name: Test backend - protected mode
    runs-on: ubuntu-latest
    needs: [build]
    strategy:
      matrix:
        test:
          - basic status batch anonymous unicode rest1 upload1 download disk
          - refs binary rm make worksheet_search worksheet_tags bundle_freeze_unfreeze worksheet_freeze_unfreeze detach perm search_time groups
          - run
          - search read kill write mimic workers
          - copy netcat
          - protected_mode
    steps:
      - name: Clear free space
        run: |
          sudo rm -rf /opt/ghc
          df -h
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: 3.9
      - uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: pip-${{ hashFiles('requirements.txt') }}
          restore-keys: |
            pip-
      - run: pip install -r requirements.txt
      - name: Setup tests
        run: |
          sudo service mysql stop
          python3 codalab_service.py build services --version ${VERSION} --pull
        env:
          VERSION: ${{ github.head_ref || 'master' }}
      - name: Run tests
        run: |
          python3 codalab_service.py start --services default --version ${VERSION} --protected-mode
          python3 test_runner.py --version ${VERSION} ${TEST}
        env:
          TEST: ${{ matrix.test }}
          VERSION: ${{ github.head_ref || 'master' }}
      - name: Save logs
        if: (failure() || cancelled())
        run: |
          mkdir /tmp/logs
          for c in $(docker ps -a --format="{{.Names}}"); do docker logs $c > /tmp/logs/$c.log 2> /tmp/logs/$c.err.log; done
      - name: Upload logs
        if: (failure() || cancelled())
        uses: actions/upload-artifact@v1
        with:
          name: logs-test-protectedmode-${{ matrix.test }}
          path: /tmp/logs

  test_backend_default_bundle_store:
    name: Test backend - default bundle store
    runs-on: ubuntu-latest
    needs: [build]
    strategy:
      matrix:
        test:
          - default_bundle_store
    steps:
      - name: Clear free space
        run: |
          sudo rm -rf /opt/ghc
          df -h
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: 3.9
      - uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: pip-${{ hashFiles('requirements.txt') }}
          restore-keys: |
            pip-
      - run: pip install -r requirements.txt
      - name: Setup tests
        run: |
          sudo service mysql stop
          python3 codalab_service.py build services --version ${VERSION} --pull
        env:
          VERSION: ${{ github.head_ref || 'master' }}
      - name: Run tests
        run: |
          CODALAB_DEFAULT_BUNDLE_STORE_NAME=store$(date +%s) python3 codalab_service.py start --services default --version ${VERSION} --protected-mode
          python3 test_runner.py --version ${VERSION} ${TEST}
        env:
          TEST: ${{ matrix.test }}
          VERSION: ${{ github.head_ref || 'master' }}
      - name: Save logs
        if: (failure() || cancelled())
        run: |
          mkdir /tmp/logs
          for c in $(docker ps -a --format="{{.Names}}"); do docker logs $c > /tmp/logs/$c.log 2> /tmp/logs/$c.err.log; done
      - name: Upload logs
        if: (failure() || cancelled())
        uses: actions/upload-artifact@v1
        with:
          name: logs-test-${{ matrix.test }}
          path: /tmp/logs

  test_backend_default_bundle_store_azure:
    name: Test backend - use azure as default bundle store 
    runs-on: ubuntu-20.04
    needs: [build]
    strategy:
      matrix:
        test:
          - upload1 upload2 upload3 upload4 download
    steps:
      - name: Clear free space
        run: |
          sudo rm -rf /opt/ghc
          df -h
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: 3.9
      - uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: pip-${{ hashFiles('requirements.txt') }}
          restore-keys: |
            pip-
      - run: pip install -r requirements.txt
      - run: pip install -e .
      - name: Setup tests
        run: |
          sudo service mysql stop
          python3 codalab_service.py build services --version ${VERSION} --pull
        env:
          VERSION: ${{ github.head_ref || 'master' }}
      - name: Run tests
        run: |
          python3 codalab_service.py start --services default azurite --version ${VERSION}
          sh ./tests/test-setup-default-store.sh
          CODALAB_DEFAULT_BUNDLE_STORE_NAME=azure-store-default python3 codalab_service.py start --services default azurite --version ${VERSION}
          python3 test_runner.py --version ${VERSION} ${TEST}
        env:
          TEST: ${{ matrix.test }}
          VERSION: ${{ github.head_ref || 'master' }}
          CODALAB_USERNAME: codalab
          CODALAB_PASSWORD: codalab
      - name: Save logs
        if: (failure() || cancelled())
        run: |
          mkdir /tmp/logs
          for c in $(docker ps -a --format="{{.Names}}"); do docker logs $c > /tmp/logs/$c.log 2> /tmp/logs/$c.err.log; done
      - name: Upload logs
        if: (failure() || cancelled())
        uses: actions/upload-artifact@v1
        with:
          name: logs-test-${{ matrix.test }}
          path: /tmp/logs

  test_backend_preemptible_worker:
    name: Test backend - preemptible workers
    runs-on: ubuntu-latest
    needs: [build]
    strategy:
      matrix:
        test:
          - preemptible
    steps:
      - name: Clear free space
        run: |
          sudo rm -rf /opt/ghc
          df -h
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: 3.9
      - uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: pip-${{ hashFiles('requirements.txt') }}
          restore-keys: |
            pip-
      - run: pip install -r requirements.txt
      - run: pip install -e .
      - name: Setup tests
        run: |
          sudo service mysql stop
          python3 codalab_service.py build services --version ${VERSION} --pull
        env:
          VERSION: ${{ github.head_ref || 'master' }}
      - name: Run tests
        run: |
          python3 codalab_service.py start --services default no-worker worker-preemptible --version ${VERSION}
          sleep 20
          python3 codalab_service.py start --services worker-preemptible2 --version ${VERSION}
          ./tests/test-setup-preemptible.sh
          python3 test_runner.py --version ${VERSION} ${TEST}
        env:
          TEST: ${{ matrix.test }}
          VERSION: ${{ github.head_ref || 'master' }}
          CODALAB_USERNAME: codalab
          CODALAB_PASSWORD: codalab
      - name: Save logs
        if: (failure() || cancelled())
        run: |
          mkdir /tmp/logs
          for c in $(docker ps -a --format="{{.Names}}"); do docker logs $c > /tmp/logs/$c.log 2> /tmp/logs/$c.err.log; done
      - name: Upload logs
        if: (failure() || cancelled())
        uses: actions/upload-artifact@v1
        with:
          name: logs-test-${{ matrix.test }}
          path: /tmp/logs

  test_backend_azure_blob:
    name: Test backend with Azure Blob Storage
    runs-on: ubuntu-latest
    needs: [build]
    strategy:
      matrix:
        test:
          - unittest gen-rest-docs gen-cli-docs gen-readthedocs basic auth status batch anonymous competition unicode rest1 upload1 upload2 upload3 upload4 download disk
          - refs binary rm make worksheet_search worksheet_tags bundle_freeze_unfreeze worksheet_freeze_unfreeze detach perm search_time groups
          - worker_manager service
          - run time
          - run2
          - search read kill write mimic workers edit_user sharing_workers
          # - search link read kill write mimic workers edit_user sharing_workers
          - resources
          - memoize
          - copy netcat netcurl
          - edit blob
          - open wopen
    steps:
      - name: Clear free space
        run: |
            sudo rm -rf /opt/ghc
            df -h
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: 3.9
      - uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: pip-${{ hashFiles('requirements.txt') }}
          restore-keys: |
            pip-
      - run: pip install -r requirements.txt
      - name: Setup tests
        run: |
          sudo service mysql stop
          python3 codalab_service.py build services --version ${VERSION} --pull
        env:
          VERSION: ${{ github.head_ref || 'master' }}
      - name: Run tests
        run: |
          python3 codalab_service.py start --services default azurite --version ${VERSION}
          python3 test_runner.py --version ${VERSION} ${TEST}
        env:
          TEST: ${{ matrix.test }}
          VERSION: ${{ github.head_ref || 'master' }}
          CODALAB_LINK_MOUNTS: /tmp
          CODALAB_ALWAYS_USE_AZURE_BLOB_BETA: 1
      - name: Save logs
        if: (failure() || cancelled())
        run: |
          mkdir /tmp/logs
          for c in $(docker ps -a --format="{{.Names}}"); do docker logs $c > /tmp/logs/$c.log 2> /tmp/logs/$c.err.log; done
      - name: Upload logs
        if: (failure() || cancelled())
        uses: actions/upload-artifact@v1
        with:
          name: logs-test-azblob-${{ matrix.test }}
          path: /tmp/logs

  test_ui:
    name: End-to-end UI Tests
    runs-on: ubuntu-latest
    needs: [build]
    strategy:
      matrix:
        test: [frontend]
    steps:
      - name: Clear free space
        run: |
            sudo rm -rf /opt/ghc
            df -h
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: 3.9
      - uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: pip-${{ hashFiles('requirements.txt') }}
          restore-keys: |
            pip-
      - run: pip install -r requirements.txt
      - name: Setup tests
        run: |
          sudo service mysql stop
          python3 codalab_service.py build services --version ${VERSION} --pull
        env:
          VERSION: ${{ github.head_ref || 'master' }}
      - name: Run tests
        run: |
          python3 codalab_service.py start --services default --version ${VERSION}
          docker exec codalab_rest-server_1 /bin/bash -c "python3 scripts/create_sample_worksheet.py --test-print"
          python3 test_runner.py --version ${VERSION} ${TEST}
        env:
          TEST: ${{ matrix.test }}
          VERSION: ${{ github.head_ref || 'master' }}
      - name: Upload screenshots on failure
        uses: actions/upload-artifact@v1
        if: failure()
        with:
          name: screenshots-test-${{ matrix.test }}
          path: tests/ui
      - name: Save logs
        if: (failure() || cancelled())
        run: |
          mkdir /tmp/logs
          for c in $(docker ps -a --format="{{.Names}}"); do docker logs $c > /tmp/logs/$c.log 2> /tmp/logs/$c.err.log; done
      - name: Upload logs
        if: (failure() || cancelled())
        uses: actions/upload-artifact@v1
        with:
          name: logs-test-${{ matrix.test }}
          path: /tmp/logs

  ci:
    name: All CI tasks complete
    runs-on: ubuntu-latest
    needs: [format, install, test_frontend, build, test_backend, test_backend_on_worker_restart, test_backend_sharedfs, test_backend_protected_mode, test_ui]
    steps:
      - uses: actions/checkout@v3
      - run: echo Done