Skip to content
Tests

Optimize delete cleanup by lazily copying deny list #9188

Sign in to view logs

Optimize delete cleanup by lazily copying deny list

Optimize delete cleanup by lazily copying deny list #9188

Workflow file for this run

.github/workflows/pull_requests.yaml at fcca580
name: Tests
on:
push:
branches:
- master
- 'stable/v*'
tags:
- '**'
pull_request:
jobs:
Run-Swagger:
name: run-swagger
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: '1.21'
cache: true
- name: Run Swagger
run: ./tools/gen-code-from-swagger.sh
- name: Error on change
run: |
# check if anything is different
CHANGED=$(git status -s | wc -l)
if [ "$CHANGED" -gt 0 ]; then
echo "Please run ./tools/gen-code-from-swagger.sh script and commit changes:"
git status -s
exit 1
else
exit 0
fi
Vulnerability-Scanning:
name: vulnerability-scanning
runs-on: ubuntu-latest
if: ${{ !github.event.pull_request.head.repo.fork }} # no PRs from fork
steps:
- uses: actions/checkout@v4
- name: Scan for Vulnerabilities in Code
uses: Templum/govulncheck-action@v1.0.0
with:
go-version: '1.21'
package: ./...
fail-on-vuln: true
Unit-Tests:
name: unit-tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: '1.21'
cache: true
- name: Unit test
run: ./test/run.sh --unit-only
- name: Archive code coverage results
uses: actions/upload-artifact@v4
with:
name: coverage-report-unit
path: coverage-unit.txt
Integration-Tests:
name: integration-tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: '1.21'
cache: true
- name: Integration test
run: ./test/run.sh --integration-only
- name: Archive code coverage results
uses: actions/upload-artifact@v4
with:
name: coverage-report-integration
path: coverage-integration.txt
Modules-Acceptance-Tests:
name: modules-acceptance-tests
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
test: ["--only-module-backup-azure", "--only-module-backup-filesystem", "--only-module-backup-gcs", "--only-module-backup-s3", "--only-module-img2vec-neural", "--only-module-many-modules", "--only-module-multi2vec-clip", "--only-module-ref2vec-centroid", "--only-module-reranker-transformers", "--only-module-text2vec-contextionary", "--only-module-text2vec-transformers"]
steps:
- uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: '1.21'
cache: true
- name: Acceptance tests (modules)
run: ./test/run.sh ${{ matrix.test }}
Modules-Acceptance-Tests-large:
name: modules-acceptance-tests-large
runs-on: ubuntu-latest-4-cores
strategy:
fail-fast: false
matrix:
test: ["--only-module-qna-transformers", "--only-module-sum-transformers"]
steps:
- uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: '1.21'
cache: true
- name: Acceptance tests Large (modules)
run: ./test/run.sh ${{ matrix.test }}
Modules-Acceptance-Tests-gcp:
name: modules-acceptance-tests-gcp
runs-on: ubuntu-latest
if: ${{ !github.event.pull_request.head.repo.fork }} # no PRs from fork
strategy:
fail-fast: false
matrix:
test: ["--only-module-multi2vec-palm", "--only-module-generative-palm", "--only-module-text2vec-palm"]
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: '1.21'
cache: true
- name: check
env:
COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
run: |
commit_message=$(git log -1 --format=%B $COMMIT_SHA)
if [[ "$commit_message" == "[gcp]"* || "$commit_message" == "prepare release"* ]]; then
echo "Run pipeline"
echo "run_pipeline=true" >> $GITHUB_ENV
else
echo "Skip pipeline. In order to run the pipeline commit title must start with: [gcp]"
echo "run_pipeline=false" >> $GITHUB_ENV
fi
- name: configure
if : ${{ env.run_pipeline == 'true' || startsWith(github.ref, 'refs/tags') }}
env:
GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
GCP_PROJECT: ${{ secrets.GCP_PROJECT }}
run: ./tools/ci/gcloud.sh
- name: Acceptance tests (modules)
env:
GCP_PROJECT: ${{ secrets.GCP_PROJECT }}
if : ${{ env.run_pipeline == 'true' || startsWith(github.ref, 'refs/tags') }}
run: PALM_APIKEY=$(gcloud auth print-access-token) ./test/run.sh ${{ matrix.test }}
Acceptance-Tests:
name: acceptance-tests
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
test: ["--acceptance-only-fast", "--acceptance-only-graphql", "--acceptance-only-replication", "--acceptance-only-python"]
steps:
- uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: '1.21'
cache: true
- name: Acceptance tests
env:
WCS_DUMMY_CI_PW: ${{ secrets.WCS_DUMMY_CI_PW }}
WCS_DUMMY_CI_PW_2: ${{ secrets.WCS_DUMMY_CI_PW_2 }}
run: ./test/run.sh ${{ matrix.test }}
Codecov:
needs: [Unit-Tests, Integration-Tests]
name: codecov
runs-on: ubuntu-latest
if: ${{ (github.ref_type == 'branch') && (github.ref_name != 'master') }}
steps:
- uses: actions/checkout@v4
- name: Download coverage artifacts integration
uses: actions/download-artifact@v4
with:
name: coverage-report-unit
- name: Download coverage unit
uses: actions/download-artifact@v4
with:
name: coverage-report-integration
- name: Codecov
uses: codecov/codecov-action@v4
with:
fail_ci_if_error: false
files: ./coverage-integration.txt, ./coverage-unit.txt
verbose: true
Compile-and-upload-binaries:
name: compile-and-upload-binaries
runs-on: ubuntu-latest-4-cores
steps:
- uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: '1.21'
cache: true
- name: goreleaser
run: |
echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | sudo tee /etc/apt/sources.list.d/goreleaser.list
sudo apt update
sudo apt install goreleaser
GIT_HASH=$(git rev-parse --short HEAD) goreleaser build --clean --snapshot
- name: Upload macos
uses: actions/upload-artifact@v4
with:
name: binaries-macos-unsigned
path: dist/weaviate_darwin_all
- name: Upload windows
uses: actions/upload-artifact@v4
with:
name: binaries-windows-amd64
path: dist/weaviate_windows_amd64_v1
- name: Upload windows
uses: actions/upload-artifact@v4
with:
name: binaries-windows-arm64
path: dist/weaviate_windows_arm64
- name: Upload linux amd64
uses: actions/upload-artifact@v4
with:
name: binaries-linux-amd64
path: dist/weaviate_linux_amd64_v1
- name: Upload linux arm64
uses: actions/upload-artifact@v4
with:
name: binaries-linux-arm64
path: dist/weaviate_linux_arm64
Acceptance-Tests-windows:
name: acceptance-tests-windows
needs: Compile-and-upload-binaries
runs-on: windows-latest
env:
AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: true
PERSISTENCE_DATA_PATH: /tmp
QUERY_DEFAULTS_LIMIT: 20
steps:
- uses: actions/checkout@v4
- name: Download binaries
uses: actions/download-artifact@v4
with:
name: binaries-windows-amd64
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: '1.21'
cache: true
- name: start weaviate
shell: bash
# Weaviate is started without a Vectorizer as running text2vec-contextionary on GH actions is difficult:
# - docker on GHA only supports windows container - which we currently are not build
# - building those containers without a windows machine is difficult to figure out
run: ./weaviate.exe --scheme http --port 8080 &
- name: run acceptance tests
shell: bash
run: go test -count 1 -race test/acceptance/actions/*.go # tests that don't need a Vectorizer
Push-Docker:
needs: [Acceptance-Tests, Modules-Acceptance-Tests, Modules-Acceptance-Tests-large, Unit-Tests, Integration-Tests, Vulnerability-Scanning, Run-Swagger]
name: push-docker
runs-on: ubuntu-latest-8-cores
if: ${{ !github.event.pull_request.head.repo.fork }} # no PRs from fork
steps:
- uses: actions/checkout@v4
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{secrets.DOCKER_USERNAME}}
password: ${{secrets.DOCKER_PASSWORD}}
- name: Push container
id: push-container
run: ./ci/push_docker.sh
env:
PR_TITLE: "${{ github.event.pull_request.title }}"
- name: Generate Report
env:
PREVIEW_TAG: "${{ steps.push-container.outputs.PREVIEW_TAG }}"
run: ./ci/generate_docker_report.sh