Don't bother hitting the DB if we got everything from cache #3827

Workflow file for this run

.github/workflows/dendrite.yml at 7b7a176

	name: Dendrite

	on:
	push:
	branches:
	- main
	paths:
	- '**.go' # only execute on changes to go files
	- 'go.sum' # or dependency updates
	- '.github/workflows/**' # or workflow changes
	pull_request:
	paths:
	- '**.go'
	- 'go.sum' # or dependency updates
	- '.github/workflows/**'
	release:
	types: [published]
	workflow_dispatch:

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	jobs:
	wasm:
	name: WASM build test
	timeout-minutes: 5
	runs-on: ubuntu-latest
	if: ${{ false }} # disable for now
	steps:
	- uses: actions/checkout@v4

	- name: Install Go
	uses: actions/setup-go@v4
	with:
	go-version: "stable"
	cache: true

	- name: Install Node
	uses: actions/setup-node@v2
	with:
	node-version: 14

	- uses: actions/cache@v4
	with:
	path: ~/.npm
	key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
	restore-keys: \|
	${{ runner.os }}-node-

	- name: Reconfigure Git to use HTTPS auth for repo packages
	run: >
	git config --global url."https://github.com/".insteadOf
	ssh://git@github.com/

	- name: Install test dependencies
	working-directory: ./test/wasm
	run: npm ci

	- name: Test
	run: ./test-dendritejs.sh

	# Run golangci-lint
	lint:
	timeout-minutes: 5
	name: Linting
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- name: Install libolm
	run: sudo apt-get install libolm-dev libolm3
	- name: Install Go
	uses: actions/setup-go@v4
	with:
	go-version: "stable"
	- name: golangci-lint
	uses: golangci/golangci-lint-action@v3

	# run go test with different go versions
	test:
	timeout-minutes: 10
	name: Unit tests
	runs-on: ubuntu-latest
	# Service containers to run with `container-job`
	services:
	# Label used to access the service container
	postgres:
	# Docker Hub image
	image: postgres:13-alpine
	# Provide the password for postgres
	env:
	POSTGRES_USER: postgres
	POSTGRES_PASSWORD: postgres
	POSTGRES_DB: dendrite
	ports:
	# Maps tcp port 5432 on service container to the host
	- 5432:5432
	# Set health checks to wait until postgres has started
	options: >-
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	steps:
	- uses: actions/checkout@v4
	- name: Install libolm
	run: sudo apt-get install libolm-dev libolm3
	- name: Setup go
	uses: actions/setup-go@v4
	with:
	go-version: "stable"
	- uses: actions/cache@v4
	# manually set up caches, as they otherwise clash with different steps using setup-go with cache=true
	with:
	path: \|
	~/.cache/go-build
	~/go/pkg/mod
	key: ${{ runner.os }}-go-stable-unit-${{ hashFiles('**/go.sum') }}
	restore-keys: \|
	${{ runner.os }}-go-stable-unit-
	- name: Set up gotestfmt
	uses: gotesttools/gotestfmt-action@v2
	with:
	# Optional: pass GITHUB_TOKEN to avoid rate limiting.
	token: ${{ secrets.GITHUB_TOKEN }}
	- run: go test -json -v ./... 2>&1 \| gotestfmt -hide all
	env:
	POSTGRES_HOST: localhost
	POSTGRES_USER: postgres
	POSTGRES_PASSWORD: postgres
	POSTGRES_DB: dendrite

	# build Dendrite for linux with different architectures and go versions
	build:
	name: Build for Linux
	timeout-minutes: 10
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	goos: ["linux"]
	goarch: ["amd64", "386"]
	steps:
	- uses: actions/checkout@v4
	- name: Setup go
	uses: actions/setup-go@v4
	with:
	go-version: "stable"
	- uses: actions/cache@v4
	with:
	path: \|
	~/.cache/go-build
	~/go/pkg/mod
	key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
	restore-keys: \|
	key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-
	- name: Install dependencies x86
	if: ${{ matrix.goarch == '386' }}
	run: sudo apt update && sudo apt-get install -y gcc-multilib
	- env:
	GOOS: ${{ matrix.goos }}
	GOARCH: ${{ matrix.goarch }}
	CGO_ENABLED: 1
	CGO_CFLAGS: -fno-stack-protector
	run: go build -trimpath -v -o "bin/" ./cmd/...

	# build for Windows 64-bit
	build_windows:
	name: Build for Windows
	timeout-minutes: 10
	runs-on: ubuntu-latest
	strategy:
	matrix:
	goos: ["windows"]
	goarch: ["amd64"]
	steps:
	- uses: actions/checkout@v4
	- name: Setup Go
	uses: actions/setup-go@v4
	with:
	go-version: "stable"
	- uses: actions/cache@v4
	with:
	path: \|
	~/.cache/go-build
	~/go/pkg/mod
	key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
	restore-keys: \|
	key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-
	- name: Install dependencies
	run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc
	- env:
	GOOS: ${{ matrix.goos }}
	GOARCH: ${{ matrix.goarch }}
	CGO_ENABLED: 1
	CC: "/usr/bin/x86_64-w64-mingw32-gcc"
	run: go build -trimpath -v -o "bin/" ./cmd/...

	# Dummy step to gate other tests on without repeating the whole list
	initial-tests-done:
	name: Initial tests passed
	needs: [lint, test, build, build_windows]
	runs-on: ubuntu-latest
	if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
	steps:
	- name: Check initial tests passed
	uses: re-actors/alls-green@release/v1
	with:
	jobs: ${{ toJSON(needs) }}

	# run go test with different go versions
	integration:
	timeout-minutes: 20
	needs: initial-tests-done
	name: Integration tests
	runs-on: ubuntu-latest
	# Service containers to run with `container-job`
	services:
	# Label used to access the service container
	postgres:
	# Docker Hub image
	image: postgres:13-alpine
	# Provide the password for postgres
	env:
	POSTGRES_USER: postgres
	POSTGRES_PASSWORD: postgres
	POSTGRES_DB: dendrite
	ports:
	# Maps tcp port 5432 on service container to the host
	- 5432:5432
	# Set health checks to wait until postgres has started
	options: >-
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	steps:
	- uses: actions/checkout@v4
	- name: Install libolm
	run: sudo apt-get install libolm-dev libolm3
	- name: Setup go
	uses: actions/setup-go@v4
	with:
	go-version: "stable"
	- name: Set up gotestfmt
	uses: gotesttools/gotestfmt-action@v2
	with:
	# Optional: pass GITHUB_TOKEN to avoid rate limiting.
	token: ${{ secrets.GITHUB_TOKEN }}
	- uses: actions/cache@v4
	with:
	path: \|
	~/.cache/go-build
	~/go/pkg/mod
	key: ${{ runner.os }}-go-stable-test-race-${{ hashFiles('**/go.sum') }}
	restore-keys: \|
	${{ runner.os }}-go-stable-test-race-
	- run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... \| grep -v /cmd/dendrite*) 2>&1 \| gotestfmt -hide all
	env:
	POSTGRES_HOST: localhost
	POSTGRES_USER: postgres
	POSTGRES_PASSWORD: postgres
	POSTGRES_DB: dendrite
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v4
	with:
	flags: unittests
	fail_ci_if_error: true
	token: ${{ secrets.CODECOV_TOKEN }}

	# run database upgrade tests
	upgrade_test:
	name: Upgrade tests
	timeout-minutes: 20
	needs: initial-tests-done
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- name: Setup go
	uses: actions/setup-go@v4
	with:
	go-version: "stable"
	cache: true
	- uses: actions/cache@v4
	with:
	path: \|
	~/.cache/go-build
	~/go/pkg/mod
	key: ${{ runner.os }}-go-upgrade-test-${{ hashFiles('**/go.sum') }}
	restore-keys: \|
	${{ runner.os }}-go-upgrade-test-
	- name: Docker version
	run: docker version
	- name: Build upgrade-tests
	run: go build ./cmd/dendrite-upgrade-tests
	- name: Test upgrade (PostgreSQL)
	run: ./dendrite-upgrade-tests --head .
	- name: Test upgrade (SQLite)
	run: ./dendrite-upgrade-tests --sqlite --head .

	# run database upgrade tests, skipping over one version
	upgrade_test_direct:
	name: Upgrade tests from HEAD-2
	timeout-minutes: 20
	needs: initial-tests-done
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- name: Setup go
	uses: actions/setup-go@v4
	with:
	go-version: "stable"
	cache: true
	- uses: actions/cache@v4
	with:
	path: \|
	~/.cache/go-build
	~/go/pkg/mod
	key: ${{ runner.os }}-go-upgrade-direct-test-${{ hashFiles('**/go.sum') }}
	restore-keys: \|
	${{ runner.os }}-go-upgrade-direct-test-
	- name: Docker version
	run: docker version
	- name: Build upgrade-tests
	run: go build ./cmd/dendrite-upgrade-tests
	- name: Test upgrade (PostgreSQL)
	run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .
	- name: Test upgrade (SQLite)
	run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .

	# run Sytest in different variations
	sytest:
	timeout-minutes: 20
	needs: initial-tests-done
	name: "Sytest (${{ matrix.label }})"
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	include:
	- label: SQLite native

	- label: SQLite Cgo
	cgo: 1

	- label: PostgreSQL
	postgres: postgres

	container:
	image: matrixdotorg/sytest-dendrite
	volumes:
	- ${{ github.workspace }}:/src
	- /root/.cache/go-build:/github/home/.cache/go-build
	- /root/.cache/go-mod:/gopath/pkg/mod
	env:
	POSTGRES: ${{ matrix.postgres && 1}}
	SYTEST_BRANCH: ${{ github.head_ref }}
	CGO_ENABLED: ${{ matrix.cgo && 1 }}
	steps:
	- uses: actions/checkout@v4
	- uses: actions/cache@v4
	with:
	path: \|
	~/.cache/go-build
	/gopath/pkg/mod
	key: ${{ runner.os }}-go-sytest-${{ hashFiles('**/go.sum') }}
	restore-keys: \|
	${{ runner.os }}-go-sytest-
	- name: Run Sytest
	run: /bootstrap.sh dendrite
	working-directory: /src
	- name: Summarise results.tap
	if: ${{ always() }}
	run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
	- name: Sytest List Maintenance
	if: ${{ always() }}
	run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist
	continue-on-error: true # not fatal
	- name: Are We Synapse Yet?
	if: ${{ always() }}
	run: /src/are-we-synapse-yet.py /logs/results.tap -v
	continue-on-error: true # not fatal
	- name: Upload Sytest logs
	uses: actions/upload-artifact@v4
	if: ${{ always() }}
	with:
	name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
	path: \|
	/logs/results.tap
	/logs/*/.log*

	# run Complement
	complement:
	name: "Complement (${{ matrix.label }})"
	timeout-minutes: 20
	needs: initial-tests-done
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	include:
	- label: SQLite native
	cgo: 0

	- label: SQLite Cgo
	cgo: 1

	- label: PostgreSQL
	postgres: Postgres
	cgo: 0
	steps:
	# Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement.
	# See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
	- name: "Set Go Version"
	run: \|
	echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
	echo "~/go/bin" >> $GITHUB_PATH
	- name: "Install Complement Dependencies"
	# We don't need to install Go because it is included on the Ubuntu 20.04 image:
	# See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
	run: \|
	sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
	go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
	- name: Run actions/checkout@v4 for dendrite
	uses: actions/checkout@v4
	with:
	path: dendrite

	# Attempt to check out the same branch of Complement as the PR. If it
	# doesn't exist, fallback to main.
	- name: Checkout complement
	shell: bash
	run: \|
	mkdir -p complement
	# Attempt to use the version of complement which best matches the current
	# build. Depending on whether this is a PR or release, etc. we need to
	# use different fallbacks.
	#
	# 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
	# for pull requests, otherwise GITHUB_REF).
	# 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
	# (GITHUB_BASE_REF for pull requests).
	# 3. Use the default complement branch ("master").
	for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
	# Skip empty branch names and merge commits.
	if [[ -z "$BRANCH_NAME" \|\| $BRANCH_NAME =~ ^refs/pull/.* ]]; then
	continue
	fi
	(wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" \| tar -xz --strip-components=1 -C complement) && break
	done
	# Build initial Dendrite image
	- run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
	working-directory: dendrite
	env:
	DOCKER_BUILDKIT: 1

	# Run Complement
	- run: \|
	set -o pipefail &&
	go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 \| gotestfmt -hide all
	shell: bash
	name: Run Complement Tests
	env:
	COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }}
	COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_
	working-directory: complement

	integration-tests-done:
	name: Integration tests passed
	needs:
	[
	initial-tests-done,
	upgrade_test,
	upgrade_test_direct,
	sytest,
	complement,
	integration
	]
	runs-on: ubuntu-latest
	if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
	steps:
	- name: Check integration tests passed
	uses: re-actors/alls-green@release/v1
	with:
	jobs: ${{ toJSON(needs) }}

	update-docker-images:
	name: Update Docker images
	permissions:
	packages: write
	contents: read
	security-events: write # To upload Trivy sarif files
	if: github.repository == 'matrix-org/dendrite' && github.ref_name == 'main'
	needs: [integration-tests-done]
	uses: matrix-org/dendrite/.github/workflows/docker.yml@main
	secrets:
	DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Don't bother hitting the DB if we got everything from cache #3827

Workflow file

Don't bother hitting the DB if we got everything from cache #3827

Jobs

Run details

Workflow file for this run