This repository has been archived by the owner on Apr 26, 2024. It is now read-only.
Delete pushers after calling on_logged_out module hook on device delete #19449
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Tests | |
on: | |
push: | |
branches: ["develop", "release-*"] | |
pull_request: | |
merge_group: | |
workflow_dispatch: | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.ref }} | |
cancel-in-progress: true | |
jobs: | |
# Job to detect what has changed so we don't run e.g. Rust checks on PRs that | |
# don't modify Rust code. | |
changes: | |
runs-on: ubuntu-latest | |
outputs: | |
rust: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.rust }} | |
steps: | |
- uses: dorny/paths-filter@v2 | |
id: filter | |
# We only check on PRs | |
if: startsWith(github.ref, 'refs/pull/') | |
with: | |
filters: | | |
rust: | |
- 'rust/**' | |
- 'Cargo.toml' | |
- 'Cargo.lock' | |
check-sampleconfig: | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Install Rust | |
uses: dtolnay/rust-toolchain@1.58.1 | |
- uses: Swatinem/rust-cache@v2 | |
- uses: matrix-org/setup-python-poetry@v1 | |
with: | |
python-version: "3.x" | |
poetry-version: "1.3.2" | |
extras: "all" | |
- run: poetry run scripts-dev/generate_sample_config.sh --check | |
- run: poetry run scripts-dev/config-lint.sh | |
check-schema-delta: | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: "3.x" | |
- run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'" | |
- run: scripts-dev/check_schema_delta.py --force-colors | |
check-lockfile: | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: "3.x" | |
- run: .ci/scripts/check_lockfile.py | |
lint: | |
runs-on: ubuntu-latest | |
steps: | |
- name: Checkout repository | |
uses: actions/checkout@v3 | |
- name: Setup Poetry | |
uses: matrix-org/setup-python-poetry@v1 | |
with: | |
install-project: "false" | |
- name: Import order (isort) | |
run: poetry run isort --check --diff . | |
- name: Code style (black) | |
run: poetry run black --check --diff . | |
- name: Semantic checks (ruff) | |
# --quiet suppresses the update check. | |
run: poetry run ruff --quiet . | |
lint-mypy: | |
runs-on: ubuntu-latest | |
name: Typechecking | |
steps: | |
- name: Checkout repository | |
uses: actions/checkout@v3 | |
- name: Setup Poetry | |
uses: matrix-org/setup-python-poetry@v1 | |
with: | |
# We want to make use of type hints in optional dependencies too. | |
extras: all | |
# We have seen odd mypy failures that were resolved when we started | |
# installing the project again: | |
# https://github.com/matrix-org/synapse/pull/15376#issuecomment-1498983775 | |
# To make CI green, err towards caution and install the project. | |
install-project: "true" | |
- name: Install Rust | |
uses: dtolnay/rust-toolchain@1.58.1 | |
- uses: Swatinem/rust-cache@v2 | |
# NB: I have two concerns with this action: | |
# 1. We occasionally see odd mypy problems that aren't reproducible | |
# locally with clean caches. I suspect some dodgy caching behaviour. | |
# 2. The action uses GHA machinery that's deprecated | |
# (https://github.com/AustinScola/mypy-cache-github-action/issues/277) | |
# It may be simpler to use actions/cache ourselves to restore .mypy_cache. | |
- name: Restore/persist mypy's cache | |
uses: AustinScola/mypy-cache-github-action@df56268388422ee282636ee2c7a9cc55ec644a41 | |
- name: Run mypy | |
run: poetry run mypy | |
lint-crlf: | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Check line endings | |
run: scripts-dev/check_line_terminators.sh | |
lint-newsfile: | |
if: ${{ (github.base_ref == 'develop' || contains(github.base_ref, 'release-')) && github.actor != 'dependabot[bot]' }} | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
with: | |
ref: ${{ github.event.pull_request.head.sha }} | |
fetch-depth: 0 | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: "3.x" | |
- run: "pip install 'towncrier>=18.6.0rc1'" | |
- run: scripts-dev/check-newsfragment.sh | |
env: | |
PULL_REQUEST_NUMBER: ${{ github.event.number }} | |
lint-pydantic: | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
with: | |
ref: ${{ github.event.pull_request.head.sha }} | |
- name: Install Rust | |
uses: dtolnay/rust-toolchain@1.58.1 | |
- uses: Swatinem/rust-cache@v2 | |
- uses: matrix-org/setup-python-poetry@v1 | |
with: | |
poetry-version: "1.3.2" | |
extras: "all" | |
- run: poetry run scripts-dev/check_pydantic_models.py | |
lint-clippy: | |
runs-on: ubuntu-latest | |
needs: changes | |
if: ${{ needs.changes.outputs.rust == 'true' }} | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Install Rust | |
uses: dtolnay/rust-toolchain@1.58.1 | |
with: | |
components: clippy | |
- uses: Swatinem/rust-cache@v2 | |
- run: cargo clippy -- -D warnings | |
# We also lint against a nightly rustc so that we can lint the benchmark | |
# suite, which requires a nightly compiler. | |
lint-clippy-nightly: | |
runs-on: ubuntu-latest | |
needs: changes | |
if: ${{ needs.changes.outputs.rust == 'true' }} | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Install Rust | |
uses: dtolnay/rust-toolchain@master | |
with: | |
toolchain: nightly-2022-12-01 | |
components: clippy | |
- uses: Swatinem/rust-cache@v2 | |
- run: cargo clippy --all-features -- -D warnings | |
lint-rustfmt: | |
runs-on: ubuntu-latest | |
needs: changes | |
if: ${{ needs.changes.outputs.rust == 'true' }} | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Install Rust | |
uses: dtolnay/rust-toolchain@master | |
with: | |
# We use nightly so that it correctly groups together imports | |
toolchain: nightly-2022-12-01 | |
components: rustfmt | |
- uses: Swatinem/rust-cache@v2 | |
- run: cargo fmt --check | |
# Dummy step to gate other tests on without repeating the whole list | |
linting-done: | |
if: ${{ !cancelled() }} # Run this even if prior jobs were skipped | |
needs: | |
- lint | |
- lint-mypy | |
- lint-crlf | |
- lint-newsfile | |
- lint-pydantic | |
- check-sampleconfig | |
- check-schema-delta | |
- check-lockfile | |
- lint-clippy | |
- lint-rustfmt | |
runs-on: ubuntu-latest | |
steps: | |
- run: "true" | |
calculate-test-jobs: | |
if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail | |
needs: linting-done | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: "3.x" | |
- id: get-matrix | |
run: .ci/scripts/calculate_jobs.py | |
outputs: | |
trial_test_matrix: ${{ steps.get-matrix.outputs.trial_test_matrix }} | |
sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }} | |
trial: | |
if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail | |
needs: calculate-test-jobs | |
runs-on: ubuntu-latest | |
strategy: | |
matrix: | |
job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }} | |
steps: | |
- uses: actions/checkout@v3 | |
- run: sudo apt-get -qq install xmlsec1 | |
- name: Set up PostgreSQL ${{ matrix.job.postgres-version }} | |
if: ${{ matrix.job.postgres-version }} | |
# 1. Mount postgres data files onto a tmpfs in-memory filesystem to reduce overhead of docker's overlayfs layer. | |
# 2. Expose the unix socket for postgres. This removes latency of using docker-proxy for connections. | |
run: | | |
docker run -d -p 5432:5432 \ | |
--tmpfs /var/lib/postgres:rw,size=6144m \ | |
--mount 'type=bind,src=/var/run/postgresql,dst=/var/run/postgresql' \ | |
-e POSTGRES_PASSWORD=postgres \ | |
-e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ | |
postgres:${{ matrix.job.postgres-version }} | |
- name: Install Rust | |
uses: dtolnay/rust-toolchain@1.58.1 | |
- uses: Swatinem/rust-cache@v2 | |
- uses: matrix-org/setup-python-poetry@v1 | |
with: | |
python-version: ${{ matrix.job.python-version }} | |
poetry-version: "1.3.2" | |
extras: ${{ matrix.job.extras }} | |
- name: Await PostgreSQL | |
if: ${{ matrix.job.postgres-version }} | |
timeout-minutes: 2 | |
run: until pg_isready -h localhost; do sleep 1; done | |
- run: poetry run trial --jobs=6 tests | |
env: | |
SYNAPSE_POSTGRES: ${{ matrix.job.database == 'postgres' || '' }} | |
SYNAPSE_POSTGRES_HOST: /var/run/postgresql | |
SYNAPSE_POSTGRES_USER: postgres | |
SYNAPSE_POSTGRES_PASSWORD: postgres | |
- name: Dump logs | |
# Logs are most useful when the command fails, always include them. | |
if: ${{ always() }} | |
# Note: Dumps to workflow logs instead of using actions/upload-artifact | |
# This keeps logs colocated with failing jobs | |
# It also ignores find's exit code; this is a best effort affair | |
run: >- | |
find _trial_temp -name '*.log' | |
-exec echo "::group::{}" \; | |
-exec cat {} \; | |
-exec echo "::endgroup::" \; | |
|| true | |
trial-olddeps: | |
# Note: sqlite only; no postgres | |
if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail | |
needs: linting-done | |
runs-on: ubuntu-20.04 | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Install Rust | |
uses: dtolnay/rust-toolchain@1.58.1 | |
- uses: Swatinem/rust-cache@v2 | |
# There aren't wheels for some of the older deps, so we need to install | |
# their build dependencies | |
- run: | | |
sudo apt-get -qq install build-essential libffi-dev python-dev \ | |
libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: '3.7' | |
# Calculating the old-deps actually takes a bunch of time, so we cache the | |
# pyproject.toml / poetry.lock. We need to cache pyproject.toml as | |
# otherwise the `poetry install` step will error due to the poetry.lock | |
# file being outdated. | |
# | |
# This caches the output of `Prepare old deps`, which should generate the | |
# same `pyproject.toml` and `poetry.lock` for a given `pyproject.toml` input. | |
- uses: actions/cache@v3 | |
id: cache-poetry-old-deps | |
name: Cache poetry.lock | |
with: | |
path: | | |
poetry.lock | |
pyproject.toml | |
key: poetry-old-deps2-${{ hashFiles('pyproject.toml') }} | |
- name: Prepare old deps | |
if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true' | |
run: .ci/scripts/prepare_old_deps.sh | |
# We only now install poetry so that `setup-python-poetry` caches the | |
# right poetry.lock's dependencies. | |
- uses: matrix-org/setup-python-poetry@v1 | |
with: | |
python-version: '3.7' | |
poetry-version: "1.3.2" | |
extras: "all test" | |
- run: poetry run trial -j6 tests | |
- name: Dump logs | |
# Logs are most useful when the command fails, always include them. | |
if: ${{ always() }} | |
# Note: Dumps to workflow logs instead of using actions/upload-artifact | |
# This keeps logs colocated with failing jobs | |
# It also ignores find's exit code; this is a best effort affair | |
run: >- | |
find _trial_temp -name '*.log' | |
-exec echo "::group::{}" \; | |
-exec cat {} \; | |
-exec echo "::endgroup::" \; | |
|| true | |
trial-pypy: | |
# Very slow; only run if the branch name includes 'pypy' | |
# Note: sqlite only; no postgres. Completely untested since poetry move. | |
if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }} | |
needs: linting-done | |
runs-on: ubuntu-latest | |
strategy: | |
matrix: | |
python-version: ["pypy-3.7"] | |
extras: ["all"] | |
steps: | |
- uses: actions/checkout@v3 | |
# Install libs necessary for PyPy to build binary wheels for dependencies | |
- run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev | |
- uses: matrix-org/setup-python-poetry@v1 | |
with: | |
python-version: ${{ matrix.python-version }} | |
poetry-version: "1.3.2" | |
extras: ${{ matrix.extras }} | |
- run: poetry run trial --jobs=2 tests | |
- name: Dump logs | |
# Logs are most useful when the command fails, always include them. | |
if: ${{ always() }} | |
# Note: Dumps to workflow logs instead of using actions/upload-artifact | |
# This keeps logs colocated with failing jobs | |
# It also ignores find's exit code; this is a best effort affair | |
run: >- | |
find _trial_temp -name '*.log' | |
-exec echo "::group::{}" \; | |
-exec cat {} \; | |
-exec echo "::endgroup::" \; | |
|| true | |
sytest: | |
if: ${{ !failure() && !cancelled() }} | |
needs: calculate-test-jobs | |
runs-on: ubuntu-latest | |
container: | |
image: matrixdotorg/sytest-synapse:${{ matrix.job.sytest-tag }} | |
volumes: | |
- ${{ github.workspace }}:/src | |
env: | |
SYTEST_BRANCH: ${{ github.head_ref }} | |
POSTGRES: ${{ matrix.job.postgres && 1}} | |
MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') && 1}} | |
ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') && 1 }} | |
WORKERS: ${{ matrix.job.workers && 1 }} | |
BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }} | |
TOP: ${{ github.workspace }} | |
strategy: | |
fail-fast: false | |
matrix: | |
job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }} | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Prepare test blacklist | |
run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers | |
- name: Install Rust | |
uses: dtolnay/rust-toolchain@1.58.1 | |
- uses: Swatinem/rust-cache@v2 | |
- name: Run SyTest | |
run: /bootstrap.sh synapse | |
working-directory: /src | |
- name: Summarise results.tap | |
if: ${{ always() }} | |
run: /sytest/scripts/tap_to_gha.pl /logs/results.tap | |
- name: Upload SyTest logs | |
uses: actions/upload-artifact@v3 | |
if: ${{ always() }} | |
with: | |
name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }}) | |
path: | | |
/logs/results.tap | |
/logs/**/*.log* | |
export-data: | |
if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail | |
needs: [linting-done, portdb] | |
runs-on: ubuntu-latest | |
env: | |
TOP: ${{ github.workspace }} | |
services: | |
postgres: | |
image: postgres | |
ports: | |
- 5432:5432 | |
env: | |
POSTGRES_PASSWORD: "postgres" | |
POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8" | |
options: >- | |
--health-cmd pg_isready | |
--health-interval 10s | |
--health-timeout 5s | |
--health-retries 5 | |
steps: | |
- uses: actions/checkout@v3 | |
- run: sudo apt-get -qq install xmlsec1 postgresql-client | |
- uses: matrix-org/setup-python-poetry@v1 | |
with: | |
poetry-version: "1.3.2" | |
extras: "postgres" | |
- run: .ci/scripts/test_export_data_command.sh | |
env: | |
PGHOST: localhost | |
PGUSER: postgres | |
PGPASSWORD: postgres | |
PGDATABASE: postgres | |
portdb: | |
if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail | |
needs: linting-done | |
runs-on: ubuntu-latest | |
strategy: | |
matrix: | |
include: | |
- python-version: "3.7" | |
postgres-version: "11" | |
- python-version: "3.11" | |
postgres-version: "15" | |
services: | |
postgres: | |
image: postgres:${{ matrix.postgres-version }} | |
ports: | |
- 5432:5432 | |
env: | |
POSTGRES_PASSWORD: "postgres" | |
POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8" | |
options: >- | |
--health-cmd pg_isready | |
--health-interval 10s | |
--health-timeout 5s | |
--health-retries 5 | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Add PostgreSQL apt repository | |
# We need a version of pg_dump that can handle the version of | |
# PostgreSQL being tested against. The Ubuntu package repository lags | |
# behind new releases, so we have to use the PostreSQL apt repository. | |
# Steps taken from https://www.postgresql.org/download/linux/ubuntu/ | |
run: | | |
sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' | |
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - | |
sudo apt-get update | |
- run: sudo apt-get -qq install xmlsec1 postgresql-client | |
- uses: matrix-org/setup-python-poetry@v1 | |
with: | |
python-version: ${{ matrix.python-version }} | |
poetry-version: "1.3.2" | |
extras: "postgres" | |
- run: .ci/scripts/test_synapse_port_db.sh | |
id: run_tester_script | |
env: | |
PGHOST: localhost | |
PGUSER: postgres | |
PGPASSWORD: postgres | |
PGDATABASE: postgres | |
- name: "Upload schema differences" | |
uses: actions/upload-artifact@v3 | |
if: ${{ failure() && !cancelled() && steps.run_tester_script.outcome == 'failure' }} | |
with: | |
name: Schema dumps | |
path: | | |
unported.sql | |
ported.sql | |
schema_diff | |
complement: | |
if: "${{ !failure() && !cancelled() }}" | |
needs: linting-done | |
runs-on: ubuntu-latest | |
strategy: | |
fail-fast: false | |
matrix: | |
include: | |
- arrangement: monolith | |
database: SQLite | |
- arrangement: monolith | |
database: Postgres | |
- arrangement: workers | |
database: Postgres | |
steps: | |
- name: Run actions/checkout@v3 for synapse | |
uses: actions/checkout@v3 | |
with: | |
path: synapse | |
- name: Install Rust | |
uses: dtolnay/rust-toolchain@1.58.1 | |
- uses: Swatinem/rust-cache@v2 | |
- uses: actions/setup-go@v4 | |
- name: Prepare Complement's Prerequisites | |
run: synapse/.ci/scripts/setup_complement_prerequisites.sh | |
- run: | | |
set -o pipefail | |
COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt | |
shell: bash | |
env: | |
POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} | |
WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} | |
name: Run Complement Tests | |
cargo-test: | |
if: ${{ needs.changes.outputs.rust == 'true' }} | |
runs-on: ubuntu-latest | |
needs: | |
- linting-done | |
- changes | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Install Rust | |
uses: dtolnay/rust-toolchain@1.58.1 | |
- uses: Swatinem/rust-cache@v2 | |
- run: cargo test | |
# We want to ensure that the cargo benchmarks still compile, which requires a | |
# nightly compiler. | |
cargo-bench: | |
if: ${{ needs.changes.outputs.rust == 'true' }} | |
runs-on: ubuntu-latest | |
needs: | |
- linting-done | |
- changes | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Install Rust | |
uses: dtolnay/rust-toolchain@master | |
with: | |
toolchain: nightly-2022-12-01 | |
- uses: Swatinem/rust-cache@v2 | |
- run: cargo bench --no-run | |
# a job which marks all the other jobs as complete, thus allowing PRs to be merged. | |
tests-done: | |
if: ${{ always() }} | |
needs: | |
- trial | |
- trial-olddeps | |
- sytest | |
- export-data | |
- portdb | |
- complement | |
- cargo-test | |
- cargo-bench | |
runs-on: ubuntu-latest | |
steps: | |
- uses: matrix-org/done-action@v2 | |
with: | |
needs: ${{ toJSON(needs) }} | |
# The newsfile lint may be skipped on non PR builds | |
# Cargo test is skipped if there is no changes on Rust code | |
skippable: | | |
lint-newsfile | |
cargo-test | |
cargo-bench |