mirror of
https://mau.dev/maunium/synapse.git
synced 2024-06-28 23:42:18 +00:00
a3865ed525
Since #11811 there has been general Complement flakiness around networking. It seems like tests are hitting the wrong containers. In an effort to diagnose the cause of this, as well as reduce its impact on this project, set the parallelsim to 1 (no parallelism) when running tests. If this fixes the flakiness then this indicates the cause and I can diagnose this further. If this doesn't fix the flakiness then that implies some kind of test pollution which also helps to diagnose this further.
429 lines
14 KiB
YAML
429 lines
14 KiB
YAML
name: Tests
|
|
|
|
on:
|
|
push:
|
|
branches: ["develop", "release-*"]
|
|
pull_request:
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
lint:
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
matrix:
|
|
toxenv:
|
|
- "check-sampleconfig"
|
|
- "check_codestyle"
|
|
- "check_isort"
|
|
- "mypy"
|
|
- "packaging"
|
|
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- uses: actions/setup-python@v2
|
|
- run: pip install tox
|
|
- run: tox -e ${{ matrix.toxenv }}
|
|
|
|
lint-crlf:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- name: Check line endings
|
|
run: scripts-dev/check_line_terminators.sh
|
|
|
|
lint-newsfile:
|
|
if: ${{ github.base_ref == 'develop' || contains(github.base_ref, 'release-') }}
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
with:
|
|
ref: ${{ github.event.pull_request.head.sha }}
|
|
fetch-depth: 0
|
|
- uses: actions/setup-python@v2
|
|
- run: pip install tox
|
|
- run: scripts-dev/check-newsfragment
|
|
env:
|
|
PULL_REQUEST_NUMBER: ${{ github.event.number }}
|
|
|
|
lint-sdist:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- uses: actions/setup-python@v2
|
|
with:
|
|
python-version: "3.x"
|
|
- run: pip install wheel
|
|
- run: python setup.py sdist bdist_wheel
|
|
- uses: actions/upload-artifact@v2
|
|
with:
|
|
name: Python Distributions
|
|
path: dist/*
|
|
|
|
# Dummy step to gate other tests on without repeating the whole list
|
|
linting-done:
|
|
if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
|
|
needs: [lint, lint-crlf, lint-newsfile, lint-sdist]
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- run: "true"
|
|
|
|
trial:
|
|
if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
|
|
needs: linting-done
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
matrix:
|
|
python-version: ["3.7", "3.8", "3.9", "3.10"]
|
|
database: ["sqlite"]
|
|
toxenv: ["py"]
|
|
include:
|
|
# Newest Python without optional deps
|
|
- python-version: "3.10"
|
|
toxenv: "py-noextras"
|
|
|
|
# Oldest Python with PostgreSQL
|
|
- python-version: "3.7"
|
|
database: "postgres"
|
|
postgres-version: "10"
|
|
toxenv: "py"
|
|
|
|
# Newest Python with newest PostgreSQL
|
|
- python-version: "3.10"
|
|
database: "postgres"
|
|
postgres-version: "14"
|
|
toxenv: "py"
|
|
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- run: sudo apt-get -qq install xmlsec1
|
|
- name: Set up PostgreSQL ${{ matrix.postgres-version }}
|
|
if: ${{ matrix.postgres-version }}
|
|
run: |
|
|
docker run -d -p 5432:5432 \
|
|
-e POSTGRES_PASSWORD=postgres \
|
|
-e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
|
|
postgres:${{ matrix.postgres-version }}
|
|
- uses: actions/setup-python@v2
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
- run: pip install tox
|
|
- name: Await PostgreSQL
|
|
if: ${{ matrix.postgres-version }}
|
|
timeout-minutes: 2
|
|
run: until pg_isready -h localhost; do sleep 1; done
|
|
- run: tox -e ${{ matrix.toxenv }}
|
|
env:
|
|
TRIAL_FLAGS: "--jobs=2"
|
|
SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
|
|
SYNAPSE_POSTGRES_HOST: localhost
|
|
SYNAPSE_POSTGRES_USER: postgres
|
|
SYNAPSE_POSTGRES_PASSWORD: postgres
|
|
- name: Dump logs
|
|
# Logs are most useful when the command fails, always include them.
|
|
if: ${{ always() }}
|
|
# Note: Dumps to workflow logs instead of using actions/upload-artifact
|
|
# This keeps logs colocated with failing jobs
|
|
# It also ignores find's exit code; this is a best effort affair
|
|
run: >-
|
|
find _trial_temp -name '*.log'
|
|
-exec echo "::group::{}" \;
|
|
-exec cat {} \;
|
|
-exec echo "::endgroup::" \;
|
|
|| true
|
|
|
|
trial-olddeps:
|
|
if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
|
|
needs: linting-done
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- name: Test with old deps
|
|
uses: docker://ubuntu:focal # For old python and sqlite
|
|
with:
|
|
workdir: /github/workspace
|
|
entrypoint: .ci/scripts/test_old_deps.sh
|
|
env:
|
|
TRIAL_FLAGS: "--jobs=2"
|
|
- name: Dump logs
|
|
# Logs are most useful when the command fails, always include them.
|
|
if: ${{ always() }}
|
|
# Note: Dumps to workflow logs instead of using actions/upload-artifact
|
|
# This keeps logs colocated with failing jobs
|
|
# It also ignores find's exit code; this is a best effort affair
|
|
run: >-
|
|
find _trial_temp -name '*.log'
|
|
-exec echo "::group::{}" \;
|
|
-exec cat {} \;
|
|
-exec echo "::endgroup::" \;
|
|
|| true
|
|
|
|
trial-pypy:
|
|
# Very slow; only run if the branch name includes 'pypy'
|
|
if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }}
|
|
needs: linting-done
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
matrix:
|
|
python-version: ["pypy-3.7"]
|
|
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
|
|
- uses: actions/setup-python@v2
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
- run: pip install tox
|
|
- run: tox -e py
|
|
env:
|
|
TRIAL_FLAGS: "--jobs=2"
|
|
- name: Dump logs
|
|
# Logs are most useful when the command fails, always include them.
|
|
if: ${{ always() }}
|
|
# Note: Dumps to workflow logs instead of using actions/upload-artifact
|
|
# This keeps logs colocated with failing jobs
|
|
# It also ignores find's exit code; this is a best effort affair
|
|
run: >-
|
|
find _trial_temp -name '*.log'
|
|
-exec echo "::group::{}" \;
|
|
-exec cat {} \;
|
|
-exec echo "::endgroup::" \;
|
|
|| true
|
|
|
|
sytest:
|
|
if: ${{ !failure() && !cancelled() }}
|
|
needs: linting-done
|
|
runs-on: ubuntu-latest
|
|
container:
|
|
image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
|
|
volumes:
|
|
- ${{ github.workspace }}:/src
|
|
env:
|
|
SYTEST_BRANCH: ${{ github.head_ref }}
|
|
POSTGRES: ${{ matrix.postgres && 1}}
|
|
MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}}
|
|
WORKERS: ${{ matrix.workers && 1 }}
|
|
REDIS: ${{ matrix.redis && 1 }}
|
|
BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
|
|
TOP: ${{ github.workspace }}
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- sytest-tag: focal
|
|
|
|
- sytest-tag: focal
|
|
postgres: postgres
|
|
|
|
- sytest-tag: testing
|
|
postgres: postgres
|
|
|
|
- sytest-tag: focal
|
|
postgres: multi-postgres
|
|
workers: workers
|
|
|
|
- sytest-tag: buster
|
|
postgres: multi-postgres
|
|
workers: workers
|
|
|
|
- sytest-tag: buster
|
|
postgres: postgres
|
|
workers: workers
|
|
redis: redis
|
|
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- name: Prepare test blacklist
|
|
run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
|
|
- name: Run SyTest
|
|
run: /bootstrap.sh synapse
|
|
working-directory: /src
|
|
- name: Summarise results.tap
|
|
if: ${{ always() }}
|
|
run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
|
|
- name: Upload SyTest logs
|
|
uses: actions/upload-artifact@v2
|
|
if: ${{ always() }}
|
|
with:
|
|
name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
|
|
path: |
|
|
/logs/results.tap
|
|
/logs/**/*.log*
|
|
|
|
export-data:
|
|
if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
|
|
needs: [linting-done, portdb]
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
TOP: ${{ github.workspace }}
|
|
|
|
services:
|
|
postgres:
|
|
image: postgres
|
|
ports:
|
|
- 5432:5432
|
|
env:
|
|
POSTGRES_PASSWORD: "postgres"
|
|
POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
|
|
options: >-
|
|
--health-cmd pg_isready
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- run: sudo apt-get -qq install xmlsec1
|
|
- uses: actions/setup-python@v2
|
|
with:
|
|
python-version: "3.9"
|
|
- run: .ci/scripts/test_export_data_command.sh
|
|
|
|
portdb:
|
|
if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
|
|
needs: linting-done
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
TOP: ${{ github.workspace }}
|
|
strategy:
|
|
matrix:
|
|
include:
|
|
- python-version: "3.7"
|
|
postgres-version: "10"
|
|
|
|
- python-version: "3.10"
|
|
postgres-version: "14"
|
|
|
|
services:
|
|
postgres:
|
|
image: postgres:${{ matrix.postgres-version }}
|
|
ports:
|
|
- 5432:5432
|
|
env:
|
|
POSTGRES_PASSWORD: "postgres"
|
|
POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
|
|
options: >-
|
|
--health-cmd pg_isready
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- run: sudo apt-get -qq install xmlsec1
|
|
- uses: actions/setup-python@v2
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
- run: .ci/scripts/test_synapse_port_db.sh
|
|
|
|
complement:
|
|
if: ${{ !failure() && !cancelled() }}
|
|
needs: linting-done
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
# The path is set via a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on the path to run Complement.
|
|
# See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
|
|
- name: "Set Go Version"
|
|
run: |
|
|
# Add Go 1.17 to the PATH: see https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#environment-variables-2
|
|
echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
|
|
# Add the Go path to the PATH: We need this so we can call gotestfmt
|
|
echo "~/go/bin" >> $GITHUB_PATH
|
|
|
|
- name: "Install Complement Dependencies"
|
|
run: |
|
|
sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
|
|
go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
|
|
|
|
- name: Run actions/checkout@v2 for synapse
|
|
uses: actions/checkout@v2
|
|
with:
|
|
path: synapse
|
|
|
|
# Attempt to check out the same branch of Complement as the PR. If it
|
|
# doesn't exist, fallback to master.
|
|
- name: Checkout complement
|
|
shell: bash
|
|
run: |
|
|
mkdir -p complement
|
|
# Attempt to use the version of complement which best matches the current
|
|
# build. Depending on whether this is a PR or release, etc. we need to
|
|
# use different fallbacks.
|
|
#
|
|
# 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
|
|
# for pull requests, otherwise GITHUB_REF).
|
|
# 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
|
|
# (GITHUB_BASE_REF for pull requests).
|
|
# 3. Use the default complement branch ("master").
|
|
for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
|
|
# Skip empty branch names and merge commits.
|
|
if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
|
|
continue
|
|
fi
|
|
|
|
(wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
|
|
done
|
|
|
|
# Build initial Synapse image
|
|
- run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
|
|
working-directory: synapse
|
|
env:
|
|
DOCKER_BUILDKIT: 1
|
|
|
|
# Build a ready-to-run Synapse image based on the initial image above.
|
|
# This new image includes a config file, keys for signing and TLS, and
|
|
# other settings to make it suitable for testing under Complement.
|
|
- run: docker build -t complement-synapse -f Synapse.Dockerfile .
|
|
working-directory: complement/dockerfiles
|
|
|
|
# Run Complement
|
|
- run: |
|
|
set -o pipefail
|
|
go test -v -json -p 1 -tags synapse_blacklist,msc2403 ./tests/... 2>&1 | gotestfmt
|
|
shell: bash
|
|
name: Run Complement Tests
|
|
env:
|
|
COMPLEMENT_BASE_IMAGE: complement-synapse:latest
|
|
working-directory: complement
|
|
|
|
# a job which marks all the other jobs as complete, thus allowing PRs to be merged.
|
|
tests-done:
|
|
if: ${{ always() }}
|
|
needs:
|
|
- lint
|
|
- lint-crlf
|
|
- lint-newsfile
|
|
- lint-sdist
|
|
- trial
|
|
- trial-olddeps
|
|
- sytest
|
|
- portdb
|
|
- complement
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Set build result
|
|
env:
|
|
NEEDS_CONTEXT: ${{ toJSON(needs) }}
|
|
# the `jq` incantation dumps out a series of "<job> <result>" lines.
|
|
# we set it to an intermediate variable to avoid a pipe, which makes it
|
|
# hard to set $rc.
|
|
run: |
|
|
rc=0
|
|
results=$(jq -r 'to_entries[] | [.key,.value.result] | join(" ")' <<< $NEEDS_CONTEXT)
|
|
while read job result ; do
|
|
# The newsfile lint may be skipped on non PR builds
|
|
if [ $result == "skipped" ] && [ $job == "lint-newsfile" ]; then
|
|
continue
|
|
fi
|
|
|
|
if [ "$result" != "success" ]; then
|
|
echo "::set-failed ::Job $job returned $result"
|
|
rc=1
|
|
fi
|
|
done <<< $results
|
|
exit $rc
|