Allow running Complement integration tests via podman (#15543)

This commit is contained in:
Andrew Morgan 2023-05-05 13:27:51 +01:00 committed by GitHub
parent 2e59e97ebd
commit 83e7fa5eee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 28 additions and 12 deletions

1
changelog.d/15543.misc Normal file
View File

@ -0,0 +1 @@
Allow running Complement tests using [podman](https://podman.io/) by adding a `PODMAN` environment variable to `scripts-dev/complement.sh`.

View File

@ -346,6 +346,7 @@ The above will run a monolithic (single-process) Synapse with SQLite as the data
A safe example would be `WORKER_TYPES="federation_inbound, federation_sender, synchrotron"`. A safe example would be `WORKER_TYPES="federation_inbound, federation_sender, synchrotron"`.
See the [worker documentation](../workers.md) for additional information on workers. See the [worker documentation](../workers.md) for additional information on workers.
- Passing `ASYNCIO_REACTOR=1` as an environment variable to use the Twisted asyncio reactor instead of the default one. - Passing `ASYNCIO_REACTOR=1` as an environment variable to use the Twisted asyncio reactor instead of the default one.
- Passing `PODMAN=1` will use the [podman](https://podman.io/) container runtime, instead of docker.
To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`, e.g: To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`, e.g:
```sh ```sh

View File

@ -11,6 +11,11 @@
# filepath of a local Complement checkout or by setting the COMPLEMENT_REF # filepath of a local Complement checkout or by setting the COMPLEMENT_REF
# environment variable to pull a different branch or commit. # environment variable to pull a different branch or commit.
# #
# To use the 'podman' command instead 'docker', set the PODMAN environment
# variable. Example:
#
# PODMAN=1 ./complement.sh
#
# By default Synapse is run in monolith mode. This can be overridden by # By default Synapse is run in monolith mode. This can be overridden by
# setting the WORKERS environment variable. # setting the WORKERS environment variable.
# #
@ -30,7 +35,6 @@
# Exit if a line returns a non-zero exit code # Exit if a line returns a non-zero exit code
set -e set -e
# Helper to emit annotations that collapse portions of the log in GitHub Actions # Helper to emit annotations that collapse portions of the log in GitHub Actions
echo_if_github() { echo_if_github() {
if [[ -n "$GITHUB_WORKFLOW" ]]; then if [[ -n "$GITHUB_WORKFLOW" ]]; then
@ -100,6 +104,16 @@ done
# enable buildkit for the docker builds # enable buildkit for the docker builds
export DOCKER_BUILDKIT=1 export DOCKER_BUILDKIT=1
# Determine whether to use the docker or podman container runtime.
if [ -n "$PODMAN" ]; then
export CONTAINER_RUNTIME=podman
export DOCKER_HOST=unix://$XDG_RUNTIME_DIR/podman/podman.sock
export BUILDAH_FORMAT=docker
export COMPLEMENT_HOSTNAME_RUNNING_COMPLEMENT=host.containers.internal
else
export CONTAINER_RUNTIME=docker
fi
# Change to the repository root # Change to the repository root
cd "$(dirname $0)/.." cd "$(dirname $0)/.."
@ -126,16 +140,16 @@ if [ -n "$use_editable_synapse" ]; then
editable_mount="$(realpath .):/editable-src:z" editable_mount="$(realpath .):/editable-src:z"
if [ -n "$rebuild_editable_synapse" ]; then if [ -n "$rebuild_editable_synapse" ]; then
unset skip_docker_build unset skip_docker_build
elif docker inspect complement-synapse-editable &>/dev/null; then elif $CONTAINER_RUNTIME inspect complement-synapse-editable &>/dev/null; then
# complement-synapse-editable already exists: see if we can still use it: # complement-synapse-editable already exists: see if we can still use it:
# - The Rust module must still be importable; it will fail to import if the Rust source has changed. # - The Rust module must still be importable; it will fail to import if the Rust source has changed.
# - The Poetry lock file must be the same (otherwise we assume dependencies have changed) # - The Poetry lock file must be the same (otherwise we assume dependencies have changed)
# First set up the module in the right place for an editable installation. # First set up the module in the right place for an editable installation.
docker run --rm -v $editable_mount --entrypoint 'cp' complement-synapse-editable -- /synapse_rust.abi3.so.bak /editable-src/synapse/synapse_rust.abi3.so $CONTAINER_RUNTIME run --rm -v $editable_mount --entrypoint 'cp' complement-synapse-editable -- /synapse_rust.abi3.so.bak /editable-src/synapse/synapse_rust.abi3.so
if (docker run --rm -v $editable_mount --entrypoint 'python' complement-synapse-editable -c 'import synapse.synapse_rust' \ if ($CONTAINER_RUNTIME run --rm -v $editable_mount --entrypoint 'python' complement-synapse-editable -c 'import synapse.synapse_rust' \
&& docker run --rm -v $editable_mount --entrypoint 'diff' complement-synapse-editable --brief /editable-src/poetry.lock /poetry.lock.bak); then && $CONTAINER_RUNTIME run --rm -v $editable_mount --entrypoint 'diff' complement-synapse-editable --brief /editable-src/poetry.lock /poetry.lock.bak); then
skip_docker_build=1 skip_docker_build=1
else else
echo "Editable Synapse image is stale. Will rebuild." echo "Editable Synapse image is stale. Will rebuild."
@ -149,25 +163,25 @@ if [ -z "$skip_docker_build" ]; then
# Build a special image designed for use in development with editable # Build a special image designed for use in development with editable
# installs. # installs.
docker build -t synapse-editable \ $CONTAINER_RUNTIME build -t synapse-editable \
-f "docker/editable.Dockerfile" . -f "docker/editable.Dockerfile" .
docker build -t synapse-workers-editable \ $CONTAINER_RUNTIME build -t synapse-workers-editable \
--build-arg FROM=synapse-editable \ --build-arg FROM=synapse-editable \
-f "docker/Dockerfile-workers" . -f "docker/Dockerfile-workers" .
docker build -t complement-synapse-editable \ $CONTAINER_RUNTIME build -t complement-synapse-editable \
--build-arg FROM=synapse-workers-editable \ --build-arg FROM=synapse-workers-editable \
-f "docker/complement/Dockerfile" "docker/complement" -f "docker/complement/Dockerfile" "docker/complement"
# Prepare the Rust module # Prepare the Rust module
docker run --rm -v $editable_mount --entrypoint 'cp' complement-synapse-editable -- /synapse_rust.abi3.so.bak /editable-src/synapse/synapse_rust.abi3.so $CONTAINER_RUNTIME run --rm -v $editable_mount --entrypoint 'cp' complement-synapse-editable -- /synapse_rust.abi3.so.bak /editable-src/synapse/synapse_rust.abi3.so
else else
# Build the base Synapse image from the local checkout # Build the base Synapse image from the local checkout
echo_if_github "::group::Build Docker image: matrixdotorg/synapse" echo_if_github "::group::Build Docker image: matrixdotorg/synapse"
docker build -t matrixdotorg/synapse \ $CONTAINER_RUNTIME build -t matrixdotorg/synapse \
--build-arg TEST_ONLY_SKIP_DEP_HASH_VERIFICATION \ --build-arg TEST_ONLY_SKIP_DEP_HASH_VERIFICATION \
--build-arg TEST_ONLY_IGNORE_POETRY_LOCKFILE \ --build-arg TEST_ONLY_IGNORE_POETRY_LOCKFILE \
-f "docker/Dockerfile" . -f "docker/Dockerfile" .
@ -175,12 +189,12 @@ if [ -z "$skip_docker_build" ]; then
# Build the workers docker image (from the base Synapse image we just built). # Build the workers docker image (from the base Synapse image we just built).
echo_if_github "::group::Build Docker image: matrixdotorg/synapse-workers" echo_if_github "::group::Build Docker image: matrixdotorg/synapse-workers"
docker build -t matrixdotorg/synapse-workers -f "docker/Dockerfile-workers" . $CONTAINER_RUNTIME build -t matrixdotorg/synapse-workers -f "docker/Dockerfile-workers" .
echo_if_github "::endgroup::" echo_if_github "::endgroup::"
# Build the unified Complement image (from the worker Synapse image we just built). # Build the unified Complement image (from the worker Synapse image we just built).
echo_if_github "::group::Build Docker image: complement/Dockerfile" echo_if_github "::group::Build Docker image: complement/Dockerfile"
docker build -t complement-synapse \ $CONTAINER_RUNTIME build -t complement-synapse \
-f "docker/complement/Dockerfile" "docker/complement" -f "docker/complement/Dockerfile" "docker/complement"
echo_if_github "::endgroup::" echo_if_github "::endgroup::"