Fix broken export-data admin command and add a test for it to CI (#11078)

Fix broken export-data admin command and add a test for it to CI
This commit is contained in:
Hillery Shay 2021-10-18 08:14:12 -07:00 committed by GitHub
parent 37b845dabc
commit 7d70582eb0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 93 additions and 8 deletions

View File

@ -0,0 +1,57 @@
#!/usr/bin/env bash
# Test for the export-data admin command against sqlite and postgres
set -xe
cd `dirname $0`/../..
echo "--- Install dependencies"
# Install dependencies for this test.
pip install psycopg2
# Install Synapse itself. This won't update any libraries.
pip install -e .
echo "--- Generate the signing key"
# Generate the server's signing key.
python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
echo "--- Prepare test database"
# Make sure the SQLite3 database is using the latest schema and has no pending background update.
scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
# Run the export-data command on the sqlite test database
python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml export-data @anon-20191002_181700-832:localhost:8800 \
--output-directory /tmp/export_data
# Test that the output directory exists and contains the rooms directory
dir="/tmp/export_data/rooms"
if [ -d "$dir" ]; then
echo "Command successful, this test passes"
else
echo "No output directories found, the command fails against a sqlite database."
exit 1
fi
# Create the PostgreSQL database.
.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
# Port the SQLite databse to postgres so we can check command works against postgres
echo "+++ Port SQLite3 databse to postgres"
scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
# Run the export-data command on postgres database
python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml export-data @anon-20191002_181700-832:localhost:8800 \
--output-directory /tmp/export_data2
# Test that the output directory exists and contains the rooms directory
dir2="/tmp/export_data2/rooms"
if [ -d "$dir2" ]; then
echo "Command successful, this test passes"
else
echo "No output directories found, the command fails against a postgres database."
exit 1
fi

View File

@ -253,6 +253,35 @@ jobs:
/logs/results.tap /logs/results.tap
/logs/**/*.log* /logs/**/*.log*
export-data:
if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
needs: [linting-done, portdb]
runs-on: ubuntu-latest
env:
TOP: ${{ github.workspace }}
services:
postgres:
image: postgres
ports:
- 5432:5432
env:
POSTGRES_PASSWORD: "postgres"
POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v2
- run: sudo apt-get -qq install xmlsec1
- uses: actions/setup-python@v2
with:
python-version: "3.9"
- run: .ci/scripts/test_export_data_command.sh
portdb: portdb:
if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
needs: linting-done needs: linting-done

1
changelog.d/11078.bugfix Normal file
View File

@ -0,0 +1 @@
Fix broken export-data admin command and add test script checking the command to CI.

View File

@ -39,6 +39,7 @@ from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore
from synapse.replication.slave.storage.receipts import SlavedReceiptsStore from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
from synapse.replication.slave.storage.registration import SlavedRegistrationStore from synapse.replication.slave.storage.registration import SlavedRegistrationStore
from synapse.server import HomeServer from synapse.server import HomeServer
from synapse.storage.databases.main.room import RoomWorkerStore
from synapse.util.logcontext import LoggingContext from synapse.util.logcontext import LoggingContext
from synapse.util.versionstring import get_version_string from synapse.util.versionstring import get_version_string
@ -58,6 +59,7 @@ class AdminCmdSlavedStore(
SlavedEventStore, SlavedEventStore,
SlavedClientIpStore, SlavedClientIpStore,
BaseSlavedStore, BaseSlavedStore,
RoomWorkerStore,
): ):
pass pass
@ -185,11 +187,7 @@ def start(config_options):
# a full worker config. # a full worker config.
config.worker.worker_app = "synapse.app.admin_cmd" config.worker.worker_app = "synapse.app.admin_cmd"
if ( if not config.worker.worker_daemonize and not config.worker.worker_log_config:
not config.worker.worker_daemonize
and not config.worker.worker_log_file
and not config.worker.worker_log_config
):
# Since we're meant to be run as a "command" let's not redirect stdio # Since we're meant to be run as a "command" let's not redirect stdio
# unless we've actually set log config. # unless we've actually set log config.
config.logging.no_redirect_stdio = True config.logging.no_redirect_stdio = True
@ -198,9 +196,9 @@ def start(config_options):
config.server.update_user_directory = False config.server.update_user_directory = False
config.worker.run_background_tasks = False config.worker.run_background_tasks = False
config.worker.start_pushers = False config.worker.start_pushers = False
config.pusher_shard_config.instances = [] config.worker.pusher_shard_config.instances = []
config.worker.send_federation = False config.worker.send_federation = False
config.federation_shard_config.instances = [] config.worker.federation_shard_config.instances = []
synapse.events.USE_FROZEN_DICTS = config.server.use_frozen_dicts synapse.events.USE_FROZEN_DICTS = config.server.use_frozen_dicts
@ -221,7 +219,7 @@ def start(config_options):
async def run(): async def run():
with LoggingContext("command"): with LoggingContext("command"):
_base.start(ss) await _base.start(ss)
await args.func(ss, args) await args.func(ss, args)
_base.start_worker_reactor( _base.start_worker_reactor(