Move scripts directory inside synapse, exposing as setuptools entry_points (#12118)

* Two scripts are basically entry_points already
* Move and rename scripts/* to synapse/_scripts/*.py
* Delete sync_room_to_group.pl
* Expose entry points in setup.py
* Update linter script and config
* Fixup scripts & docs mentioning scripts that moved

Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
This commit is contained in:
David Robertson 2022-03-02 13:00:16 +00:00 committed by GitHub
parent 6adb89ff00
commit f3f0ab10fe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 77 additions and 135 deletions

View File

@ -21,7 +21,7 @@ python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
echo "--- Prepare test database" echo "--- Prepare test database"
# Make sure the SQLite3 database is using the latest schema and has no pending background update. # Make sure the SQLite3 database is using the latest schema and has no pending background update.
scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
# Run the export-data command on the sqlite test database # Run the export-data command on the sqlite test database
python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml export-data @anon-20191002_181700-832:localhost:8800 \ python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml export-data @anon-20191002_181700-832:localhost:8800 \
@ -41,7 +41,7 @@ fi
# Port the SQLite databse to postgres so we can check command works against postgres # Port the SQLite databse to postgres so we can check command works against postgres
echo "+++ Port SQLite3 databse to postgres" echo "+++ Port SQLite3 databse to postgres"
scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
# Run the export-data command on postgres database # Run the export-data command on postgres database
python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml export-data @anon-20191002_181700-832:localhost:8800 \ python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml export-data @anon-20191002_181700-832:localhost:8800 \

View File

@ -25,17 +25,19 @@ python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
echo "--- Prepare test database" echo "--- Prepare test database"
# Make sure the SQLite3 database is using the latest schema and has no pending background update. # Make sure the SQLite3 database is using the latest schema and has no pending background update.
scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
# Create the PostgreSQL database. # Create the PostgreSQL database.
.ci/scripts/postgres_exec.py "CREATE DATABASE synapse" .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
echo "+++ Run synapse_port_db against test database" echo "+++ Run synapse_port_db against test database"
coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml # TODO: this invocation of synapse_port_db (and others below) used to be prepended with `coverage run`,
# but coverage seems unable to find the entrypoints installed by `pip install -e .`.
synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
# We should be able to run twice against the same database. # We should be able to run twice against the same database.
echo "+++ Run synapse_port_db a second time" echo "+++ Run synapse_port_db a second time"
coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
##### #####
@ -46,7 +48,7 @@ echo "--- Prepare empty SQLite database"
# we do this by deleting the sqlite db, and then doing the same again. # we do this by deleting the sqlite db, and then doing the same again.
rm .ci/test_db.db rm .ci/test_db.db
scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
# re-create the PostgreSQL database. # re-create the PostgreSQL database.
.ci/scripts/postgres_exec.py \ .ci/scripts/postgres_exec.py \
@ -54,4 +56,4 @@ scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-b
"CREATE DATABASE synapse" "CREATE DATABASE synapse"
echo "+++ Run synapse_port_db against empty database" echo "+++ Run synapse_port_db against empty database"
coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml

View File

@ -3,7 +3,6 @@
# things to include # things to include
!docker !docker
!scripts
!synapse !synapse
!MANIFEST.in !MANIFEST.in
!README.rst !README.rst

View File

@ -17,7 +17,6 @@ recursive-include synapse/storage *.txt
recursive-include synapse/storage *.md recursive-include synapse/storage *.md
recursive-include docs * recursive-include docs *
recursive-include scripts *
recursive-include scripts-dev * recursive-include scripts-dev *
recursive-include synapse *.pyi recursive-include synapse *.pyi
recursive-include tests *.py recursive-include tests *.py

1
changelog.d/12118.misc Normal file
View File

@ -0,0 +1 @@
Move scripts to Synapse package and expose as setuptools entry points.

View File

@ -46,7 +46,6 @@ RUN \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
# Copy just what we need to pip install # Copy just what we need to pip install
COPY scripts /synapse/scripts/
COPY MANIFEST.in README.rst setup.py synctl /synapse/ COPY MANIFEST.in README.rst setup.py synctl /synapse/
COPY synapse/__init__.py /synapse/synapse/__init__.py COPY synapse/__init__.py /synapse/synapse/__init__.py
COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py

View File

@ -158,9 +158,9 @@ same as integers.
There are three separate aspects to this: There are three separate aspects to this:
* Any new boolean column must be added to the `BOOLEAN_COLUMNS` list in * Any new boolean column must be added to the `BOOLEAN_COLUMNS` list in
`scripts/synapse_port_db`. This tells the port script to cast the integer `synapse/_scripts/synapse_port_db.py`. This tells the port script to cast
value from SQLite to a boolean before writing the value to the postgres the integer value from SQLite to a boolean before writing the value to the
database. postgres database.
* Before SQLite 3.23, `TRUE` and `FALSE` were not recognised as constants by * Before SQLite 3.23, `TRUE` and `FALSE` were not recognised as constants by
SQLite, and the `IS [NOT] TRUE`/`IS [NOT] FALSE` operators were not SQLite, and the `IS [NOT] TRUE`/`IS [NOT] FALSE` operators were not

View File

@ -12,7 +12,7 @@ UPDATE users SET admin = 1 WHERE name = '@foo:bar.com';
``` ```
A new server admin user can also be created using the `register_new_matrix_user` A new server admin user can also be created using the `register_new_matrix_user`
command. This is a script that is located in the `scripts/` directory, or possibly command. This is a script that is distributed as part of synapse. It is possibly
already on your `$PATH` depending on how Synapse was installed. already on your `$PATH` depending on how Synapse was installed.
Finding your user's `access_token` is client-dependent, but will usually be shown in the client's settings. Finding your user's `access_token` is client-dependent, but will usually be shown in the client's settings.

View File

@ -23,6 +23,10 @@ files =
# https://docs.python.org/3/library/re.html#re.X # https://docs.python.org/3/library/re.html#re.X
exclude = (?x) exclude = (?x)
^( ^(
|synapse/_scripts/export_signing_key.py
|synapse/_scripts/move_remote_media_to_new_store.py
|synapse/_scripts/synapse_port_db.py
|synapse/_scripts/update_synapse_database.py
|synapse/storage/databases/__init__.py |synapse/storage/databases/__init__.py
|synapse/storage/databases/main/__init__.py |synapse/storage/databases/main/__init__.py
|synapse/storage/databases/main/cache.py |synapse/storage/databases/main/cache.py

View File

@ -10,19 +10,19 @@ SAMPLE_CONFIG="docs/sample_config.yaml"
SAMPLE_LOG_CONFIG="docs/sample_log_config.yaml" SAMPLE_LOG_CONFIG="docs/sample_log_config.yaml"
check() { check() {
diff -u "$SAMPLE_LOG_CONFIG" <(./scripts/generate_log_config) >/dev/null || return 1 diff -u "$SAMPLE_LOG_CONFIG" <(synapse/_scripts/generate_log_config.py) >/dev/null || return 1
} }
if [ "$1" == "--check" ]; then if [ "$1" == "--check" ]; then
diff -u "$SAMPLE_CONFIG" <(./scripts/generate_config --header-file docs/.sample_config_header.yaml) >/dev/null || { diff -u "$SAMPLE_CONFIG" <(synapse/_scripts/generate_config.py --header-file docs/.sample_config_header.yaml) >/dev/null || {
echo -e "\e[1m\e[31m$SAMPLE_CONFIG is not up-to-date. Regenerate it with \`scripts-dev/generate_sample_config\`.\e[0m" >&2 echo -e "\e[1m\e[31m$SAMPLE_CONFIG is not up-to-date. Regenerate it with \`scripts-dev/generate_sample_config\`.\e[0m" >&2
exit 1 exit 1
} }
diff -u "$SAMPLE_LOG_CONFIG" <(./scripts/generate_log_config) >/dev/null || { diff -u "$SAMPLE_LOG_CONFIG" <(synapse/_scripts/generate_log_config.py) >/dev/null || {
echo -e "\e[1m\e[31m$SAMPLE_LOG_CONFIG is not up-to-date. Regenerate it with \`scripts-dev/generate_sample_config\`.\e[0m" >&2 echo -e "\e[1m\e[31m$SAMPLE_LOG_CONFIG is not up-to-date. Regenerate it with \`scripts-dev/generate_sample_config\`.\e[0m" >&2
exit 1 exit 1
} }
else else
./scripts/generate_config --header-file docs/.sample_config_header.yaml -o "$SAMPLE_CONFIG" synapse/_scripts/generate_config.py --header-file docs/.sample_config_header.yaml -o "$SAMPLE_CONFIG"
./scripts/generate_log_config -o "$SAMPLE_LOG_CONFIG" synapse/_scripts/generate_log_config.py -o "$SAMPLE_LOG_CONFIG"
fi fi

View File

@ -84,13 +84,6 @@ else
files=( files=(
"synapse" "docker" "tests" "synapse" "docker" "tests"
# annoyingly, black doesn't find these so we have to list them # annoyingly, black doesn't find these so we have to list them
"scripts/export_signing_key"
"scripts/generate_config"
"scripts/generate_log_config"
"scripts/hash_password"
"scripts/register_new_matrix_user"
"scripts/synapse_port_db"
"scripts/update_synapse_database"
"scripts-dev" "scripts-dev"
"scripts-dev/build_debian_packages" "scripts-dev/build_debian_packages"
"scripts-dev/sign_json" "scripts-dev/sign_json"

View File

@ -147,7 +147,7 @@ python -m synapse.app.homeserver --generate-keys -c "$SQLITE_CONFIG"
# Make sure the SQLite3 database is using the latest schema and has no pending background update. # Make sure the SQLite3 database is using the latest schema and has no pending background update.
echo "Running db background jobs..." echo "Running db background jobs..."
scripts/update_synapse_database --database-config --run-background-updates "$SQLITE_CONFIG" synapse/_scripts/update_synapse_database.py --database-config --run-background-updates "$SQLITE_CONFIG"
# Create the PostgreSQL database. # Create the PostgreSQL database.
echo "Creating postgres database..." echo "Creating postgres database..."
@ -156,10 +156,10 @@ createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_DB_NAME"
echo "Copying data from SQLite3 to Postgres with synapse_port_db..." echo "Copying data from SQLite3 to Postgres with synapse_port_db..."
if [ -z "$COVERAGE" ]; then if [ -z "$COVERAGE" ]; then
# No coverage needed # No coverage needed
scripts/synapse_port_db --sqlite-database "$SQLITE_DB" --postgres-config "$POSTGRES_CONFIG" synapse/_scripts/synapse_port_db.py --sqlite-database "$SQLITE_DB" --postgres-config "$POSTGRES_CONFIG"
else else
# Coverage desired # Coverage desired
coverage run scripts/synapse_port_db --sqlite-database "$SQLITE_DB" --postgres-config "$POSTGRES_CONFIG" coverage run synapse/_scripts/synapse_port_db.py --sqlite-database "$SQLITE_DB" --postgres-config "$POSTGRES_CONFIG"
fi fi
# Delete schema_version, applied_schema_deltas and applied_module_schemas tables # Delete schema_version, applied_schema_deltas and applied_module_schemas tables

View File

@ -1,19 +0,0 @@
#!/usr/bin/env python
# Copyright 2015, 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from synapse._scripts.register_new_matrix_user import main
if __name__ == "__main__":
main()

View File

@ -1,19 +0,0 @@
#!/usr/bin/env python
# Copyright 2021 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from synapse._scripts.review_recent_signups import main
if __name__ == "__main__":
main()

View File

@ -1,45 +0,0 @@
#!/usr/bin/env perl
use strict;
use warnings;
use JSON::XS;
use LWP::UserAgent;
use URI::Escape;
if (@ARGV < 4) {
die "usage: $0 <homeserver url> <access_token> <room_id|room_alias> <group_id>\n";
}
my ($hs, $access_token, $room_id, $group_id) = @ARGV;
my $ua = LWP::UserAgent->new();
$ua->timeout(10);
if ($room_id =~ /^#/) {
$room_id = uri_escape($room_id);
$room_id = decode_json($ua->get("${hs}/_matrix/client/r0/directory/room/${room_id}?access_token=${access_token}")->decoded_content)->{room_id};
}
my $room_users = [ keys %{decode_json($ua->get("${hs}/_matrix/client/r0/rooms/${room_id}/joined_members?access_token=${access_token}")->decoded_content)->{joined}} ];
my $group_users = [
(map { $_->{user_id} } @{decode_json($ua->get("${hs}/_matrix/client/unstable/groups/${group_id}/users?access_token=${access_token}" )->decoded_content)->{chunk}}),
(map { $_->{user_id} } @{decode_json($ua->get("${hs}/_matrix/client/unstable/groups/${group_id}/invited_users?access_token=${access_token}" )->decoded_content)->{chunk}}),
];
die "refusing to sync from empty room" unless (@$room_users);
die "refusing to sync to empty group" unless (@$group_users);
my $diff = {};
foreach my $user (@$room_users) { $diff->{$user}++ }
foreach my $user (@$group_users) { $diff->{$user}-- }
foreach my $user (keys %$diff) {
if ($diff->{$user} == 1) {
warn "inviting $user";
print STDERR $ua->put("${hs}/_matrix/client/unstable/groups/${group_id}/admin/users/invite/${user}?access_token=${access_token}", Content=>'{}')->status_line."\n";
}
elsif ($diff->{$user} == -1) {
warn "removing $user";
print STDERR $ua->put("${hs}/_matrix/client/unstable/groups/${group_id}/admin/users/remove/${user}?access_token=${access_token}", Content=>'{}')->status_line."\n";
}
}

View File

@ -15,7 +15,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import glob
import os import os
from typing import Any, Dict from typing import Any, Dict
@ -153,8 +152,19 @@ setup(
python_requires="~=3.7", python_requires="~=3.7",
entry_points={ entry_points={
"console_scripts": [ "console_scripts": [
# Application
"synapse_homeserver = synapse.app.homeserver:main", "synapse_homeserver = synapse.app.homeserver:main",
"synapse_worker = synapse.app.generic_worker:main", "synapse_worker = synapse.app.generic_worker:main",
# Scripts
"export_signing_key = synapse._scripts.export_signing_key:main",
"generate_config = synapse._scripts.generate_config:main",
"generate_log_config = synapse._scripts.generate_log_config:main",
"generate_signing_key = synapse._scripts.generate_signing_key:main",
"hash_password = synapse._scripts.hash_password:main",
"register_new_matrix_user = synapse._scripts.register_new_matrix_user:main",
"synapse_port_db = synapse._scripts.synapse_port_db:main",
"synapse_review_recent_signups = synapse._scripts.review_recent_signups:main",
"update_synapse_database = synapse._scripts.update_synapse_database:main",
] ]
}, },
classifiers=[ classifiers=[
@ -167,6 +177,6 @@ setup(
"Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.10",
], ],
scripts=["synctl"] + glob.glob("scripts/*"), scripts=["synctl"],
cmdclass={"test": TestCommand}, cmdclass={"test": TestCommand},
) )

View File

@ -20,7 +20,7 @@ apps:
generate-config: generate-config:
command: generate_config command: generate_config
generate-signing-key: generate-signing-key:
command: generate_signing_key.py command: generate_signing_key
register-new-matrix-user: register-new-matrix-user:
command: register_new_matrix_user command: register_new_matrix_user
plugs: [network] plugs: [network]

View File

@ -50,7 +50,7 @@ def format_for_config(public_key: nacl.signing.VerifyKey, expiry_ts: int):
) )
if __name__ == "__main__": def main():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument( parser.add_argument(
@ -85,7 +85,6 @@ if __name__ == "__main__":
else format_plain else format_plain
) )
keys = []
for file in args.key_file: for file in args.key_file:
try: try:
res = read_signing_keys(file) res = read_signing_keys(file)
@ -98,3 +97,7 @@ if __name__ == "__main__":
res = [] res = []
for key in res: for key in res:
formatter(get_verify_key(key)) formatter(get_verify_key(key))
if __name__ == "__main__":
main()

View File

@ -6,7 +6,8 @@ import sys
from synapse.config.homeserver import HomeServerConfig from synapse.config.homeserver import HomeServerConfig
if __name__ == "__main__":
def main():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument( parser.add_argument(
"--config-dir", "--config-dir",
@ -76,3 +77,7 @@ if __name__ == "__main__":
shutil.copyfileobj(args.header_file, args.output_file) shutil.copyfileobj(args.header_file, args.output_file)
args.output_file.write(conf) args.output_file.write(conf)
if __name__ == "__main__":
main()

View File

@ -19,7 +19,8 @@ import sys
from synapse.config.logger import DEFAULT_LOG_CONFIG from synapse.config.logger import DEFAULT_LOG_CONFIG
if __name__ == "__main__":
def main():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument( parser.add_argument(
@ -42,3 +43,7 @@ if __name__ == "__main__":
out = args.output_file out = args.output_file
out.write(DEFAULT_LOG_CONFIG.substitute(log_file=args.log_file)) out.write(DEFAULT_LOG_CONFIG.substitute(log_file=args.log_file))
out.flush() out.flush()
if __name__ == "__main__":
main()

View File

@ -19,7 +19,8 @@ from signedjson.key import generate_signing_key, write_signing_keys
from synapse.util.stringutils import random_string from synapse.util.stringutils import random_string
if __name__ == "__main__":
def main():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument( parser.add_argument(
@ -34,3 +35,7 @@ if __name__ == "__main__":
key_id = "a_" + random_string(4) key_id = "a_" + random_string(4)
key = (generate_signing_key(key_id),) key = (generate_signing_key(key_id),)
write_signing_keys(args.output_file, key) write_signing_keys(args.output_file, key)
if __name__ == "__main__":
main()

View File

@ -8,9 +8,6 @@ import unicodedata
import bcrypt import bcrypt
import yaml import yaml
bcrypt_rounds = 12
password_pepper = ""
def prompt_for_pass(): def prompt_for_pass():
password = getpass.getpass("Password: ") password = getpass.getpass("Password: ")
@ -26,7 +23,10 @@ def prompt_for_pass():
return password return password
if __name__ == "__main__": def main():
bcrypt_rounds = 12
password_pepper = ""
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description=( description=(
"Calculate the hash of a new password, so that passwords can be reset" "Calculate the hash of a new password, so that passwords can be reset"
@ -77,3 +77,7 @@ if __name__ == "__main__":
).decode("ascii") ).decode("ascii")
print(hashed) print(hashed)
if __name__ == "__main__":
main()

View File

@ -28,7 +28,7 @@ This can be extracted from postgres with::
To use, pipe the above into:: To use, pipe the above into::
PYTHON_PATH=. ./scripts/move_remote_media_to_new_store.py <source repo> <dest repo> PYTHON_PATH=. synapse/_scripts/move_remote_media_to_new_store.py <source repo> <dest repo>
""" """
import argparse import argparse

View File

@ -1146,7 +1146,7 @@ class TerminalProgress(Progress):
############################################## ##############################################
if __name__ == "__main__": def main():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="A script to port an existing synapse SQLite database to" description="A script to port an existing synapse SQLite database to"
" a new PostgreSQL database." " a new PostgreSQL database."
@ -1251,3 +1251,7 @@ if __name__ == "__main__":
sys.stderr.write(end_error) sys.stderr.write(end_error)
sys.exit(5) sys.exit(5)
if __name__ == "__main__":
main()

View File

@ -383,7 +383,7 @@ class RootConfig:
Build a default configuration file Build a default configuration file
This is used when the user explicitly asks us to generate a config file This is used when the user explicitly asks us to generate a config file
(eg with --generate_config). (eg with --generate-config).
Args: Args:
config_dir_path: The path where the config files are kept. Used to config_dir_path: The path where the config files are kept. Used to

View File

@ -38,15 +38,7 @@ lint_targets =
setup.py setup.py
synapse synapse
tests tests
scripts
# annoyingly, black doesn't find these so we have to list them # annoyingly, black doesn't find these so we have to list them
scripts/export_signing_key
scripts/generate_config
scripts/generate_log_config
scripts/hash_password
scripts/register_new_matrix_user
scripts/synapse_port_db
scripts/update_synapse_database
scripts-dev scripts-dev
scripts-dev/build_debian_packages scripts-dev/build_debian_packages
scripts-dev/sign_json scripts-dev/sign_json