mirror of
https://git.anonymousland.org/anonymousland/synapse-product.git
synced 2024-10-01 08:25:44 -04:00
961ee75a9b
Of note: * No untyped defs in `register_new_matrix_user` This one might be contraversial. `request_registration` has three dependency-injection arguments used for testing. I'm removing the injection of the `requests` module and using `unitest.mock.patch` in the test cases instead. Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the change: ``` synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any" synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any" ``` And after: ``` synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType" synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response" ``` * Drive-by comment in `synapse.storage.types` * No untyped defs in `synapse_port_db` This was by far the most painful. I'm happy to break this up into smaller pieces for review if it's not managable as-is.
123 lines
3.6 KiB
Python
Executable File
123 lines
3.6 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# Copyright 2017 New Vector Ltd
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""
|
|
Moves a list of remote media from one media store to another.
|
|
|
|
The input should be a list of media files to be moved, one per line. Each line
|
|
should be formatted::
|
|
|
|
<origin server>|<file id>
|
|
|
|
This can be extracted from postgres with::
|
|
|
|
psql --tuples-only -A -c "select media_origin, filesystem_id from
|
|
matrix.remote_media_cache where ..."
|
|
|
|
To use, pipe the above into::
|
|
|
|
PYTHON_PATH=. synapse/_scripts/move_remote_media_to_new_store.py <source repo> <dest repo>
|
|
"""
|
|
|
|
import argparse
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import sys
|
|
|
|
from synapse.rest.media.v1.filepath import MediaFilePaths
|
|
|
|
logger = logging.getLogger()
|
|
|
|
|
|
def main(src_repo: str, dest_repo: str) -> None:
|
|
src_paths = MediaFilePaths(src_repo)
|
|
dest_paths = MediaFilePaths(dest_repo)
|
|
for line in sys.stdin:
|
|
line = line.strip()
|
|
parts = line.split("|")
|
|
if len(parts) != 2:
|
|
print("Unable to parse input line %s" % line, file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
move_media(parts[0], parts[1], src_paths, dest_paths)
|
|
|
|
|
|
def move_media(
|
|
origin_server: str,
|
|
file_id: str,
|
|
src_paths: MediaFilePaths,
|
|
dest_paths: MediaFilePaths,
|
|
) -> None:
|
|
"""Move the given file, and any thumbnails, to the dest repo
|
|
|
|
Args:
|
|
origin_server:
|
|
file_id:
|
|
src_paths:
|
|
dest_paths:
|
|
"""
|
|
logger.info("%s/%s", origin_server, file_id)
|
|
|
|
# check that the original exists
|
|
original_file = src_paths.remote_media_filepath(origin_server, file_id)
|
|
if not os.path.exists(original_file):
|
|
logger.warning(
|
|
"Original for %s/%s (%s) does not exist",
|
|
origin_server,
|
|
file_id,
|
|
original_file,
|
|
)
|
|
else:
|
|
mkdir_and_move(
|
|
original_file, dest_paths.remote_media_filepath(origin_server, file_id)
|
|
)
|
|
|
|
# now look for thumbnails
|
|
original_thumb_dir = src_paths.remote_media_thumbnail_dir(origin_server, file_id)
|
|
if not os.path.exists(original_thumb_dir):
|
|
return
|
|
|
|
mkdir_and_move(
|
|
original_thumb_dir,
|
|
dest_paths.remote_media_thumbnail_dir(origin_server, file_id),
|
|
)
|
|
|
|
|
|
def mkdir_and_move(original_file: str, dest_file: str) -> None:
|
|
dirname = os.path.dirname(dest_file)
|
|
if not os.path.exists(dirname):
|
|
logger.debug("mkdir %s", dirname)
|
|
os.makedirs(dirname)
|
|
logger.debug("mv %s %s", original_file, dest_file)
|
|
shutil.move(original_file, dest_file)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(
|
|
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
|
|
)
|
|
parser.add_argument("-v", action="store_true", help="enable debug logging")
|
|
parser.add_argument("src_repo", help="Path to source content repo")
|
|
parser.add_argument("dest_repo", help="Path to source content repo")
|
|
args = parser.parse_args()
|
|
|
|
logging.basicConfig(
|
|
level=logging.DEBUG if args.v else logging.INFO,
|
|
format="%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s",
|
|
)
|
|
|
|
main(args.src_repo, args.dest_repo)
|