2015-05-06 10:41:19 +00:00
|
|
|
#!/usr/bin/env python
|
2015-04-10 17:47:20 +00:00
|
|
|
#
|
2023-11-21 20:29:58 +00:00
|
|
|
# This file is licensed under the Affero General Public License (AGPL) version 3.
|
|
|
|
#
|
2024-01-23 11:26:48 +00:00
|
|
|
# Copyright 2019 The Matrix.org Foundation C.I.C.
|
|
|
|
# Copyright 2015, 2016 OpenMarket Ltd
|
2023-11-21 20:29:58 +00:00
|
|
|
# Copyright (C) 2023 New Vector, Ltd
|
|
|
|
#
|
|
|
|
# This program is free software: you can redistribute it and/or modify
|
|
|
|
# it under the terms of the GNU Affero General Public License as
|
|
|
|
# published by the Free Software Foundation, either version 3 of the
|
|
|
|
# License, or (at your option) any later version.
|
|
|
|
#
|
|
|
|
# See the GNU Affero General Public License for more details:
|
|
|
|
# <https://www.gnu.org/licenses/agpl-3.0.html>.
|
|
|
|
#
|
|
|
|
# Originally licensed under the Apache License, Version 2.0:
|
|
|
|
# <http://www.apache.org/licenses/LICENSE-2.0>.
|
|
|
|
#
|
|
|
|
# [This file includes modifications made by New Vector Limited]
|
2015-04-10 17:47:20 +00:00
|
|
|
#
|
|
|
|
#
|
|
|
|
|
|
|
|
import argparse
|
2015-04-17 10:13:05 +00:00
|
|
|
import curses
|
2015-04-10 17:47:20 +00:00
|
|
|
import logging
|
2023-03-22 15:36:42 +00:00
|
|
|
import os
|
2015-04-17 10:13:05 +00:00
|
|
|
import sys
|
|
|
|
import time
|
|
|
|
import traceback
|
2022-04-08 14:00:12 +00:00
|
|
|
from types import TracebackType
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
from typing import (
|
|
|
|
Any,
|
|
|
|
Awaitable,
|
|
|
|
Callable,
|
|
|
|
Dict,
|
|
|
|
Generator,
|
|
|
|
Iterable,
|
|
|
|
List,
|
|
|
|
NoReturn,
|
|
|
|
Optional,
|
|
|
|
Set,
|
|
|
|
Tuple,
|
|
|
|
Type,
|
|
|
|
TypeVar,
|
|
|
|
cast,
|
|
|
|
)
|
2015-04-10 17:47:20 +00:00
|
|
|
|
2018-10-20 00:16:55 +00:00
|
|
|
import yaml
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
from typing_extensions import TypedDict
|
2018-10-20 00:16:55 +00:00
|
|
|
|
2022-04-08 14:00:12 +00:00
|
|
|
from twisted.internet import defer, reactor as reactor_
|
2018-10-20 00:16:55 +00:00
|
|
|
|
2019-12-18 10:45:12 +00:00
|
|
|
from synapse.config.database import DatabaseConnectionConfig
|
2019-10-23 14:31:59 +00:00
|
|
|
from synapse.config.homeserver import HomeServerConfig
|
2020-01-21 19:04:58 +00:00
|
|
|
from synapse.logging.context import (
|
|
|
|
LoggingContext,
|
|
|
|
make_deferred_yieldable,
|
|
|
|
run_in_background,
|
|
|
|
)
|
2023-01-20 18:02:18 +00:00
|
|
|
from synapse.notifier import ReplicationNotifier
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
from synapse.storage.database import DatabasePool, LoggingTransaction, make_conn
|
2024-03-28 15:44:07 +00:00
|
|
|
from synapse.storage.databases.main import FilteringWorkerStore
|
2022-02-02 11:37:18 +00:00
|
|
|
from synapse.storage.databases.main.account_data import AccountDataWorkerStore
|
2020-08-05 20:38:57 +00:00
|
|
|
from synapse.storage.databases.main.client_ips import ClientIpBackgroundUpdateStore
|
|
|
|
from synapse.storage.databases.main.deviceinbox import DeviceInboxBackgroundUpdateStore
|
|
|
|
from synapse.storage.databases.main.devices import DeviceBackgroundUpdateStore
|
2023-04-04 20:16:08 +00:00
|
|
|
from synapse.storage.databases.main.e2e_room_keys import EndToEndRoomKeyBackgroundStore
|
2020-11-02 13:55:56 +00:00
|
|
|
from synapse.storage.databases.main.end_to_end_keys import EndToEndKeyBackgroundStore
|
2023-07-05 09:43:19 +00:00
|
|
|
from synapse.storage.databases.main.event_federation import EventFederationWorkerStore
|
2022-06-30 14:05:49 +00:00
|
|
|
from synapse.storage.databases.main.event_push_actions import EventPushActionsStore
|
2020-08-05 20:38:57 +00:00
|
|
|
from synapse.storage.databases.main.events_bg_updates import (
|
2019-10-24 12:31:49 +00:00
|
|
|
EventsBackgroundUpdatesStore,
|
2019-10-23 14:54:17 +00:00
|
|
|
)
|
2020-08-05 20:38:57 +00:00
|
|
|
from synapse.storage.databases.main.media_repository import (
|
2019-10-24 12:31:49 +00:00
|
|
|
MediaRepositoryBackgroundUpdateStore,
|
2019-10-23 14:54:17 +00:00
|
|
|
)
|
2021-09-03 16:16:56 +00:00
|
|
|
from synapse.storage.databases.main.presence import PresenceBackgroundUpdateStore
|
2023-04-26 23:03:26 +00:00
|
|
|
from synapse.storage.databases.main.profile import ProfileWorkerStore
|
2024-03-28 15:44:07 +00:00
|
|
|
from synapse.storage.databases.main.push_rule import PusherWorkerStore
|
|
|
|
from synapse.storage.databases.main.pusher import PusherBackgroundUpdatesStore
|
2022-09-13 07:16:37 +00:00
|
|
|
from synapse.storage.databases.main.receipts import ReceiptsBackgroundUpdateStore
|
2020-08-05 20:38:57 +00:00
|
|
|
from synapse.storage.databases.main.registration import (
|
2019-10-24 12:31:49 +00:00
|
|
|
RegistrationBackgroundUpdateStore,
|
2020-07-16 10:46:44 +00:00
|
|
|
find_max_generated_user_id_localpart,
|
2019-10-23 14:54:17 +00:00
|
|
|
)
|
2022-10-13 12:02:11 +00:00
|
|
|
from synapse.storage.databases.main.relations import RelationsWorkerStore
|
2020-08-05 20:38:57 +00:00
|
|
|
from synapse.storage.databases.main.room import RoomBackgroundUpdateStore
|
|
|
|
from synapse.storage.databases.main.roommember import RoomMemberBackgroundUpdateStore
|
|
|
|
from synapse.storage.databases.main.search import SearchBackgroundUpdateStore
|
|
|
|
from synapse.storage.databases.main.state import MainStateBackgroundUpdateStore
|
|
|
|
from synapse.storage.databases.main.stats import StatsStore
|
|
|
|
from synapse.storage.databases.main.user_directory import (
|
2019-10-24 12:31:49 +00:00
|
|
|
UserDirectoryBackgroundUpdateStore,
|
2019-10-23 14:54:17 +00:00
|
|
|
)
|
2020-08-05 20:38:57 +00:00
|
|
|
from synapse.storage.databases.state.bg_updates import StateBackgroundUpdateStore
|
2019-10-24 12:31:49 +00:00
|
|
|
from synapse.storage.engines import create_engine
|
|
|
|
from synapse.storage.prepare_database import prepare_database
|
2022-04-08 14:00:12 +00:00
|
|
|
from synapse.types import ISynapseReactor
|
2022-06-07 14:24:11 +00:00
|
|
|
from synapse.util import SYNAPSE_VERSION, Clock
|
2015-04-10 17:47:20 +00:00
|
|
|
|
2022-04-08 14:00:12 +00:00
|
|
|
# Cast safety: Twisted does some naughty magic which replaces the
|
|
|
|
# twisted.internet.reactor module with a Reactor instance at runtime.
|
|
|
|
reactor = cast(ISynapseReactor, reactor_)
|
2015-09-03 06:51:42 +00:00
|
|
|
logger = logging.getLogger("synapse_port_db")
|
2015-04-10 17:47:20 +00:00
|
|
|
|
|
|
|
|
2023-02-20 13:35:24 +00:00
|
|
|
# SQLite doesn't have a dedicated boolean type (it stores True/False as 1/0). This means
|
|
|
|
# portdb will read sqlite bools as integers, then try to insert them into postgres
|
|
|
|
# boolean columns---which fails. Lacking some Python-parseable metaschema, we must
|
|
|
|
# specify which integer columns should be inserted as booleans into postgres.
|
2015-04-15 09:23:24 +00:00
|
|
|
BOOLEAN_COLUMNS = {
|
2023-02-20 13:35:24 +00:00
|
|
|
"access_tokens": ["used"],
|
|
|
|
"account_validity": ["email_sent"],
|
|
|
|
"device_lists_changes_in_room": ["converted_to_destinations"],
|
|
|
|
"device_lists_outbound_pokes": ["sent"],
|
|
|
|
"devices": ["hidden"],
|
|
|
|
"e2e_fallback_keys_json": ["used"],
|
|
|
|
"e2e_room_keys": ["is_verified"],
|
2015-04-15 09:23:24 +00:00
|
|
|
"event_edges": ["is_state"],
|
2023-02-20 13:35:24 +00:00
|
|
|
"events": ["processed", "outlier", "contains_url"],
|
2024-07-22 09:33:17 +00:00
|
|
|
"local_media_repository": ["safe_from_quarantine", "authenticated"],
|
|
|
|
"per_user_experimental_features": ["enabled"],
|
2015-04-15 09:23:24 +00:00
|
|
|
"presence_list": ["accepted"],
|
2016-04-06 13:08:18 +00:00
|
|
|
"presence_stream": ["currently_active"],
|
2016-09-27 10:19:45 +00:00
|
|
|
"public_room_list_stream": ["visibility"],
|
2023-02-20 13:35:24 +00:00
|
|
|
"pushers": ["enabled"],
|
2019-10-18 09:13:59 +00:00
|
|
|
"redactions": ["have_censored"],
|
2024-07-22 09:33:17 +00:00
|
|
|
"remote_media_cache": ["authenticated"],
|
2019-10-18 09:13:59 +00:00
|
|
|
"room_stats_state": ["is_federatable"],
|
2023-02-20 13:35:24 +00:00
|
|
|
"rooms": ["is_public", "has_auth_chain_index"],
|
Sliding Sync: Pre-populate room data for quick filtering/sorting (#17512)
Pre-populate room data for quick filtering/sorting in the Sliding Sync
API
Spawning from
https://github.com/element-hq/synapse/pull/17450#discussion_r1697335578
This PR is acting as the Synapse version `N+1` step in the gradual
migration being tracked by
https://github.com/element-hq/synapse/issues/17623
Adding two new database tables:
- `sliding_sync_joined_rooms`: A table for storing room meta data that
the local server is still participating in. The info here can be shared
across all `Membership.JOIN`. Keyed on `(room_id)` and updated when the
relevant room current state changes or a new event is sent in the room.
- `sliding_sync_membership_snapshots`: A table for storing a snapshot of
room meta data at the time of the local user's membership. Keyed on
`(room_id, user_id)` and only updated when a user's membership in a room
changes.
Also adds background updates to populate these tables with all of the
existing data.
We want to have the guarantee that if a row exists in the sliding sync
tables, we are able to rely on it (accurate data). And if a row doesn't
exist, we use a fallback to get the same info until the background
updates fill in the rows or a new event comes in triggering it to be
fully inserted. This means we need a couple extra things in place until
we bump `SCHEMA_COMPAT_VERSION` and run the foreground update in the
`N+2` part of the gradual migration. For context on why we can't rely on
the tables without these things see [1].
1. On start-up, block until we clear out any rows for the rooms that
have had events since the max-`stream_ordering` of the
`sliding_sync_joined_rooms` table (compare to max-`stream_ordering` of
the `events` table). For `sliding_sync_membership_snapshots`, we can
compare to the max-`stream_ordering` of `local_current_membership`
- This accounts for when someone downgrades their Synapse version and
then upgrades it again. This will ensure that we don't have any
stale/out-of-date data in the
`sliding_sync_joined_rooms`/`sliding_sync_membership_snapshots` tables
since any new events sent in rooms would have also needed to be written
to the sliding sync tables. For example a new event needs to bump
`event_stream_ordering` in `sliding_sync_joined_rooms` table or some
state in the room changing (like the room name). Or another example of
someone's membership changing in a room affecting
`sliding_sync_membership_snapshots`.
1. Add another background update that will catch-up with any rows that
were just deleted from the sliding sync tables (based on the activity in
the `events`/`local_current_membership`). The rooms that need
recalculating are added to the
`sliding_sync_joined_rooms_to_recalculate` table.
1. Making sure rows are fully inserted. Instead of partially inserting,
we need to check if the row already exists and fully insert all data if
not.
All of this extra functionality can be removed once the
`SCHEMA_COMPAT_VERSION` is bumped with support for the new sliding sync
tables so people can no longer downgrade (the `N+2` part of the gradual
migration).
<details>
<summary><sup>[1]</sup></summary>
For `sliding_sync_joined_rooms`, since we partially insert rows as state
comes in, we can't rely on the existence of the row for a given
`room_id`. We can't even rely on looking at whether the background
update has finished. There could still be partial rows from when someone
reverted their Synapse version after the background update finished, had
some state changes (or new rooms), then upgraded again and more state
changes happen leaving a partial row.
For `sliding_sync_membership_snapshots`, we insert items as a whole
except for the `forgotten` column ~~so we can rely on rows existing and
just need to always use a fallback for the `forgotten` data. We can't
use the `forgotten` column in the table for the same reasons above about
`sliding_sync_joined_rooms`.~~ We could have an out-of-date membership
from when someone reverted their Synapse version. (same problems as
outlined for `sliding_sync_joined_rooms` above)
Discussed in an [internal
meeting](https://docs.google.com/document/d/1MnuvPkaCkT_wviSQZ6YKBjiWciCBFMd-7hxyCO-OCbQ/edit#bookmark=id.dz5x6ef4mxz7)
</details>
### TODO
- [x] Update `stream_ordering`/`bump_stamp`
- [x] Handle remote invites
- [x] Handle state resets
- [x] Consider adding `sender` so we can filter `LEAVE` memberships and
distinguish from kicks.
- [x] We should add it to be able to tell leaves from kicks
- [x] Consider adding `tombstone` state to help address
https://github.com/element-hq/synapse/issues/17540
- [x] We should add it `tombstone_successor_room_id`
- [x] Consider adding `forgotten` status to avoid extra
lookup/table-join on `room_memberships`
- [x] We should add it
- [x] Background update to fill in values for all joined rooms and
non-join membership
- [x] Clean-up tables when room is deleted
- [ ] Make sure tables are useful to our use case
- First explored in
https://github.com/element-hq/synapse/compare/erikj/ss_use_new_tables
- Also explored in
https://github.com/element-hq/synapse/commit/76b5a576eb363496315dfd39510cad7d02b0fc73
- [x] Plan for how can we use this with a fallback
- See plan discussed above in main area of the issue description
- Discussed in an [internal
meeting](https://docs.google.com/document/d/1MnuvPkaCkT_wviSQZ6YKBjiWciCBFMd-7hxyCO-OCbQ/edit#bookmark=id.dz5x6ef4mxz7)
- [x] Plan for how we can rely on this new table without a fallback
- Synapse version `N+1`: (this PR) Bump `SCHEMA_VERSION` to `87`. Add
new tables and background update to backfill all rows. Since this is a
new table, we don't have to add any `NOT VALID` constraints and validate
them when the background update completes. Read from new tables with a
fallback in cases where the rows aren't filled in yet.
- Synapse version `N+2`: Bump `SCHEMA_VERSION` to `88` and bump
`SCHEMA_COMPAT_VERSION` to `87` because we don't want people to
downgrade and miss writes while they are on an older version. Add a
foreground update to finish off the backfill so we can read from new
tables without the fallback. Application code can now rely on the new
tables being populated.
- Discussed in an [internal
meeting](https://docs.google.com/document/d/1MnuvPkaCkT_wviSQZ6YKBjiWciCBFMd-7hxyCO-OCbQ/edit#bookmark=id.hh7shg4cxdhj)
### Dev notes
```
SYNAPSE_TEST_LOG_LEVEL=INFO poetry run trial tests.storage.test_events.SlidingSyncPrePopulatedTablesTestCase
SYNAPSE_POSTGRES=1 SYNAPSE_POSTGRES_USER=postgres SYNAPSE_TEST_LOG_LEVEL=INFO poetry run trial tests.storage.test_events.SlidingSyncPrePopulatedTablesTestCase
```
```
SYNAPSE_TEST_LOG_LEVEL=INFO poetry run trial tests.handlers.test_sliding_sync.FilterRoomsTestCase
```
Reference:
- [Development docs on background updates and worked examples of gradual
migrations
](https://github.com/element-hq/synapse/blob/1dfa59b238cee0dc62163588cc9481896c288979/docs/development/database_schema.md#background-updates)
- A real example of a gradual migration:
https://github.com/matrix-org/synapse/pull/15649#discussion_r1213779514
- Adding `rooms.creator` field that needed a background update to
backfill data, https://github.com/matrix-org/synapse/pull/10697
- Adding `rooms.room_version` that needed a background update to
backfill data, https://github.com/matrix-org/synapse/pull/6729
- Adding `room_stats_state.room_type` that needed a background update to
backfill data, https://github.com/matrix-org/synapse/pull/13031
- Tables from MSC2716: `insertion_events`, `insertion_event_edges`,
`insertion_event_extremities`, `batch_events`
- `current_state_events` updated in
`synapse/storage/databases/main/events.py`
---
```
persist_event (adds to queue)
_persist_event_batch
_persist_events_and_state_updates (assigns `stream_ordering` to events)
_persist_events_txn
_store_event_txn
_update_metadata_tables_txn
_store_room_members_txn
_update_current_state_txn
```
---
> Concatenated Indexes [...] (also known as multi-column, composite or
combined index)
>
> [...] key consists of multiple columns.
>
> We can take advantage of the fact that the first index column is
always usable for searching
>
> *--
https://use-the-index-luke.com/sql/where-clause/the-equals-operator/concatenated-keys*
---
Dealing with `portdb` (`synapse/_scripts/synapse_port_db.py`),
https://github.com/element-hq/synapse/pull/17512#discussion_r1725998219
---
<details>
<summary>SQL queries:</summary>
Both of these are equivalent and work in SQLite and Postgres
Options 1:
```sql
WITH data_table (room_id, user_id, membership_event_id, membership, event_stream_ordering, {", ".join(insert_keys)}) AS (
VALUES (
?, ?, ?,
(SELECT membership FROM room_memberships WHERE event_id = ?),
(SELECT stream_ordering FROM events WHERE event_id = ?),
{", ".join("?" for _ in insert_values)}
)
)
INSERT INTO sliding_sync_non_join_memberships
(room_id, user_id, membership_event_id, membership, event_stream_ordering, {", ".join(insert_keys)})
SELECT * FROM data_table
WHERE membership != ?
ON CONFLICT (room_id, user_id)
DO UPDATE SET
membership_event_id = EXCLUDED.membership_event_id,
membership = EXCLUDED.membership,
event_stream_ordering = EXCLUDED.event_stream_ordering,
{", ".join(f"{key} = EXCLUDED.{key}" for key in insert_keys)}
```
Option 2:
```sql
INSERT INTO sliding_sync_non_join_memberships
(room_id, user_id, membership_event_id, membership, event_stream_ordering, {", ".join(insert_keys)})
SELECT
column1 as room_id,
column2 as user_id,
column3 as membership_event_id,
column4 as membership,
column5 as event_stream_ordering,
{", ".join("column" + str(i) for i in range(6, 6 + len(insert_keys)))}
FROM (
VALUES (
?, ?, ?,
(SELECT membership FROM room_memberships WHERE event_id = ?),
(SELECT stream_ordering FROM events WHERE event_id = ?),
{", ".join("?" for _ in insert_values)}
)
) as v
WHERE membership != ?
ON CONFLICT (room_id, user_id)
DO UPDATE SET
membership_event_id = EXCLUDED.membership_event_id,
membership = EXCLUDED.membership,
event_stream_ordering = EXCLUDED.event_stream_ordering,
{", ".join(f"{key} = EXCLUDED.{key}" for key in insert_keys)}
```
If we don't need the `membership` condition, we could use:
```sql
INSERT INTO sliding_sync_non_join_memberships
(room_id, membership_event_id, user_id, membership, event_stream_ordering, {", ".join(insert_keys)})
VALUES (
?, ?, ?,
(SELECT membership FROM room_memberships WHERE event_id = ?),
(SELECT stream_ordering FROM events WHERE event_id = ?),
{", ".join("?" for _ in insert_values)}
)
ON CONFLICT (room_id, user_id)
DO UPDATE SET
membership_event_id = EXCLUDED.membership_event_id,
membership = EXCLUDED.membership,
event_stream_ordering = EXCLUDED.event_stream_ordering,
{", ".join(f"{key} = EXCLUDED.{key}" for key in insert_keys)}
```
</details>
### Pull Request Checklist
<!-- Please read
https://element-hq.github.io/synapse/latest/development/contributing_guide.html
before submitting your pull request -->
* [x] Pull request is based on the develop branch
* [x] Pull request includes a [changelog
file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog).
The entry should:
- Be a short description of your change which makes sense to users.
"Fixed a bug that prevented receiving messages from other servers."
instead of "Moved X method from `EventStore` to `EventWorkerStore`.".
- Use markdown where necessary, mostly for `code blocks`.
- End with either a period (.) or an exclamation mark (!).
- Start with a capital letter.
- Feel free to credit yourself, by adding a sentence "Contributed by
@github_username." or "Contributed by [Your Name]." to the end of the
entry.
* [x] [Code
style](https://element-hq.github.io/synapse/latest/code_style.html) is
correct
(run the
[linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters))
---------
Co-authored-by: Erik Johnston <erik@matrix.org>
2024-08-29 15:09:51 +00:00
|
|
|
"sliding_sync_joined_rooms": ["is_encrypted"],
|
|
|
|
"sliding_sync_membership_snapshots": [
|
|
|
|
"has_known_state",
|
|
|
|
"is_encrypted",
|
|
|
|
],
|
2024-05-01 16:45:17 +00:00
|
|
|
"users": ["shadow_banned", "approved", "locked", "suspended"],
|
2023-02-20 13:35:24 +00:00
|
|
|
"un_partial_stated_event_stream": ["rejection_status_changed"],
|
|
|
|
"users_who_share_rooms": ["share_private"],
|
2015-04-15 09:23:24 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2023-02-20 13:35:24 +00:00
|
|
|
# These tables are never deleted from in normal operation [*], so we can resume porting
|
|
|
|
# over rows from a previous attempt rather than starting from scratch.
|
|
|
|
#
|
|
|
|
# [*]: We do delete from many of these tables when purging a room, and
|
|
|
|
# presumably when purging old events. So we might e.g.
|
|
|
|
#
|
|
|
|
# 1. Run portdb and port half of some table.
|
|
|
|
# 2. Stop portdb.
|
|
|
|
# 3. Purge something, deleting some of the rows we've ported over.
|
|
|
|
# 4. Restart portdb. The rows deleted from sqlite are still present in postgres.
|
|
|
|
#
|
|
|
|
# But this isn't the end of the world: we should be able to repeat the purge
|
|
|
|
# on the postgres DB when porting completes.
|
2015-04-13 12:45:04 +00:00
|
|
|
APPEND_ONLY_TABLES = [
|
2023-02-20 13:35:24 +00:00
|
|
|
"cache_invalidation_stream_by_instance",
|
|
|
|
"event_auth",
|
|
|
|
"event_edges",
|
|
|
|
"event_json",
|
2015-04-13 12:45:04 +00:00
|
|
|
"event_reference_hashes",
|
2023-02-20 13:35:24 +00:00
|
|
|
"event_search",
|
|
|
|
"event_to_state_groups",
|
2015-04-13 12:45:04 +00:00
|
|
|
"events",
|
2023-02-20 13:35:24 +00:00
|
|
|
"ex_outlier_stream",
|
2015-04-13 12:45:04 +00:00
|
|
|
"local_media_repository",
|
|
|
|
"local_media_repository_thumbnails",
|
2023-02-20 13:35:24 +00:00
|
|
|
"presence_stream",
|
|
|
|
"public_room_list_stream",
|
|
|
|
"push_rules_stream",
|
|
|
|
"received_transactions",
|
|
|
|
"redactions",
|
|
|
|
"rejections",
|
2015-04-13 12:45:04 +00:00
|
|
|
"remote_media_cache",
|
|
|
|
"remote_media_cache_thumbnails",
|
2023-02-20 13:35:24 +00:00
|
|
|
"room_memberships",
|
|
|
|
"room_names",
|
|
|
|
"rooms",
|
2015-04-13 12:45:04 +00:00
|
|
|
"sent_transactions",
|
2023-02-20 13:35:24 +00:00
|
|
|
"state_events",
|
|
|
|
"state_group_edges",
|
2015-04-13 12:45:04 +00:00
|
|
|
"state_groups",
|
|
|
|
"state_groups_state",
|
2016-09-27 10:19:45 +00:00
|
|
|
"stream_ordering_to_exterm",
|
2023-02-20 13:35:24 +00:00
|
|
|
"topics",
|
|
|
|
"transaction_id_to_pdu",
|
|
|
|
"un_partial_stated_event_stream",
|
|
|
|
"users",
|
2015-04-13 12:45:04 +00:00
|
|
|
]
|
|
|
|
|
|
|
|
|
2020-06-17 14:01:18 +00:00
|
|
|
IGNORED_TABLES = {
|
2020-06-23 11:33:25 +00:00
|
|
|
# We don't port these tables, as they're a faff and we can regenerate
|
|
|
|
# them anyway.
|
2020-06-17 14:01:18 +00:00
|
|
|
"user_directory",
|
|
|
|
"user_directory_search",
|
2020-06-23 11:33:25 +00:00
|
|
|
"user_directory_search_content",
|
|
|
|
"user_directory_search_docsize",
|
|
|
|
"user_directory_search_segdir",
|
|
|
|
"user_directory_search_segments",
|
|
|
|
"user_directory_search_stat",
|
|
|
|
"user_directory_search_pos",
|
|
|
|
"users_who_share_private_rooms",
|
2023-10-27 19:14:02 +00:00
|
|
|
"users_in_public_rooms",
|
2020-06-23 11:33:25 +00:00
|
|
|
# UI auth sessions have foreign keys so additional care needs to be taken,
|
|
|
|
# the sessions are transient anyway, so ignore them.
|
2020-06-17 14:01:18 +00:00
|
|
|
"ui_auth_sessions",
|
|
|
|
"ui_auth_sessions_credentials",
|
2020-09-28 14:35:02 +00:00
|
|
|
"ui_auth_sessions_ips",
|
2023-07-05 16:25:00 +00:00
|
|
|
# Ignore the worker locks table, as a) there shouldn't be any acquired locks
|
|
|
|
# after porting, and b) the circular foreign key constraints make it hard to
|
|
|
|
# port.
|
|
|
|
"worker_read_write_locks_mode",
|
|
|
|
"worker_read_write_locks",
|
2020-06-17 14:01:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2020-01-21 19:04:58 +00:00
|
|
|
# Error returned by the run function. Used at the top-level part of the script to
|
|
|
|
# handle errors and return codes.
|
2022-04-08 14:00:12 +00:00
|
|
|
end_error: Optional[str] = None
|
2020-01-21 19:04:58 +00:00
|
|
|
# The exec_info for the error, if any. If error is defined but not exec_info the script
|
|
|
|
# will show only the error message without the stacktrace, if exec_info is defined but
|
|
|
|
# not the error then the script will show nothing outside of what's printed in the run
|
|
|
|
# function. If both are defined, the script will print both the error and the stacktrace.
|
2022-04-08 14:00:12 +00:00
|
|
|
end_error_exec_info: Optional[
|
|
|
|
Tuple[Type[BaseException], BaseException, TracebackType]
|
|
|
|
] = None
|
2015-04-17 10:13:05 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
R = TypeVar("R")
|
|
|
|
|
2015-04-17 10:13:05 +00:00
|
|
|
|
2019-10-23 14:31:59 +00:00
|
|
|
class Store(
|
2022-06-30 14:05:49 +00:00
|
|
|
EventPushActionsStore,
|
2019-10-23 14:31:59 +00:00
|
|
|
ClientIpBackgroundUpdateStore,
|
|
|
|
DeviceInboxBackgroundUpdateStore,
|
|
|
|
DeviceBackgroundUpdateStore,
|
|
|
|
EventsBackgroundUpdatesStore,
|
|
|
|
MediaRepositoryBackgroundUpdateStore,
|
|
|
|
RegistrationBackgroundUpdateStore,
|
2019-12-04 17:57:35 +00:00
|
|
|
RoomBackgroundUpdateStore,
|
2019-10-23 14:31:59 +00:00
|
|
|
RoomMemberBackgroundUpdateStore,
|
|
|
|
SearchBackgroundUpdateStore,
|
|
|
|
StateBackgroundUpdateStore,
|
2019-12-20 10:48:24 +00:00
|
|
|
MainStateBackgroundUpdateStore,
|
2019-10-23 14:31:59 +00:00
|
|
|
UserDirectoryBackgroundUpdateStore,
|
2020-11-02 13:55:56 +00:00
|
|
|
EndToEndKeyBackgroundStore,
|
2023-04-04 20:16:08 +00:00
|
|
|
EndToEndRoomKeyBackgroundStore,
|
2019-10-23 14:31:59 +00:00
|
|
|
StatsStore,
|
2022-02-02 11:37:18 +00:00
|
|
|
AccountDataWorkerStore,
|
2023-04-26 23:03:26 +00:00
|
|
|
FilteringWorkerStore,
|
|
|
|
ProfileWorkerStore,
|
2021-03-02 14:05:01 +00:00
|
|
|
PusherWorkerStore,
|
2023-03-24 15:09:39 +00:00
|
|
|
PusherBackgroundUpdatesStore,
|
2021-09-03 16:16:56 +00:00
|
|
|
PresenceBackgroundUpdateStore,
|
2022-09-13 07:16:37 +00:00
|
|
|
ReceiptsBackgroundUpdateStore,
|
2022-10-13 12:02:11 +00:00
|
|
|
RelationsWorkerStore,
|
2023-07-05 09:43:19 +00:00
|
|
|
EventFederationWorkerStore,
|
2019-10-23 14:31:59 +00:00
|
|
|
):
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def execute(self, f: Callable[..., R], *args: Any, **kwargs: Any) -> Awaitable[R]:
|
2020-08-05 20:38:57 +00:00
|
|
|
return self.db_pool.runInteraction(f.__name__, f, *args, **kwargs)
|
2015-04-17 10:13:05 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def execute_sql(self, sql: str, *args: object) -> Awaitable[List[Tuple]]:
|
|
|
|
def r(txn: LoggingTransaction) -> List[Tuple]:
|
2015-04-28 10:16:44 +00:00
|
|
|
txn.execute(sql, args)
|
|
|
|
return txn.fetchall()
|
2018-10-20 00:16:55 +00:00
|
|
|
|
2020-08-05 20:38:57 +00:00
|
|
|
return self.db_pool.runInteraction("execute_sql", r)
|
2015-04-28 10:16:44 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def insert_many_txn(
|
|
|
|
self, txn: LoggingTransaction, table: str, headers: List[str], rows: List[Tuple]
|
|
|
|
) -> None:
|
2015-04-10 17:47:20 +00:00
|
|
|
sql = "INSERT INTO %s (%s) VALUES (%s)" % (
|
|
|
|
table,
|
|
|
|
", ".join(k for k in headers),
|
2018-10-20 00:16:55 +00:00
|
|
|
", ".join("%s" for _ in headers),
|
2015-04-10 17:47:20 +00:00
|
|
|
)
|
|
|
|
|
2015-04-15 09:23:24 +00:00
|
|
|
try:
|
|
|
|
txn.executemany(sql, rows)
|
2018-10-20 00:16:55 +00:00
|
|
|
except Exception:
|
|
|
|
logger.exception("Failed to insert: %s", table)
|
2015-04-15 09:23:24 +00:00
|
|
|
raise
|
2015-04-10 17:47:20 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
# Note: the parent method is an `async def`.
|
|
|
|
def set_room_is_public(self, room_id: str, is_public: bool) -> NoReturn:
|
2020-01-07 14:18:43 +00:00
|
|
|
raise Exception(
|
|
|
|
"Attempt to set room_is_public during port_db: database not empty?"
|
|
|
|
)
|
|
|
|
|
2015-04-10 17:47:20 +00:00
|
|
|
|
2019-10-23 14:31:59 +00:00
|
|
|
class MockHomeserver:
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def __init__(self, config: HomeServerConfig):
|
2019-10-23 14:31:59 +00:00
|
|
|
self.clock = Clock(reactor)
|
|
|
|
self.config = config
|
2021-10-06 14:47:41 +00:00
|
|
|
self.hostname = config.server.server_name
|
2022-06-07 14:24:11 +00:00
|
|
|
self.version_string = SYNAPSE_VERSION
|
2019-10-23 14:31:59 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def get_clock(self) -> Clock:
|
2019-10-23 14:31:59 +00:00
|
|
|
return self.clock
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def get_reactor(self) -> ISynapseReactor:
|
2019-12-18 10:45:12 +00:00
|
|
|
return reactor
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def get_instance_name(self) -> str:
|
2020-05-22 15:11:35 +00:00
|
|
|
return "master"
|
|
|
|
|
2022-06-29 10:32:38 +00:00
|
|
|
def should_send_federation(self) -> bool:
|
|
|
|
return False
|
|
|
|
|
2023-01-20 18:02:18 +00:00
|
|
|
def get_replication_notifier(self) -> ReplicationNotifier:
|
|
|
|
return ReplicationNotifier()
|
|
|
|
|
2019-10-23 14:31:59 +00:00
|
|
|
|
2022-04-08 14:00:12 +00:00
|
|
|
class Porter:
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def __init__(
|
|
|
|
self,
|
|
|
|
sqlite_config: Dict[str, Any],
|
|
|
|
progress: "Progress",
|
|
|
|
batch_size: int,
|
|
|
|
hs_config: HomeServerConfig,
|
|
|
|
):
|
2022-04-08 14:00:12 +00:00
|
|
|
self.sqlite_config = sqlite_config
|
|
|
|
self.progress = progress
|
|
|
|
self.batch_size = batch_size
|
|
|
|
self.hs_config = hs_config
|
2015-04-15 09:23:24 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
async def setup_table(self, table: str) -> Tuple[str, int, int, int, int]:
|
2015-04-28 10:16:44 +00:00
|
|
|
if table in APPEND_ONLY_TABLES:
|
2015-04-17 10:13:05 +00:00
|
|
|
# It's safe to just carry on inserting.
|
2020-08-05 20:38:57 +00:00
|
|
|
row = await self.postgres_store.db_pool.simple_select_one(
|
2015-04-17 10:13:05 +00:00
|
|
|
table="port_from_sqlite3",
|
|
|
|
keyvalues={"table_name": table},
|
2016-08-04 10:28:02 +00:00
|
|
|
retcols=("forward_rowid", "backward_rowid"),
|
2015-04-17 10:13:05 +00:00
|
|
|
allow_none=True,
|
|
|
|
)
|
2015-04-10 17:47:20 +00:00
|
|
|
|
2015-04-28 10:16:44 +00:00
|
|
|
total_to_port = None
|
2016-08-04 10:28:02 +00:00
|
|
|
if row is None:
|
2015-04-28 10:16:44 +00:00
|
|
|
if table == "sent_transactions":
|
2019-12-04 13:52:46 +00:00
|
|
|
(
|
|
|
|
forward_chunk,
|
|
|
|
already_ported,
|
|
|
|
total_to_port,
|
2020-01-21 19:04:58 +00:00
|
|
|
) = await self._setup_sent_transactions()
|
2016-08-04 10:28:02 +00:00
|
|
|
backward_chunk = 0
|
2015-04-28 10:16:44 +00:00
|
|
|
else:
|
2020-08-05 20:38:57 +00:00
|
|
|
await self.postgres_store.db_pool.simple_insert(
|
2015-04-28 10:16:44 +00:00
|
|
|
table="port_from_sqlite3",
|
2016-08-04 10:28:02 +00:00
|
|
|
values={
|
|
|
|
"table_name": table,
|
|
|
|
"forward_rowid": 1,
|
|
|
|
"backward_rowid": 0,
|
2018-10-20 00:16:55 +00:00
|
|
|
},
|
2015-04-28 10:16:44 +00:00
|
|
|
)
|
2015-04-13 12:45:04 +00:00
|
|
|
|
2016-08-04 10:28:02 +00:00
|
|
|
forward_chunk = 1
|
|
|
|
backward_chunk = 0
|
2015-04-28 10:16:44 +00:00
|
|
|
already_ported = 0
|
2016-08-04 10:28:02 +00:00
|
|
|
else:
|
2023-11-09 16:13:31 +00:00
|
|
|
forward_chunk, backward_chunk = row
|
2015-04-27 16:36:37 +00:00
|
|
|
|
2015-04-28 10:16:44 +00:00
|
|
|
if total_to_port is None:
|
2020-01-21 19:04:58 +00:00
|
|
|
already_ported, total_to_port = await self._get_total_count_to_port(
|
2016-08-04 10:28:02 +00:00
|
|
|
table, forward_chunk, backward_chunk
|
2015-04-28 10:16:44 +00:00
|
|
|
)
|
2015-04-17 10:13:05 +00:00
|
|
|
else:
|
2018-10-20 00:16:55 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def delete_all(txn: LoggingTransaction) -> None:
|
2015-04-28 10:16:44 +00:00
|
|
|
txn.execute(
|
2018-10-20 00:16:55 +00:00
|
|
|
"DELETE FROM port_from_sqlite3 WHERE table_name = %s", (table,)
|
2015-04-28 10:16:44 +00:00
|
|
|
)
|
|
|
|
txn.execute("TRUNCATE %s CASCADE" % (table,))
|
|
|
|
|
2020-01-21 19:04:58 +00:00
|
|
|
await self.postgres_store.execute(delete_all)
|
2015-04-28 10:16:44 +00:00
|
|
|
|
2020-08-05 20:38:57 +00:00
|
|
|
await self.postgres_store.db_pool.simple_insert(
|
2015-04-27 16:36:37 +00:00
|
|
|
table="port_from_sqlite3",
|
2018-10-20 00:16:55 +00:00
|
|
|
values={"table_name": table, "forward_rowid": 1, "backward_rowid": 0},
|
2015-04-27 16:36:37 +00:00
|
|
|
)
|
2015-04-15 09:23:24 +00:00
|
|
|
|
2016-08-04 10:28:02 +00:00
|
|
|
forward_chunk = 1
|
|
|
|
backward_chunk = 0
|
2015-04-17 10:13:05 +00:00
|
|
|
|
2020-01-21 19:04:58 +00:00
|
|
|
already_ported, total_to_port = await self._get_total_count_to_port(
|
2016-08-04 10:28:02 +00:00
|
|
|
table, forward_chunk, backward_chunk
|
2015-04-28 10:16:44 +00:00
|
|
|
)
|
2015-04-17 10:13:05 +00:00
|
|
|
|
2020-01-21 19:04:58 +00:00
|
|
|
return table, already_ported, total_to_port, forward_chunk, backward_chunk
|
2015-04-27 16:53:40 +00:00
|
|
|
|
2020-11-11 15:07:34 +00:00
|
|
|
async def get_table_constraints(self) -> Dict[str, Set[str]]:
|
2021-05-14 10:46:35 +00:00
|
|
|
"""Returns a map of tables that have foreign key constraints to tables they depend on."""
|
2020-11-11 15:07:34 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def _get_constraints(txn: LoggingTransaction) -> Dict[str, Set[str]]:
|
2020-11-11 15:07:34 +00:00
|
|
|
# We can pull the information about foreign key constraints out from
|
|
|
|
# the postgres schema tables.
|
|
|
|
sql = """
|
|
|
|
SELECT DISTINCT
|
|
|
|
tc.table_name,
|
|
|
|
ccu.table_name AS foreign_table_name
|
|
|
|
FROM
|
|
|
|
information_schema.table_constraints AS tc
|
|
|
|
INNER JOIN information_schema.constraint_column_usage AS ccu
|
|
|
|
USING (table_schema, constraint_name)
|
2021-06-24 13:33:20 +00:00
|
|
|
WHERE tc.constraint_type = 'FOREIGN KEY'
|
|
|
|
AND tc.table_name != ccu.table_name;
|
2020-11-11 15:07:34 +00:00
|
|
|
"""
|
|
|
|
txn.execute(sql)
|
|
|
|
|
2022-04-08 14:00:12 +00:00
|
|
|
results: Dict[str, Set[str]] = {}
|
2020-11-11 15:07:34 +00:00
|
|
|
for table, foreign_table in txn:
|
|
|
|
results.setdefault(table, set()).add(foreign_table)
|
|
|
|
return results
|
|
|
|
|
|
|
|
return await self.postgres_store.db_pool.runInteraction(
|
|
|
|
"get_table_constraints", _get_constraints
|
|
|
|
)
|
|
|
|
|
2020-01-21 19:04:58 +00:00
|
|
|
async def handle_table(
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
self,
|
|
|
|
table: str,
|
|
|
|
postgres_size: int,
|
|
|
|
table_size: int,
|
|
|
|
forward_chunk: int,
|
|
|
|
backward_chunk: int,
|
|
|
|
) -> None:
|
2018-04-06 12:47:11 +00:00
|
|
|
logger.info(
|
|
|
|
"Table %s: %i/%i (rows %i-%i) already ported",
|
2018-10-20 00:16:55 +00:00
|
|
|
table,
|
|
|
|
postgres_size,
|
|
|
|
table_size,
|
|
|
|
backward_chunk + 1,
|
|
|
|
forward_chunk - 1,
|
2018-04-06 12:47:11 +00:00
|
|
|
)
|
|
|
|
|
2015-04-17 10:13:05 +00:00
|
|
|
if not table_size:
|
2015-04-10 17:47:20 +00:00
|
|
|
return
|
|
|
|
|
2015-04-17 10:13:05 +00:00
|
|
|
self.progress.add_table(table, postgres_size, table_size)
|
2015-04-10 17:47:20 +00:00
|
|
|
|
2016-04-22 09:35:53 +00:00
|
|
|
if table == "event_search":
|
2020-01-21 19:04:58 +00:00
|
|
|
await self.handle_search_table(
|
2016-08-04 10:28:02 +00:00
|
|
|
postgres_size, table_size, forward_chunk, backward_chunk
|
|
|
|
)
|
2016-04-22 09:35:53 +00:00
|
|
|
return
|
|
|
|
|
2020-06-17 14:01:18 +00:00
|
|
|
if table in IGNORED_TABLES:
|
2017-07-20 09:47:01 +00:00
|
|
|
self.progress.update(table, table_size) # Mark table as done
|
|
|
|
return
|
|
|
|
|
|
|
|
if table == "user_directory_stream_pos":
|
2017-07-20 15:16:29 +00:00
|
|
|
# We need to make sure there is a single row, `(X, null), as that is
|
|
|
|
# what synapse expects to be there.
|
2020-08-05 20:38:57 +00:00
|
|
|
await self.postgres_store.db_pool.simple_insert(
|
2018-10-20 00:16:55 +00:00
|
|
|
table=table, values={"stream_id": None}
|
2017-07-20 09:47:01 +00:00
|
|
|
)
|
|
|
|
self.progress.update(table, table_size) # Mark table as done
|
|
|
|
return
|
|
|
|
|
2022-07-11 09:36:18 +00:00
|
|
|
# We sweep over rowids in two directions: one forwards (rowids 1, 2, 3, ...)
|
|
|
|
# and another backwards (rowids 0, -1, -2, ...).
|
2016-08-04 10:28:02 +00:00
|
|
|
forward_select = (
|
2018-10-20 00:16:55 +00:00
|
|
|
"SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,)
|
2015-04-17 10:13:05 +00:00
|
|
|
)
|
2015-04-10 17:47:20 +00:00
|
|
|
|
2016-08-04 10:28:02 +00:00
|
|
|
backward_select = (
|
2022-07-11 09:36:18 +00:00
|
|
|
"SELECT rowid, * FROM %s WHERE rowid <= ? ORDER BY rowid DESC LIMIT ?"
|
|
|
|
% (table,)
|
2016-08-04 10:28:02 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
do_forward = [True]
|
|
|
|
do_backward = [True]
|
|
|
|
|
2015-04-17 10:13:05 +00:00
|
|
|
while True:
|
2018-10-20 00:16:55 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def r(
|
|
|
|
txn: LoggingTransaction,
|
|
|
|
) -> Tuple[Optional[List[str]], List[Tuple], List[Tuple]]:
|
2016-08-04 10:28:02 +00:00
|
|
|
forward_rows = []
|
|
|
|
backward_rows = []
|
|
|
|
if do_forward[0]:
|
2018-10-20 00:16:55 +00:00
|
|
|
txn.execute(forward_select, (forward_chunk, self.batch_size))
|
2016-08-04 10:28:02 +00:00
|
|
|
forward_rows = txn.fetchall()
|
|
|
|
if not forward_rows:
|
|
|
|
do_forward[0] = False
|
|
|
|
|
|
|
|
if do_backward[0]:
|
2018-10-20 00:16:55 +00:00
|
|
|
txn.execute(backward_select, (backward_chunk, self.batch_size))
|
2016-08-04 10:28:02 +00:00
|
|
|
backward_rows = txn.fetchall()
|
|
|
|
if not backward_rows:
|
|
|
|
do_backward[0] = False
|
|
|
|
|
|
|
|
if forward_rows or backward_rows:
|
2023-08-29 14:38:56 +00:00
|
|
|
assert txn.description is not None
|
|
|
|
headers: Optional[List[str]] = [
|
|
|
|
column[0] for column in txn.description
|
|
|
|
]
|
2016-08-04 10:28:02 +00:00
|
|
|
else:
|
|
|
|
headers = None
|
2015-04-10 17:47:20 +00:00
|
|
|
|
2016-08-04 10:28:02 +00:00
|
|
|
return headers, forward_rows, backward_rows
|
2015-04-10 17:47:20 +00:00
|
|
|
|
2020-08-05 20:38:57 +00:00
|
|
|
headers, frows, brows = await self.sqlite_store.db_pool.runInteraction(
|
2019-12-05 11:20:49 +00:00
|
|
|
"select", r
|
|
|
|
)
|
2015-04-10 17:47:20 +00:00
|
|
|
|
2016-08-04 10:28:02 +00:00
|
|
|
if frows or brows:
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
assert headers is not None
|
2016-08-04 10:28:02 +00:00
|
|
|
if frows:
|
|
|
|
forward_chunk = max(row[0] for row in frows) + 1
|
|
|
|
if brows:
|
|
|
|
backward_chunk = min(row[0] for row in brows) - 1
|
2015-04-17 10:13:05 +00:00
|
|
|
|
2016-08-04 10:28:02 +00:00
|
|
|
rows = frows + brows
|
2017-10-31 16:58:49 +00:00
|
|
|
rows = self._convert_rows(table, headers, rows)
|
2015-04-10 17:47:20 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def insert(txn: LoggingTransaction) -> None:
|
|
|
|
assert headers is not None
|
2018-10-20 00:16:55 +00:00
|
|
|
self.postgres_store.insert_many_txn(txn, table, headers[1:], rows)
|
2015-04-10 17:47:20 +00:00
|
|
|
|
2020-08-05 20:38:57 +00:00
|
|
|
self.postgres_store.db_pool.simple_update_one_txn(
|
2016-04-22 09:35:53 +00:00
|
|
|
txn,
|
|
|
|
table="port_from_sqlite3",
|
|
|
|
keyvalues={"table_name": table},
|
2016-08-04 10:28:02 +00:00
|
|
|
updatevalues={
|
|
|
|
"forward_rowid": forward_chunk,
|
|
|
|
"backward_rowid": backward_chunk,
|
|
|
|
},
|
2016-04-22 09:35:53 +00:00
|
|
|
)
|
|
|
|
|
2020-01-21 19:04:58 +00:00
|
|
|
await self.postgres_store.execute(insert)
|
2016-04-22 09:35:53 +00:00
|
|
|
|
|
|
|
postgres_size += len(rows)
|
|
|
|
|
|
|
|
self.progress.update(table, postgres_size)
|
|
|
|
else:
|
|
|
|
return
|
|
|
|
|
2020-01-21 19:04:58 +00:00
|
|
|
async def handle_search_table(
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
self,
|
|
|
|
postgres_size: int,
|
|
|
|
table_size: int,
|
|
|
|
forward_chunk: int,
|
|
|
|
backward_chunk: int,
|
|
|
|
) -> None:
|
2016-04-22 09:35:53 +00:00
|
|
|
select = (
|
|
|
|
"SELECT es.rowid, es.*, e.origin_server_ts, e.stream_ordering"
|
|
|
|
" FROM event_search as es"
|
|
|
|
" INNER JOIN events AS e USING (event_id, room_id)"
|
|
|
|
" WHERE es.rowid >= ?"
|
|
|
|
" ORDER BY es.rowid LIMIT ?"
|
|
|
|
)
|
2015-11-18 15:54:50 +00:00
|
|
|
|
2016-04-22 09:35:53 +00:00
|
|
|
while True:
|
2018-10-20 00:16:55 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def r(txn: LoggingTransaction) -> Tuple[List[str], List[Tuple]]:
|
2018-10-20 00:16:55 +00:00
|
|
|
txn.execute(select, (forward_chunk, self.batch_size))
|
2016-04-22 09:35:53 +00:00
|
|
|
rows = txn.fetchall()
|
2023-08-29 14:38:56 +00:00
|
|
|
assert txn.description is not None
|
2016-04-22 09:35:53 +00:00
|
|
|
headers = [column[0] for column in txn.description]
|
|
|
|
|
|
|
|
return headers, rows
|
|
|
|
|
2020-08-05 20:38:57 +00:00
|
|
|
headers, rows = await self.sqlite_store.db_pool.runInteraction("select", r)
|
2016-04-22 09:35:53 +00:00
|
|
|
|
|
|
|
if rows:
|
2016-08-04 10:28:02 +00:00
|
|
|
forward_chunk = rows[-1][0] + 1
|
2016-04-22 09:35:53 +00:00
|
|
|
|
|
|
|
# We have to treat event_search differently since it has a
|
|
|
|
# different structure in the two different databases.
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def insert(txn: LoggingTransaction) -> None:
|
2016-04-22 09:35:53 +00:00
|
|
|
sql = (
|
|
|
|
"INSERT INTO event_search (event_id, room_id, key,"
|
|
|
|
" sender, vector, origin_server_ts, stream_ordering)"
|
|
|
|
" VALUES (?,?,?,?,to_tsvector('english', ?),?,?)"
|
|
|
|
)
|
|
|
|
|
2017-10-01 23:53:32 +00:00
|
|
|
rows_dict = []
|
|
|
|
for row in rows:
|
|
|
|
d = dict(zip(headers, row))
|
2019-12-04 13:52:46 +00:00
|
|
|
if "\0" in d["value"]:
|
|
|
|
logger.warning("dropping search row %s", d)
|
2017-10-01 23:53:32 +00:00
|
|
|
else:
|
|
|
|
rows_dict.append(d)
|
2016-04-22 09:35:53 +00:00
|
|
|
|
2018-10-20 00:16:55 +00:00
|
|
|
txn.executemany(
|
|
|
|
sql,
|
|
|
|
[
|
|
|
|
(
|
|
|
|
row["event_id"],
|
|
|
|
row["room_id"],
|
|
|
|
row["key"],
|
|
|
|
row["sender"],
|
|
|
|
row["value"],
|
|
|
|
row["origin_server_ts"],
|
|
|
|
row["stream_ordering"],
|
|
|
|
)
|
|
|
|
for row in rows_dict
|
|
|
|
],
|
|
|
|
)
|
2016-04-22 09:35:53 +00:00
|
|
|
|
2020-08-05 20:38:57 +00:00
|
|
|
self.postgres_store.db_pool.simple_update_one_txn(
|
2016-04-22 09:35:53 +00:00
|
|
|
txn,
|
|
|
|
table="port_from_sqlite3",
|
|
|
|
keyvalues={"table_name": "event_search"},
|
2016-08-04 10:28:02 +00:00
|
|
|
updatevalues={
|
|
|
|
"forward_rowid": forward_chunk,
|
|
|
|
"backward_rowid": backward_chunk,
|
|
|
|
},
|
2016-04-22 09:35:53 +00:00
|
|
|
)
|
2015-04-10 17:47:20 +00:00
|
|
|
|
2020-01-21 19:04:58 +00:00
|
|
|
await self.postgres_store.execute(insert)
|
2015-04-10 17:47:20 +00:00
|
|
|
|
2015-04-17 10:13:05 +00:00
|
|
|
postgres_size += len(rows)
|
2015-04-10 17:47:20 +00:00
|
|
|
|
2016-04-22 09:35:53 +00:00
|
|
|
self.progress.update("event_search", postgres_size)
|
|
|
|
|
2015-04-17 10:13:05 +00:00
|
|
|
else:
|
|
|
|
return
|
|
|
|
|
2020-01-09 17:46:52 +00:00
|
|
|
def build_db_store(
|
2021-05-14 10:46:35 +00:00
|
|
|
self,
|
|
|
|
db_config: DatabaseConnectionConfig,
|
|
|
|
allow_outdated_version: bool = False,
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
) -> Store:
|
2019-10-23 14:31:59 +00:00
|
|
|
"""Builds and returns a database store using the provided configuration.
|
2015-04-10 17:47:20 +00:00
|
|
|
|
2019-10-23 14:31:59 +00:00
|
|
|
Args:
|
2020-01-09 17:46:52 +00:00
|
|
|
db_config: The database configuration
|
|
|
|
allow_outdated_version: True to suppress errors about the database server
|
|
|
|
version being too old to run a complete synapse
|
2019-10-23 14:31:59 +00:00
|
|
|
|
|
|
|
Returns:
|
|
|
|
The built Store object.
|
|
|
|
"""
|
2019-12-18 10:45:12 +00:00
|
|
|
self.progress.set_state("Preparing %s" % db_config.config["name"])
|
2019-10-23 14:31:59 +00:00
|
|
|
|
2019-12-18 10:45:12 +00:00
|
|
|
engine = create_engine(db_config.config)
|
2019-10-23 14:31:59 +00:00
|
|
|
|
2019-12-18 10:45:12 +00:00
|
|
|
hs = MockHomeserver(self.hs_config)
|
2019-10-23 14:31:59 +00:00
|
|
|
|
2020-10-02 14:20:45 +00:00
|
|
|
with make_conn(db_config, engine, "portdb") as db_conn:
|
2020-01-09 17:46:52 +00:00
|
|
|
engine.check_database(
|
|
|
|
db_conn, allow_outdated_version=allow_outdated_version
|
|
|
|
)
|
2020-01-15 14:59:33 +00:00
|
|
|
prepare_database(db_conn, engine, config=self.hs_config)
|
2022-04-08 14:00:12 +00:00
|
|
|
# Type safety: ignore that we're using Mock homeservers here.
|
|
|
|
store = Store(DatabasePool(hs, db_config, engine), db_conn, hs) # type: ignore[arg-type]
|
2020-01-09 17:21:30 +00:00
|
|
|
db_conn.commit()
|
2015-04-17 10:13:05 +00:00
|
|
|
|
2019-10-23 14:31:59 +00:00
|
|
|
return store
|
2015-04-17 10:13:05 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
async def run_background_updates_on_postgres(self) -> None:
|
2019-10-23 14:31:59 +00:00
|
|
|
# Manually apply all background updates on the PostgreSQL database.
|
2019-12-05 11:20:49 +00:00
|
|
|
postgres_ready = (
|
2020-08-05 20:38:57 +00:00
|
|
|
await self.postgres_store.db_pool.updates.has_completed_background_updates()
|
2019-12-05 11:20:49 +00:00
|
|
|
)
|
2019-10-23 14:31:59 +00:00
|
|
|
|
|
|
|
if not postgres_ready:
|
|
|
|
# Only say that we're running background updates when there are background
|
|
|
|
# updates to run.
|
|
|
|
self.progress.set_state("Running background updates on PostgreSQL")
|
|
|
|
|
|
|
|
while not postgres_ready:
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
await self.postgres_store.db_pool.updates.do_next_background_update(True)
|
2020-01-21 19:04:58 +00:00
|
|
|
postgres_ready = await (
|
2020-08-05 20:38:57 +00:00
|
|
|
self.postgres_store.db_pool.updates.has_completed_background_updates()
|
2019-10-23 14:31:59 +00:00
|
|
|
)
|
2015-04-17 10:13:05 +00:00
|
|
|
|
2022-07-07 10:08:04 +00:00
|
|
|
@staticmethod
|
|
|
|
def _is_sqlite_autovacuum_enabled(txn: LoggingTransaction) -> bool:
|
|
|
|
"""
|
|
|
|
Returns true if auto_vacuum is enabled in SQLite.
|
|
|
|
https://www.sqlite.org/pragma.html#pragma_auto_vacuum
|
|
|
|
|
|
|
|
Vacuuming changes the rowids on rows in the database.
|
|
|
|
Auto-vacuuming is therefore dangerous when used in conjunction with this script.
|
|
|
|
|
|
|
|
Note that the auto_vacuum setting can't be changed without performing
|
|
|
|
a VACUUM after trying to change the pragma.
|
|
|
|
"""
|
|
|
|
txn.execute("PRAGMA auto_vacuum")
|
|
|
|
row = txn.fetchone()
|
|
|
|
assert row is not None, "`PRAGMA auto_vacuum` did not give a row."
|
|
|
|
(autovacuum_setting,) = row
|
|
|
|
# 0 means off. 1 means full. 2 means incremental.
|
|
|
|
return autovacuum_setting != 0
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
async def run(self) -> None:
|
2020-01-21 19:04:58 +00:00
|
|
|
"""Ports the SQLite database to a PostgreSQL database.
|
|
|
|
|
|
|
|
When a fatal error is met, its message is assigned to the global "end_error"
|
|
|
|
variable. When this error comes with a stacktrace, its exec_info is assigned to
|
|
|
|
the global "end_error_exec_info" variable.
|
|
|
|
"""
|
|
|
|
global end_error
|
|
|
|
|
2019-10-23 14:31:59 +00:00
|
|
|
try:
|
2020-01-09 17:46:52 +00:00
|
|
|
# we allow people to port away from outdated versions of sqlite.
|
2020-01-09 17:21:30 +00:00
|
|
|
self.sqlite_store = self.build_db_store(
|
2020-01-09 17:46:52 +00:00
|
|
|
DatabaseConnectionConfig("master-sqlite", self.sqlite_config),
|
|
|
|
allow_outdated_version=True,
|
2019-12-18 10:45:12 +00:00
|
|
|
)
|
2019-10-23 14:31:59 +00:00
|
|
|
|
2022-07-07 10:08:04 +00:00
|
|
|
# For safety, ensure auto_vacuums are disabled.
|
|
|
|
if await self.sqlite_store.db_pool.runInteraction(
|
|
|
|
"is_sqlite_autovacuum_enabled", self._is_sqlite_autovacuum_enabled
|
|
|
|
):
|
|
|
|
end_error = (
|
|
|
|
"auto_vacuum is enabled in the SQLite database."
|
|
|
|
" (This is not the default configuration.)\n"
|
|
|
|
" This script relies on rowids being consistent and must not"
|
|
|
|
" be used if the database could be vacuumed between re-runs.\n"
|
|
|
|
" To disable auto_vacuum, you need to stop Synapse and run the following SQL:\n"
|
|
|
|
" PRAGMA auto_vacuum=off;\n"
|
|
|
|
" VACUUM;"
|
|
|
|
)
|
|
|
|
return
|
|
|
|
|
2019-10-23 14:31:59 +00:00
|
|
|
# Check if all background updates are done, abort if not.
|
2024-09-02 11:39:04 +00:00
|
|
|
updates_complete = await self.sqlite_store.db_pool.updates.has_completed_background_updates()
|
2019-10-23 14:31:59 +00:00
|
|
|
if not updates_complete:
|
2020-01-21 19:04:58 +00:00
|
|
|
end_error = (
|
2019-10-23 14:31:59 +00:00
|
|
|
"Pending background updates exist in the SQLite3 database."
|
|
|
|
" Please start Synapse again and wait until every update has finished"
|
|
|
|
" before running this script.\n"
|
|
|
|
)
|
2020-01-21 19:04:58 +00:00
|
|
|
return
|
2015-04-29 10:42:28 +00:00
|
|
|
|
2020-01-09 17:21:30 +00:00
|
|
|
self.postgres_store = self.build_db_store(
|
2021-10-06 14:47:41 +00:00
|
|
|
self.hs_config.database.get_single_database()
|
2019-10-23 14:31:59 +00:00
|
|
|
)
|
2015-04-17 10:13:05 +00:00
|
|
|
|
2020-01-21 19:04:58 +00:00
|
|
|
await self.run_background_updates_on_postgres()
|
2015-04-17 10:13:05 +00:00
|
|
|
|
2018-04-06 12:47:11 +00:00
|
|
|
self.progress.set_state("Creating port tables")
|
2018-10-20 00:16:55 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def create_port_table(txn: LoggingTransaction) -> None:
|
2015-04-17 10:13:05 +00:00
|
|
|
txn.execute(
|
2018-04-06 12:45:10 +00:00
|
|
|
"CREATE TABLE IF NOT EXISTS port_from_sqlite3 ("
|
2015-04-17 10:13:05 +00:00
|
|
|
" table_name varchar(100) NOT NULL UNIQUE,"
|
2016-08-04 10:28:02 +00:00
|
|
|
" forward_rowid bigint NOT NULL,"
|
|
|
|
" backward_rowid bigint NOT NULL"
|
2015-04-17 10:13:05 +00:00
|
|
|
")"
|
|
|
|
)
|
|
|
|
|
2016-08-04 10:35:49 +00:00
|
|
|
# The old port script created a table with just a "rowid" column.
|
|
|
|
# We want people to be able to rerun this script from an old port
|
|
|
|
# so that they can pick up any missing events that were not
|
|
|
|
# ported across.
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def alter_table(txn: LoggingTransaction) -> None:
|
2016-08-04 10:35:49 +00:00
|
|
|
txn.execute(
|
|
|
|
"ALTER TABLE IF EXISTS port_from_sqlite3"
|
|
|
|
" RENAME rowid TO forward_rowid"
|
|
|
|
)
|
|
|
|
txn.execute(
|
|
|
|
"ALTER TABLE IF EXISTS port_from_sqlite3"
|
|
|
|
" ADD backward_rowid bigint NOT NULL DEFAULT 0"
|
|
|
|
)
|
|
|
|
|
|
|
|
try:
|
2020-08-05 20:38:57 +00:00
|
|
|
await self.postgres_store.db_pool.runInteraction(
|
|
|
|
"alter_table", alter_table
|
|
|
|
)
|
2018-10-24 09:35:01 +00:00
|
|
|
except Exception:
|
|
|
|
# On Error Resume Next
|
2018-04-06 12:45:10 +00:00
|
|
|
pass
|
2016-08-04 10:35:49 +00:00
|
|
|
|
2020-08-05 20:38:57 +00:00
|
|
|
await self.postgres_store.db_pool.runInteraction(
|
2018-04-06 12:45:10 +00:00
|
|
|
"create_port_table", create_port_table
|
|
|
|
)
|
2015-04-17 10:13:05 +00:00
|
|
|
|
2020-11-13 11:53:51 +00:00
|
|
|
# Step 2. Set up sequences
|
|
|
|
#
|
2023-08-01 14:36:33 +00:00
|
|
|
# We do this before porting the tables so that even if we fail half
|
2020-11-13 11:53:51 +00:00
|
|
|
# way through the postgres DB always have sequences that are greater
|
|
|
|
# than their respective tables. If we don't then creating the
|
|
|
|
# `DataStore` object will fail due to the inconsistency.
|
|
|
|
self.progress.set_state("Setting up sequence generators")
|
|
|
|
await self._setup_state_group_id_seq()
|
|
|
|
await self._setup_user_id_seq()
|
|
|
|
await self._setup_events_stream_seqs()
|
2023-08-01 14:36:33 +00:00
|
|
|
await self._setup_sequence(
|
|
|
|
"un_partial_stated_event_stream_sequence",
|
2024-05-30 11:07:32 +00:00
|
|
|
[("un_partial_stated_event_stream", "stream_id")],
|
2023-08-01 14:36:33 +00:00
|
|
|
)
|
2021-02-23 12:33:24 +00:00
|
|
|
await self._setup_sequence(
|
2024-05-30 11:07:32 +00:00
|
|
|
"device_inbox_sequence",
|
|
|
|
[
|
|
|
|
("device_inbox", "stream_id"),
|
|
|
|
("device_federation_outbox", "stream_id"),
|
|
|
|
],
|
2021-02-23 12:33:24 +00:00
|
|
|
)
|
|
|
|
await self._setup_sequence(
|
2021-04-23 11:21:55 +00:00
|
|
|
"account_data_sequence",
|
2024-05-30 11:07:32 +00:00
|
|
|
[
|
|
|
|
("room_account_data", "stream_id"),
|
|
|
|
("room_tags_revisions", "stream_id"),
|
|
|
|
("account_data", "stream_id"),
|
|
|
|
],
|
|
|
|
)
|
|
|
|
await self._setup_sequence(
|
|
|
|
"receipts_sequence",
|
|
|
|
[
|
|
|
|
("receipts_linearized", "stream_id"),
|
|
|
|
],
|
|
|
|
)
|
|
|
|
await self._setup_sequence(
|
|
|
|
"presence_stream_sequence",
|
|
|
|
[
|
|
|
|
("presence_stream", "stream_id"),
|
|
|
|
],
|
2021-04-23 11:21:55 +00:00
|
|
|
)
|
2021-02-23 12:33:24 +00:00
|
|
|
await self._setup_auth_chain_sequence()
|
2023-08-01 14:36:33 +00:00
|
|
|
await self._setup_sequence(
|
|
|
|
"application_services_txn_id_seq",
|
2024-05-30 11:07:32 +00:00
|
|
|
[
|
|
|
|
(
|
|
|
|
"application_services_txns",
|
|
|
|
"txn_id",
|
|
|
|
)
|
|
|
|
],
|
|
|
|
)
|
|
|
|
await self._setup_sequence(
|
|
|
|
"device_lists_sequence",
|
|
|
|
[
|
|
|
|
("device_lists_stream", "stream_id"),
|
|
|
|
("user_signature_stream", "stream_id"),
|
|
|
|
("device_lists_outbound_pokes", "stream_id"),
|
|
|
|
("device_lists_changes_in_room", "stream_id"),
|
|
|
|
("device_lists_remote_pending", "stream_id"),
|
|
|
|
("device_lists_changes_converted_stream_position", "stream_id"),
|
|
|
|
],
|
|
|
|
)
|
|
|
|
await self._setup_sequence(
|
|
|
|
"e2e_cross_signing_keys_sequence",
|
|
|
|
[
|
|
|
|
("e2e_cross_signing_keys", "stream_id"),
|
|
|
|
],
|
|
|
|
)
|
|
|
|
await self._setup_sequence(
|
|
|
|
"push_rules_stream_sequence",
|
|
|
|
[
|
|
|
|
("push_rules_stream", "stream_id"),
|
|
|
|
],
|
|
|
|
)
|
|
|
|
await self._setup_sequence(
|
|
|
|
"pushers_sequence",
|
|
|
|
[
|
|
|
|
("pushers", "id"),
|
|
|
|
("deleted_pushers", "stream_id"),
|
|
|
|
],
|
2023-08-01 14:36:33 +00:00
|
|
|
)
|
2020-11-13 11:53:51 +00:00
|
|
|
|
|
|
|
# Step 3. Get tables.
|
2018-04-06 12:47:11 +00:00
|
|
|
self.progress.set_state("Fetching tables")
|
2020-08-05 20:38:57 +00:00
|
|
|
sqlite_tables = await self.sqlite_store.db_pool.simple_select_onecol(
|
2018-10-20 00:16:55 +00:00
|
|
|
table="sqlite_master", keyvalues={"type": "table"}, retcol="name"
|
2018-04-06 12:47:11 +00:00
|
|
|
)
|
|
|
|
|
2020-08-05 20:38:57 +00:00
|
|
|
postgres_tables = await self.postgres_store.db_pool.simple_select_onecol(
|
2018-04-06 12:47:11 +00:00
|
|
|
table="information_schema.tables",
|
|
|
|
keyvalues={},
|
|
|
|
retcol="distinct table_name",
|
|
|
|
)
|
|
|
|
|
|
|
|
tables = set(sqlite_tables) & set(postgres_tables)
|
|
|
|
logger.info("Found %d tables", len(tables))
|
2015-04-27 16:53:40 +00:00
|
|
|
|
2020-11-13 11:53:51 +00:00
|
|
|
# Step 4. Figure out what still needs copying
|
2018-04-06 12:47:11 +00:00
|
|
|
self.progress.set_state("Checking on port progress")
|
2020-01-21 19:04:58 +00:00
|
|
|
setup_res = await make_deferred_yieldable(
|
|
|
|
defer.gatherResults(
|
|
|
|
[
|
|
|
|
run_in_background(self.setup_table, table)
|
|
|
|
for table in tables
|
|
|
|
if table not in ["schema_version", "applied_schema_deltas"]
|
|
|
|
and not table.startswith("sqlite_")
|
|
|
|
],
|
|
|
|
consumeErrors=True,
|
|
|
|
)
|
2015-04-17 10:13:05 +00:00
|
|
|
)
|
2020-11-11 15:07:34 +00:00
|
|
|
# Map from table name to args passed to `handle_table`, i.e. a tuple
|
|
|
|
# of: `postgres_size`, `table_size`, `forward_chunk`, `backward_chunk`.
|
2023-07-05 16:25:00 +00:00
|
|
|
tables_to_port_info_map = {
|
|
|
|
r[0]: r[1:] for r in setup_res if r[0] not in IGNORED_TABLES
|
|
|
|
}
|
2015-04-17 10:13:05 +00:00
|
|
|
|
2020-11-13 11:53:51 +00:00
|
|
|
# Step 5. Do the copying.
|
2020-11-11 15:07:34 +00:00
|
|
|
#
|
|
|
|
# This is slightly convoluted as we need to ensure tables are ported
|
|
|
|
# in the correct order due to foreign key constraints.
|
2018-04-06 12:47:11 +00:00
|
|
|
self.progress.set_state("Copying to postgres")
|
2020-11-11 15:07:34 +00:00
|
|
|
|
|
|
|
constraints = await self.get_table_constraints()
|
|
|
|
tables_ported = set() # type: Set[str]
|
|
|
|
|
|
|
|
while tables_to_port_info_map:
|
|
|
|
# Pulls out all tables that are still to be ported and which
|
|
|
|
# only depend on tables that are already ported (if any).
|
|
|
|
tables_to_port = [
|
|
|
|
table
|
|
|
|
for table in tables_to_port_info_map
|
|
|
|
if not constraints.get(table, set()) - tables_ported
|
|
|
|
]
|
|
|
|
|
|
|
|
await make_deferred_yieldable(
|
|
|
|
defer.gatherResults(
|
|
|
|
[
|
|
|
|
run_in_background(
|
|
|
|
self.handle_table,
|
|
|
|
table,
|
|
|
|
*tables_to_port_info_map.pop(table),
|
|
|
|
)
|
|
|
|
for table in tables_to_port
|
|
|
|
],
|
|
|
|
consumeErrors=True,
|
|
|
|
)
|
2020-01-21 19:04:58 +00:00
|
|
|
)
|
2020-11-11 15:07:34 +00:00
|
|
|
|
|
|
|
tables_ported.update(tables_to_port)
|
2015-04-27 16:53:40 +00:00
|
|
|
|
2015-04-17 10:13:05 +00:00
|
|
|
self.progress.done()
|
2020-01-21 19:04:58 +00:00
|
|
|
except Exception as e:
|
2015-04-17 10:13:05 +00:00
|
|
|
global end_error_exec_info
|
2020-10-20 10:47:24 +00:00
|
|
|
end_error = str(e)
|
2022-04-08 14:00:12 +00:00
|
|
|
# Type safety: we're in an exception handler, so the exc_info() tuple
|
|
|
|
# will not be (None, None, None).
|
|
|
|
end_error_exec_info = sys.exc_info() # type: ignore[assignment]
|
2015-04-17 10:13:05 +00:00
|
|
|
logger.exception("")
|
|
|
|
finally:
|
|
|
|
reactor.stop()
|
2015-04-10 17:47:20 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def _convert_rows(
|
|
|
|
self, table: str, headers: List[str], rows: List[Tuple]
|
|
|
|
) -> List[Tuple]:
|
2015-04-28 10:16:44 +00:00
|
|
|
bool_col_names = BOOLEAN_COLUMNS.get(table, [])
|
|
|
|
|
2018-10-20 00:16:55 +00:00
|
|
|
bool_cols = [i for i, h in enumerate(headers) if h in bool_col_names]
|
2015-04-28 10:16:44 +00:00
|
|
|
|
2017-10-31 16:58:49 +00:00
|
|
|
class BadValueException(Exception):
|
|
|
|
pass
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def conv(j: int, col: object) -> object:
|
2015-04-28 10:16:44 +00:00
|
|
|
if j in bool_cols:
|
|
|
|
return bool(col)
|
2019-10-23 14:31:59 +00:00
|
|
|
if isinstance(col, bytes):
|
|
|
|
return bytearray(col)
|
2020-06-16 12:51:47 +00:00
|
|
|
elif isinstance(col, str) and "\0" in col:
|
2019-10-31 10:23:24 +00:00
|
|
|
logger.warning(
|
2018-10-20 00:16:55 +00:00
|
|
|
"DROPPING ROW: NUL value in table %s col %s: %r",
|
|
|
|
table,
|
|
|
|
headers[j],
|
|
|
|
col,
|
|
|
|
)
|
|
|
|
raise BadValueException()
|
2015-04-28 10:16:44 +00:00
|
|
|
return col
|
|
|
|
|
2017-10-31 16:58:49 +00:00
|
|
|
outrows = []
|
2021-05-14 10:46:35 +00:00
|
|
|
for row in rows:
|
2017-10-31 16:58:49 +00:00
|
|
|
try:
|
2018-10-20 00:16:55 +00:00
|
|
|
outrows.append(
|
|
|
|
tuple(conv(j, col) for j, col in enumerate(row) if j > 0)
|
|
|
|
)
|
2017-10-31 16:58:49 +00:00
|
|
|
except BadValueException:
|
|
|
|
pass
|
|
|
|
|
|
|
|
return outrows
|
2015-04-28 10:16:44 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
async def _setup_sent_transactions(self) -> Tuple[int, int, int]:
|
2015-04-28 10:16:44 +00:00
|
|
|
# Only save things from the last day
|
2016-08-04 10:28:02 +00:00
|
|
|
yesterday = int(time.time() * 1000) - 86400000
|
2015-04-28 10:16:44 +00:00
|
|
|
|
|
|
|
# And save the max transaction id from each destination
|
|
|
|
select = (
|
|
|
|
"SELECT rowid, * FROM sent_transactions WHERE rowid IN ("
|
|
|
|
"SELECT max(rowid) FROM sent_transactions"
|
|
|
|
" GROUP BY destination"
|
|
|
|
")"
|
|
|
|
)
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def r(txn: LoggingTransaction) -> Tuple[List[str], List[Tuple]]:
|
2015-04-28 10:16:44 +00:00
|
|
|
txn.execute(select)
|
|
|
|
rows = txn.fetchall()
|
2023-08-29 14:38:56 +00:00
|
|
|
assert txn.description is not None
|
|
|
|
headers = [column[0] for column in txn.description]
|
2015-04-28 10:16:44 +00:00
|
|
|
|
2019-12-04 13:52:46 +00:00
|
|
|
ts_ind = headers.index("ts")
|
2015-04-28 10:16:44 +00:00
|
|
|
|
|
|
|
return headers, [r for r in rows if r[ts_ind] < yesterday]
|
|
|
|
|
2020-08-05 20:38:57 +00:00
|
|
|
headers, rows = await self.sqlite_store.db_pool.runInteraction("select", r)
|
2015-04-28 10:16:44 +00:00
|
|
|
|
2017-10-31 16:58:49 +00:00
|
|
|
rows = self._convert_rows("sent_transactions", headers, rows)
|
2015-04-28 10:16:44 +00:00
|
|
|
|
|
|
|
inserted_rows = len(rows)
|
2015-09-02 10:11:11 +00:00
|
|
|
if inserted_rows:
|
|
|
|
max_inserted_rowid = max(r[0] for r in rows)
|
2015-04-28 10:16:44 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def insert(txn: LoggingTransaction) -> None:
|
2015-09-02 10:11:11 +00:00
|
|
|
self.postgres_store.insert_many_txn(
|
|
|
|
txn, "sent_transactions", headers[1:], rows
|
|
|
|
)
|
2015-04-28 10:16:44 +00:00
|
|
|
|
2020-01-21 19:04:58 +00:00
|
|
|
await self.postgres_store.execute(insert)
|
2015-09-02 10:11:11 +00:00
|
|
|
else:
|
|
|
|
max_inserted_rowid = 0
|
2015-04-28 10:16:44 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def get_start_id(txn: LoggingTransaction) -> int:
|
2015-04-28 10:16:44 +00:00
|
|
|
txn.execute(
|
|
|
|
"SELECT rowid FROM sent_transactions WHERE ts >= ?"
|
|
|
|
" ORDER BY rowid ASC LIMIT 1",
|
2018-10-20 00:16:55 +00:00
|
|
|
(yesterday,),
|
2015-04-28 10:16:44 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
rows = txn.fetchall()
|
|
|
|
if rows:
|
|
|
|
return rows[0][0]
|
|
|
|
else:
|
|
|
|
return 1
|
|
|
|
|
2020-01-21 19:04:58 +00:00
|
|
|
next_chunk = await self.sqlite_store.execute(get_start_id)
|
2015-04-28 10:16:44 +00:00
|
|
|
next_chunk = max(max_inserted_rowid + 1, next_chunk)
|
|
|
|
|
2020-08-05 20:38:57 +00:00
|
|
|
await self.postgres_store.db_pool.simple_insert(
|
2015-04-28 10:16:44 +00:00
|
|
|
table="port_from_sqlite3",
|
2016-08-04 10:28:02 +00:00
|
|
|
values={
|
|
|
|
"table_name": "sent_transactions",
|
|
|
|
"forward_rowid": next_chunk,
|
|
|
|
"backward_rowid": 0,
|
2018-10-20 00:16:55 +00:00
|
|
|
},
|
2015-04-28 10:16:44 +00:00
|
|
|
)
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def get_sent_table_size(txn: LoggingTransaction) -> int:
|
2015-04-28 10:16:44 +00:00
|
|
|
txn.execute(
|
2018-10-20 00:16:55 +00:00
|
|
|
"SELECT count(*) FROM sent_transactions" " WHERE ts >= ?", (yesterday,)
|
2015-04-28 10:16:44 +00:00
|
|
|
)
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
result = txn.fetchone()
|
|
|
|
assert result is not None
|
|
|
|
return int(result[0])
|
2015-04-28 10:16:44 +00:00
|
|
|
|
2020-01-21 19:04:58 +00:00
|
|
|
remaining_count = await self.sqlite_store.execute(get_sent_table_size)
|
2015-04-28 10:16:44 +00:00
|
|
|
|
|
|
|
total_count = remaining_count + inserted_rows
|
|
|
|
|
2020-01-21 19:04:58 +00:00
|
|
|
return next_chunk, inserted_rows, total_count
|
2015-04-28 10:16:44 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
async def _get_remaining_count_to_port(
|
|
|
|
self, table: str, forward_chunk: int, backward_chunk: int
|
|
|
|
) -> int:
|
|
|
|
frows = cast(
|
|
|
|
List[Tuple[int]],
|
|
|
|
await self.sqlite_store.execute_sql(
|
|
|
|
"SELECT count(*) FROM %s WHERE rowid >= ?" % (table,), forward_chunk
|
|
|
|
),
|
2015-04-28 10:16:44 +00:00
|
|
|
)
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
brows = cast(
|
|
|
|
List[Tuple[int]],
|
|
|
|
await self.sqlite_store.execute_sql(
|
|
|
|
"SELECT count(*) FROM %s WHERE rowid <= ?" % (table,), backward_chunk
|
|
|
|
),
|
2016-08-04 10:28:02 +00:00
|
|
|
)
|
|
|
|
|
2020-01-21 19:04:58 +00:00
|
|
|
return frows[0][0] + brows[0][0]
|
2015-04-28 10:16:44 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
async def _get_already_ported_count(self, table: str) -> int:
|
2020-01-21 19:04:58 +00:00
|
|
|
rows = await self.postgres_store.execute_sql(
|
2018-10-20 00:16:55 +00:00
|
|
|
"SELECT count(*) FROM %s" % (table,)
|
2015-04-28 10:16:44 +00:00
|
|
|
)
|
|
|
|
|
2020-01-21 19:04:58 +00:00
|
|
|
return rows[0][0]
|
2015-04-28 10:16:44 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
async def _get_total_count_to_port(
|
|
|
|
self, table: str, forward_chunk: int, backward_chunk: int
|
|
|
|
) -> Tuple[int, int]:
|
2020-01-21 19:04:58 +00:00
|
|
|
remaining, done = await make_deferred_yieldable(
|
|
|
|
defer.gatherResults(
|
|
|
|
[
|
|
|
|
run_in_background(
|
|
|
|
self._get_remaining_count_to_port,
|
|
|
|
table,
|
|
|
|
forward_chunk,
|
|
|
|
backward_chunk,
|
|
|
|
),
|
|
|
|
run_in_background(self._get_already_ported_count, table),
|
|
|
|
],
|
|
|
|
)
|
2015-04-28 10:16:44 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
remaining = int(remaining) if remaining else 0
|
|
|
|
done = int(done) if done else 0
|
|
|
|
|
2020-01-21 19:04:58 +00:00
|
|
|
return done, remaining + done
|
2015-04-28 10:16:44 +00:00
|
|
|
|
2021-02-23 12:33:24 +00:00
|
|
|
async def _setup_state_group_id_seq(self) -> None:
|
2024-09-02 11:39:04 +00:00
|
|
|
curr_id: Optional[
|
|
|
|
int
|
|
|
|
] = await self.sqlite_store.db_pool.simple_select_one_onecol(
|
|
|
|
table="state_groups", keyvalues={}, retcol="MAX(id)", allow_none=True
|
2020-11-13 11:53:51 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
if not curr_id:
|
|
|
|
return
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def r(txn: LoggingTransaction) -> None:
|
|
|
|
assert curr_id is not None
|
2019-12-02 18:23:41 +00:00
|
|
|
next_id = curr_id + 1
|
2018-10-20 00:16:55 +00:00
|
|
|
txn.execute("ALTER SEQUENCE state_group_id_seq RESTART WITH %s", (next_id,))
|
|
|
|
|
2020-11-13 11:53:51 +00:00
|
|
|
await self.postgres_store.db_pool.runInteraction("setup_state_group_id_seq", r)
|
|
|
|
|
2021-02-23 12:33:24 +00:00
|
|
|
async def _setup_user_id_seq(self) -> None:
|
2020-11-13 11:53:51 +00:00
|
|
|
curr_id = await self.sqlite_store.db_pool.runInteraction(
|
|
|
|
"setup_user_id_seq", find_max_generated_user_id_localpart
|
|
|
|
)
|
2018-04-06 12:48:40 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def r(txn: LoggingTransaction) -> None:
|
2020-11-13 11:53:51 +00:00
|
|
|
next_id = curr_id + 1
|
2020-07-16 10:46:44 +00:00
|
|
|
txn.execute("ALTER SEQUENCE user_id_seq RESTART WITH %s", (next_id,))
|
|
|
|
|
2021-02-23 12:33:24 +00:00
|
|
|
await self.postgres_store.db_pool.runInteraction("setup_user_id_seq", r)
|
2020-07-16 10:46:44 +00:00
|
|
|
|
2021-02-23 12:33:24 +00:00
|
|
|
async def _setup_events_stream_seqs(self) -> None:
|
2021-05-14 10:46:35 +00:00
|
|
|
"""Set the event stream sequences to the correct values."""
|
2020-11-13 11:53:51 +00:00
|
|
|
|
|
|
|
# We get called before we've ported the events table, so we need to
|
|
|
|
# fetch the current positions from the SQLite store.
|
|
|
|
curr_forward_id = await self.sqlite_store.db_pool.simple_select_one_onecol(
|
|
|
|
table="events", keyvalues={}, retcol="MAX(stream_ordering)", allow_none=True
|
|
|
|
)
|
|
|
|
|
|
|
|
curr_backward_id = await self.sqlite_store.db_pool.simple_select_one_onecol(
|
|
|
|
table="events",
|
|
|
|
keyvalues={},
|
|
|
|
retcol="MAX(-MIN(stream_ordering), 1)",
|
|
|
|
allow_none=True,
|
|
|
|
)
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def _setup_events_stream_seqs_set_pos(txn: LoggingTransaction) -> None:
|
2020-11-13 11:53:51 +00:00
|
|
|
if curr_forward_id:
|
2020-09-24 12:43:49 +00:00
|
|
|
txn.execute(
|
2020-11-13 11:53:51 +00:00
|
|
|
"ALTER SEQUENCE events_stream_seq RESTART WITH %s",
|
|
|
|
(curr_forward_id + 1,),
|
2020-09-24 12:43:49 +00:00
|
|
|
)
|
|
|
|
|
2021-05-10 12:02:55 +00:00
|
|
|
if curr_backward_id:
|
|
|
|
txn.execute(
|
|
|
|
"ALTER SEQUENCE events_backfill_stream_seq RESTART WITH %s",
|
|
|
|
(curr_backward_id + 1,),
|
|
|
|
)
|
2020-09-24 12:43:49 +00:00
|
|
|
|
2021-02-23 12:33:24 +00:00
|
|
|
await self.postgres_store.db_pool.runInteraction(
|
2021-05-14 10:46:35 +00:00
|
|
|
"_setup_events_stream_seqs",
|
|
|
|
_setup_events_stream_seqs_set_pos,
|
2020-09-24 12:43:49 +00:00
|
|
|
)
|
|
|
|
|
2021-05-14 10:46:35 +00:00
|
|
|
async def _setup_sequence(
|
2023-08-01 14:36:33 +00:00
|
|
|
self,
|
|
|
|
sequence_name: str,
|
2024-05-30 11:07:32 +00:00
|
|
|
stream_id_tables: Iterable[Tuple[str, str]],
|
2021-05-14 10:46:35 +00:00
|
|
|
) -> None:
|
|
|
|
"""Set a sequence to the correct value."""
|
2021-02-23 12:33:24 +00:00
|
|
|
current_stream_ids = []
|
2024-05-30 11:07:32 +00:00
|
|
|
for stream_id_table, column_name in stream_id_tables:
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
max_stream_id = cast(
|
|
|
|
int,
|
|
|
|
await self.sqlite_store.db_pool.simple_select_one_onecol(
|
|
|
|
table=stream_id_table,
|
|
|
|
keyvalues={},
|
2023-08-01 14:36:33 +00:00
|
|
|
retcol=f"COALESCE(MAX({column_name}), 1)",
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
allow_none=True,
|
|
|
|
),
|
2021-02-23 12:33:24 +00:00
|
|
|
)
|
|
|
|
current_stream_ids.append(max_stream_id)
|
2021-01-07 20:19:26 +00:00
|
|
|
|
2021-02-23 12:33:24 +00:00
|
|
|
next_id = max(current_stream_ids) + 1
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def r(txn: LoggingTransaction) -> None:
|
2021-05-14 10:46:35 +00:00
|
|
|
sql = "ALTER SEQUENCE %s RESTART WITH" % (sequence_name,)
|
|
|
|
txn.execute(sql + " %s", (next_id,))
|
2021-01-07 20:19:26 +00:00
|
|
|
|
2021-05-14 10:46:35 +00:00
|
|
|
await self.postgres_store.db_pool.runInteraction(
|
|
|
|
"_setup_%s" % (sequence_name,), r
|
|
|
|
)
|
2021-02-23 12:33:24 +00:00
|
|
|
|
|
|
|
async def _setup_auth_chain_sequence(self) -> None:
|
2024-09-02 11:39:04 +00:00
|
|
|
curr_chain_id: Optional[
|
|
|
|
int
|
|
|
|
] = await self.sqlite_store.db_pool.simple_select_one_onecol(
|
|
|
|
table="event_auth_chains",
|
|
|
|
keyvalues={},
|
|
|
|
retcol="MAX(chain_id)",
|
|
|
|
allow_none=True,
|
2021-02-23 12:33:24 +00:00
|
|
|
)
|
2021-01-07 20:19:26 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def r(txn: LoggingTransaction) -> None:
|
|
|
|
# Presumably there is at least one row in event_auth_chains.
|
|
|
|
assert curr_chain_id is not None
|
2021-01-07 20:19:26 +00:00
|
|
|
txn.execute(
|
2021-02-23 12:33:24 +00:00
|
|
|
"ALTER SEQUENCE event_auth_chain_id RESTART WITH %s",
|
2021-05-24 13:03:00 +00:00
|
|
|
(curr_chain_id + 1,),
|
2021-01-07 20:19:26 +00:00
|
|
|
)
|
|
|
|
|
2021-05-10 12:02:55 +00:00
|
|
|
if curr_chain_id is not None:
|
|
|
|
await self.postgres_store.db_pool.runInteraction(
|
|
|
|
"_setup_event_auth_chain_id",
|
|
|
|
r,
|
|
|
|
)
|
2021-01-07 20:19:26 +00:00
|
|
|
|
2015-04-10 17:47:20 +00:00
|
|
|
|
2015-04-27 16:53:15 +00:00
|
|
|
##############################################
|
2018-10-20 00:16:55 +00:00
|
|
|
# The following is simply UI stuff
|
2015-04-27 16:53:15 +00:00
|
|
|
##############################################
|
|
|
|
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
class TableProgress(TypedDict):
|
|
|
|
start: int
|
|
|
|
num_done: int
|
|
|
|
total: int
|
|
|
|
perc: int
|
|
|
|
|
|
|
|
|
|
|
|
class Progress:
|
2021-05-14 10:46:35 +00:00
|
|
|
"""Used to report progress of the port"""
|
2018-10-20 00:16:55 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def __init__(self) -> None:
|
|
|
|
self.tables: Dict[str, TableProgress] = {}
|
2015-04-27 16:53:15 +00:00
|
|
|
|
|
|
|
self.start_time = int(time.time())
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def add_table(self, table: str, cur: int, size: int) -> None:
|
2015-04-27 16:53:15 +00:00
|
|
|
self.tables[table] = {
|
|
|
|
"start": cur,
|
|
|
|
"num_done": cur,
|
|
|
|
"total": size,
|
|
|
|
"perc": int(cur * 100 / size),
|
|
|
|
}
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def update(self, table: str, num_done: int) -> None:
|
2015-04-27 16:53:15 +00:00
|
|
|
data = self.tables[table]
|
|
|
|
data["num_done"] = num_done
|
|
|
|
data["perc"] = int(num_done * 100 / data["total"])
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def done(self) -> None:
|
|
|
|
pass
|
|
|
|
|
|
|
|
def set_state(self, state: str) -> None:
|
2015-04-27 16:53:15 +00:00
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
class CursesProgress(Progress):
|
2021-05-14 10:46:35 +00:00
|
|
|
"""Reports progress to a curses window"""
|
2018-10-20 00:16:55 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def __init__(self, stdscr: "curses.window"):
|
2015-04-27 16:53:15 +00:00
|
|
|
self.stdscr = stdscr
|
|
|
|
|
|
|
|
curses.use_default_colors()
|
|
|
|
curses.curs_set(0)
|
|
|
|
|
|
|
|
curses.init_pair(1, curses.COLOR_RED, -1)
|
|
|
|
curses.init_pair(2, curses.COLOR_GREEN, -1)
|
|
|
|
|
2022-04-08 14:00:12 +00:00
|
|
|
self.last_update = 0.0
|
2015-04-27 16:53:15 +00:00
|
|
|
|
|
|
|
self.finished = False
|
|
|
|
|
2015-04-28 11:55:29 +00:00
|
|
|
self.total_processed = 0
|
|
|
|
self.total_remaining = 0
|
|
|
|
|
2023-08-15 12:11:20 +00:00
|
|
|
super().__init__()
|
2015-04-27 16:53:15 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def update(self, table: str, num_done: int) -> None:
|
2023-08-15 12:11:20 +00:00
|
|
|
super().update(table, num_done)
|
2015-04-27 16:53:15 +00:00
|
|
|
|
2015-04-28 11:55:29 +00:00
|
|
|
self.total_processed = 0
|
|
|
|
self.total_remaining = 0
|
2021-05-14 10:46:35 +00:00
|
|
|
for data in self.tables.values():
|
2015-04-28 11:55:29 +00:00
|
|
|
self.total_processed += data["num_done"] - data["start"]
|
|
|
|
self.total_remaining += data["total"] - data["num_done"]
|
|
|
|
|
2015-04-27 16:53:15 +00:00
|
|
|
self.render()
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def render(self, force: bool = False) -> None:
|
2015-04-27 16:53:15 +00:00
|
|
|
now = time.time()
|
|
|
|
|
|
|
|
if not force and now - self.last_update < 0.2:
|
|
|
|
# reactor.callLater(1, self.render)
|
|
|
|
return
|
|
|
|
|
|
|
|
self.stdscr.clear()
|
|
|
|
|
|
|
|
rows, cols = self.stdscr.getmaxyx()
|
|
|
|
|
|
|
|
duration = int(now) - int(self.start_time)
|
|
|
|
|
|
|
|
minutes, seconds = divmod(duration, 60)
|
2019-12-04 13:52:46 +00:00
|
|
|
duration_str = "%02dm %02ds" % (minutes, seconds)
|
2015-04-27 16:53:15 +00:00
|
|
|
|
|
|
|
if self.finished:
|
|
|
|
status = "Time spent: %s (Done!)" % (duration_str,)
|
|
|
|
else:
|
2015-04-28 11:55:29 +00:00
|
|
|
if self.total_processed > 0:
|
|
|
|
left = float(self.total_remaining) / self.total_processed
|
|
|
|
|
|
|
|
est_remaining = (int(now) - self.start_time) * left
|
2019-12-04 13:52:46 +00:00
|
|
|
est_remaining_str = "%02dm %02ds remaining" % divmod(est_remaining, 60)
|
2015-04-27 16:53:15 +00:00
|
|
|
else:
|
|
|
|
est_remaining_str = "Unknown"
|
2018-10-20 00:16:55 +00:00
|
|
|
status = "Time spent: %s (est. remaining: %s)" % (
|
|
|
|
duration_str,
|
|
|
|
est_remaining_str,
|
2015-04-27 16:53:15 +00:00
|
|
|
)
|
|
|
|
|
2018-10-20 00:16:55 +00:00
|
|
|
self.stdscr.addstr(0, 0, status, curses.A_BOLD)
|
2015-04-27 16:53:15 +00:00
|
|
|
|
2021-10-11 15:28:29 +00:00
|
|
|
max_len = max(len(t) for t in self.tables.keys())
|
2015-04-27 16:53:15 +00:00
|
|
|
|
|
|
|
left_margin = 5
|
|
|
|
middle_space = 1
|
|
|
|
|
2022-04-08 14:00:12 +00:00
|
|
|
items = sorted(self.tables.items(), key=lambda i: (i[1]["perc"], i[0]))
|
2015-04-27 16:53:15 +00:00
|
|
|
|
|
|
|
for i, (table, data) in enumerate(items):
|
|
|
|
if i + 2 >= rows:
|
|
|
|
break
|
|
|
|
|
|
|
|
perc = data["perc"]
|
|
|
|
|
|
|
|
color = curses.color_pair(2) if perc == 100 else curses.color_pair(1)
|
|
|
|
|
|
|
|
self.stdscr.addstr(
|
2018-10-20 00:16:55 +00:00
|
|
|
i + 2, left_margin + max_len - len(table), table, curses.A_BOLD | color
|
2015-04-27 16:53:15 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
size = 20
|
|
|
|
|
|
|
|
progress = "[%s%s]" % (
|
2016-08-04 10:28:02 +00:00
|
|
|
"#" * int(perc * size / 100),
|
|
|
|
" " * (size - int(perc * size / 100)),
|
2015-04-27 16:53:15 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
self.stdscr.addstr(
|
2018-10-20 00:16:55 +00:00
|
|
|
i + 2,
|
|
|
|
left_margin + max_len + middle_space,
|
2015-04-27 16:53:15 +00:00
|
|
|
"%s %3d%% (%d/%d)" % (progress, perc, data["num_done"], data["total"]),
|
|
|
|
)
|
|
|
|
|
|
|
|
if self.finished:
|
2018-10-20 00:16:55 +00:00
|
|
|
self.stdscr.addstr(rows - 1, 0, "Press any key to exit...")
|
2015-04-27 16:53:15 +00:00
|
|
|
|
|
|
|
self.stdscr.refresh()
|
|
|
|
self.last_update = time.time()
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def done(self) -> None:
|
2015-04-27 16:53:15 +00:00
|
|
|
self.finished = True
|
|
|
|
self.render(True)
|
|
|
|
self.stdscr.getch()
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def set_state(self, state: str) -> None:
|
2015-04-27 16:53:15 +00:00
|
|
|
self.stdscr.clear()
|
2018-10-20 00:16:55 +00:00
|
|
|
self.stdscr.addstr(0, 0, state + "...", curses.A_BOLD)
|
2015-04-27 16:53:15 +00:00
|
|
|
self.stdscr.refresh()
|
|
|
|
|
|
|
|
|
|
|
|
class TerminalProgress(Progress):
|
2021-05-14 10:46:35 +00:00
|
|
|
"""Just prints progress to the terminal"""
|
2018-10-20 00:16:55 +00:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def update(self, table: str, num_done: int) -> None:
|
2023-08-15 12:11:20 +00:00
|
|
|
super().update(table, num_done)
|
2015-04-27 16:53:15 +00:00
|
|
|
|
|
|
|
data = self.tables[table]
|
|
|
|
|
2018-10-20 00:16:55 +00:00
|
|
|
print(
|
|
|
|
"%s: %d%% (%d/%d)" % (table, data["perc"], data["num_done"], data["total"])
|
2015-04-27 16:53:15 +00:00
|
|
|
)
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def set_state(self, state: str) -> None:
|
2018-10-20 00:16:55 +00:00
|
|
|
print(state + "...")
|
2015-04-27 16:53:15 +00:00
|
|
|
|
|
|
|
|
|
|
|
##############################################
|
|
|
|
##############################################
|
|
|
|
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def main() -> None:
|
2015-04-28 16:42:36 +00:00
|
|
|
parser = argparse.ArgumentParser(
|
|
|
|
description="A script to port an existing synapse SQLite database to"
|
2018-10-20 00:16:55 +00:00
|
|
|
" a new PostgreSQL database."
|
2015-04-28 16:42:36 +00:00
|
|
|
)
|
2019-12-04 13:52:46 +00:00
|
|
|
parser.add_argument("-v", action="store_true")
|
2015-04-10 17:47:20 +00:00
|
|
|
parser.add_argument(
|
2018-10-20 00:16:55 +00:00
|
|
|
"--sqlite-database",
|
|
|
|
required=True,
|
2015-04-28 16:42:36 +00:00
|
|
|
help="The snapshot of the SQLite database file. This must not be"
|
2018-10-20 00:16:55 +00:00
|
|
|
" currently used by a running synapse server",
|
2015-04-28 16:42:36 +00:00
|
|
|
)
|
|
|
|
parser.add_argument(
|
2018-10-20 00:16:55 +00:00
|
|
|
"--postgres-config",
|
2019-12-04 13:52:46 +00:00
|
|
|
type=argparse.FileType("r"),
|
2018-10-20 00:16:55 +00:00
|
|
|
required=True,
|
|
|
|
help="The database config file for the PostgreSQL database",
|
2015-04-28 16:42:36 +00:00
|
|
|
)
|
|
|
|
parser.add_argument(
|
2019-12-04 13:52:46 +00:00
|
|
|
"--curses", action="store_true", help="display a curses based progress UI"
|
2015-04-10 17:47:20 +00:00
|
|
|
)
|
|
|
|
|
2015-04-28 16:42:36 +00:00
|
|
|
parser.add_argument(
|
2018-10-20 00:16:55 +00:00
|
|
|
"--batch-size",
|
|
|
|
type=int,
|
|
|
|
default=1000,
|
2015-04-28 16:42:36 +00:00
|
|
|
help="The number of rows to select from the SQLite table each"
|
2018-10-20 00:16:55 +00:00
|
|
|
" iteration [default=1000]",
|
2015-04-28 16:42:36 +00:00
|
|
|
)
|
2015-04-17 10:13:05 +00:00
|
|
|
|
2015-04-10 17:47:20 +00:00
|
|
|
args = parser.parse_args()
|
2015-04-17 10:13:05 +00:00
|
|
|
|
2022-04-08 14:00:12 +00:00
|
|
|
logging.basicConfig(
|
|
|
|
level=logging.DEBUG if args.v else logging.INFO,
|
|
|
|
format="%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s",
|
|
|
|
filename="port-synapse.log" if args.curses else None,
|
|
|
|
)
|
2015-04-10 17:47:20 +00:00
|
|
|
|
2023-03-22 15:36:42 +00:00
|
|
|
if not os.path.isfile(args.sqlite_database):
|
|
|
|
sys.stderr.write(
|
|
|
|
"The sqlite database you specified does not exist, please check that you have the"
|
|
|
|
"correct path."
|
|
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
|
2015-04-10 17:47:20 +00:00
|
|
|
sqlite_config = {
|
|
|
|
"name": "sqlite3",
|
|
|
|
"args": {
|
2023-03-21 17:49:25 +00:00
|
|
|
"database": args.sqlite_database,
|
2015-04-10 17:47:20 +00:00
|
|
|
"cp_min": 1,
|
|
|
|
"cp_max": 1,
|
|
|
|
"check_same_thread": False,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
2019-10-23 14:31:59 +00:00
|
|
|
hs_config = yaml.safe_load(args.postgres_config)
|
2015-04-10 17:47:20 +00:00
|
|
|
|
2019-10-23 14:31:59 +00:00
|
|
|
if "database" not in hs_config:
|
|
|
|
sys.stderr.write("The configuration file must have a 'database' section.\n")
|
|
|
|
sys.exit(4)
|
|
|
|
|
|
|
|
postgres_config = hs_config["database"]
|
2015-05-08 12:45:58 +00:00
|
|
|
|
2015-04-28 16:42:36 +00:00
|
|
|
if "name" not in postgres_config:
|
2019-10-23 14:31:59 +00:00
|
|
|
sys.stderr.write("Malformed database config: no 'name'\n")
|
2015-04-28 16:42:36 +00:00
|
|
|
sys.exit(2)
|
|
|
|
if postgres_config["name"] != "psycopg2":
|
2019-10-23 14:31:59 +00:00
|
|
|
sys.stderr.write("Database must use the 'psycopg2' connector.\n")
|
2015-04-28 16:42:36 +00:00
|
|
|
sys.exit(3)
|
|
|
|
|
2023-06-21 13:20:46 +00:00
|
|
|
# Don't run the background tasks that get started by the data stores.
|
|
|
|
hs_config["run_background_tasks_on"] = "some_other_process"
|
|
|
|
|
2019-10-23 14:31:59 +00:00
|
|
|
config = HomeServerConfig()
|
|
|
|
config.parse_config_dict(hs_config, "", "")
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def start(stdscr: Optional["curses.window"] = None) -> None:
|
2022-04-08 14:00:12 +00:00
|
|
|
progress: Progress
|
2015-04-17 10:13:05 +00:00
|
|
|
if stdscr:
|
|
|
|
progress = CursesProgress(stdscr)
|
|
|
|
else:
|
|
|
|
progress = TerminalProgress()
|
|
|
|
|
|
|
|
porter = Porter(
|
|
|
|
sqlite_config=sqlite_config,
|
|
|
|
progress=progress,
|
|
|
|
batch_size=args.batch_size,
|
2019-10-23 14:31:59 +00:00
|
|
|
hs_config=config,
|
2015-04-17 10:13:05 +00:00
|
|
|
)
|
|
|
|
|
2020-01-21 19:04:58 +00:00
|
|
|
@defer.inlineCallbacks
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 11:41:55 +00:00
|
|
|
def run() -> Generator["defer.Deferred[Any]", Any, None]:
|
2020-01-21 19:04:58 +00:00
|
|
|
with LoggingContext("synapse_port_db_run"):
|
|
|
|
yield defer.ensureDeferred(porter.run())
|
|
|
|
|
|
|
|
reactor.callWhenRunning(run)
|
2015-04-17 10:13:05 +00:00
|
|
|
|
|
|
|
reactor.run()
|
|
|
|
|
|
|
|
if args.curses:
|
|
|
|
curses.wrapper(start)
|
|
|
|
else:
|
|
|
|
start()
|
2015-04-10 17:47:20 +00:00
|
|
|
|
2020-01-21 19:04:58 +00:00
|
|
|
if end_error:
|
|
|
|
if end_error_exec_info:
|
|
|
|
exc_type, exc_value, exc_traceback = end_error_exec_info
|
|
|
|
traceback.print_exception(exc_type, exc_value, exc_traceback)
|
|
|
|
|
|
|
|
sys.stderr.write(end_error)
|
|
|
|
|
2019-12-04 17:48:23 +00:00
|
|
|
sys.exit(5)
|
2022-03-02 13:00:16 +00:00
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
main()
|