From 89036579ed475464ccc8619344db513042e1d792 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Mar 2015 11:18:49 +0000 Subject: [PATCH 001/104] Update schema to work with mariadb --- synapse/storage/schema/delta/12/v12.sql | 48 +++++---- synapse/storage/schema/delta/13/v13.sql | 19 ++-- synapse/storage/schema/delta/14/v14.sql | 6 +- .../schema/full_schemas/11/event_edges.sql | 52 +++++----- .../full_schemas/11/event_signatures.sql | 46 ++++----- synapse/storage/schema/full_schemas/11/im.sql | 98 +++++++++---------- .../storage/schema/full_schemas/11/keys.sql | 16 +-- .../full_schemas/11/media_repository.sql | 46 ++++----- .../schema/full_schemas/11/presence.sql | 18 ++-- .../schema/full_schemas/11/profiles.sql | 8 +- .../schema/full_schemas/11/redactions.sql | 6 +- .../schema/full_schemas/11/room_aliases.sql | 12 +-- .../storage/schema/full_schemas/11/state.sql | 29 +++--- .../schema/full_schemas/11/transactions.sql | 32 +++--- .../storage/schema/full_schemas/11/users.sql | 29 +++--- synapse/storage/schema/schema_version.sql | 15 ++- 16 files changed, 224 insertions(+), 256 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index b87ef1fe7..fbe8486b9 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -14,54 +14,50 @@ */ CREATE TABLE IF NOT EXISTS rejections( - event_id TEXT NOT NULL, - reason TEXT NOT NULL, - last_check TEXT NOT NULL, - CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE + event_id VARCHAR(255) NOT NULL, + reason VARCHAR(255) NOT NULL, + last_check VARCHAR(255) NOT NULL, + UNIQUE (event_id) ); -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name TEXT NOT NULL, - profile_tag varchar(32) NOT NULL, - kind varchar(8) NOT NULL, - app_id varchar(64) NOT NULL, - app_display_name varchar(64) NOT NULL, - device_display_name varchar(128) NOT NULL, - pushkey blob NOT NULL, + id INTEGER PRIMARY KEY, + user_name VARCHAR(255) NOT NULL, + profile_tag VARCHAR(32) NOT NULL, + kind VARCHAR(8) NOT NULL, + app_id VARCHAR(64) NOT NULL, + app_display_name VARCHAR(64) NOT NULL, + device_display_name VARCHAR(128) NOT NULL, + pushkey VARBINARY(512) NOT NULL, ts BIGINT NOT NULL, - lang varchar(8), - data blob, + lang VARCHAR(8), + data BLOB, last_token TEXT, last_success BIGINT, failing_since BIGINT, - FOREIGN KEY(user_name) REFERENCES users(name), UNIQUE (app_id, pushkey) ); CREATE TABLE IF NOT EXISTS push_rules ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name TEXT NOT NULL, - rule_id TEXT NOT NULL, + id INTEGER PRIMARY KEY, + user_name VARCHAR(255) NOT NULL, + rule_id VARCHAR(255) NOT NULL, priority_class TINYINT NOT NULL, priority INTEGER NOT NULL DEFAULT 0, - conditions TEXT NOT NULL, - actions TEXT NOT NULL, + conditions VARCHAR(255) NOT NULL, + actions VARCHAR(255) NOT NULL, UNIQUE(user_name, rule_id) ); CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( - user_id TEXT, + user_id VARCHAR(255), filter_id INTEGER, - filter_json TEXT, - FOREIGN KEY(user_id) REFERENCES users(id) + filter_json BLOB ); CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( - user_id, filter_id + user_id, filter_id ); - -PRAGMA user_version = 12; diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index e491ad5ae..68d4ebd89 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -14,21 +14,18 @@ */ CREATE TABLE IF NOT EXISTS application_services( - id INTEGER PRIMARY KEY AUTOINCREMENT, - url TEXT, - token TEXT, - hs_token TEXT, - sender TEXT, - UNIQUE(token) ON CONFLICT ROLLBACK + id INTEGER PRIMARY KEY, + url VARCHAR(255), + token VARCHAR(255), + hs_token VARCHAR(255), + sender VARCHAR(255), + UNIQUE(token) ); CREATE TABLE IF NOT EXISTS application_services_regex( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY, as_id INTEGER NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ - regex TEXT, + regex VARCHAR(255), FOREIGN KEY(as_id) REFERENCES application_services(id) ); - - - diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index 021272644..2b50adbd6 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -1,7 +1,7 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name TEXT NOT NULL, - rule_id TEXT NOT NULL, + id INTEGER PRIMARY KEY, + user_name VARCHAR(255) NOT NULL, + rule_id VARCHAR(255) NOT NULL, enabled TINYINT, UNIQUE(user_name, rule_id) ); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 1e766d6db..823653f2e 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -14,9 +14,9 @@ */ CREATE TABLE IF NOT EXISTS event_forward_extremities( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + UNIQUE (event_id, room_id) ); CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); @@ -24,9 +24,9 @@ CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_backward_extremities( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + UNIQUE (event_id, room_id) ); CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); @@ -34,11 +34,11 @@ CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id CREATE TABLE IF NOT EXISTS event_edges( - event_id TEXT NOT NULL, - prev_event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - is_state INTEGER NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, prev_event_id, room_id, is_state) + event_id VARCHAR(255) NOT NULL, + prev_event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + is_state BOOL NOT NULL, + UNIQUE (event_id, prev_event_id, room_id, is_state) ); CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); @@ -46,30 +46,30 @@ CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); CREATE TABLE IF NOT EXISTS room_depth( - room_id TEXT NOT NULL, + room_id VARCHAR(255) NOT NULL, min_depth INTEGER NOT NULL, - CONSTRAINT uniqueness UNIQUE (room_id) + UNIQUE (room_id) ); CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( - event_id TEXT NOT NULL, - destination TEXT NOT NULL, + event_id VARCHAR(255) NOT NULL, + destination VARCHAR(255) NOT NULL, delivered_ts INTEGER DEFAULT 0, -- or 0 if not delivered - CONSTRAINT uniqueness UNIQUE (event_id, destination) ON CONFLICT REPLACE + UNIQUE (event_id, destination) ); CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); CREATE TABLE IF NOT EXISTS state_forward_extremities( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + type VARCHAR(255) NOT NULL, + state_key VARCHAR(255) NOT NULL, + UNIQUE (event_id, room_id) ); CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( @@ -79,11 +79,11 @@ CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_auth( - event_id TEXT NOT NULL, - auth_id TEXT NOT NULL, - room_id TEXT NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, auth_id, room_id) + event_id VARCHAR(255) NOT NULL, + auth_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + UNIQUE (event_id, auth_id, room_id) ); CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); -CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); \ No newline at end of file +CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index c28c39c48..a7c3acf77 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -14,52 +14,42 @@ */ CREATE TABLE IF NOT EXISTS event_content_hashes ( - event_id TEXT, - algorithm TEXT, + event_id VARCHAR(255), + algorithm VARCHAR(255), hash BLOB, - CONSTRAINT uniqueness UNIQUE (event_id, algorithm) + UNIQUE (event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes( - event_id -); +CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); CREATE TABLE IF NOT EXISTS event_reference_hashes ( - event_id TEXT, - algorithm TEXT, + event_id VARCHAR(255), + algorithm VARCHAR(255), hash BLOB, - CONSTRAINT uniqueness UNIQUE (event_id, algorithm) + UNIQUE (event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes ( - event_id -); +CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); CREATE TABLE IF NOT EXISTS event_signatures ( - event_id TEXT, - signature_name TEXT, - key_id TEXT, + event_id VARCHAR(255), + signature_name VARCHAR(255), + key_id VARCHAR(255), signature BLOB, - CONSTRAINT uniqueness UNIQUE (event_id, signature_name, key_id) + UNIQUE (event_id, signature_name, key_id) ); -CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures ( - event_id -); +CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); CREATE TABLE IF NOT EXISTS event_edge_hashes( - event_id TEXT, - prev_event_id TEXT, - algorithm TEXT, + event_id VARCHAR(255), + prev_event_id VARCHAR(255), + algorithm VARCHAR(255), hash BLOB, - CONSTRAINT uniqueness UNIQUE ( - event_id, prev_event_id, algorithm - ) + UNIQUE (event_id, prev_event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes( - event_id -); +CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index dd00c1cd2..8feacdbe0 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -14,17 +14,17 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, + stream_ordering INTEGER PRIMARY KEY, topological_ordering INTEGER NOT NULL, - event_id TEXT NOT NULL, - type TEXT NOT NULL, - room_id TEXT NOT NULL, - content TEXT NOT NULL, - unrecognized_keys TEXT, + event_id VARCHAR(255) NOT NULL, + type VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + content BLOB NOT NULL, + unrecognized_keys BLOB, processed BOOL NOT NULL, outlier BOOL NOT NULL, depth INTEGER DEFAULT 0 NOT NULL, - CONSTRAINT ev_uniq UNIQUE (event_id) + UNIQUE (event_id) ); CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id); @@ -34,92 +34,92 @@ CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - internal_metadata NOT NULL, + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + internal_metadata BLOB NOT NULL, json BLOB NOT NULL, - CONSTRAINT ev_j_uniq UNIQUE (event_id) + UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS event_json_id ON event_json(event_id); CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); -CREATE TABLE IF NOT EXISTS state_events( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - prev_state TEXT +CREATE TABLE state_events( + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + type VARCHAR(255) NOT NULL, + state_key VARCHAR(255) NOT NULL, + prev_state VARCHAR(255), + UNIQUE (event_id) ); -CREATE UNIQUE INDEX IF NOT EXISTS state_events_event_id ON state_events (event_id); CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); CREATE TABLE IF NOT EXISTS current_state_events( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - CONSTRAINT curr_uniq UNIQUE (room_id, type, state_key) ON CONFLICT REPLACE + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + type VARCHAR(255) NOT NULL, + state_key VARCHAR(255) NOT NULL, + UNIQUE (event_id), + UNIQUE (room_id, type, state_key) ); -CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id); CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); CREATE TABLE IF NOT EXISTS room_memberships( - event_id TEXT NOT NULL, - user_id TEXT NOT NULL, - sender TEXT NOT NULL, - room_id TEXT NOT NULL, - membership TEXT NOT NULL + event_id VARCHAR(255) NOT NULL, + user_id VARCHAR(255) NOT NULL, + sender VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + membership VARCHAR(255) NOT NULL, + UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id); CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); CREATE TABLE IF NOT EXISTS feedback( - event_id TEXT NOT NULL, - feedback_type TEXT, - target_event_id TEXT, - sender TEXT, - room_id TEXT + event_id VARCHAR(255) NOT NULL, + feedback_type VARCHAR(255), + target_event_id VARCHAR(255), + sender VARCHAR(255), + room_id VARCHAR(255), + UNIQUE (event_id) ); CREATE TABLE IF NOT EXISTS topics( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - topic TEXT NOT NULL + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + topic VARCHAR(255) NOT NULL, + UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id); CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - name TEXT NOT NULL + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + name VARCHAR(255) NOT NULL, + UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id); CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( - room_id TEXT PRIMARY KEY NOT NULL, + room_id VARCHAR(255) PRIMARY KEY NOT NULL, is_public INTEGER, - creator TEXT + creator VARCHAR(255) ); CREATE TABLE IF NOT EXISTS room_hosts( - room_id TEXT NOT NULL, - host TEXT NOT NULL, - CONSTRAINT room_hosts_uniq UNIQUE (room_id, host) ON CONFLICT IGNORE + room_id VARCHAR(255) NOT NULL, + host VARCHAR(255) NOT NULL, + UNIQUE (room_id, host) ); CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index a9e0a4fe0..7a5afe094 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -13,19 +13,19 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS server_tls_certificates( - server_name TEXT, -- Server name. - fingerprint TEXT, -- Certificate fingerprint. - from_server TEXT, -- Which key server the certificate was fetched from. + server_name VARCHAR(255), -- Server name. + fingerprint VARCHAR(255), -- Certificate fingerprint. + from_server VARCHAR(255), -- Which key server the certificate was fetched from. ts_added_ms INTEGER, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. - CONSTRAINT uniqueness UNIQUE (server_name, fingerprint) + UNIQUE (server_name, fingerprint) ); CREATE TABLE IF NOT EXISTS server_signature_keys( - server_name TEXT, -- Server name. - key_id TEXT, -- Key version. - from_server TEXT, -- Which key server the key was fetched form. + server_name VARCHAR(255), -- Server name. + key_id VARCHAR(255), -- Key version. + from_server VARCHAR(255), -- Which key server the key was fetched form. ts_added_ms INTEGER, -- When the key was added. verify_key BLOB, -- NACL verification key. - CONSTRAINT uniqueness UNIQUE (server_name, key_id) + UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index afdf48cbf..1e84c1d96 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -14,23 +14,23 @@ */ CREATE TABLE IF NOT EXISTS local_media_repository ( - media_id TEXT, -- The id used to refer to the media. - media_type TEXT, -- The MIME-type of the media. + media_id VARCHAR(255), -- The id used to refer to the media. + media_type VARCHAR(255), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. created_ts INTEGER, -- When the content was uploaded in ms. - upload_name TEXT, -- The name the media was uploaded with. - user_id TEXT, -- The user who uploaded the file. - CONSTRAINT uniqueness UNIQUE (media_id) + upload_name VARCHAR(255), -- The name the media was uploaded with. + user_id VARCHAR(255), -- The user who uploaded the file. + UNIQUE (media_id) ); CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( - media_id TEXT, -- The id used to refer to the media. + media_id VARCHAR(255), -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_type TEXT, -- The MIME-type of the thumbnail. - thumbnail_method TEXT, -- The method used to make the thumbnail. + thumbnail_type VARCHAR(255), -- The MIME-type of the thumbnail. + thumbnail_method VARCHAR(255), -- The method used to make the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - CONSTRAINT uniqueness UNIQUE ( + UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type ) ); @@ -39,29 +39,29 @@ CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); CREATE TABLE IF NOT EXISTS remote_media_cache ( - media_origin TEXT, -- The remote HS the media came from. - media_id TEXT, -- The id used to refer to the media on that server. - media_type TEXT, -- The MIME-type of the media. + media_origin VARCHAR(255), -- The remote HS the media came from. + media_id VARCHAR(255), -- The id used to refer to the media on that server. + media_type VARCHAR(255), -- The MIME-type of the media. created_ts INTEGER, -- When the content was uploaded in ms. - upload_name TEXT, -- The name the media was uploaded with. + upload_name VARCHAR(255), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. - filesystem_id TEXT, -- The name used to store the media on disk. - CONSTRAINT uniqueness UNIQUE (media_origin, media_id) + filesystem_id VARCHAR(255), -- The name used to store the media on disk. + UNIQUE (media_origin, media_id) ); CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( - media_origin TEXT, -- The remote HS the media came from. - media_id TEXT, -- The id used to refer to the media. + media_origin VARCHAR(255), -- The remote HS the media came from. + media_id VARCHAR(255), -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_method TEXT, -- The method used to make the thumbnail - thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_method VARCHAR(255), -- The method used to make the thumbnail + thumbnail_type VARCHAR(255), -- The MIME-type of the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - filesystem_id TEXT, -- The name used to store the media on disk. - CONSTRAINT uniqueness UNIQUE ( + filesystem_id VARCHAR(255), -- The name used to store the media on disk. + UNIQUE ( media_origin, media_id, thumbnail_width, thumbnail_height, - thumbnail_type, thumbnail_type - ) + thumbnail_type + ) ); CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index f9f8db969..519812349 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -13,26 +13,26 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS presence( - user_id INTEGER NOT NULL, + user_id VARCHAR(255) NOT NULL, state INTEGER, - status_msg TEXT, + status_msg VARCHAR(255), mtime INTEGER, -- miliseconds since last state change - FOREIGN KEY(user_id) REFERENCES users(id) + UNIQUE(user_id) ); -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state CREATE TABLE IF NOT EXISTS presence_allow_inbound( - observed_user_id INTEGER NOT NULL, - observer_user_id TEXT, -- a UserID, - FOREIGN KEY(observed_user_id) REFERENCES users(id) + observed_user_id VARCHAR(255) NOT NULL, + observer_user_id VARCHAR(255), -- a UserID, + UNIQUE(observed_user_id) ); -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? CREATE TABLE IF NOT EXISTS presence_list( - user_id INTEGER NOT NULL, - observed_user_id TEXT, -- a UserID, + user_id VARCHAR(255) NOT NULL, + observed_user_id VARCHAR(255), -- a UserID, accepted BOOLEAN, - FOREIGN KEY(user_id) REFERENCES users(id) + UNIQUE(user_id) ); diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index f06a528b4..9b22c9b1b 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS profiles( - user_id INTEGER NOT NULL, - displayname TEXT, - avatar_url TEXT, - FOREIGN KEY(user_id) REFERENCES users(id) + user_id VARCHAR(255) NOT NULL, + displayname VARCHAR(255), + avatar_url VARCHAR(255), + UNIQUE(user_id) ); diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index 5011d95db..774584d7c 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -13,9 +13,9 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS redactions ( - event_id TEXT NOT NULL, - redacts TEXT NOT NULL, - CONSTRAINT ev_uniq UNIQUE (event_id) + event_id VARCHAR(255) NOT NULL, + redacts VARCHAR(255) NOT NULL, + UNIQUE (event_id) ); CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index 0d2df0160..dd51c31c1 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -14,14 +14,12 @@ */ CREATE TABLE IF NOT EXISTS room_aliases( - room_alias TEXT NOT NULL, - room_id TEXT NOT NULL + room_alias VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + UNIQUE (room_alias) ); CREATE TABLE IF NOT EXISTS room_alias_servers( - room_alias TEXT NOT NULL, - server TEXT NOT NULL + room_alias VARCHAR(255) NOT NULL, + server VARCHAR(255) NOT NULL ); - - - diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index 1fe8f1e43..408a2d391 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -15,33 +15,26 @@ CREATE TABLE IF NOT EXISTS state_groups( id INTEGER PRIMARY KEY, - room_id TEXT NOT NULL, - event_id TEXT NOT NULL + room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(255) NOT NULL ); CREATE TABLE IF NOT EXISTS state_groups_state( state_group INTEGER NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - event_id TEXT NOT NULL + room_id VARCHAR(255) NOT NULL, + type VARCHAR(255) NOT NULL, + state_key VARCHAR(255) NOT NULL, + event_id VARCHAR(255) NOT NULL ); CREATE TABLE IF NOT EXISTS event_to_state_groups( - event_id TEXT NOT NULL, + event_id VARCHAR(255) NOT NULL, state_group INTEGER NOT NULL, - CONSTRAINT event_to_state_groups_uniq UNIQUE (event_id) + UNIQUE (event_id) ); CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); -CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state( - state_group -); -CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state( - room_id, type, state_key -); - -CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups( - event_id -); \ No newline at end of file +CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(state_group); +CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(room_id, type, state_key); +CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id); \ No newline at end of file diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 2d30f99b0..0158308a3 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -14,34 +14,31 @@ */ -- Stores what transaction ids we have received and what our response was CREATE TABLE IF NOT EXISTS received_transactions( - transaction_id TEXT, - origin TEXT, + transaction_id VARCHAR(255), + origin VARCHAR(255), ts INTEGER, response_code INTEGER, - response_json TEXT, + response_json BLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx - CONSTRAINT uniquesss UNIQUE (transaction_id, origin) ON CONFLICT REPLACE + UNIQUE (transaction_id, origin) ); -CREATE UNIQUE INDEX IF NOT EXISTS transactions_txid ON received_transactions(transaction_id, origin); CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; -- Stores what transactions we've sent, what their response was (if we got one) and whether we have -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( - id INTEGER PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering - transaction_id TEXT, - destination TEXT, + id INTEGER PRIMARY KEY, -- This is used to apply insertion ordering + transaction_id VARCHAR(255), + destination VARCHAR(255), response_code INTEGER DEFAULT 0, - response_json TEXT, + response_json BLOB, ts INTEGER ); CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); -CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions( - destination -); +CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions(destination); CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); -- So that we can do an efficient look up of all transactions that have yet to be successfully -- sent. @@ -51,18 +48,17 @@ CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_c -- For sent transactions only. CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( transaction_id INTEGER, - destination TEXT, - pdu_id TEXT, - pdu_origin TEXT + destination VARCHAR(255), + pdu_id VARCHAR(255), + pdu_origin VARCHAR(255), + UNIQUE (transaction_id, destination) ); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination); CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_index ON transaction_id_to_pdu(transaction_id, destination); -- To track destination health CREATE TABLE IF NOT EXISTS destinations( - destination TEXT PRIMARY KEY, + destination VARCHAR(255) PRIMARY KEY, retry_last_ts INTEGER, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 08ccfdac0..49a122eed 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -13,33 +13,32 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS users( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT, - password_hash TEXT, + id INTEGER PRIMARY KEY, + name VARCHAR(255), + password_hash VARBINARY(255), creation_ts INTEGER, admin BOOL DEFAULT 0 NOT NULL, - UNIQUE(name) ON CONFLICT ROLLBACK + UNIQUE(name) ); CREATE TABLE IF NOT EXISTS access_tokens( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY, user_id INTEGER NOT NULL, - device_id TEXT, - token TEXT NOT NULL, + device_id VARCHAR(255), + token VARCHAR(255) NOT NULL, last_used INTEGER, FOREIGN KEY(user_id) REFERENCES users(id), - UNIQUE(token) ON CONFLICT ROLLBACK + UNIQUE(token) ); CREATE TABLE IF NOT EXISTS user_ips ( - user TEXT NOT NULL, - access_token TEXT NOT NULL, - device_id TEXT, - ip TEXT NOT NULL, - user_agent TEXT NOT NULL, + user VARCHAR(255) NOT NULL, + access_token VARCHAR(255) NOT NULL, + device_id VARCHAR(255), + ip VARCHAR(255) NOT NULL, + user_agent VARCHAR(255) NOT NULL, last_seen INTEGER NOT NULL, - CONSTRAINT user_ip UNIQUE (user, access_token, ip, user_agent) ON CONFLICT REPLACE + UNIQUE (user, access_token, ip, user_agent) ); CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); - diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql index 0431e2d05..28762861e 100644 --- a/synapse/storage/schema/schema_version.sql +++ b/synapse/storage/schema/schema_version.sql @@ -14,17 +14,16 @@ */ CREATE TABLE IF NOT EXISTS schema_version( - Lock char(1) NOT NULL DEFAULT 'X', -- Makes sure this table only has one row. - version INTEGER NOT NULL, - upgraded BOOL NOT NULL, -- Whether we reached this version from an upgrade or an initial schema. - CONSTRAINT schema_version_lock_x CHECK (Lock='X') - CONSTRAINT schema_version_lock_uniq UNIQUE (Lock) + `Lock` CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row. + `version` INTEGER NOT NULL, + `upgraded` BOOL NOT NULL, -- Whether we reached this version from an upgrade or an initial schema. + CHECK (`Lock`='X') ); CREATE TABLE IF NOT EXISTS applied_schema_deltas( - version INTEGER NOT NULL, - file TEXT NOT NULL, - CONSTRAINT schema_deltas_ver_file UNIQUE (version, file) ON CONFLICT IGNORE + `version` INTEGER NOT NULL, + `file` VARCHAR(255) NOT NULL, + UNIQUE(version, file) ); CREATE INDEX IF NOT EXISTS schema_deltas_ver ON applied_schema_deltas(version); From fae059cc18c0aab25b79d5352aa98ccfda8c637a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Mar 2015 13:42:39 +0000 Subject: [PATCH 002/104] Fix up schemas to work with mariadb --- synapse/storage/schema/delta/12/v12.sql | 16 +++++------ synapse/storage/schema/delta/13/v13.sql | 10 +++---- synapse/storage/schema/delta/14/v14.sql | 4 +-- .../schema/full_schemas/11/event_edges.sql | 18 ++++++------ .../full_schemas/11/event_signatures.sql | 8 +++--- synapse/storage/schema/full_schemas/11/im.sql | 28 +++++++++---------- .../storage/schema/full_schemas/11/keys.sql | 8 +++--- .../full_schemas/11/media_repository.sql | 12 ++++---- .../schema/full_schemas/11/presence.sql | 10 +++---- .../schema/full_schemas/11/profiles.sql | 2 +- .../schema/full_schemas/11/redactions.sql | 2 +- .../schema/full_schemas/11/room_aliases.sql | 4 +-- .../storage/schema/full_schemas/11/state.sql | 12 ++++---- .../schema/full_schemas/11/transactions.sql | 14 +++++----- .../storage/schema/full_schemas/11/users.sql | 18 ++++++------ 15 files changed, 82 insertions(+), 84 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index fbe8486b9..42ea1b2ff 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -18,11 +18,11 @@ CREATE TABLE IF NOT EXISTS rejections( reason VARCHAR(255) NOT NULL, last_check VARCHAR(255) NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( - id INTEGER PRIMARY KEY, + id BIGINT PRIMARY KEY, user_name VARCHAR(255) NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, @@ -37,10 +37,10 @@ CREATE TABLE IF NOT EXISTS pushers ( last_success BIGINT, failing_since BIGINT, UNIQUE (app_id, pushkey) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS push_rules ( - id INTEGER PRIMARY KEY, + id BIGINT PRIMARY KEY, user_name VARCHAR(255) NOT NULL, rule_id VARCHAR(255) NOT NULL, priority_class TINYINT NOT NULL, @@ -48,16 +48,16 @@ CREATE TABLE IF NOT EXISTS push_rules ( conditions VARCHAR(255) NOT NULL, actions VARCHAR(255) NOT NULL, UNIQUE(user_name, rule_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(255), - filter_id INTEGER, + filter_id BIGINT, filter_json BLOB -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( user_id, filter_id -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index 68d4ebd89..f0a5daf44 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -14,18 +14,18 @@ */ CREATE TABLE IF NOT EXISTS application_services( - id INTEGER PRIMARY KEY, + id BIGINT PRIMARY KEY, url VARCHAR(255), token VARCHAR(255), hs_token VARCHAR(255), sender VARCHAR(255), UNIQUE(token) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS application_services_regex( - id INTEGER PRIMARY KEY, - as_id INTEGER NOT NULL, + id BIGINT PRIMARY KEY, + as_id BIGINT NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(255), FOREIGN KEY(as_id) REFERENCES application_services(id) -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index 2b50adbd6..a1260c5c1 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -1,9 +1,9 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( - id INTEGER PRIMARY KEY, + id BIGINT PRIMARY KEY, user_name VARCHAR(255) NOT NULL, rule_id VARCHAR(255) NOT NULL, enabled TINYINT, UNIQUE(user_name, rule_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 823653f2e..910e25dfc 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (event_id, room_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); @@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (event_id, room_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_edges( room_id VARCHAR(255) NOT NULL, is_state BOOL NOT NULL, UNIQUE (event_id, prev_event_id, room_id, is_state) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); @@ -49,7 +49,7 @@ CREATE TABLE IF NOT EXISTS room_depth( room_id VARCHAR(255) NOT NULL, min_depth INTEGER NOT NULL, UNIQUE (room_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); @@ -57,9 +57,9 @@ CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( event_id VARCHAR(255) NOT NULL, destination VARCHAR(255) NOT NULL, - delivered_ts INTEGER DEFAULT 0, -- or 0 if not delivered + delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); @@ -70,11 +70,11 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities( type VARCHAR(255) NOT NULL, state_key VARCHAR(255) NOT NULL, UNIQUE (event_id, room_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( room_id, type, state_key -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); @@ -83,7 +83,7 @@ CREATE TABLE IF NOT EXISTS event_auth( auth_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (event_id, auth_id, room_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index a7c3acf77..334d7c868 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( algorithm VARCHAR(255), hash BLOB, UNIQUE (event_id, algorithm) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); @@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS event_reference_hashes ( algorithm VARCHAR(255), hash BLOB, UNIQUE (event_id, algorithm) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_signatures ( key_id VARCHAR(255), signature BLOB, UNIQUE (event_id, signature_name, key_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); @@ -50,6 +50,6 @@ CREATE TABLE IF NOT EXISTS event_edge_hashes( algorithm VARCHAR(255), hash BLOB, UNIQUE (event_id, prev_event_id, algorithm) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index 8feacdbe0..d58106def 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering INTEGER PRIMARY KEY, - topological_ordering INTEGER NOT NULL, + stream_ordering BIGINT PRIMARY KEY, + topological_ordering BIGINT NOT NULL, event_id VARCHAR(255) NOT NULL, type VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, @@ -23,9 +23,9 @@ CREATE TABLE IF NOT EXISTS events( unrecognized_keys BLOB, processed BOOL NOT NULL, outlier BOOL NOT NULL, - depth INTEGER DEFAULT 0 NOT NULL, + depth BIGINT DEFAULT 0 NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id); CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); @@ -39,19 +39,19 @@ CREATE TABLE IF NOT EXISTS event_json( internal_metadata BLOB NOT NULL, json BLOB NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); -CREATE TABLE state_events( +CREATE TABLE IF NOT EXISTS state_events( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, type VARCHAR(255) NOT NULL, state_key VARCHAR(255) NOT NULL, prev_state VARCHAR(255), UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); @@ -65,7 +65,7 @@ CREATE TABLE IF NOT EXISTS current_state_events( state_key VARCHAR(255) NOT NULL, UNIQUE (event_id), UNIQUE (room_id, type, state_key) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); @@ -78,7 +78,7 @@ CREATE TABLE IF NOT EXISTS room_memberships( room_id VARCHAR(255) NOT NULL, membership VARCHAR(255) NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); @@ -90,14 +90,14 @@ CREATE TABLE IF NOT EXISTS feedback( sender VARCHAR(255), room_id VARCHAR(255), UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS topics( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, topic VARCHAR(255) NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); @@ -112,14 +112,14 @@ CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( room_id VARCHAR(255) PRIMARY KEY NOT NULL, - is_public INTEGER, + is_public BOOL, creator VARCHAR(255) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS room_hosts( room_id VARCHAR(255) NOT NULL, host VARCHAR(255) NOT NULL, UNIQUE (room_id, host) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index 7a5afe094..c0f2ec29b 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -16,16 +16,16 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( server_name VARCHAR(255), -- Server name. fingerprint VARCHAR(255), -- Certificate fingerprint. from_server VARCHAR(255), -- Which key server the certificate was fetched from. - ts_added_ms INTEGER, -- When the certifcate was added. + ts_added_ms BIGINT, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(255), -- Server name. key_id VARCHAR(255), -- Key version. from_server VARCHAR(255), -- Which key server the key was fetched form. - ts_added_ms INTEGER, -- When the key was added. + ts_added_ms BIGINT, -- When the key was added. verify_key BLOB, -- NACL verification key. UNIQUE (server_name, key_id) -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index 1e84c1d96..8bc84dc24 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -17,11 +17,11 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( media_id VARCHAR(255), -- The id used to refer to the media. media_type VARCHAR(255), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. - created_ts INTEGER, -- When the content was uploaded in ms. + created_ts BIGINT, -- When the content was uploaded in ms. upload_name VARCHAR(255), -- The name the media was uploaded with. user_id VARCHAR(255), -- The user who uploaded the file. UNIQUE (media_id) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( media_id VARCHAR(255), -- The id used to refer to the media. @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type ) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); @@ -42,12 +42,12 @@ CREATE TABLE IF NOT EXISTS remote_media_cache ( media_origin VARCHAR(255), -- The remote HS the media came from. media_id VARCHAR(255), -- The id used to refer to the media on that server. media_type VARCHAR(255), -- The MIME-type of the media. - created_ts INTEGER, -- When the content was uploaded in ms. + created_ts BIGINT, -- When the content was uploaded in ms. upload_name VARCHAR(255), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(255), -- The name used to store the media on disk. UNIQUE (media_origin, media_id) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( media_origin VARCHAR(255), -- The remote HS the media came from. @@ -62,7 +62,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type ) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id ON local_media_repository_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index 519812349..803132108 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -14,11 +14,11 @@ */ CREATE TABLE IF NOT EXISTS presence( user_id VARCHAR(255) NOT NULL, - state INTEGER, + state VARCHAR(20), status_msg VARCHAR(255), - mtime INTEGER, -- miliseconds since last state change + mtime BIGINT, -- miliseconds since last state change UNIQUE(user_id) -); +) ENGINE = INNODB; -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS presence_allow_inbound( observed_user_id VARCHAR(255) NOT NULL, observer_user_id VARCHAR(255), -- a UserID, UNIQUE(observed_user_id) -); +) ENGINE = INNODB; -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? @@ -35,4 +35,4 @@ CREATE TABLE IF NOT EXISTS presence_list( observed_user_id VARCHAR(255), -- a UserID, accepted BOOLEAN, UNIQUE(user_id) -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index 9b22c9b1b..32defe2f7 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -17,4 +17,4 @@ CREATE TABLE IF NOT EXISTS profiles( displayname VARCHAR(255), avatar_url VARCHAR(255), UNIQUE(user_id) -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index 774584d7c..ba93e860f 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS redactions ( event_id VARCHAR(255) NOT NULL, redacts VARCHAR(255) NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index dd51c31c1..1e706aac2 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -17,9 +17,9 @@ CREATE TABLE IF NOT EXISTS room_aliases( room_alias VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (room_alias) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS room_alias_servers( room_alias VARCHAR(255) NOT NULL, server VARCHAR(255) NOT NULL -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index 408a2d391..be9dc2920 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -14,24 +14,24 @@ */ CREATE TABLE IF NOT EXISTS state_groups( - id INTEGER PRIMARY KEY, + id VARCHAR(20) PRIMARY KEY, room_id VARCHAR(255) NOT NULL, event_id VARCHAR(255) NOT NULL -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS state_groups_state( - state_group INTEGER NOT NULL, + state_group VARCHAR(20) NOT NULL, room_id VARCHAR(255) NOT NULL, type VARCHAR(255) NOT NULL, state_key VARCHAR(255) NOT NULL, event_id VARCHAR(255) NOT NULL -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS event_to_state_groups( event_id VARCHAR(255) NOT NULL, - state_group INTEGER NOT NULL, + state_group VARCHAR(255) NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 0158308a3..cde6919e4 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -16,12 +16,12 @@ CREATE TABLE IF NOT EXISTS received_transactions( transaction_id VARCHAR(255), origin VARCHAR(255), - ts INTEGER, + ts BIGINT, response_code INTEGER, response_json BLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; @@ -29,13 +29,13 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin -- Stores what transactions we've sent, what their response was (if we got one) and whether we have -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( - id INTEGER PRIMARY KEY, -- This is used to apply insertion ordering + id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering transaction_id VARCHAR(255), destination VARCHAR(255), response_code INTEGER DEFAULT 0, response_json BLOB, ts INTEGER -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions(destination); @@ -52,13 +52,13 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( pdu_id VARCHAR(255), pdu_origin VARCHAR(255), UNIQUE (transaction_id, destination) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(255) PRIMARY KEY, - retry_last_ts INTEGER, + retry_last_ts BIGINT, retry_interval INTEGER -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 49a122eed..55bffb22f 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -13,23 +13,21 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS users( - id INTEGER PRIMARY KEY, name VARCHAR(255), password_hash VARBINARY(255), - creation_ts INTEGER, + creation_ts BIGINT, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS access_tokens( - id INTEGER PRIMARY KEY, - user_id INTEGER NOT NULL, + id INTEGER PRIMARY KEY AUTO_INCREMENT, + user_id VARCHAR(255) NOT NULL, device_id VARCHAR(255), token VARCHAR(255) NOT NULL, - last_used INTEGER, - FOREIGN KEY(user_id) REFERENCES users(id), + last_used BIGINT, UNIQUE(token) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS user_ips ( user VARCHAR(255) NOT NULL, @@ -37,8 +35,8 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(255), ip VARCHAR(255) NOT NULL, user_agent VARCHAR(255) NOT NULL, - last_seen INTEGER NOT NULL, + last_seen BIGINT NOT NULL, UNIQUE (user, access_token, ip, user_agent) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); From 58ed393235381b37182212547af7d65e2f6fa384 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Mar 2015 15:12:05 +0000 Subject: [PATCH 003/104] Remove redundant key --- synapse/storage/schema/full_schemas/11/im.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index d58106def..9849e969b 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -27,7 +27,6 @@ CREATE TABLE IF NOT EXISTS events( UNIQUE (event_id) ) ENGINE = INNODB; -CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id); CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); From d7a0496f3ec534076121632352f44733253e1e16 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Mar 2015 15:59:48 +0000 Subject: [PATCH 004/104] Convert storage layer to be mysql compatible --- synapse/storage/__init__.py | 164 ++++++++++++++++++++-------- synapse/storage/_base.py | 30 ++--- synapse/storage/appservice.py | 4 +- synapse/storage/directory.py | 4 +- synapse/storage/event_federation.py | 25 ++--- synapse/storage/presence.py | 1 - synapse/storage/push_rule.py | 4 +- synapse/storage/registration.py | 14 +-- synapse/storage/room.py | 4 +- synapse/storage/roommember.py | 2 +- synapse/storage/state.py | 10 +- synapse/storage/stream.py | 4 +- synapse/storage/transactions.py | 6 +- 13 files changed, 171 insertions(+), 101 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4b16f445d..30cba4771 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -51,6 +51,8 @@ import logging import os import re +import threading + logger = logging.getLogger(__name__) @@ -89,6 +91,9 @@ class DataStore(RoomMemberStore, RoomStore, self.min_token_deferred = self._get_min_token() self.min_token = None + self._next_stream_id_lock = threading.Lock() + self._next_stream_id = int(hs.get_clock().time_msec()) * 1000 + @defer.inlineCallbacks @log_function def persist_event(self, event, context, backfilled=False, @@ -172,7 +177,6 @@ class DataStore(RoomMemberStore, RoomStore, "type": s.type, "state_key": s.state_key, }, - or_replace=True, ) if event.is_state() and is_new_state: @@ -186,7 +190,6 @@ class DataStore(RoomMemberStore, RoomStore, "type": event.type, "state_key": event.state_key, }, - or_replace=True, ) for prev_state_id, _ in event.prev_state: @@ -285,7 +288,6 @@ class DataStore(RoomMemberStore, RoomStore, "internal_metadata": metadata_json.decode("UTF-8"), "json": encode_canonical_json(event_dict).decode("UTF-8"), }, - or_replace=True, ) content = encode_canonical_json( @@ -303,8 +305,9 @@ class DataStore(RoomMemberStore, RoomStore, "depth": event.depth, } - if stream_ordering is not None: - vals["stream_ordering"] = stream_ordering + if stream_ordering is None: + stream_ordering = self.get_next_stream_id() + unrec = { k: v @@ -322,21 +325,18 @@ class DataStore(RoomMemberStore, RoomStore, unrec ).decode("UTF-8") - try: - self._simple_insert_txn( - txn, - "events", - vals, - or_replace=(not outlier), - or_ignore=bool(outlier), - ) - except: - logger.warn( - "Failed to persist, probably duplicate: %s", - event.event_id, - exc_info=True, - ) - raise _RollbackButIsFineException("_persist_event") + sql = ( + "INSERT INTO events" + " (stream_ordering, topological_ordering, event_id, type," + " room_id, content, processed, outlier, depth)" + " VALUES (%s,?,?,?,?,?,?,?,?)" + ) % (stream_ordering,) + + txn.execute( + sql, + (event.depth, event.event_id, event.type, event.room_id, + content, True, outlier, event.depth) + ) if context.rejected: self._store_rejections_txn(txn, event.event_id, context.rejected) @@ -357,7 +357,6 @@ class DataStore(RoomMemberStore, RoomStore, txn, "state_events", vals, - or_replace=True, ) if is_new_state and not context.rejected: @@ -370,7 +369,6 @@ class DataStore(RoomMemberStore, RoomStore, "type": event.type, "state_key": event.state_key, }, - or_replace=True, ) for e_id, h in event.prev_state: @@ -383,7 +381,6 @@ class DataStore(RoomMemberStore, RoomStore, "room_id": event.room_id, "is_state": 1, }, - or_ignore=True, ) for hash_alg, hash_base64 in event.hashes.items(): @@ -408,7 +405,6 @@ class DataStore(RoomMemberStore, RoomStore, "room_id": event.room_id, "auth_id": auth_id, }, - or_ignore=True, ) (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) @@ -420,8 +416,7 @@ class DataStore(RoomMemberStore, RoomStore, # invalidate the cache for the redacted event self._get_event_cache.pop(event.redacts) txn.execute( - "INSERT OR IGNORE INTO redactions " - "(event_id, redacts) VALUES (?,?)", + "INSERT INTO redactions (event_id, redacts) VALUES (?,?)", (event.event_id, event.redacts) ) @@ -515,7 +510,8 @@ class DataStore(RoomMemberStore, RoomStore, "ip": ip, "user_agent": user_agent, "last_seen": int(self._clock.time_msec()), - } + }, + or_replace=True, ) def get_user_ip_and_agents(self, user): @@ -559,6 +555,12 @@ class DataStore(RoomMemberStore, RoomStore, "have_events", f, ) + def get_next_stream_id(self): + with self._next_stream_id_lock: + i = self._next_stream_id + self._next_stream_id += 1 + return i + def read_schema(path): """ Read the named database schema. @@ -594,7 +596,7 @@ def prepare_database(db_conn): else: _setup_new_database(cur) - cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,)) + # cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,)) cur.close() db_conn.commit() @@ -657,19 +659,17 @@ def _setup_new_database(cur): directory_entries = os.listdir(sql_dir) - sql_script = "BEGIN TRANSACTION;\n" for filename in fnmatch.filter(directory_entries, "*.sql"): sql_loc = os.path.join(sql_dir, filename) logger.debug("Applying schema %s", sql_loc) - sql_script += read_schema(sql_loc) - sql_script += "\n" - sql_script += "COMMIT TRANSACTION;" - cur.executescript(sql_script) + executescript(cur, sql_loc) cur.execute( - "INSERT OR REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)", - (max_current_ver, False) + _convert_param_style( + "REPLACE INTO schema_version (version, upgraded)" + " VALUES (?,?)" + ), + (max_current_ver, False,) ) _upgrade_existing_database( @@ -737,6 +737,8 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, if not upgraded: start_ver += 1 + logger.debug("applied_delta_files: %s", applied_delta_files) + for v in range(start_ver, SCHEMA_VERSION + 1): logger.debug("Upgrading schema to v%d", v) @@ -753,6 +755,7 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, directory_entries.sort() for file_name in directory_entries: relative_path = os.path.join(str(v), file_name) + logger.debug("Found file: %s", relative_path) if relative_path in applied_delta_files: continue @@ -774,9 +777,8 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, module.run_upgrade(cur) elif ext == ".sql": # A plain old .sql file, just read and execute it - delta_schema = read_schema(absolute_path) logger.debug("Applying schema %s", relative_path) - cur.executescript(delta_schema) + executescript(cur, absolute_path) else: # Not a valid delta file. logger.warn( @@ -788,24 +790,85 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, # Mark as done. cur.execute( - "INSERT INTO applied_schema_deltas (version, file)" - " VALUES (?,?)", + _convert_param_style( + "INSERT INTO applied_schema_deltas (version, file)" + " VALUES (?,?)" + ), (v, relative_path) ) cur.execute( - "INSERT OR REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)", + _convert_param_style( + "REPLACE INTO schema_version (version, upgraded)" + " VALUES (?,?)" + ), (v, True) ) +def _convert_param_style(sql): + return sql.replace("?", "%s") + + +def get_statements(f): + statement_buffer = "" + in_comment = False # If we're in a /* ... */ style comment + + for line in f: + line = line.strip() + + if in_comment: + # Check if this line contains an end to the comment + comments = line.split("*/", 1) + if len(comments) == 1: + continue + line = comments[1] + in_comment = False + + # Remove inline block comments + line = re.sub(r"/\*.*\*/", " ", line) + + # Does this line start a comment? + comments = line.split("/*", 1) + if len(comments) > 1: + line = comments[0] + in_comment = True + + # Deal with line comments + line = line.split("--", 1)[0] + line = line.split("//", 1)[0] + + # Find *all* semicolons. We need to treat first and last entry + # specially. + statements = line.split(";") + + # We must prepend statement_buffer to the first statement + first_statement = "%s %s" % ( + statement_buffer.strip(), + statements[0].strip() + ) + statements[0] = first_statement + + # Every entry, except the last, is a full statement + for statement in statements[:-1]: + yield statement.strip() + + # The last entry did *not* end in a semicolon, so we store it for the + # next semicolon we find + statement_buffer = statements[-1].strip() + + +def executescript(txn, schema_path): + with open(schema_path, 'r') as f: + for statement in get_statements(f): + txn.execute(statement) + + def _get_or_create_schema_state(txn): schema_path = os.path.join( dir_path, "schema", "schema_version.sql", ) - create_schema = read_schema(schema_path) - txn.executescript(create_schema) + executescript(txn, schema_path) txn.execute("SELECT version, upgraded FROM schema_version") row = txn.fetchone() @@ -814,10 +877,13 @@ def _get_or_create_schema_state(txn): if current_version: txn.execute( - "SELECT file FROM applied_schema_deltas WHERE version >= ?", + _convert_param_style( + "SELECT file FROM applied_schema_deltas WHERE version >= ?" + ), (current_version,) ) - return current_version, txn.fetchall(), upgraded + applied_deltas = [d for d, in txn.fetchall()] + return current_version, applied_deltas, upgraded return None @@ -849,7 +915,9 @@ def prepare_sqlite3_database(db_conn): if row and row[0]: db_conn.execute( - "INSERT OR REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)", + _convert_param_style( + "REPLACE INTO schema_version (version, upgraded)" + " VALUES (?,?)" + ), (row[0], False) ) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 2979a8352..24ff872da 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -102,6 +102,10 @@ def cached(max_entries=1000): return wrap +def _convert_param_style(sql): + return sql.replace("?", "%s") + + class LoggingTransaction(object): """An object that almost-transparently proxies for the 'txn' object passed to the constructor. Adds logging and metrics to the .execute() @@ -122,6 +126,8 @@ class LoggingTransaction(object): # TODO(paul): Maybe use 'info' and 'debug' for values? sql_logger.debug("[SQL] {%s} %s", self.name, sql) + sql = _convert_param_style(sql) + try: if args and args[0]: values = args[0] @@ -305,11 +311,11 @@ class SQLBaseStore(object): The result of decoder(results) """ def interaction(txn): - cursor = txn.execute(query, args) + txn.execute(query, args) if decoder: - return decoder(cursor) + return decoder(txn) else: - return cursor.fetchall() + return txn.fetchall() return self.runInteraction(desc, interaction) @@ -337,8 +343,7 @@ class SQLBaseStore(object): def _simple_insert_txn(self, txn, table, values, or_replace=False, or_ignore=False): sql = "%s INTO %s (%s) VALUES(%s)" % ( - ("INSERT OR REPLACE" if or_replace else - "INSERT OR IGNORE" if or_ignore else "INSERT"), + ("REPLACE" if or_replace else "INSERT"), table, ", ".join(k for k in values), ", ".join("?" for k in values) @@ -448,8 +453,7 @@ class SQLBaseStore(object): def _simple_select_onecol_txn(self, txn, table, keyvalues, retcol): sql = ( - "SELECT %(retcol)s FROM %(table)s WHERE %(where)s " - "ORDER BY rowid asc" + "SELECT %(retcol)s FROM %(table)s WHERE %(where)s" ) % { "retcol": retcol, "table": table, @@ -505,14 +509,14 @@ class SQLBaseStore(object): retcols : list of strings giving the names of the columns to return """ if keyvalues: - sql = "SELECT %s FROM %s WHERE %s ORDER BY rowid asc" % ( + sql = "SELECT %s FROM %s WHERE %s" % ( ", ".join(retcols), table, " AND ".join("%s = ?" % (k, ) for k in keyvalues) ) txn.execute(sql, keyvalues.values()) else: - sql = "SELECT %s FROM %s ORDER BY rowid asc" % ( + sql = "SELECT %s FROM %s" % ( ", ".join(retcols), table ) @@ -546,7 +550,7 @@ class SQLBaseStore(object): retcols=None, allow_none=False): """ Combined SELECT then UPDATE.""" if retcols: - select_sql = "SELECT %s FROM %s WHERE %s ORDER BY rowid asc" % ( + select_sql = "SELECT %s FROM %s WHERE %s" % ( ", ".join(retcols), table, " AND ".join("%s = ?" % (k) for k in keyvalues) @@ -580,8 +584,8 @@ class SQLBaseStore(object): updatevalues.values() + keyvalues.values() ) - if txn.rowcount == 0: - raise StoreError(404, "No row found") + # if txn.rowcount == 0: + # raise StoreError(404, "No row found") if txn.rowcount > 1: raise StoreError(500, "More than one row matched") @@ -802,7 +806,7 @@ class Table(object): _select_where_clause = "SELECT %s FROM %s WHERE %s" _select_clause = "SELECT %s FROM %s" - _insert_clause = "INSERT OR REPLACE INTO %s (%s) VALUES (%s)" + _insert_clause = "REPLACE INTO %s (%s) VALUES (%s)" @classmethod def select_statement(cls, where_clause=None): diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 850676ce6..375265d66 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -147,11 +147,11 @@ class ApplicationServiceStore(SQLBaseStore): return True def _get_as_id_txn(self, txn, token): - cursor = txn.execute( + txn.execute( "SELECT id FROM application_services WHERE token=?", (token,) ) - res = cursor.fetchone() + res = txn.fetchone() if res: return res[0] diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index 68b7d5969..0c2adffbb 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -111,12 +111,12 @@ class DirectoryStore(SQLBaseStore): ) def _delete_room_alias_txn(self, txn, room_alias): - cursor = txn.execute( + txn.execute( "SELECT room_id FROM room_aliases WHERE room_alias = ?", (room_alias.to_string(),) ) - res = cursor.fetchone() + res = txn.fetchone() if res: room_id = res[0] else: diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 2deda8ac5..5d66b2f24 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -242,7 +242,6 @@ class EventFederationStore(SQLBaseStore): "room_id": room_id, "min_depth": depth, }, - or_replace=True, ) def _handle_prev_events(self, txn, outlier, event_id, prev_events, @@ -262,7 +261,6 @@ class EventFederationStore(SQLBaseStore): "room_id": room_id, "is_state": 0, }, - or_ignore=True, ) # Update the extremities table if this is not an outlier. @@ -281,19 +279,19 @@ class EventFederationStore(SQLBaseStore): # We only insert as a forward extremity the new event if there are # no other events that reference it as a prev event query = ( - "INSERT OR IGNORE INTO %(table)s (event_id, room_id) " - "SELECT ?, ? WHERE NOT EXISTS (" - "SELECT 1 FROM %(event_edges)s WHERE " - "prev_event_id = ? " - ")" - ) % { - "table": "event_forward_extremities", - "event_edges": "event_edges", - } + "SELECT 1 FROM event_edges WHERE prev_event_id = ?" + ) - logger.debug("query: %s", query) + txn.execute(query, (event_id,)) - txn.execute(query, (event_id, room_id, event_id)) + if not txn.fetchone(): + query = ( + "INSERT INTO event_forward_extremities" + " (event_id, room_id)" + " VALUES (?, ?)" + ) + + txn.execute(query, (event_id, room_id)) # Insert all the prev_events as a backwards thing, they'll get # deleted in a second if they're incorrect anyway. @@ -306,7 +304,6 @@ class EventFederationStore(SQLBaseStore): "event_id": e_id, "room_id": room_id, }, - or_ignore=True, ) # Also delete from the backwards extremities table all ones that diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index 1dcd34723..0084d67e5 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -45,7 +45,6 @@ class PresenceStore(SQLBaseStore): updatevalues={"state": new_state["state"], "status_msg": new_state["status_msg"], "mtime": self._clock.time_msec()}, - retcols=["state"], ) def allow_presence_visible(self, observed_localpart, observer_userid): diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index d769db2c7..27a071632 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -153,7 +153,7 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, (user_name, priority_class, new_rule_priority)) # now insert the new rule - sql = "INSERT OR REPLACE INTO "+PushRuleTable.table_name+" (" + sql = "INSERT INTO "+PushRuleTable.table_name+" (" sql += ",".join(new_rule.keys())+") VALUES (" sql += ", ".join(["?" for _ in new_rule.keys()])+")" @@ -182,7 +182,7 @@ class PushRuleStore(SQLBaseStore): new_rule['priority_class'] = priority_class new_rule['priority'] = new_prio - sql = "INSERT OR REPLACE INTO "+PushRuleTable.table_name+" (" + sql = "INSERT INTO "+PushRuleTable.table_name+" (" sql += ",".join(new_rule.keys())+") VALUES (" sql += ", ".join(["?" for _ in new_rule.keys()])+")" diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index adc8fc079..344dd3aaa 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -39,14 +39,10 @@ class RegistrationStore(SQLBaseStore): Raises: StoreError if there was a problem adding this. """ - row = yield self._simple_select_one("users", {"name": user_id}, ["id"]) - if not row: - raise StoreError(400, "Bad user ID supplied.") - row_id = row["id"] yield self._simple_insert( "access_tokens", { - "user_id": row_id, + "user_id": user_id, "token": token } ) @@ -82,7 +78,7 @@ class RegistrationStore(SQLBaseStore): # it's possible for this to get a conflict, but only for a single user # since tokens are namespaced based on their user ID txn.execute("INSERT INTO access_tokens(user_id, token) " + - "VALUES (?,?)", [txn.lastrowid, token]) + "VALUES (?,?)", [user_id, token]) def get_user_by_id(self, user_id): query = ("SELECT users.name, users.password_hash FROM users" @@ -124,12 +120,12 @@ class RegistrationStore(SQLBaseStore): "SELECT users.name, users.admin," " access_tokens.device_id, access_tokens.id as token_id" " FROM users" - " INNER JOIN access_tokens on users.id = access_tokens.user_id" + " INNER JOIN access_tokens on users.name = access_tokens.user_id" " WHERE token = ?" ) - cursor = txn.execute(sql, (token,)) - rows = self.cursor_to_dict(cursor) + txn.execute(sql, (token,)) + rows = self.cursor_to_dict(txn) if rows: return rows[0] diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 549c9af39..3c23f2921 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -114,9 +114,9 @@ class RoomStore(SQLBaseStore): "name": name_subquery, } - c = txn.execute(sql, (is_public,)) + txn.execute(sql, (is_public,)) - return c.fetchall() + return txn.fetchall() rows = yield self.runInteraction( "get_rooms", f diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 65ffb4627..e8ede14cd 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -68,7 +68,7 @@ class RoomMemberStore(SQLBaseStore): # Update room hosts table if event.membership == Membership.JOIN: sql = ( - "INSERT OR IGNORE INTO room_hosts (room_id, host) " + "REPLACE INTO room_hosts (room_id, host) " "VALUES (?, ?)" ) txn.execute(sql, (event.room_id, domain)) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 456e4bd45..888837cd1 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -15,6 +15,8 @@ from ._base import SQLBaseStore +from synapse.util.stringutils import random_string + import logging logger = logging.getLogger(__name__) @@ -89,14 +91,15 @@ class StateStore(SQLBaseStore): state_group = context.state_group if not state_group: + group = _make_group_id(self._clock) state_group = self._simple_insert_txn( txn, table="state_groups", values={ + "id": group, "room_id": event.room_id, "event_id": event.event_id, }, - or_ignore=True, ) for state in state_events.values(): @@ -110,7 +113,6 @@ class StateStore(SQLBaseStore): "state_key": state.state_key, "event_id": state.event_id, }, - or_ignore=True, ) self._simple_insert_txn( @@ -122,3 +124,7 @@ class StateStore(SQLBaseStore): }, or_replace=True, ) + + +def _make_group_id(clock): + return str(int(clock.time_msec())) + random_string(5) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 09bc52221..64adb0c7f 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -110,7 +110,7 @@ class _StreamToken(namedtuple("_StreamToken", "topological stream")): if self.topological is None: return "(%d < %s)" % (self.stream, "stream_ordering") else: - return "(%d < %s OR (%d == %s AND %d < %s))" % ( + return "(%d < %s OR (%d = %s AND %d < %s))" % ( self.topological, "topological_ordering", self.topological, "topological_ordering", self.stream, "stream_ordering", @@ -120,7 +120,7 @@ class _StreamToken(namedtuple("_StreamToken", "topological stream")): if self.topological is None: return "(%d >= %s)" % (self.stream, "stream_ordering") else: - return "(%d > %s OR (%d == %s AND %d >= %s))" % ( + return "(%d > %s OR (%d = %s AND %d >= %s))" % ( self.topological, "topological_ordering", self.topological, "topological_ordering", self.stream, "stream_ordering", diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 0b8a3b7a0..b5ed5453d 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -121,8 +121,8 @@ class TransactionStore(SQLBaseStore): SentTransactions.select_statement("destination = ?"), ) - results = txn.execute(query, (destination,)) - results = SentTransactions.decode_results(results) + txn.execute(query, (destination,)) + results = SentTransactions.decode_results(txn) prev_txns = [r.transaction_id for r in results] @@ -266,7 +266,7 @@ class TransactionStore(SQLBaseStore): retry_last_ts, retry_interval): query = ( - "INSERT OR REPLACE INTO %s " + "REPLACE INTO %s " "(destination, retry_last_ts, retry_interval) " "VALUES (?, ?, ?) " ) % DestinationsTable.table_name From 0d0610870ddd3c47fe397f1d026bf1ca01dbdf74 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 10:55:31 +0000 Subject: [PATCH 005/104] Fix up schemas some more --- synapse/storage/schema/delta/12/v12.sql | 2 +- synapse/storage/schema/full_schemas/11/event_edges.sql | 2 +- synapse/storage/schema/full_schemas/11/transactions.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index 42ea1b2ff..b526109e6 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -60,4 +60,4 @@ CREATE TABLE IF NOT EXISTS user_filters( CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( user_id, filter_id -) ENGINE = INNODB; +); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 910e25dfc..0f53488e9 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -74,7 +74,7 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities( CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( room_id, type, state_key -) ENGINE = INNODB; +); CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index cde6919e4..0570bf95d 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -34,7 +34,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( destination VARCHAR(255), response_code INTEGER DEFAULT 0, response_json BLOB, - ts INTEGER + ts BIGINT ) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); From 455579ca90dd5479dae785b5a1b9bdd201654ea6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 10:55:55 +0000 Subject: [PATCH 006/104] Make database selection configurable --- synapse/app/homeserver.py | 44 ++++++++++++++++++++++++++++---------- synapse/config/database.py | 9 ++++++++ 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 15c454af7..a2fca2e02 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -61,6 +61,7 @@ import resource import subprocess import sqlite3 import syweb +import yaml logger = logging.getLogger(__name__) @@ -108,14 +109,14 @@ class SynapseHomeServer(HomeServer): return None def build_db_pool(self): - return adbapi.ConnectionPool( - "sqlite3", self.get_db_name(), - check_same_thread=False, - cp_min=1, - cp_max=1, - cp_openfun=prepare_database, # Prepare the database for each conn - # so that :memory: sqlite works - ) + name = self.db_config.pop("name", None) + if name == "MySQLdb": + return adbapi.ConnectionPool( + name, + **self.db_config + ) + + raise RuntimeError("Unsupported database type") def create_resource_tree(self, redirect_root_to_web_client): """Create the resource tree for this Home Server. @@ -357,11 +358,29 @@ def setup(config_options): tls_context_factory = context_factory.ServerContextFactory(config) + if config.database_config: + with open(config.database_config, 'r') as f: + db_config = yaml.safe_load(f) + + name = db_config.get("name", None) + if name == "MySQLdb": + db_config.update({ + "sql_mode": "TRADITIONAL", + "charset": "utf8", + "use_unicode": True, + }) + else: + db_config = { + "name": "sqlite3", + "database": config.database_path, + } + hs = SynapseHomeServer( config.server_name, domain_with_port=domain_with_port, upload_dir=os.path.abspath("uploads"), db_name=config.database_path, + db_config=db_config, tls_context_factory=tls_context_factory, config=config, content_addr=config.content_addr, @@ -377,9 +396,12 @@ def setup(config_options): logger.info("Preparing database: %s...", db_name) try: - with sqlite3.connect(db_name) as db_conn: - prepare_sqlite3_database(db_conn) - prepare_database(db_conn) + # with sqlite3.connect(db_name) as db_conn: + # prepare_sqlite3_database(db_conn) + # prepare_database(db_conn) + import MySQLdb + db_conn = MySQLdb.connect(**db_config) + prepare_database(db_conn) except UpgradeDatabaseException: sys.stderr.write( "\nFailed to upgrade database.\n" diff --git a/synapse/config/database.py b/synapse/config/database.py index 87efe5464..8dc9873f8 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -26,6 +26,11 @@ class DatabaseConfig(Config): self.database_path = self.abspath(args.database_path) self.event_cache_size = self.parse_size(args.event_cache_size) + if args.database_config: + self.database_config = self.abspath(args.database_config) + else: + self.database_config = None + @classmethod def add_arguments(cls, parser): super(DatabaseConfig, cls).add_arguments(parser) @@ -38,6 +43,10 @@ class DatabaseConfig(Config): "--event-cache-size", default="100K", help="Number of events to cache in memory." ) + db_group.add_argument( + "--database-config", default=None, + help="Location of the database configuration file." + ) @classmethod def generate_config(cls, args, config_dir_path): From d4f50f3ae5a23886eb271ad8cee92d5fb5aae8b0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 10:57:08 +0000 Subject: [PATCH 007/104] decode_result takes an iterable --- synapse/storage/room.py | 5 ++++- synapse/storage/transactions.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 3c23f2921..b5031f5c7 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -68,7 +68,10 @@ class RoomStore(SQLBaseStore): """ query = RoomsTable.select_statement("room_id=?") return self._execute( - "get_room", RoomsTable.decode_single_result, query, room_id, + "get_room", + lambda txn: RoomsTable.decode_single_result(txn.fetchall()), + query, + room_id, ) @defer.inlineCallbacks diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index b5ed5453d..92eec69ef 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -122,7 +122,7 @@ class TransactionStore(SQLBaseStore): ) txn.execute(query, (destination,)) - results = SentTransactions.decode_results(txn) + results = SentTransactions.decode_results(txn.fetchall()) prev_txns = [r.transaction_id for r in results] From d288d273e15c3686ff73b8cfd8c71707174dc173 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 10:57:44 +0000 Subject: [PATCH 008/104] Generate transaction id in code --- synapse/storage/transactions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 92eec69ef..08e481a36 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -130,7 +130,7 @@ class TransactionStore(SQLBaseStore): query = SentTransactions.insert_statement() txn.execute(query, SentTransactions.EntryType( - None, + self.get_next_stream_id(), transaction_id=transaction_id, destination=destination, ts=origin_server_ts, From cb8162d3d17a97574073d49bd6eef51c93b68157 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 13:52:56 +0000 Subject: [PATCH 009/104] Rearrange storage modules --- synapse/storage/__init__.py | 466 +----------------------------------- synapse/storage/_base.py | 7 + synapse/storage/events.py | 387 ++++++++++++++++++++++++++++++ synapse/storage/feedback.py | 47 ---- synapse/storage/room.py | 37 +++ synapse/storage/state.py | 32 +++ synapse/storage/stream.py | 19 ++ 7 files changed, 486 insertions(+), 509 deletions(-) create mode 100644 synapse/storage/events.py delete mode 100644 synapse/storage/feedback.py diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 30cba4771..8604746fb 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -15,12 +15,9 @@ from twisted.internet import defer -from synapse.util.logutils import log_function -from synapse.api.constants import EventTypes - from .appservice import ApplicationServiceStore from .directory import DirectoryStore -from .feedback import FeedbackStore +from .events import EventsStore from .presence import PresenceStore from .profile import ProfileStore from .registration import RegistrationStore @@ -39,11 +36,6 @@ from .state import StateStore from .signatures import SignatureStore from .filtering import FilteringStore -from syutil.base64util import decode_base64 -from syutil.jsonutil import encode_canonical_json - -from synapse.crypto.event_signing import compute_event_reference_hash - import fnmatch import imp @@ -64,15 +56,8 @@ SCHEMA_VERSION = 14 dir_path = os.path.abspath(os.path.dirname(__file__)) -class _RollbackButIsFineException(Exception): - """ This exception is used to rollback a transaction without implying - something went wrong. - """ - pass - - class DataStore(RoomMemberStore, RoomStore, - RegistrationStore, StreamStore, ProfileStore, FeedbackStore, + RegistrationStore, StreamStore, ProfileStore, PresenceStore, TransactionStore, DirectoryStore, KeyStore, StateStore, SignatureStore, ApplicationServiceStore, @@ -81,7 +66,8 @@ class DataStore(RoomMemberStore, RoomStore, RejectionsStore, FilteringStore, PusherStore, - PushRuleStore + PushRuleStore, + EventsStore, ): def __init__(self, hs): @@ -94,412 +80,6 @@ class DataStore(RoomMemberStore, RoomStore, self._next_stream_id_lock = threading.Lock() self._next_stream_id = int(hs.get_clock().time_msec()) * 1000 - @defer.inlineCallbacks - @log_function - def persist_event(self, event, context, backfilled=False, - is_new_state=True, current_state=None): - stream_ordering = None - if backfilled: - if not self.min_token_deferred.called: - yield self.min_token_deferred - self.min_token -= 1 - stream_ordering = self.min_token - - try: - yield self.runInteraction( - "persist_event", - self._persist_event_txn, - event=event, - context=context, - backfilled=backfilled, - stream_ordering=stream_ordering, - is_new_state=is_new_state, - current_state=current_state, - ) - except _RollbackButIsFineException: - pass - - @defer.inlineCallbacks - def get_event(self, event_id, check_redacted=True, - get_prev_content=False, allow_rejected=False, - allow_none=False): - """Get an event from the database by event_id. - - Args: - event_id (str): The event_id of the event to fetch - check_redacted (bool): If True, check if event has been redacted - and redact it. - get_prev_content (bool): If True and event is a state event, - include the previous states content in the unsigned field. - allow_rejected (bool): If True return rejected events. - allow_none (bool): If True, return None if no event found, if - False throw an exception. - - Returns: - Deferred : A FrozenEvent. - """ - event = yield self.runInteraction( - "get_event", self._get_event_txn, - event_id, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - allow_rejected=allow_rejected, - ) - - if not event and not allow_none: - raise RuntimeError("Could not find event %s" % (event_id,)) - - defer.returnValue(event) - - @log_function - def _persist_event_txn(self, txn, event, context, backfilled, - stream_ordering=None, is_new_state=True, - current_state=None): - - # Remove the any existing cache entries for the event_id - self._get_event_cache.pop(event.event_id) - - # We purposefully do this first since if we include a `current_state` - # key, we *want* to update the `current_state_events` table - if current_state: - txn.execute( - "DELETE FROM current_state_events WHERE room_id = ?", - (event.room_id,) - ) - - for s in current_state: - self._simple_insert_txn( - txn, - "current_state_events", - { - "event_id": s.event_id, - "room_id": s.room_id, - "type": s.type, - "state_key": s.state_key, - }, - ) - - if event.is_state() and is_new_state: - if not backfilled and not context.rejected: - self._simple_insert_txn( - txn, - table="state_forward_extremities", - values={ - "event_id": event.event_id, - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - }, - ) - - for prev_state_id, _ in event.prev_state: - self._simple_delete_txn( - txn, - table="state_forward_extremities", - keyvalues={ - "event_id": prev_state_id, - } - ) - - outlier = event.internal_metadata.is_outlier() - - if not outlier: - self._store_state_groups_txn(txn, event, context) - - self._update_min_depth_for_room_txn( - txn, - event.room_id, - event.depth - ) - - self._handle_prev_events( - txn, - outlier=outlier, - event_id=event.event_id, - prev_events=event.prev_events, - room_id=event.room_id, - ) - - have_persisted = self._simple_select_one_onecol_txn( - txn, - table="event_json", - keyvalues={"event_id": event.event_id}, - retcol="event_id", - allow_none=True, - ) - - metadata_json = encode_canonical_json( - event.internal_metadata.get_dict() - ) - - # If we have already persisted this event, we don't need to do any - # more processing. - # The processing above must be done on every call to persist event, - # since they might not have happened on previous calls. For example, - # if we are persisting an event that we had persisted as an outlier, - # but is no longer one. - if have_persisted: - if not outlier: - sql = ( - "UPDATE event_json SET internal_metadata = ?" - " WHERE event_id = ?" - ) - txn.execute( - sql, - (metadata_json.decode("UTF-8"), event.event_id,) - ) - - sql = ( - "UPDATE events SET outlier = 0" - " WHERE event_id = ?" - ) - txn.execute( - sql, - (event.event_id,) - ) - return - - if event.type == EventTypes.Member: - self._store_room_member_txn(txn, event) - elif event.type == EventTypes.Feedback: - self._store_feedback_txn(txn, event) - elif event.type == EventTypes.Name: - self._store_room_name_txn(txn, event) - elif event.type == EventTypes.Topic: - self._store_room_topic_txn(txn, event) - elif event.type == EventTypes.Redaction: - self._store_redaction(txn, event) - - event_dict = { - k: v - for k, v in event.get_dict().items() - if k not in [ - "redacted", - "redacted_because", - ] - } - - self._simple_insert_txn( - txn, - table="event_json", - values={ - "event_id": event.event_id, - "room_id": event.room_id, - "internal_metadata": metadata_json.decode("UTF-8"), - "json": encode_canonical_json(event_dict).decode("UTF-8"), - }, - ) - - content = encode_canonical_json( - event.content - ).decode("UTF-8") - - vals = { - "topological_ordering": event.depth, - "event_id": event.event_id, - "type": event.type, - "room_id": event.room_id, - "content": content, - "processed": True, - "outlier": outlier, - "depth": event.depth, - } - - if stream_ordering is None: - stream_ordering = self.get_next_stream_id() - - - unrec = { - k: v - for k, v in event.get_dict().items() - if k not in vals.keys() and k not in [ - "redacted", - "redacted_because", - "signatures", - "hashes", - "prev_events", - ] - } - - vals["unrecognized_keys"] = encode_canonical_json( - unrec - ).decode("UTF-8") - - sql = ( - "INSERT INTO events" - " (stream_ordering, topological_ordering, event_id, type," - " room_id, content, processed, outlier, depth)" - " VALUES (%s,?,?,?,?,?,?,?,?)" - ) % (stream_ordering,) - - txn.execute( - sql, - (event.depth, event.event_id, event.type, event.room_id, - content, True, outlier, event.depth) - ) - - if context.rejected: - self._store_rejections_txn(txn, event.event_id, context.rejected) - - if event.is_state(): - vals = { - "event_id": event.event_id, - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - } - - # TODO: How does this work with backfilling? - if hasattr(event, "replaces_state"): - vals["prev_state"] = event.replaces_state - - self._simple_insert_txn( - txn, - "state_events", - vals, - ) - - if is_new_state and not context.rejected: - self._simple_insert_txn( - txn, - "current_state_events", - { - "event_id": event.event_id, - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - }, - ) - - for e_id, h in event.prev_state: - self._simple_insert_txn( - txn, - table="event_edges", - values={ - "event_id": event.event_id, - "prev_event_id": e_id, - "room_id": event.room_id, - "is_state": 1, - }, - ) - - for hash_alg, hash_base64 in event.hashes.items(): - hash_bytes = decode_base64(hash_base64) - self._store_event_content_hash_txn( - txn, event.event_id, hash_alg, hash_bytes, - ) - - for prev_event_id, prev_hashes in event.prev_events: - for alg, hash_base64 in prev_hashes.items(): - hash_bytes = decode_base64(hash_base64) - self._store_prev_event_hash_txn( - txn, event.event_id, prev_event_id, alg, hash_bytes - ) - - for auth_id, _ in event.auth_events: - self._simple_insert_txn( - txn, - table="event_auth", - values={ - "event_id": event.event_id, - "room_id": event.room_id, - "auth_id": auth_id, - }, - ) - - (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) - self._store_event_reference_hash_txn( - txn, event.event_id, ref_alg, ref_hash_bytes - ) - - def _store_redaction(self, txn, event): - # invalidate the cache for the redacted event - self._get_event_cache.pop(event.redacts) - txn.execute( - "INSERT INTO redactions (event_id, redacts) VALUES (?,?)", - (event.event_id, event.redacts) - ) - - @defer.inlineCallbacks - def get_current_state(self, room_id, event_type=None, state_key=""): - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = e.event_id " - "LIMIT 1" - ) - - sql = ( - "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " - "INNER JOIN current_state_events as c ON e.event_id = c.event_id " - "INNER JOIN state_events as s ON e.event_id = s.event_id " - "WHERE c.room_id = ? " - ) % { - "redacted": del_sql, - } - - if event_type and state_key is not None: - sql += " AND s.type = ? AND s.state_key = ? " - args = (room_id, event_type, state_key) - elif event_type: - sql += " AND s.type = ?" - args = (room_id, event_type) - else: - args = (room_id, ) - - results = yield self._execute_and_decode("get_current_state", sql, *args) - - events = yield self._parse_events(results) - defer.returnValue(events) - - @defer.inlineCallbacks - def get_room_name_and_aliases(self, room_id): - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = e.event_id " - "LIMIT 1" - ) - - sql = ( - "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " - "INNER JOIN current_state_events as c ON e.event_id = c.event_id " - "INNER JOIN state_events as s ON e.event_id = s.event_id " - "WHERE c.room_id = ? " - ) % { - "redacted": del_sql, - } - - sql += " AND ((s.type = 'm.room.name' AND s.state_key = '')" - sql += " OR s.type = 'm.room.aliases')" - args = (room_id,) - - results = yield self._execute_and_decode("get_current_state", sql, *args) - - events = yield self._parse_events(results) - - name = None - aliases = [] - - for e in events: - if e.type == 'm.room.name': - if 'name' in e.content: - name = e.content['name'] - elif e.type == 'm.room.aliases': - if 'aliases' in e.content: - aliases.extend(e.content['aliases']) - - defer.returnValue((name, aliases)) - - @defer.inlineCallbacks - def _get_min_token(self): - row = yield self._execute( - "_get_min_token", None, "SELECT MIN(stream_ordering) FROM events" - ) - - self.min_token = row[0][0] if row and row[0] and row[0][0] else -1 - self.min_token = min(self.min_token, -1) - - logger.debug("min_token is: %s", self.min_token) - - defer.returnValue(self.min_token) - def insert_client_ip(self, user, access_token, device_id, ip, user_agent): return self._simple_insert( "user_ips", @@ -523,44 +103,6 @@ class DataStore(RoomMemberStore, RoomStore, ], ) - def have_events(self, event_ids): - """Given a list of event ids, check if we have already processed them. - - Returns: - dict: Has an entry for each event id we already have seen. Maps to - the rejected reason string if we rejected the event, else maps to - None. - """ - if not event_ids: - return defer.succeed({}) - - def f(txn): - sql = ( - "SELECT e.event_id, reason FROM events as e " - "LEFT JOIN rejections as r ON e.event_id = r.event_id " - "WHERE e.event_id = ?" - ) - - res = {} - for event_id in event_ids: - txn.execute(sql, (event_id,)) - row = txn.fetchone() - if row: - _, rejected = row - res[event_id] = rejected - - return res - - return self.runInteraction( - "have_events", f, - ) - - def get_next_stream_id(self): - with self._next_stream_id_lock: - i = self._next_stream_id - self._next_stream_id += 1 - return i - def read_schema(path): """ Read the named database schema. diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 24ff872da..37bb28e6c 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -791,6 +791,13 @@ class SQLBaseStore(object): return result[0] if result else None +class _RollbackButIsFineException(Exception): + """ This exception is used to rollback a transaction without implying + something went wrong. + """ + pass + + class Table(object): """ A base class used to store information about a particular table. """ diff --git a/synapse/storage/events.py b/synapse/storage/events.py new file mode 100644 index 000000000..b222dfb4a --- /dev/null +++ b/synapse/storage/events.py @@ -0,0 +1,387 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from _base import SQLBaseStore, _RollbackButIsFineException + +from twisted.internet import defer + +from synapse.util.logutils import log_function +from synapse.api.constants import EventTypes +from synapse.crypto.event_signing import compute_event_reference_hash + +from syutil.base64util import decode_base64 +from syutil.jsonutil import encode_canonical_json + +import logging + +logger = logging.getLogger(__name__) + + +class EventsStore(SQLBaseStore): + @defer.inlineCallbacks + @log_function + def persist_event(self, event, context, backfilled=False, + is_new_state=True, current_state=None): + stream_ordering = None + if backfilled: + if not self.min_token_deferred.called: + yield self.min_token_deferred + self.min_token -= 1 + stream_ordering = self.min_token + + try: + yield self.runInteraction( + "persist_event", + self._persist_event_txn, + event=event, + context=context, + backfilled=backfilled, + stream_ordering=stream_ordering, + is_new_state=is_new_state, + current_state=current_state, + ) + except _RollbackButIsFineException: + pass + + @defer.inlineCallbacks + def get_event(self, event_id, check_redacted=True, + get_prev_content=False, allow_rejected=False, + allow_none=False): + """Get an event from the database by event_id. + + Args: + event_id (str): The event_id of the event to fetch + check_redacted (bool): If True, check if event has been redacted + and redact it. + get_prev_content (bool): If True and event is a state event, + include the previous states content in the unsigned field. + allow_rejected (bool): If True return rejected events. + allow_none (bool): If True, return None if no event found, if + False throw an exception. + + Returns: + Deferred : A FrozenEvent. + """ + event = yield self.runInteraction( + "get_event", self._get_event_txn, + event_id, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, + ) + + if not event and not allow_none: + raise RuntimeError("Could not find event %s" % (event_id,)) + + defer.returnValue(event) + + @log_function + def _persist_event_txn(self, txn, event, context, backfilled, + stream_ordering=None, is_new_state=True, + current_state=None): + + # Remove the any existing cache entries for the event_id + self._get_event_cache.pop(event.event_id) + + # We purposefully do this first since if we include a `current_state` + # key, we *want* to update the `current_state_events` table + if current_state: + txn.execute( + "DELETE FROM current_state_events WHERE room_id = ?", + (event.room_id,) + ) + + for s in current_state: + self._simple_insert_txn( + txn, + "current_state_events", + { + "event_id": s.event_id, + "room_id": s.room_id, + "type": s.type, + "state_key": s.state_key, + }, + ) + + if event.is_state() and is_new_state: + if not backfilled and not context.rejected: + self._simple_insert_txn( + txn, + table="state_forward_extremities", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + }, + ) + + for prev_state_id, _ in event.prev_state: + self._simple_delete_txn( + txn, + table="state_forward_extremities", + keyvalues={ + "event_id": prev_state_id, + } + ) + + outlier = event.internal_metadata.is_outlier() + + if not outlier: + self._store_state_groups_txn(txn, event, context) + + self._update_min_depth_for_room_txn( + txn, + event.room_id, + event.depth + ) + + self._handle_prev_events( + txn, + outlier=outlier, + event_id=event.event_id, + prev_events=event.prev_events, + room_id=event.room_id, + ) + + have_persisted = self._simple_select_one_onecol_txn( + txn, + table="event_json", + keyvalues={"event_id": event.event_id}, + retcol="event_id", + allow_none=True, + ) + + metadata_json = encode_canonical_json( + event.internal_metadata.get_dict() + ) + + # If we have already persisted this event, we don't need to do any + # more processing. + # The processing above must be done on every call to persist event, + # since they might not have happened on previous calls. For example, + # if we are persisting an event that we had persisted as an outlier, + # but is no longer one. + if have_persisted: + if not outlier: + sql = ( + "UPDATE event_json SET internal_metadata = ?" + " WHERE event_id = ?" + ) + txn.execute( + sql, + (metadata_json.decode("UTF-8"), event.event_id,) + ) + + sql = ( + "UPDATE events SET outlier = 0" + " WHERE event_id = ?" + ) + txn.execute( + sql, + (event.event_id,) + ) + return + + if event.type == EventTypes.Member: + self._store_room_member_txn(txn, event) + elif event.type == EventTypes.Name: + self._store_room_name_txn(txn, event) + elif event.type == EventTypes.Topic: + self._store_room_topic_txn(txn, event) + elif event.type == EventTypes.Redaction: + self._store_redaction(txn, event) + + event_dict = { + k: v + for k, v in event.get_dict().items() + if k not in [ + "redacted", + "redacted_because", + ] + } + + self._simple_insert_txn( + txn, + table="event_json", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "internal_metadata": metadata_json.decode("UTF-8"), + "json": encode_canonical_json(event_dict).decode("UTF-8"), + }, + ) + + content = encode_canonical_json( + event.content + ).decode("UTF-8") + + vals = { + "topological_ordering": event.depth, + "event_id": event.event_id, + "type": event.type, + "room_id": event.room_id, + "content": content, + "processed": True, + "outlier": outlier, + "depth": event.depth, + } + + if stream_ordering is None: + stream_ordering = self.get_next_stream_id() + + + unrec = { + k: v + for k, v in event.get_dict().items() + if k not in vals.keys() and k not in [ + "redacted", + "redacted_because", + "signatures", + "hashes", + "prev_events", + ] + } + + vals["unrecognized_keys"] = encode_canonical_json( + unrec + ).decode("UTF-8") + + sql = ( + "INSERT INTO events" + " (stream_ordering, topological_ordering, event_id, type," + " room_id, content, processed, outlier, depth)" + " VALUES (%s,?,?,?,?,?,?,?,?)" + ) % (stream_ordering,) + + txn.execute( + sql, + (event.depth, event.event_id, event.type, event.room_id, + content, True, outlier, event.depth) + ) + + if context.rejected: + self._store_rejections_txn(txn, event.event_id, context.rejected) + + if event.is_state(): + vals = { + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + } + + # TODO: How does this work with backfilling? + if hasattr(event, "replaces_state"): + vals["prev_state"] = event.replaces_state + + self._simple_insert_txn( + txn, + "state_events", + vals, + ) + + if is_new_state and not context.rejected: + self._simple_insert_txn( + txn, + "current_state_events", + { + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + }, + ) + + for e_id, h in event.prev_state: + self._simple_insert_txn( + txn, + table="event_edges", + values={ + "event_id": event.event_id, + "prev_event_id": e_id, + "room_id": event.room_id, + "is_state": 1, + }, + ) + + for hash_alg, hash_base64 in event.hashes.items(): + hash_bytes = decode_base64(hash_base64) + self._store_event_content_hash_txn( + txn, event.event_id, hash_alg, hash_bytes, + ) + + for prev_event_id, prev_hashes in event.prev_events: + for alg, hash_base64 in prev_hashes.items(): + hash_bytes = decode_base64(hash_base64) + self._store_prev_event_hash_txn( + txn, event.event_id, prev_event_id, alg, hash_bytes + ) + + for auth_id, _ in event.auth_events: + self._simple_insert_txn( + txn, + table="event_auth", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "auth_id": auth_id, + }, + ) + + (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) + self._store_event_reference_hash_txn( + txn, event.event_id, ref_alg, ref_hash_bytes + ) + + def _store_redaction(self, txn, event): + # invalidate the cache for the redacted event + self._get_event_cache.pop(event.redacts) + txn.execute( + "INSERT INTO redactions (event_id, redacts) VALUES (?,?)", + (event.event_id, event.redacts) + ) + + def have_events(self, event_ids): + """Given a list of event ids, check if we have already processed them. + + Returns: + dict: Has an entry for each event id we already have seen. Maps to + the rejected reason string if we rejected the event, else maps to + None. + """ + if not event_ids: + return defer.succeed({}) + + def f(txn): + sql = ( + "SELECT e.event_id, reason FROM events as e " + "LEFT JOIN rejections as r ON e.event_id = r.event_id " + "WHERE e.event_id = ?" + ) + + res = {} + for event_id in event_ids: + txn.execute(sql, (event_id,)) + row = txn.fetchone() + if row: + _, rejected = row + res[event_id] = rejected + + return res + + return self.runInteraction( + "have_events", f, + ) diff --git a/synapse/storage/feedback.py b/synapse/storage/feedback.py deleted file mode 100644 index 8eab769b7..000000000 --- a/synapse/storage/feedback.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from twisted.internet import defer - -from ._base import SQLBaseStore - - -class FeedbackStore(SQLBaseStore): - - def _store_feedback_txn(self, txn, event): - self._simple_insert_txn(txn, "feedback", { - "event_id": event.event_id, - "feedback_type": event.content["type"], - "room_id": event.room_id, - "target_event_id": event.content["target_event_id"], - "sender": event.user_id, - }) - - @defer.inlineCallbacks - def get_feedback_for_event(self, event_id): - sql = ( - "SELECT events.* FROM events INNER JOIN feedback " - "ON events.event_id = feedback.event_id " - "WHERE feedback.target_event_id = ? " - ) - - rows = yield self._execute_and_decode("get_feedback_for_event", sql, event_id) - - defer.returnValue( - [ - (yield self._parse_events(r)) - for r in rows - ] - ) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index b5031f5c7..c64f8f53a 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -161,6 +161,43 @@ class RoomStore(SQLBaseStore): } ) + @defer.inlineCallbacks + def get_room_name_and_aliases(self, room_id): + del_sql = ( + "SELECT event_id FROM redactions WHERE redacts = e.event_id " + "LIMIT 1" + ) + + sql = ( + "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " + "INNER JOIN current_state_events as c ON e.event_id = c.event_id " + "INNER JOIN state_events as s ON e.event_id = s.event_id " + "WHERE c.room_id = ? " + ) % { + "redacted": del_sql, + } + + sql += " AND ((s.type = 'm.room.name' AND s.state_key = '')" + sql += " OR s.type = 'm.room.aliases')" + args = (room_id,) + + results = yield self._execute_and_decode("get_current_state", sql, *args) + + events = yield self._parse_events(results) + + name = None + aliases = [] + + for e in events: + if e.type == 'm.room.name': + if 'name' in e.content: + name = e.content['name'] + elif e.type == 'm.room.aliases': + if 'aliases' in e.content: + aliases.extend(e.content['aliases']) + + defer.returnValue((name, aliases)) + class RoomsTable(Table): table_name = "rooms" diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 888837cd1..012144302 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -15,6 +15,8 @@ from ._base import SQLBaseStore +from twisted.internet import defer + from synapse.util.stringutils import random_string import logging @@ -125,6 +127,36 @@ class StateStore(SQLBaseStore): or_replace=True, ) + @defer.inlineCallbacks + def get_current_state(self, room_id, event_type=None, state_key=""): + del_sql = ( + "SELECT event_id FROM redactions WHERE redacts = e.event_id " + "LIMIT 1" + ) + + sql = ( + "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " + "INNER JOIN current_state_events as c ON e.event_id = c.event_id " + "INNER JOIN state_events as s ON e.event_id = s.event_id " + "WHERE c.room_id = ? " + ) % { + "redacted": del_sql, + } + + if event_type and state_key is not None: + sql += " AND s.type = ? AND s.state_key = ? " + args = (room_id, event_type, state_key) + elif event_type: + sql += " AND s.type = ?" + args = (room_id, event_type) + else: + args = (room_id, ) + + results = yield self._execute_and_decode("get_current_state", sql, *args) + + events = yield self._parse_events(results) + defer.returnValue(events) + def _make_group_id(clock): return str(int(clock.time_msec())) + random_string(5) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 64adb0c7f..9565fc77c 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -419,6 +419,25 @@ class StreamStore(SQLBaseStore): self._get_room_events_max_id_txn ) + @defer.inlineCallbacks + def _get_min_token(self): + row = yield self._execute( + "_get_min_token", None, "SELECT MIN(stream_ordering) FROM events" + ) + + self.min_token = row[0][0] if row and row[0] and row[0][0] else -1 + self.min_token = min(self.min_token, -1) + + logger.debug("min_token is: %s", self.min_token) + + defer.returnValue(self.min_token) + + def get_next_stream_id(self): + with self._next_stream_id_lock: + i = self._next_stream_id + self._next_stream_id += 1 + return i + def _get_room_events_max_id_txn(self, txn): txn.execute( "SELECT MAX(stream_ordering) as m FROM events" From 278149f53392d5e3b08f72106517afe2711c856c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Mar 2015 13:43:21 +0000 Subject: [PATCH 010/104] Sanitize TransactionStore --- synapse/handlers/federation.py | 2 +- synapse/storage/transactions.py | 189 +++++++++++++++++--------------- 2 files changed, 104 insertions(+), 87 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 15ba417e0..9a4773ac0 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -179,7 +179,7 @@ class FederationHandler(BaseHandler): # it's probably a good idea to mark it as not in retry-state # for sending (although this is a bit of a leap) retry_timings = yield self.store.get_destination_retry_timings(origin) - if (retry_timings and retry_timings.retry_last_ts): + if retry_timings and retry_timings["retry_last_ts"]: self.store.set_destination_retry_timings(origin, 0, 0) room = yield self.store.get_room(event.room_id) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 7d2239244..9dec58c21 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._base import SQLBaseStore, Table, cached +from ._base import SQLBaseStore, cached from collections import namedtuple @@ -84,13 +84,18 @@ class TransactionStore(SQLBaseStore): def _set_received_txn_response(self, txn, transaction_id, origin, code, response_json): - query = ( - "UPDATE %s " - "SET response_code = ?, response_json = ? " - "WHERE transaction_id = ? AND origin = ?" - ) % ReceivedTransactionsTable.table_name - - txn.execute(query, (code, response_json, transaction_id, origin)) + self._simple_update_one_txn( + txn, + table=ReceivedTransactionsTable.table_name, + keyvalues={ + "transaction_id": transaction_id, + "origin": origin, + }, + updatevalues={ + "response_code": code, + "response_json": response_json, + } + ) def prep_send_transaction(self, transaction_id, destination, origin_server_ts): @@ -121,38 +126,32 @@ class TransactionStore(SQLBaseStore): # First we find out what the prev_txns should be. # Since we know that we are only sending one transaction at a time, # we can simply take the last one. - query = "%s ORDER BY id DESC LIMIT 1" % ( - SentTransactions.select_statement("destination = ?"), - ) + query = ( + "SELECT * FROM sent_transactions" + " WHERE destination = ?" + " ORDER BY id DESC LIMIT 1" + ) txn.execute(query, (destination,)) - results = SentTransactions.decode_results(txn.fetchall()) + results = self.cursor_to_dict(txn) - prev_txns = [r.transaction_id for r in results] + prev_txns = [r["transaction_id"] for r in results] # Actually add the new transaction to the sent_transactions table. - query = SentTransactions.insert_statement() - txn.execute(query, SentTransactions.EntryType( - self.get_next_stream_id(), - transaction_id=transaction_id, - destination=destination, - ts=origin_server_ts, - response_code=0, - response_json=None - )) + self._simple_insert_txn( + txn, + table=SentTransactions.table_name, + values={ + "transaction_id": self.get_next_stream_id(), + "destination": destination, + "ts": origin_server_ts, + "response_code": 0, + "response_json": None, + } + ) - # Update the tx id -> pdu id mapping - - # values = [ - # (transaction_id, destination, pdu[0], pdu[1]) - # for pdu in pdu_list - # ] - # - # logger.debug("Inserting: %s", repr(values)) - # - # query = TransactionsToPduTable.insert_statement() - # txn.executemany(query, values) + # TODO Update the tx id -> pdu id mapping return prev_txns @@ -171,15 +170,20 @@ class TransactionStore(SQLBaseStore): transaction_id, destination, code, response_dict ) - def _delivered_txn(cls, txn, transaction_id, destination, + def _delivered_txn(self, txn, transaction_id, destination, code, response_json): - query = ( - "UPDATE %s " - "SET response_code = ?, response_json = ? " - "WHERE transaction_id = ? AND destination = ?" - ) % SentTransactions.table_name - - txn.execute(query, (code, response_json, transaction_id, destination)) + self._simple_update_one_txn( + txn, + table=SentTransactions.table_name, + keyvalues={ + "transaction_id": transaction_id, + "destination": destination, + }, + updatevalues={ + "response_code": code, + "response_json": response_json, + } + ) def get_transactions_after(self, transaction_id, destination): """Get all transactions after a given local transaction_id. @@ -189,25 +193,26 @@ class TransactionStore(SQLBaseStore): destination (str) Returns: - list: A list of `ReceivedTransactionsTable.EntryType` + list: A list of dicts """ return self.runInteraction( "get_transactions_after", self._get_transactions_after, transaction_id, destination ) - def _get_transactions_after(cls, txn, transaction_id, destination): - where = ( - "destination = ? AND id > (select id FROM %s WHERE " - "transaction_id = ? AND destination = ?)" - ) % ( - SentTransactions.table_name + def _get_transactions_after(self, txn, transaction_id, destination): + query = ( + "SELECT * FROM sent_transactions" + " WHERE destination = ? AND id >" + " (" + " SELECT id FROM sent_transactions" + " WHERE transaction_id = ? AND destination = ?" + " )" ) - query = SentTransactions.select_statement(where) txn.execute(query, (destination, transaction_id, destination)) - return ReceivedTransactionsTable.decode_results(txn.fetchall()) + return self.cursor_to_dict(txn) @cached() def get_destination_retry_timings(self, destination): @@ -218,22 +223,27 @@ class TransactionStore(SQLBaseStore): Returns: None if not retrying - Otherwise a DestinationsTable.EntryType for the retry scheme + Otherwise a dict for the retry scheme """ return self.runInteraction( "get_destination_retry_timings", self._get_destination_retry_timings, destination) - def _get_destination_retry_timings(cls, txn, destination): - query = DestinationsTable.select_statement("destination = ?") - txn.execute(query, (destination,)) - result = txn.fetchall() - if result: - result = DestinationsTable.decode_single_result(result) - if result.retry_last_ts > 0: - return result - else: - return None + def _get_destination_retry_timings(self, txn, destination): + result = self._simple_select_one_txn( + txn, + table=DestinationsTable.table_name, + keyvalues={ + "destination": destination, + }, + retcols=DestinationsTable.fields, + allow_none=True, + ) + + if result["retry_last_ts"] > 0: + return result + else: + return None def set_destination_retry_timings(self, destination, retry_last_ts, retry_interval): @@ -249,11 +259,11 @@ class TransactionStore(SQLBaseStore): # As this is the new value, we might as well prefill the cache self.get_destination_retry_timings.prefill( destination, - DestinationsTable.EntryType( - destination, - retry_last_ts, - retry_interval - ) + { + "destination": destination, + "retry_last_ts": retry_last_ts, + "retry_interval": retry_interval + }, ) # XXX: we could chose to not bother persisting this if our cache thinks @@ -270,18 +280,27 @@ class TransactionStore(SQLBaseStore): retry_last_ts, retry_interval): query = ( - "REPLACE INTO %s " - "(destination, retry_last_ts, retry_interval) " - "VALUES (?, ?, ?) " - ) % DestinationsTable.table_name + "INSERT INTO destinations" + " (destination, retry_last_ts, retry_interval)" + " VALUES (?, ?, ?)" + " ON DUPLICATE KEY UPDATE" + " retry_last_ts=?, retry_interval=?" + ) - txn.execute(query, (destination, retry_last_ts, retry_interval)) + txn.execute( + query, + ( + destination, + retry_last_ts, retry_interval, + retry_last_ts, retry_interval, + ) + ) def get_destinations_needing_retry(self): """Get all destinations which are due a retry for sending a transaction. Returns: - list: A list of `DestinationsTable.EntryType` + list: A list of dicts """ return self.runInteraction( @@ -289,14 +308,17 @@ class TransactionStore(SQLBaseStore): self._get_destinations_needing_retry ) - def _get_destinations_needing_retry(cls, txn): - where = "retry_last_ts > 0 and retry_next_ts < now()" - query = DestinationsTable.select_statement(where) - txn.execute(query) - return DestinationsTable.decode_results(txn.fetchall()) + def _get_destinations_needing_retry(self, txn): + query = ( + "SELECT * FROM destinations" + " WHERE retry_last_ts > 0 and retry_next_ts < ?" + ) + + txn.execute(query, (self._clock.time_msec(),)) + return self.cursor_to_dict(txn) -class ReceivedTransactionsTable(Table): +class ReceivedTransactionsTable(object): table_name = "received_transactions" fields = [ @@ -308,10 +330,8 @@ class ReceivedTransactionsTable(Table): "has_been_referenced", ] - EntryType = namedtuple("ReceivedTransactionsEntry", fields) - -class SentTransactions(Table): +class SentTransactions(object): table_name = "sent_transactions" fields = [ @@ -326,7 +346,7 @@ class SentTransactions(Table): EntryType = namedtuple("SentTransactionsEntry", fields) -class TransactionsToPduTable(Table): +class TransactionsToPduTable(object): table_name = "transaction_id_to_pdu" fields = [ @@ -336,10 +356,8 @@ class TransactionsToPduTable(Table): "pdu_origin", ] - EntryType = namedtuple("TransactionsToPduEntry", fields) - -class DestinationsTable(Table): +class DestinationsTable(object): table_name = "destinations" fields = [ @@ -348,4 +366,3 @@ class DestinationsTable(Table): "retry_interval", ] - EntryType = namedtuple("DestinationsEntry", fields) From d5272b1d2cf170d2b41d9aa991c3a37e9f7aac34 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Mar 2015 14:02:34 +0000 Subject: [PATCH 011/104] Use 'update or insert' rather than on 'conflict replace' --- synapse/storage/transactions.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 9dec58c21..1a5bb41cb 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -276,26 +276,33 @@ class TransactionStore(SQLBaseStore): retry_interval, ) - def _set_destination_retry_timings(cls, txn, destination, + def _set_destination_retry_timings(self, txn, destination, retry_last_ts, retry_interval): - query = ( - "INSERT INTO destinations" - " (destination, retry_last_ts, retry_interval)" - " VALUES (?, ?, ?)" - " ON DUPLICATE KEY UPDATE" - " retry_last_ts=?, retry_interval=?" + "UPDATE destinations" + " SET retry_last_ts = ?, retry_interval = ?" + " WHERE destinations = ?" ) txn.execute( query, ( - destination, - retry_last_ts, retry_interval, - retry_last_ts, retry_interval, + retry_last_ts, retry_interval, destination, ) ) + if txn.rowcount == 0: + # destination wasn't already in table. Insert it. + self._simple_insert_txn( + txn, + table="destinations", + values={ + "destination": destination, + "retry_last_ts": retry_last_ts, + "retry_interval": retry_interval, + } + ) + def get_destinations_needing_retry(self): """Get all destinations which are due a retry for sending a transaction. From 9a7f4962982e309877acb777eb358178579cacc9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Mar 2015 15:29:04 +0000 Subject: [PATCH 012/104] Sanitize RoomMemberStore --- synapse/handlers/room.py | 19 ------ synapse/storage/roommember.py | 124 +++++++++++++++++----------------- 2 files changed, 61 insertions(+), 82 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 823affc38..bc7f1c240 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -310,25 +310,6 @@ class RoomMemberHandler(BaseHandler): # paginating defer.returnValue(chunk_data) - @defer.inlineCallbacks - def get_room_member(self, room_id, member_user_id, auth_user_id): - """Retrieve a room member from a room. - - Args: - room_id : The room the member is in. - member_user_id : The member's user ID - auth_user_id : The user ID of the user making this request. - Returns: - The room member, or None if this member does not exist. - Raises: - SynapseError if something goes wrong. - """ - yield self.auth.check_joined_room(room_id, auth_user_id) - - member = yield self.store.get_room_member(user_id=member_user_id, - room_id=room_id) - defer.returnValue(member) - @defer.inlineCallbacks def change_membership(self, event, context, do_auth=True): """ Change the membership status of a user in a room. diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 17ee4bb9e..a22950520 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -74,23 +74,10 @@ class RoomMemberStore(SQLBaseStore): txn.execute(sql, (event.room_id, domain)) elif event.membership != Membership.INVITE: # Check if this was the last person to have left. - member_events = self._get_members_query_txn( - txn, - where_clause=("c.room_id = ? AND m.membership = ?" - " AND m.user_id != ?"), - where_values=(event.room_id, Membership.JOIN, target_user_id,) + joined_domains = self._get_joined_hosts_for_room_txn( + txn, event.room_id ) - joined_domains = set() - for e in member_events: - try: - joined_domains.add( - UserID.from_string(e.state_key).domain - ) - except: - # FIXME: How do we deal with invalid user ids in the db? - logger.exception("Invalid user_id: %s", event.state_key) - if domain not in joined_domains: sql = ( "DELETE FROM room_hosts WHERE room_id = ? AND host = ?" @@ -100,7 +87,6 @@ class RoomMemberStore(SQLBaseStore): self.get_rooms_for_user.invalidate(target_user_id) - @defer.inlineCallbacks def get_room_member(self, user_id, room_id): """Retrieve the current state of a room member. @@ -110,41 +96,27 @@ class RoomMemberStore(SQLBaseStore): Returns: Deferred: Results in a MembershipEvent or None. """ - rows = yield self._get_members_by_dict({ - "e.room_id": room_id, - "m.user_id": user_id, - }) + def f(txn): + events = self._get_members_events_txn( + txn, + room_id, + user_id=user_id, + ) - defer.returnValue(rows[0] if rows else None) + return events[0] if events else None - def _get_room_member(self, txn, user_id, room_id): - sql = ( - "SELECT e.* FROM events as e" - " INNER JOIN room_memberships as m" - " ON e.event_id = m.event_id" - " INNER JOIN current_state_events as c" - " ON m.event_id = c.event_id" - " WHERE m.user_id = ? and e.room_id = ?" - " LIMIT 1" - ) - txn.execute(sql, (user_id, room_id)) - rows = self.cursor_to_dict(txn) - if rows: - return self._parse_events_txn(txn, rows)[0] - else: - return None + return self.runInteraction("get_room_member", f) def get_users_in_room(self, room_id): def f(txn): - sql = ( - "SELECT m.user_id FROM room_memberships as m" - " INNER JOIN current_state_events as c" - " ON m.event_id = c.event_id" - " WHERE m.membership = ? AND m.room_id = ?" + + rows = self._get_members_rows_txn( + txn, + room_id=room_id, + membership=Membership.JOIN, ) - txn.execute(sql, (Membership.JOIN, room_id)) - return [r[0] for r in txn.fetchall()] + return [r["user_id"] for r in rows] return self.runInteraction("get_users_in_room", f) def get_room_members(self, room_id, membership=None): @@ -159,11 +131,14 @@ class RoomMemberStore(SQLBaseStore): list of namedtuples representing the members in this room. """ - where = {"m.room_id": room_id} - if membership: - where["m.membership"] = membership + def f(txn): + return self._get_members_events_txn( + txn, + room_id, + membership=membership, + ) - return self._get_members_by_dict(where) + return self.runInteraction("get_room_members", f) def get_rooms_for_user_where_membership_is(self, user_id, membership_list): """ Get all the rooms for this user where the membership for this user @@ -209,28 +184,52 @@ class RoomMemberStore(SQLBaseStore): ] def get_joined_hosts_for_room(self, room_id): - return self._simple_select_onecol( - "room_hosts", - {"room_id": room_id}, - "host", - desc="get_joined_hosts_for_room", + return self.runInteraction( + "get_joined_hosts_for_room", + self._get_joined_hosts_for_room_txn, + room_id, ) - def _get_members_by_dict(self, where_dict): - clause = " AND ".join("%s = ?" % k for k in where_dict.keys()) - vals = where_dict.values() - return self._get_members_query(clause, vals) + def _get_joined_hosts_for_room_txn(self, txn, room_id): + rows = self._get_members_rows_txn( + txn, + room_id, membership=Membership.JOIN + ) + + joined_domains = set( + UserID.from_string(r["user_id"]).domain + for r in rows + ) + + return joined_domains def _get_members_query(self, where_clause, where_values): return self.runInteraction( - "get_members_query", self._get_members_query_txn, + "get_members_query", self._get_members_events_txn, where_clause, where_values ) - def _get_members_query_txn(self, txn, where_clause, where_values): + def _get_members_events_txn(self, txn, room_id, membership=None, user_id=None): + rows = self._get_members_rows_txn( + txn, + room_id, membership, user_id, + ) + return self._get_events_txn(txn, [r["event_id"] for r in rows]) + + def _get_members_rows_txn(self, txn, room_id, membership=None, user_id=None): + where_clause = "c.room_id = ?" + where_values = [room_id] + + if membership: + where_clause += " AND m.membership = ?" + where_values.append(membership) + + if user_id: + where_clause += " AND m.user_id = ?" + where_values.append(user_id) + sql = ( - "SELECT e.* FROM events as e " - "INNER JOIN room_memberships as m " + "SELECT m.* FROM room_memberships as m " "ON e.event_id = m.event_id " "INNER JOIN current_state_events as c " "ON m.event_id = c.event_id " @@ -242,8 +241,7 @@ class RoomMemberStore(SQLBaseStore): txn.execute(sql, where_values) rows = self.cursor_to_dict(txn) - results = self._parse_events_txn(txn, rows) - return results + return rows @cached() def get_rooms_for_user(self, user_id): From 6e7131f02f33695b6423db9a377f130ffebbdb67 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Mar 2015 15:38:56 +0000 Subject: [PATCH 013/104] Remove uses of REPLACE and ON CONFLICT IGNORE to make the SQL more portable. --- synapse/storage/_base.py | 13 ++++--------- synapse/storage/keys.py | 2 -- synapse/storage/signatures.py | 4 ---- synapse/storage/state.py | 1 - 4 files changed, 4 insertions(+), 16 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index d038c5509..9214e0c49 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -341,26 +341,21 @@ class SQLBaseStore(object): # "Simple" SQL API methods that operate on a single table with no JOINs, # no complex WHERE clauses, just a dict of values for columns. - def _simple_insert(self, table, values, or_replace=False, or_ignore=False, - desc="_simple_insert"): + def _simple_insert(self, table, values, desc="_simple_insert"): """Executes an INSERT query on the named table. Args: table : string giving the table name values : dict of new column names and values for them - or_replace : bool; if True performs an INSERT OR REPLACE """ return self.runInteraction( desc, - self._simple_insert_txn, table, values, or_replace=or_replace, - or_ignore=or_ignore, + self._simple_insert_txn, table, values, ) @log_function - def _simple_insert_txn(self, txn, table, values, or_replace=False, - or_ignore=False): - sql = "%s INTO %s (%s) VALUES(%s)" % ( - ("REPLACE" if or_replace else "INSERT"), + def _simple_insert_txn(self, txn, table, values): + sql = "INSERT INTO %s (%s) VALUES(%s)" % ( table, ", ".join(k for k in values), ", ".join("?" for k in values) diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 09d1e6365..25fef7943 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -66,7 +66,6 @@ class KeyStore(SQLBaseStore): "ts_added_ms": time_now_ms, "tls_certificate": buffer(tls_certificate_bytes), }, - or_ignore=True, ) @defer.inlineCallbacks @@ -116,5 +115,4 @@ class KeyStore(SQLBaseStore): "ts_added_ms": time_now_ms, "verify_key": buffer(verify_key.encode()), }, - or_ignore=True, ) diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index d0d53770f..13ce33510 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -56,7 +56,6 @@ class SignatureStore(SQLBaseStore): "algorithm": algorithm, "hash": buffer(hash_bytes), }, - or_ignore=True, ) def get_event_reference_hashes(self, event_ids): @@ -119,7 +118,6 @@ class SignatureStore(SQLBaseStore): "algorithm": algorithm, "hash": buffer(hash_bytes), }, - or_ignore=True, ) def _get_event_signatures_txn(self, txn, event_id): @@ -164,7 +162,6 @@ class SignatureStore(SQLBaseStore): "key_id": key_id, "signature": buffer(signature_bytes), }, - or_ignore=True, ) def _get_prev_event_hashes_txn(self, txn, event_id): @@ -198,5 +195,4 @@ class SignatureStore(SQLBaseStore): "algorithm": algorithm, "hash": buffer(hash_bytes), }, - or_ignore=True, ) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 721f2862c..65ea9c4d8 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -124,7 +124,6 @@ class StateStore(SQLBaseStore): "state_group": state_group, "event_id": event.event_id, }, - or_replace=True, ) @defer.inlineCallbacks From 9115421ace4cd6cd1080ab99c997994541aea30b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Mar 2015 16:17:39 +0000 Subject: [PATCH 014/104] Use _simple_upsert --- synapse/storage/__init__.py | 8 +++++--- synapse/storage/transactions.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4877f45dc..e03d55b00 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -81,14 +81,16 @@ class DataStore(RoomMemberStore, RoomStore, self._next_stream_id = int(hs.get_clock().time_msec()) * 1000 def insert_client_ip(self, user, access_token, device_id, ip, user_agent): - return self._simple_insert( + return self._simple_upsert( "user_ips", - { + keyvalues={ "user": user.to_string(), "access_token": access_token, - "device_id": device_id, "ip": ip, "user_agent": user_agent, + }, + values={ + "device_id": device_id, "last_seen": int(self._clock.time_msec()), }, desc="insert_client_ip", diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 1a5bb41cb..c6e00ae3f 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -84,14 +84,14 @@ class TransactionStore(SQLBaseStore): def _set_received_txn_response(self, txn, transaction_id, origin, code, response_json): - self._simple_update_one_txn( + self._simple_upsert_txn( txn, table=ReceivedTransactionsTable.table_name, keyvalues={ "transaction_id": transaction_id, "origin": origin, }, - updatevalues={ + values={ "response_code": code, "response_json": response_json, } From 9e98f1022a927adef959e0a76dde2fc6f708917d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Mar 2015 16:19:01 +0000 Subject: [PATCH 015/104] Don't order by rowid --- synapse/storage/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 9214e0c49..1ea39bc0a 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -585,7 +585,7 @@ class SQLBaseStore(object): def _simple_select_one_txn(self, txn, table, keyvalues, retcols, allow_none=False): - select_sql = "SELECT %s FROM %s WHERE %s ORDER BY rowid asc" % ( + select_sql = "SELECT %s FROM %s WHERE %s" % ( ", ".join(retcols), table, " AND ".join("%s = ?" % (k) for k in keyvalues) From 5a447098dd8442c7c69b2db22d42e6ce40c2dc6e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Mar 2015 16:19:24 +0000 Subject: [PATCH 016/104] Don't use room hosts table --- synapse/storage/roommember.py | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index a22950520..df707f812 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -65,26 +65,6 @@ class RoomMemberStore(SQLBaseStore): } ) - # Update room hosts table - if event.membership == Membership.JOIN: - sql = ( - "REPLACE INTO room_hosts (room_id, host) " - "VALUES (?, ?)" - ) - txn.execute(sql, (event.room_id, domain)) - elif event.membership != Membership.INVITE: - # Check if this was the last person to have left. - joined_domains = self._get_joined_hosts_for_room_txn( - txn, event.room_id - ) - - if domain not in joined_domains: - sql = ( - "DELETE FROM room_hosts WHERE room_id = ? AND host = ?" - ) - - txn.execute(sql, (event.room_id, domain)) - self.get_rooms_for_user.invalidate(target_user_id) def get_room_member(self, user_id, room_id): @@ -229,11 +209,10 @@ class RoomMemberStore(SQLBaseStore): where_values.append(user_id) sql = ( - "SELECT m.* FROM room_memberships as m " - "ON e.event_id = m.event_id " - "INNER JOIN current_state_events as c " - "ON m.event_id = c.event_id " - "WHERE %(where)s " + "SELECT m.* FROM room_memberships as m" + " INNER JOIN current_state_events as c" + " ON m.event_id = c.event_id" + " WHERE %(where)s" ) % { "where": where_clause, } From 1aa44939fc94ba1062d53a74dbd9629c32a2ae20 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Mar 2015 16:20:05 +0000 Subject: [PATCH 017/104] Fix bugs in transactions storage --- synapse/storage/transactions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index c6e00ae3f..03e1e3b80 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -143,7 +143,8 @@ class TransactionStore(SQLBaseStore): txn, table=SentTransactions.table_name, values={ - "transaction_id": self.get_next_stream_id(), + "id": self.get_next_stream_id(), + "transaction_id": transaction_id, "destination": destination, "ts": origin_server_ts, "response_code": 0, @@ -240,7 +241,7 @@ class TransactionStore(SQLBaseStore): allow_none=True, ) - if result["retry_last_ts"] > 0: + if result and result["retry_last_ts"] > 0: return result else: return None From f488293d96dc690ce7a4b3163fe8c4e0d3e31899 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Mar 2015 16:20:26 +0000 Subject: [PATCH 018/104] Don't reinsert into event_edges --- synapse/storage/events.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 542559c59..4d636d3f4 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -149,14 +149,6 @@ class EventsStore(SQLBaseStore): event.depth ) - self._handle_prev_events( - txn, - outlier=outlier, - event_id=event.event_id, - prev_events=event.prev_events, - room_id=event.room_id, - ) - have_persisted = self._simple_select_one_onecol_txn( txn, table="event_json", @@ -196,6 +188,14 @@ class EventsStore(SQLBaseStore): ) return + self._handle_prev_events( + txn, + outlier=outlier, + event_id=event.event_id, + prev_events=event.prev_events, + room_id=event.room_id, + ) + if event.type == EventTypes.Member: self._store_room_member_txn(txn, event) elif event.type == EventTypes.Name: From 0ba393924a823698eb5263a29ce6c4b50488ac4b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Mar 2015 16:31:52 +0000 Subject: [PATCH 019/104] Escape non printing ascii character --- synapse/storage/room.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 864103332..501e947ad 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -102,10 +102,10 @@ class RoomStore(SQLBaseStore): "ON c.event_id = room_names.event_id " ) - # We use non printing ascii character US () as a seperator + # We use non printing ascii character US (\x1F) as a separator sql = ( "SELECT r.room_id, n.name, t.topic, " - "group_concat(a.room_alias, '') " + "group_concat(a.room_alias, '\x1F') " "FROM rooms AS r " "LEFT JOIN (%(topic)s) AS t ON t.room_id = r.room_id " "LEFT JOIN (%(name)s) AS n ON n.room_id = r.room_id " @@ -130,7 +130,7 @@ class RoomStore(SQLBaseStore): "room_id": r[0], "name": r[1], "topic": r[2], - "aliases": r[3].split(""), + "aliases": r[3].split("\x1F"), } for r in rows ] From 0e8f5095c7e7075b249ad53a9f60a4d2fdeeaaed Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 25 Mar 2015 17:15:20 +0000 Subject: [PATCH 020/104] Fix unicode database support --- synapse/app/homeserver.py | 47 ++++++++++++------- synapse/handlers/login.py | 2 +- synapse/rest/client/v1/profile.py | 7 ++- synapse/storage/__init__.py | 12 +++-- synapse/storage/_base.py | 4 ++ synapse/storage/events.py | 8 ++-- synapse/storage/keys.py | 4 +- synapse/storage/profile.py | 12 ++++- synapse/storage/registration.py | 18 +++++-- synapse/storage/room.py | 1 + .../full_schemas/11/media_repository.sql | 2 +- .../schema/full_schemas/11/profiles.sql | 2 +- .../schema/full_schemas/11/transactions.sql | 1 - synapse/storage/signatures.py | 10 ++-- synapse/storage/transactions.py | 2 +- 15 files changed, 88 insertions(+), 44 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 394e93e6c..beab6ffc7 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -110,14 +110,12 @@ class SynapseHomeServer(HomeServer): return None def build_db_pool(self): - name = self.db_config.pop("name", None) - if name == "MySQLdb": - return adbapi.ConnectionPool( - name, - **self.db_config - ) + name = self.db_config["name"] - raise RuntimeError("Unsupported database type") + return adbapi.ConnectionPool( + name, + **self.db_config.get("args", {}) + ) def create_resource_tree(self, redirect_root_to_web_client): """Create the resource tree for this Home Server. @@ -323,7 +321,7 @@ def change_resource_limit(soft_file_no): resource.setrlimit(resource.RLIMIT_NOFILE, (soft_file_no, hard)) logger.info("Set file limit to: %d", soft_file_no) - except (ValueError, resource.error) as e: + except ( ValueError, resource.error) as e: logger.warn("Failed to set file limit: %s", e) @@ -363,20 +361,33 @@ def setup(config_options): if config.database_config: with open(config.database_config, 'r') as f: db_config = yaml.safe_load(f) - - name = db_config.get("name", None) - if name == "MySQLdb": - db_config.update({ - "sql_mode": "TRADITIONAL", - "charset": "utf8", - "use_unicode": True, - }) else: db_config = { "name": "sqlite3", "database": config.database_path, } + db_config = { + k: v for k, v in db_config.items() + if not k.startswith("cp_") + } + + name = db_config.get("name", None) + if name in ["MySQLdb", "mysql.connector"]: + db_config.setdefault("args", {}).update({ + "sql_mode": "TRADITIONAL", + "charset": "utf8", + "use_unicode": True, + }) + elif name == "sqlite3": + db_config.setdefault("args", {}).update({ + "cp_min": 1, + "cp_max": 1, + "cp_openfun": prepare_database, + }) + else: + raise RuntimeError("Unsupported database type '%s'" % (name,)) + hs = SynapseHomeServer( config.server_name, domain_with_port=domain_with_port, @@ -401,8 +412,8 @@ def setup(config_options): # with sqlite3.connect(db_name) as db_conn: # prepare_sqlite3_database(db_conn) # prepare_database(db_conn) - import MySQLdb - db_conn = MySQLdb.connect(**db_config) + import mysql.connector + db_conn = mysql.connector.connect(**db_config.get("args", {})) prepare_database(db_conn) except UpgradeDatabaseException: sys.stderr.write( diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 744780046..76647c794 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -57,7 +57,7 @@ class LoginHandler(BaseHandler): logger.warn("Attempted to login as %s but they do not exist", user) raise LoginError(403, "", errcode=Codes.FORBIDDEN) - stored_hash = user_info[0]["password_hash"] + stored_hash = user_info["password_hash"] if bcrypt.checkpw(password, stored_hash): # generate an access token and store it. token = self.reg_handler._generate_token(user) diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index 1e77eb49c..7387b4adb 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -19,9 +19,13 @@ from twisted.internet import defer from .base import ClientV1RestServlet, client_path_pattern from synapse.types import UserID +import logging import simplejson as json +logger = logging.getLogger(__name__) + + class ProfileDisplaynameRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/profile/(?P[^/]*)/displayname") @@ -47,7 +51,8 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): defer.returnValue((400, "Unable to parse name")) yield self.handlers.profile_handler.set_displayname( - user, auth_user, new_name) + user, auth_user, new_name + ) defer.returnValue((200, {})) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index e03d55b00..abde7d0df 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -410,10 +410,14 @@ def executescript(txn, schema_path): def _get_or_create_schema_state(txn): - schema_path = os.path.join( - dir_path, "schema", "schema_version.sql", - ) - executescript(txn, schema_path) + try: + # Bluntly try creating the schema_version tables. + schema_path = os.path.join( + dir_path, "schema", "schema_version.sql", + ) + executescript(txn, schema_path) + except: + pass txn.execute("SELECT version, upgraded FROM schema_version") row = txn.fetchone() diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 1ea39bc0a..76ec3ee93 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -755,6 +755,8 @@ class SQLBaseStore(object): return None internal_metadata, js, redacted, rejected_reason = res + js = js.decode("utf8") + internal_metadata = internal_metadata.decode("utf8") start_time = update_counter("select_event", start_time) @@ -779,9 +781,11 @@ class SQLBaseStore(object): sql_getevents_timer.inc_by(curr_time - last_time, desc) return curr_time + logger.debug("Got js: %r", js) d = json.loads(js) start_time = update_counter("decode_json", start_time) + logger.debug("Got internal_metadata: %r", internal_metadata) internal_metadata = json.loads(internal_metadata) start_time = update_counter("decode_internal", start_time) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 4d636d3f4..69f598967 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -294,15 +294,17 @@ class EventsStore(SQLBaseStore): ) if is_new_state and not context.rejected: - self._simple_insert_txn( + self._simple_upsert_txn( txn, "current_state_events", - { - "event_id": event.event_id, + keyvalues={ "room_id": event.room_id, "type": event.type, "state_key": event.state_key, }, + values={ + "event_id": event.event_id, + } ) for e_id, h in event.prev_state: diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 25fef7943..e6975a945 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -64,7 +64,7 @@ class KeyStore(SQLBaseStore): "fingerprint": fingerprint, "from_server": from_server, "ts_added_ms": time_now_ms, - "tls_certificate": buffer(tls_certificate_bytes), + "tls_certificate": tls_certificate_bytes, }, ) @@ -113,6 +113,6 @@ class KeyStore(SQLBaseStore): "key_id": "%s:%s" % (verify_key.alg, verify_key.version), "from_server": from_server, "ts_added_ms": time_now_ms, - "verify_key": buffer(verify_key.encode()), + "verify_key": verify_key.encode(), }, ) diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py index a6e52cb24..09778045b 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer + from ._base import SQLBaseStore @@ -24,19 +26,25 @@ class ProfileStore(SQLBaseStore): desc="create_profile", ) + @defer.inlineCallbacks def get_profile_displayname(self, user_localpart): - return self._simple_select_one_onecol( + name = yield self._simple_select_one_onecol( table="profiles", keyvalues={"user_id": user_localpart}, retcol="displayname", desc="get_profile_displayname", ) + if name: + name = name.decode("utf8") + + defer.returnValue(name) + def set_profile_displayname(self, user_localpart, new_displayname): return self._simple_update_one( table="profiles", keyvalues={"user_id": user_localpart}, - updatevalues={"displayname": new_displayname}, + updatevalues={"displayname": new_displayname.encode("utf8")}, desc="set_profile_displayname", ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index fe26d6d62..7258f7b2a 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -81,13 +81,23 @@ class RegistrationStore(SQLBaseStore): txn.execute("INSERT INTO access_tokens(user_id, token) " + "VALUES (?,?)", [user_id, token]) + @defer.inlineCallbacks def get_user_by_id(self, user_id): - query = ("SELECT users.name, users.password_hash FROM users" - " WHERE users.name = ?") - return self._execute( - "get_user_by_id", self.cursor_to_dict, query, user_id + user_info = yield self._simple_select_one( + table="users", + keyvalues={ + "name": user_id, + }, + retcols=["name", "password_hash"], + allow_none=True, ) + if user_info: + user_info["password_hash"] = user_info["password_hash"].decode("utf8") + + defer.returnValue(user_info) + + @cached() # TODO(paul): Currently there's no code to invalidate this cache. That # means if/when we ever add internal ways to invalidate access tokens or diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 501e947ad..a1a76280f 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -72,6 +72,7 @@ class RoomStore(SQLBaseStore): keyvalues={"room_id": room_id}, retcols=RoomsTable.fields, desc="get_room", + allow_none=True, ) @defer.inlineCallbacks diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index 8bc84dc24..d9559f590 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -65,4 +65,4 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( ) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id - ON local_media_repository_thumbnails (media_id); + ON remote_media_cache_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index 32defe2f7..552645c56 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS profiles( user_id VARCHAR(255) NOT NULL, - displayname VARCHAR(255), + displayname VARBINARY(255), avatar_url VARCHAR(255), UNIQUE(user_id) ) ENGINE = INNODB; diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 0570bf95d..bd13bba8c 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -38,7 +38,6 @@ CREATE TABLE IF NOT EXISTS sent_transactions( ) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); -CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions(destination); CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); -- So that we can do an efficient look up of all transactions that have yet to be successfully -- sent. diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 13ce33510..35bba854f 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -54,7 +54,7 @@ class SignatureStore(SQLBaseStore): { "event_id": event_id, "algorithm": algorithm, - "hash": buffer(hash_bytes), + "hash": hash_bytes, }, ) @@ -99,7 +99,7 @@ class SignatureStore(SQLBaseStore): " WHERE event_id = ?" ) txn.execute(query, (event_id, )) - return dict(txn.fetchall()) + return {k: v for k, v in txn.fetchall()} def _store_event_reference_hash_txn(self, txn, event_id, algorithm, hash_bytes): @@ -116,7 +116,7 @@ class SignatureStore(SQLBaseStore): { "event_id": event_id, "algorithm": algorithm, - "hash": buffer(hash_bytes), + "hash": hash_bytes, }, ) @@ -160,7 +160,7 @@ class SignatureStore(SQLBaseStore): "event_id": event_id, "signature_name": signature_name, "key_id": key_id, - "signature": buffer(signature_bytes), + "signature": signature_bytes, }, ) @@ -193,6 +193,6 @@ class SignatureStore(SQLBaseStore): "event_id": event_id, "prev_event_id": prev_event_id, "algorithm": algorithm, - "hash": buffer(hash_bytes), + "hash": hash_bytes, }, ) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 03e1e3b80..e3e484fb2 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -282,7 +282,7 @@ class TransactionStore(SQLBaseStore): query = ( "UPDATE destinations" " SET retry_last_ts = ?, retry_interval = ?" - " WHERE destinations = ?" + " WHERE destination = ?" ) txn.execute( From 9236136f3a4f0d8119d4a6333f37378f8e259e4a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 1 Apr 2015 14:12:33 +0100 Subject: [PATCH 021/104] Make work in both Maria and SQLite. Fix tests --- synapse/app/homeserver.py | 20 ++++++--- synapse/storage/__init__.py | 43 +++++++++---------- synapse/storage/_base.py | 30 +++++++++---- synapse/storage/engines/__init__.py | 35 +++++++++++++++ synapse/storage/engines/maria.py | 30 +++++++++++++ synapse/storage/engines/sqlite3.py | 25 +++++++++++ synapse/storage/keys.py | 4 +- synapse/storage/registration.py | 8 +++- synapse/storage/schema/delta/12/v12.sql | 8 ++-- synapse/storage/schema/delta/13/v13.sql | 4 +- synapse/storage/schema/delta/14/v14.sql | 2 +- .../schema/full_schemas/11/event_edges.sql | 14 +++--- .../full_schemas/11/event_signatures.sql | 8 ++-- synapse/storage/schema/full_schemas/11/im.sql | 18 ++++---- .../storage/schema/full_schemas/11/keys.sql | 4 +- .../full_schemas/11/media_repository.sql | 8 ++-- .../schema/full_schemas/11/presence.sql | 6 +-- .../schema/full_schemas/11/profiles.sql | 4 +- .../schema/full_schemas/11/redactions.sql | 2 +- .../schema/full_schemas/11/room_aliases.sql | 8 ++-- .../storage/schema/full_schemas/11/state.sql | 6 +-- .../schema/full_schemas/11/transactions.sql | 8 ++-- .../storage/schema/full_schemas/11/users.sql | 10 ++--- synapse/storage/signatures.py | 8 ++-- synapse/storage/stream.py | 6 --- synapse/util/retryutils.py | 2 +- tests/federation/test_federation.py | 10 +++-- tests/handlers/test_federation.py | 9 ++++ tests/handlers/test_presence.py | 7 ++- tests/handlers/test_typing.py | 7 ++- tests/rest/client/v1/test_events.py | 9 ---- tests/storage/test_appservice.py | 12 ++---- tests/storage/test_base.py | 20 +++++---- tests/storage/test_registration.py | 36 ++++++++++------ tests/storage/test_roommember.py | 14 +++--- tests/utils.py | 11 ++++- 36 files changed, 296 insertions(+), 160 deletions(-) create mode 100644 synapse/storage/engines/__init__.py create mode 100644 synapse/storage/engines/maria.py create mode 100644 synapse/storage/engines/sqlite3.py diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index beab6ffc7..b185b2f56 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -20,6 +20,7 @@ sys.dont_write_bytecode = True from synapse.storage import ( prepare_database, prepare_sqlite3_database, UpgradeDatabaseException, ) +from synapse.storage.engines import create_engine from synapse.server import HomeServer @@ -376,7 +377,7 @@ def setup(config_options): if name in ["MySQLdb", "mysql.connector"]: db_config.setdefault("args", {}).update({ "sql_mode": "TRADITIONAL", - "charset": "utf8", + "charset": "utf8mb4", "use_unicode": True, }) elif name == "sqlite3": @@ -388,6 +389,8 @@ def setup(config_options): else: raise RuntimeError("Unsupported database type '%s'" % (name,)) + database_engine = create_engine(name) + hs = SynapseHomeServer( config.server_name, domain_with_port=domain_with_port, @@ -398,6 +401,7 @@ def setup(config_options): config=config, content_addr=config.content_addr, version_string=version_string, + database_engine=database_engine, ) hs.create_resource_tree( @@ -409,12 +413,14 @@ def setup(config_options): logger.info("Preparing database: %s...", db_name) try: - # with sqlite3.connect(db_name) as db_conn: - # prepare_sqlite3_database(db_conn) - # prepare_database(db_conn) - import mysql.connector - db_conn = mysql.connector.connect(**db_config.get("args", {})) - prepare_database(db_conn) + db_conn = database_engine.module.connect(**db_config.get("args", {})) + + if name == "sqlite3": + prepare_sqlite3_database(db_conn) + + prepare_database(db_conn, database_engine) + + db_conn.commit() except UpgradeDatabaseException: sys.stderr.write( "\nFailed to upgrade database.\n" diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index abde7d0df..f8053484c 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -77,9 +77,6 @@ class DataStore(RoomMemberStore, RoomStore, self.min_token_deferred = self._get_min_token() self.min_token = None - self._next_stream_id_lock = threading.Lock() - self._next_stream_id = int(hs.get_clock().time_msec()) * 1000 - def insert_client_ip(self, user, access_token, device_id, ip, user_agent): return self._simple_upsert( "user_ips", @@ -127,19 +124,21 @@ class UpgradeDatabaseException(PrepareDatabaseException): pass -def prepare_database(db_conn): +def prepare_database(db_conn, database_engine): """Prepares a database for usage. Will either create all necessary tables or upgrade from an older schema version. """ try: cur = db_conn.cursor() - version_info = _get_or_create_schema_state(cur) + version_info = _get_or_create_schema_state(cur, database_engine) if version_info: user_version, delta_files, upgraded = version_info - _upgrade_existing_database(cur, user_version, delta_files, upgraded) + _upgrade_existing_database( + cur, user_version, delta_files, upgraded, database_engine + ) else: - _setup_new_database(cur) + _setup_new_database(cur, database_engine) # cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,)) @@ -150,7 +149,7 @@ def prepare_database(db_conn): raise -def _setup_new_database(cur): +def _setup_new_database(cur, database_engine): """Sets up the database by finding a base set of "full schemas" and then applying any necessary deltas. @@ -210,7 +209,7 @@ def _setup_new_database(cur): executescript(cur, sql_loc) cur.execute( - _convert_param_style( + database_engine.convert_param_style( "REPLACE INTO schema_version (version, upgraded)" " VALUES (?,?)" ), @@ -221,12 +220,13 @@ def _setup_new_database(cur): cur, current_version=max_current_ver, applied_delta_files=[], - upgraded=False + upgraded=False, + database_engine=database_engine, ) def _upgrade_existing_database(cur, current_version, applied_delta_files, - upgraded): + upgraded, database_engine): """Upgrades an existing database. Delta files can either be SQL stored in *.sql files, or python modules @@ -335,26 +335,22 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, # Mark as done. cur.execute( - _convert_param_style( + database_engine.convert_param_style( "INSERT INTO applied_schema_deltas (version, file)" - " VALUES (?,?)" + " VALUES (?,?)", ), (v, relative_path) ) cur.execute( - _convert_param_style( + database_engine.convert_param_style( "REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)" + " VALUES (?,?)", ), (v, True) ) -def _convert_param_style(sql): - return sql.replace("?", "%s") - - def get_statements(f): statement_buffer = "" in_comment = False # If we're in a /* ... */ style comment @@ -409,7 +405,7 @@ def executescript(txn, schema_path): txn.execute(statement) -def _get_or_create_schema_state(txn): +def _get_or_create_schema_state(txn, database_engine): try: # Bluntly try creating the schema_version tables. schema_path = os.path.join( @@ -426,7 +422,7 @@ def _get_or_create_schema_state(txn): if current_version: txn.execute( - _convert_param_style( + database_engine.convert_param_style( "SELECT file FROM applied_schema_deltas WHERE version >= ?" ), (current_version,) @@ -446,6 +442,8 @@ def prepare_sqlite3_database(db_conn): new. This only affects sqlite databases since they were the only ones supported at the time. """ + import sqlite3 + with db_conn: schema_path = os.path.join( dir_path, "schema", "schema_version.sql", @@ -466,7 +464,8 @@ def prepare_sqlite3_database(db_conn): db_conn.execute( _convert_param_style( "REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)" + " VALUES (?,?)", + sqlite3 ), (row[0], False) ) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 76ec3ee93..047d100f4 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -29,6 +29,7 @@ import functools import simplejson as json import sys import time +import threading logger = logging.getLogger(__name__) @@ -118,19 +119,16 @@ def cached(max_entries=1000, num_args=1): return wrap -def _convert_param_style(sql): - return sql.replace("?", "%s") - - class LoggingTransaction(object): """An object that almost-transparently proxies for the 'txn' object passed to the constructor. Adds logging and metrics to the .execute() method.""" - __slots__ = ["txn", "name"] + __slots__ = ["txn", "name", "database_engine"] - def __init__(self, txn, name): + def __init__(self, txn, name, database_engine): object.__setattr__(self, "txn", txn) object.__setattr__(self, "name", name) + object.__setattr__(self, "database_engine", database_engine) def __getattr__(self, name): return getattr(self.txn, name) @@ -142,7 +140,7 @@ class LoggingTransaction(object): # TODO(paul): Maybe use 'info' and 'debug' for values? sql_logger.debug("[SQL] {%s} %s", self.name, sql) - sql = _convert_param_style(sql) + sql = self.database_engine.convert_param_style(sql) try: if args and args[0]: @@ -227,9 +225,14 @@ class SQLBaseStore(object): self._get_event_cache = LruCache(hs.config.event_cache_size) + self.database_engine = hs.database_engine + # Pretend the getEventCache is just another named cache caches_by_name["*getEvent*"] = self._get_event_cache + self._next_stream_id_lock = threading.Lock() + self._next_stream_id = int(hs.get_clock().time_msec()) * 1000 + def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -281,7 +284,10 @@ class SQLBaseStore(object): sql_scheduling_timer.inc_by(time.time() * 1000 - start_time) transaction_logger.debug("[TXN START] {%s}", name) try: - return func(LoggingTransaction(txn, name), *args, **kwargs) + return func( + LoggingTransaction(txn, name, self.database_engine), + *args, **kwargs + ) except: logger.exception("[TXN FAIL] {%s}", name) raise @@ -588,7 +594,7 @@ class SQLBaseStore(object): select_sql = "SELECT %s FROM %s WHERE %s" % ( ", ".join(retcols), table, - " AND ".join("%s = ?" % (k) for k in keyvalues) + " AND ".join("%s = ?" % (k,) for k in keyvalues) ) txn.execute(select_sql, keyvalues.values()) @@ -836,6 +842,12 @@ class SQLBaseStore(object): result = txn.fetchone() return result[0] if result else None + def get_next_stream_id(self): + with self._next_stream_id_lock: + i = self._next_stream_id + self._next_stream_id += 1 + return i + class _RollbackButIsFineException(Exception): """ This exception is used to rollback a transaction without implying diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py new file mode 100644 index 000000000..709b6f88a --- /dev/null +++ b/synapse/storage/engines/__init__.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .maria import MariaEngine +from .sqlite3 import Sqlite3Engine + + +SUPPORTED_MODULE = { + "sqlite3": Sqlite3Engine, + "mysql.connector": MariaEngine, +} + + +def create_engine(name): + engine_class = SUPPORTED_MODULE.get(name, None) + + if engine_class: + module = __import__(name) + return engine_class(module) + + raise RuntimeError( + "Unsupported database engine '%s'" % (name,) + ) diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py new file mode 100644 index 000000000..df4776364 --- /dev/null +++ b/synapse/storage/engines/maria.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import types + + +class MariaEngine(object): + def __init__(self, database_module): + self.module = database_module + + def convert_param_style(self, sql): + return sql.replace("?", "%s") + + def encode_parameter(self, param): + if isinstance(param, types.BufferType): + return str(param) + return param diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py new file mode 100644 index 000000000..639cdea41 --- /dev/null +++ b/synapse/storage/engines/sqlite3.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class Sqlite3Engine(object): + def __init__(self, database_module): + self.module = database_module + + def convert_param_style(self, sql): + return sql + + def encode_parameter(self, param): + return param diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index e6975a945..25fef7943 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -64,7 +64,7 @@ class KeyStore(SQLBaseStore): "fingerprint": fingerprint, "from_server": from_server, "ts_added_ms": time_now_ms, - "tls_certificate": tls_certificate_bytes, + "tls_certificate": buffer(tls_certificate_bytes), }, ) @@ -113,6 +113,6 @@ class KeyStore(SQLBaseStore): "key_id": "%s:%s" % (verify_key.alg, verify_key.version), "from_server": from_server, "ts_added_ms": time_now_ms, - "verify_key": verify_key.encode(), + "verify_key": buffer(verify_key.encode()), }, ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 7258f7b2a..0c785ec98 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -42,6 +42,7 @@ class RegistrationStore(SQLBaseStore): yield self._simple_insert( "access_tokens", { + "id": self.get_next_stream_id(), "user_id": user_id, "token": token }, @@ -78,8 +79,11 @@ class RegistrationStore(SQLBaseStore): # it's possible for this to get a conflict, but only for a single user # since tokens are namespaced based on their user ID - txn.execute("INSERT INTO access_tokens(user_id, token) " + - "VALUES (?,?)", [user_id, token]) + txn.execute( + "INSERT INTO access_tokens(id, user_id, token)" + " VALUES (?,?,?)", + (self.get_next_stream_id(), user_id, token,) + ) @defer.inlineCallbacks def get_user_by_id(self, user_id): diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index b526109e6..90ac47485 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS rejections( reason VARCHAR(255) NOT NULL, last_check VARCHAR(255) NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( @@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS pushers ( last_success BIGINT, failing_since BIGINT, UNIQUE (app_id, pushkey) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS push_rules ( id BIGINT PRIMARY KEY, @@ -48,7 +48,7 @@ CREATE TABLE IF NOT EXISTS push_rules ( conditions VARCHAR(255) NOT NULL, actions VARCHAR(255) NOT NULL, UNIQUE(user_name, rule_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); @@ -56,7 +56,7 @@ CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(255), filter_id BIGINT, filter_json BLOB -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( user_id, filter_id diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index f0a5daf44..4953b6323 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS application_services( hs_token VARCHAR(255), sender VARCHAR(255), UNIQUE(token) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS application_services_regex( id BIGINT PRIMARY KEY, @@ -28,4 +28,4 @@ CREATE TABLE IF NOT EXISTS application_services_regex( namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(255), FOREIGN KEY(as_id) REFERENCES application_services(id) -) ENGINE = INNODB; +) ; diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index a1260c5c1..3bda073c9 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -4,6 +4,6 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( rule_id VARCHAR(255) NOT NULL, enabled TINYINT, UNIQUE(user_name, rule_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 0f53488e9..336cd563d 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (event_id, room_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); @@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (event_id, room_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_edges( room_id VARCHAR(255) NOT NULL, is_state BOOL NOT NULL, UNIQUE (event_id, prev_event_id, room_id, is_state) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); @@ -49,7 +49,7 @@ CREATE TABLE IF NOT EXISTS room_depth( room_id VARCHAR(255) NOT NULL, min_depth INTEGER NOT NULL, UNIQUE (room_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); @@ -59,7 +59,7 @@ create TABLE IF NOT EXISTS event_destinations( destination VARCHAR(255) NOT NULL, delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); @@ -70,7 +70,7 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities( type VARCHAR(255) NOT NULL, state_key VARCHAR(255) NOT NULL, UNIQUE (event_id, room_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( room_id, type, state_key @@ -83,7 +83,7 @@ CREATE TABLE IF NOT EXISTS event_auth( auth_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (event_id, auth_id, room_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index 334d7c868..11e611598 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( algorithm VARCHAR(255), hash BLOB, UNIQUE (event_id, algorithm) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); @@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS event_reference_hashes ( algorithm VARCHAR(255), hash BLOB, UNIQUE (event_id, algorithm) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_signatures ( key_id VARCHAR(255), signature BLOB, UNIQUE (event_id, signature_name, key_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); @@ -50,6 +50,6 @@ CREATE TABLE IF NOT EXISTS event_edge_hashes( algorithm VARCHAR(255), hash BLOB, UNIQUE (event_id, prev_event_id, algorithm) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index 9849e969b..a0fb33762 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS events( outlier BOOL NOT NULL, depth BIGINT DEFAULT 0 NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); @@ -38,7 +38,7 @@ CREATE TABLE IF NOT EXISTS event_json( internal_metadata BLOB NOT NULL, json BLOB NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); @@ -50,7 +50,7 @@ CREATE TABLE IF NOT EXISTS state_events( state_key VARCHAR(255) NOT NULL, prev_state VARCHAR(255), UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); @@ -64,7 +64,7 @@ CREATE TABLE IF NOT EXISTS current_state_events( state_key VARCHAR(255) NOT NULL, UNIQUE (event_id), UNIQUE (room_id, type, state_key) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); @@ -77,7 +77,7 @@ CREATE TABLE IF NOT EXISTS room_memberships( room_id VARCHAR(255) NOT NULL, membership VARCHAR(255) NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); @@ -89,14 +89,14 @@ CREATE TABLE IF NOT EXISTS feedback( sender VARCHAR(255), room_id VARCHAR(255), UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS topics( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, topic VARCHAR(255) NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); @@ -113,12 +113,12 @@ CREATE TABLE IF NOT EXISTS rooms( room_id VARCHAR(255) PRIMARY KEY NOT NULL, is_public BOOL, creator VARCHAR(255) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS room_hosts( room_id VARCHAR(255) NOT NULL, host VARCHAR(255) NOT NULL, UNIQUE (room_id, host) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index c0f2ec29b..a785cdb4c 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -19,7 +19,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( ts_added_ms BIGINT, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(255), -- Server name. @@ -28,4 +28,4 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( ts_added_ms BIGINT, -- When the key was added. verify_key BLOB, -- NACL verification key. UNIQUE (server_name, key_id) -) ENGINE = INNODB; +) ; diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index d9559f590..27fe297af 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( upload_name VARCHAR(255), -- The name the media was uploaded with. user_id VARCHAR(255), -- The user who uploaded the file. UNIQUE (media_id) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( media_id VARCHAR(255), -- The id used to refer to the media. @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type ) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache ( media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(255), -- The name used to store the media on disk. UNIQUE (media_origin, media_id) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( media_origin VARCHAR(255), -- The remote HS the media came from. @@ -62,7 +62,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type ) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id ON remote_media_cache_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index 803132108..b48b110ae 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS presence( status_msg VARCHAR(255), mtime BIGINT, -- miliseconds since last state change UNIQUE(user_id) -) ENGINE = INNODB; +) ; -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS presence_allow_inbound( observed_user_id VARCHAR(255) NOT NULL, observer_user_id VARCHAR(255), -- a UserID, UNIQUE(observed_user_id) -) ENGINE = INNODB; +) ; -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? @@ -35,4 +35,4 @@ CREATE TABLE IF NOT EXISTS presence_list( observed_user_id VARCHAR(255), -- a UserID, accepted BOOLEAN, UNIQUE(user_id) -) ENGINE = INNODB; +) ; diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index 552645c56..92da48f97 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS profiles( user_id VARCHAR(255) NOT NULL, - displayname VARBINARY(255), + displayname VARCHAR(255), avatar_url VARCHAR(255), UNIQUE(user_id) -) ENGINE = INNODB; +) ; diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index ba93e860f..9b52a2012 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS redactions ( event_id VARCHAR(255) NOT NULL, redacts VARCHAR(255) NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index 1e706aac2..220df8757 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -14,12 +14,12 @@ */ CREATE TABLE IF NOT EXISTS room_aliases( - room_alias VARCHAR(255) NOT NULL, + room_alias VARBINARY(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (room_alias) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS room_alias_servers( - room_alias VARCHAR(255) NOT NULL, + room_alias VARBINARY(255) NOT NULL, server VARCHAR(255) NOT NULL -) ENGINE = INNODB; +) ; diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index be9dc2920..40584a325 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS state_groups( id VARCHAR(20) PRIMARY KEY, room_id VARCHAR(255) NOT NULL, event_id VARCHAR(255) NOT NULL -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS state_groups_state( state_group VARCHAR(20) NOT NULL, @@ -25,13 +25,13 @@ CREATE TABLE IF NOT EXISTS state_groups_state( type VARCHAR(255) NOT NULL, state_key VARCHAR(255) NOT NULL, event_id VARCHAR(255) NOT NULL -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS event_to_state_groups( event_id VARCHAR(255) NOT NULL, state_group VARCHAR(255) NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index bd13bba8c..d33bdfb30 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( response_json BLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( response_code INTEGER DEFAULT 0, response_json BLOB, ts BIGINT -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); @@ -51,7 +51,7 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( pdu_id VARCHAR(255), pdu_origin VARCHAR(255), UNIQUE (transaction_id, destination) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); @@ -60,4 +60,4 @@ CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(255) PRIMARY KEY, retry_last_ts BIGINT, retry_interval INTEGER -) ENGINE = INNODB; +) ; diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 55bffb22f..28909f580 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -14,20 +14,20 @@ */ CREATE TABLE IF NOT EXISTS users( name VARCHAR(255), - password_hash VARBINARY(255), + password_hash VARCHAR(255), creation_ts BIGINT, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS access_tokens( - id INTEGER PRIMARY KEY AUTO_INCREMENT, + id BIGINT PRIMARY KEY, user_id VARCHAR(255) NOT NULL, device_id VARCHAR(255), token VARCHAR(255) NOT NULL, last_used BIGINT, UNIQUE(token) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS user_ips ( user VARCHAR(255) NOT NULL, @@ -37,6 +37,6 @@ CREATE TABLE IF NOT EXISTS user_ips ( user_agent VARCHAR(255) NOT NULL, last_seen BIGINT NOT NULL, UNIQUE (user, access_token, ip, user_agent) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 35bba854f..f05182863 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -54,7 +54,7 @@ class SignatureStore(SQLBaseStore): { "event_id": event_id, "algorithm": algorithm, - "hash": hash_bytes, + "hash": buffer(hash_bytes), }, ) @@ -116,7 +116,7 @@ class SignatureStore(SQLBaseStore): { "event_id": event_id, "algorithm": algorithm, - "hash": hash_bytes, + "hash": buffer(hash_bytes), }, ) @@ -160,7 +160,7 @@ class SignatureStore(SQLBaseStore): "event_id": event_id, "signature_name": signature_name, "key_id": key_id, - "signature": signature_bytes, + "signature": buffer(signature_bytes), }, ) @@ -193,6 +193,6 @@ class SignatureStore(SQLBaseStore): "event_id": event_id, "prev_event_id": prev_event_id, "algorithm": algorithm, - "hash": hash_bytes, + "hash": buffer(hash_bytes), }, ) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 3a310cd00..e6bb5a807 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -433,12 +433,6 @@ class StreamStore(SQLBaseStore): defer.returnValue(self.min_token) - def get_next_stream_id(self): - with self._next_stream_id_lock: - i = self._next_stream_id - self._next_stream_id += 1 - return i - def _get_room_events_max_id_txn(self, txn): txn.execute( "SELECT MAX(stream_ordering) as m FROM events" diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index 4e8223279..a42138f55 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -60,7 +60,7 @@ def get_retry_limiter(destination, clock, store, **kwargs): if retry_timings: retry_last_ts, retry_interval = ( - retry_timings.retry_last_ts, retry_timings.retry_interval + retry_timings["retry_last_ts"], retry_timings["retry_interval"] ) now = int(clock.time_msec()) diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 2ecd00d2a..a4ef60b91 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -24,8 +24,6 @@ from ..utils import MockHttpResource, MockClock, setup_test_homeserver from synapse.federation import initialize_http_replication from synapse.events import FrozenEvent -from synapse.storage.transactions import DestinationsTable - def make_pdu(prev_pdus=[], **kwargs): """Provide some default fields for making a PduTuple.""" @@ -57,8 +55,14 @@ class FederationTestCase(unittest.TestCase): self.mock_persistence.get_received_txn_response.return_value = ( defer.succeed(None) ) + + retry_timings_res = { + "destination": "", + "retry_last_ts": 0, + "retry_interval": 0, + } self.mock_persistence.get_destination_retry_timings.return_value = ( - defer.succeed(DestinationsTable.EntryType("", 0, 0)) + defer.succeed(retry_timings_res) ) self.mock_persistence.get_auth_chain.return_value = [] self.clock = MockClock() diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index c13ade328..08d2404b6 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -87,6 +87,15 @@ class FederationTestCase(unittest.TestCase): self.datastore.get_room.return_value = defer.succeed(True) self.auth.check_host_in_room.return_value = defer.succeed(True) + retry_timings_res = { + "destination": "", + "retry_last_ts": 0, + "retry_interval": 0, + } + self.datastore.get_destination_retry_timings.return_value = ( + defer.succeed(retry_timings_res) + ) + def have_events(event_ids): return defer.succeed({}) self.datastore.have_events.side_effect = have_events diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 04eba4289..9b0e60691 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -194,8 +194,13 @@ class MockedDatastorePresenceTestCase(PresenceTestCase): return datastore def setUp_datastore_federation_mocks(self, datastore): + retry_timings_res = { + "destination": "", + "retry_last_ts": 0, + "retry_interval": 0, + } datastore.get_destination_retry_timings.return_value = ( - defer.succeed(DestinationsTable.EntryType("", 0, 0)) + defer.succeed(retry_timings_res) ) def get_received_txn_response(*args): diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index bf34b7ccb..2d76b2356 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -96,8 +96,13 @@ class TypingNotificationsTestCase(unittest.TestCase): self.event_source = hs.get_event_sources().sources["typing"] self.datastore = hs.get_datastore() + retry_timings_res = { + "destination": "", + "retry_last_ts": 0, + "retry_interval": 0, + } self.datastore.get_destination_retry_timings.return_value = ( - defer.succeed(DestinationsTable.EntryType("", 0, 0)) + defer.succeed(retry_timings_res) ) def get_received_txn_response(*args): diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py index 36b0f2ff6..445272e32 100644 --- a/tests/rest/client/v1/test_events.py +++ b/tests/rest/client/v1/test_events.py @@ -115,12 +115,6 @@ class EventStreamPermissionsTestCase(RestTestCase): hs = yield setup_test_homeserver( http_client=None, replication_layer=Mock(), - clock=Mock(spec=[ - "call_later", - "cancel_call_later", - "time_msec", - "time" - ]), ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), @@ -132,9 +126,6 @@ class EventStreamPermissionsTestCase(RestTestCase): hs.get_handlers().federation_handler = Mock() - hs.get_clock().time_msec.return_value = 1000000 - hs.get_clock().time.return_value = 1000 - synapse.rest.client.v1.register.register_servlets(hs, self.mock_resource) synapse.rest.client.v1.events.register_servlets(hs, self.mock_resource) synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index ca5b92ec8..2ad55c846 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -16,22 +16,18 @@ from tests import unittest from twisted.internet import defer from synapse.appservice import ApplicationService -from synapse.server import HomeServer from synapse.storage.appservice import ApplicationServiceStore -from mock import Mock -from tests.utils import SQLiteMemoryDbPool, MockClock +from tests.utils import setup_test_homeserver class ApplicationServiceStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def setUp(self): - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - hs = HomeServer( - "test", db_pool=db_pool, clock=MockClock(), config=Mock() - ) + hs = yield setup_test_homeserver() + db_pool = hs.get_db_pool() + self.as_token = "token1" db_pool.runQuery( "INSERT INTO application_services(token) VALUES(?)", diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 7f5845cf0..5c17d3014 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -24,6 +24,7 @@ from collections import OrderedDict from synapse.server import HomeServer from synapse.storage._base import SQLBaseStore +from synapse.storage.engines import create_engine class SQLBaseStoreTestCase(unittest.TestCase): @@ -40,7 +41,12 @@ class SQLBaseStoreTestCase(unittest.TestCase): config = Mock() config.event_cache_size = 1 - hs = HomeServer("test", db_pool=self.db_pool, config=config) + hs = HomeServer( + "test", + db_pool=self.db_pool, + config=config, + database_engine=create_engine("sqlite3"), + ) self.datastore = SQLBaseStore(hs) @@ -86,8 +92,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): self.assertEquals("Value", value) self.mock_txn.execute.assert_called_with( - "SELECT retcol FROM tablename WHERE keycol = ? " - "ORDER BY rowid asc", + "SELECT retcol FROM tablename WHERE keycol = ?", ["TheKey"] ) @@ -104,8 +109,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): self.assertEquals({"colA": 1, "colB": 2, "colC": 3}, ret) self.mock_txn.execute.assert_called_with( - "SELECT colA, colB, colC FROM tablename WHERE keycol = ? " - "ORDER BY rowid asc", + "SELECT colA, colB, colC FROM tablename WHERE keycol = ?", ["TheKey"] ) @@ -139,8 +143,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): self.assertEquals([{"colA": 1}, {"colA": 2}, {"colA": 3}], ret) self.mock_txn.execute.assert_called_with( - "SELECT colA FROM tablename WHERE keycol = ? " - "ORDER BY rowid asc", + "SELECT colA FROM tablename WHERE keycol = ?", ["A set"] ) @@ -189,8 +192,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): self.assertEquals({"columname": "Old Value"}, ret) self.mock_txn.execute.assert_has_calls([ - call('SELECT columname FROM tablename WHERE keycol = ? ' - 'ORDER BY rowid asc', + call('SELECT columname FROM tablename WHERE keycol = ?', ['TheKey']), call("UPDATE tablename SET columname = ? WHERE keycol = ?", ["New Value", "TheKey"]) diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index e0b81f2b5..78f600420 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -42,28 +42,38 @@ class RegistrationStoreTestCase(unittest.TestCase): self.assertEquals( # TODO(paul): Surely this field should be 'user_id', not 'name' # Additionally surely it shouldn't come in a 1-element list - [{"name": self.user_id, "password_hash": self.pwhash}], + {"name": self.user_id, "password_hash": self.pwhash}, (yield self.store.get_user_by_id(self.user_id)) ) - self.assertEquals( - {"admin": 0, - "device_id": None, - "name": self.user_id, - "token_id": 1}, - (yield self.store.get_user_by_token(self.tokens[0])) + result = yield self.store.get_user_by_token(self.tokens[1]) + + self.assertDictContainsSubset( + { + "admin": 0, + "device_id": None, + "name": self.user_id, + }, + result ) + self.assertTrue("token_id" in result) + @defer.inlineCallbacks def test_add_tokens(self): yield self.store.register(self.user_id, self.tokens[0], self.pwhash) yield self.store.add_access_token_to_user(self.user_id, self.tokens[1]) - self.assertEquals( - {"admin": 0, - "device_id": None, - "name": self.user_id, - "token_id": 2}, - (yield self.store.get_user_by_token(self.tokens[1])) + result = yield self.store.get_user_by_token(self.tokens[1]) + + self.assertDictContainsSubset( + { + "admin": 0, + "device_id": None, + "name": self.user_id, + }, + result ) + self.assertTrue("token_id" in result) + diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index 811fea544..785953cc8 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -119,7 +119,7 @@ class RoomMemberStoreTestCase(unittest.TestCase): yield self.inject_room_member(self.room, self.u_alice, Membership.JOIN) self.assertEquals( - ["test"], + {"test"}, (yield self.store.get_joined_hosts_for_room(self.room.to_string())) ) @@ -127,7 +127,7 @@ class RoomMemberStoreTestCase(unittest.TestCase): yield self.inject_room_member(self.room, self.u_bob, Membership.JOIN) self.assertEquals( - ["test"], + {"test"}, (yield self.store.get_joined_hosts_for_room(self.room.to_string())) ) @@ -136,9 +136,9 @@ class RoomMemberStoreTestCase(unittest.TestCase): self.assertEquals( {"test", "elsewhere"}, - set((yield + (yield self.store.get_joined_hosts_for_room(self.room.to_string()) - )) + ) ) # Should still have both hosts @@ -146,15 +146,15 @@ class RoomMemberStoreTestCase(unittest.TestCase): self.assertEquals( {"test", "elsewhere"}, - set((yield + (yield self.store.get_joined_hosts_for_room(self.room.to_string()) - )) + ) ) # Should have only one host after other leaves yield self.inject_room_member(self.room, self.u_charlie, Membership.LEAVE) self.assertEquals( - ["test"], + {"test"}, (yield self.store.get_joined_hosts_for_room(self.room.to_string())) ) diff --git a/tests/utils.py b/tests/utils.py index 81e82a80d..cc038fecf 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -17,6 +17,7 @@ from synapse.http.server import HttpServer from synapse.api.errors import cs_error, CodeMessageException, StoreError from synapse.api.constants import EventTypes from synapse.storage import prepare_database +from synapse.storage.engines import create_engine from synapse.server import HomeServer from synapse.util.logcontext import LoggingContext @@ -44,18 +45,23 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): config.event_cache_size = 1 config.disable_registration = False + if "clock" not in kargs: + kargs["clock"] = MockClock() + if datastore is None: db_pool = SQLiteMemoryDbPool() yield db_pool.prepare() hs = HomeServer( name, db_pool=db_pool, config=config, version_string="Synapse/tests", + database_engine=create_engine("sqlite3"), **kargs ) else: hs = HomeServer( name, db_pool=None, datastore=datastore, config=config, version_string="Synapse/tests", + database_engine=create_engine("sqlite3"), **kargs ) @@ -227,7 +233,10 @@ class SQLiteMemoryDbPool(ConnectionPool, object): ) def prepare(self): - return self.runWithConnection(prepare_database) + engine = create_engine("sqlite3") + return self.runWithConnection( + lambda conn: prepare_database(conn, engine) + ) class MemoryDataStore(object): From e24c32e6f3c0d7c75529d05762645fe613085bec Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 1 Apr 2015 15:09:51 +0100 Subject: [PATCH 022/104] Fix SQLite support --- synapse/app/homeserver.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index b185b2f56..1ab6effd5 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -365,7 +365,9 @@ def setup(config_options): else: db_config = { "name": "sqlite3", - "database": config.database_path, + "args": { + "database": config.database_path, + }, } db_config = { @@ -381,10 +383,12 @@ def setup(config_options): "use_unicode": True, }) elif name == "sqlite3": + def open_fun(conn): + prepare_database(conn, database_engine) db_config.setdefault("args", {}).update({ "cp_min": 1, "cp_max": 1, - "cp_openfun": prepare_database, + "cp_openfun": open_fun, }) else: raise RuntimeError("Unsupported database type '%s'" % (name,)) @@ -413,7 +417,12 @@ def setup(config_options): logger.info("Preparing database: %s...", db_name) try: - db_conn = database_engine.module.connect(**db_config.get("args", {})) + db_conn = database_engine.module.connect( + **{ + k: v for k, v in db_config.get("args", {}).items() + if not k.startswith("cp_") + } + ) if name == "sqlite3": prepare_sqlite3_database(db_conn) From 779f7b0f443bfab351702161763b62b7bb6a27d8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 2 Apr 2015 10:06:22 +0100 Subject: [PATCH 023/104] Fix unicode support --- synapse/app/homeserver.py | 12 ++-- synapse/storage/_base.py | 22 +++--- synapse/storage/engines/__init__.py | 4 +- synapse/storage/engines/maria.py | 12 ++++ synapse/storage/engines/sqlite3.py | 9 +++ synapse/storage/keys.py | 14 ++-- synapse/storage/schema/delta/12/v12.sql | 18 ++--- synapse/storage/schema/delta/13/v13.sql | 10 +-- synapse/storage/schema/delta/14/v14.sql | 4 +- .../schema/full_schemas/11/event_edges.sql | 34 +++++----- .../full_schemas/11/event_signatures.sql | 20 +++--- synapse/storage/schema/full_schemas/11/im.sql | 68 +++++++++---------- .../storage/schema/full_schemas/11/keys.sql | 12 ++-- .../full_schemas/11/media_repository.sql | 34 +++++----- .../schema/full_schemas/11/presence.sql | 16 ++--- .../schema/full_schemas/11/profiles.sql | 6 +- .../schema/full_schemas/11/redactions.sql | 4 +- .../schema/full_schemas/11/room_aliases.sql | 8 +-- .../storage/schema/full_schemas/11/state.sql | 16 ++--- .../schema/full_schemas/11/transactions.sql | 16 ++--- .../storage/schema/full_schemas/11/users.sql | 20 +++--- synapse/storage/schema/schema_version.sql | 2 +- 22 files changed, 193 insertions(+), 168 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 1ab6effd5..9d7a58080 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -18,7 +18,7 @@ import sys sys.dont_write_bytecode = True from synapse.storage import ( - prepare_database, prepare_sqlite3_database, UpgradeDatabaseException, + prepare_database, UpgradeDatabaseException, ) from synapse.storage.engines import create_engine @@ -381,19 +381,18 @@ def setup(config_options): "sql_mode": "TRADITIONAL", "charset": "utf8mb4", "use_unicode": True, + "collation": "utf8mb4_general_ci", }) elif name == "sqlite3": - def open_fun(conn): - prepare_database(conn, database_engine) db_config.setdefault("args", {}).update({ "cp_min": 1, "cp_max": 1, - "cp_openfun": open_fun, }) else: raise RuntimeError("Unsupported database type '%s'" % (name,)) database_engine = create_engine(name) + db_config["args"]["cp_openfun"] = database_engine.on_new_connection hs = SynapseHomeServer( config.server_name, @@ -424,10 +423,7 @@ def setup(config_options): } ) - if name == "sqlite3": - prepare_sqlite3_database(db_conn) - - prepare_database(db_conn, database_engine) + database_engine.prepare_database(db_conn) db_conn.commit() except UpgradeDatabaseException: diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 047d100f4..de4f66197 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -142,19 +142,23 @@ class LoggingTransaction(object): sql = self.database_engine.convert_param_style(sql) - try: - if args and args[0]: - values = args[0] + if args and args[0]: + args = list(args) + args[0] = [ + self.database_engine.encode_parameter(a) for a in args[0] + ] + try: sql_logger.debug( - "[SQL values] {%s} " + ", ".join(("<%r>",) * len(values)), + "[SQL values] {%s} " + ", ".join(("<%r>",) * len(args[0])), self.name, - *values + *args[0] ) - except: - # Don't let logging failures stop SQL from working - pass + except: + # Don't let logging failures stop SQL from working + pass start = time.time() * 1000 + try: return self.txn.execute( sql, *args, **kwargs @@ -761,8 +765,6 @@ class SQLBaseStore(object): return None internal_metadata, js, redacted, rejected_reason = res - js = js.decode("utf8") - internal_metadata = internal_metadata.decode("utf8") start_time = update_counter("select_event", start_time) diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index 709b6f88a..29702be92 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -16,6 +16,8 @@ from .maria import MariaEngine from .sqlite3 import Sqlite3Engine +import importlib + SUPPORTED_MODULE = { "sqlite3": Sqlite3Engine, @@ -27,7 +29,7 @@ def create_engine(name): engine_class = SUPPORTED_MODULE.get(name, None) if engine_class: - module = __import__(name) + module = importlib.import_module(name) return engine_class(module) raise RuntimeError( diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py index df4776364..7fcb706a6 100644 --- a/synapse/storage/engines/maria.py +++ b/synapse/storage/engines/maria.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from synapse.storage import prepare_database import types @@ -28,3 +29,14 @@ class MariaEngine(object): if isinstance(param, types.BufferType): return str(param) return param + + def on_new_connection(self, db_conn): + pass + + def prepare_database(self, db_conn): + cur = db_conn.cursor() + cur.execute( + "ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci" + ) + db_conn.commit() + prepare_database(db_conn, self) diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 639cdea41..e802b5d5f 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from synapse.storage import prepare_database, prepare_sqlite3_database + class Sqlite3Engine(object): def __init__(self, database_module): @@ -23,3 +25,10 @@ class Sqlite3Engine(object): def encode_parameter(self, param): return param + + def on_new_connection(self, db_conn): + self.prepare_database(db_conn) + + def prepare_database(self, db_conn): + prepare_sqlite3_database(db_conn) + prepare_database(db_conn, self) diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 25fef7943..d3b9b3866 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -57,15 +57,18 @@ class KeyStore(SQLBaseStore): OpenSSL.crypto.FILETYPE_ASN1, tls_certificate ) fingerprint = hashlib.sha256(tls_certificate_bytes).hexdigest() - return self._simple_insert( + return self._simple_upsert( table="server_tls_certificates", - values={ + keyvalues={ "server_name": server_name, "fingerprint": fingerprint, + }, + values={ "from_server": from_server, "ts_added_ms": time_now_ms, "tls_certificate": buffer(tls_certificate_bytes), }, + desc="store_server_certificate", ) @defer.inlineCallbacks @@ -106,13 +109,16 @@ class KeyStore(SQLBaseStore): ts_now_ms (int): The time now in milliseconds verification_key (VerifyKey): The NACL verify key. """ - return self._simple_insert( + return self._simple_upsert( table="server_signature_keys", - values={ + keyvalues={ "server_name": server_name, "key_id": "%s:%s" % (verify_key.alg, verify_key.version), + }, + values={ "from_server": from_server, "ts_added_ms": time_now_ms, "verify_key": buffer(verify_key.encode()), }, + desc="store_server_verify_key", ) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index 90ac47485..717d289f7 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -14,16 +14,16 @@ */ CREATE TABLE IF NOT EXISTS rejections( - event_id VARCHAR(255) NOT NULL, - reason VARCHAR(255) NOT NULL, - last_check VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + reason VARCHAR(150) NOT NULL, + last_check VARCHAR(150) NOT NULL, UNIQUE (event_id) ) ; -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( id BIGINT PRIMARY KEY, - user_name VARCHAR(255) NOT NULL, + user_name VARCHAR(150) NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, app_id VARCHAR(64) NOT NULL, @@ -41,19 +41,19 @@ CREATE TABLE IF NOT EXISTS pushers ( CREATE TABLE IF NOT EXISTS push_rules ( id BIGINT PRIMARY KEY, - user_name VARCHAR(255) NOT NULL, - rule_id VARCHAR(255) NOT NULL, + user_name VARCHAR(150) NOT NULL, + rule_id VARCHAR(150) NOT NULL, priority_class TINYINT NOT NULL, priority INTEGER NOT NULL DEFAULT 0, - conditions VARCHAR(255) NOT NULL, - actions VARCHAR(255) NOT NULL, + conditions VARCHAR(150) NOT NULL, + actions VARCHAR(150) NOT NULL, UNIQUE(user_name, rule_id) ) ; CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( - user_id VARCHAR(255), + user_id VARCHAR(150), filter_id BIGINT, filter_json BLOB ) ; diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index 4953b6323..f5275a59b 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -15,10 +15,10 @@ CREATE TABLE IF NOT EXISTS application_services( id BIGINT PRIMARY KEY, - url VARCHAR(255), - token VARCHAR(255), - hs_token VARCHAR(255), - sender VARCHAR(255), + url VARCHAR(150), + token VARCHAR(150), + hs_token VARCHAR(150), + sender VARCHAR(150), UNIQUE(token) ) ; @@ -26,6 +26,6 @@ CREATE TABLE IF NOT EXISTS application_services_regex( id BIGINT PRIMARY KEY, as_id BIGINT NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ - regex VARCHAR(255), + regex VARCHAR(150), FOREIGN KEY(as_id) REFERENCES application_services(id) ) ; diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index 3bda073c9..1d582cc62 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -1,7 +1,7 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( id BIGINT PRIMARY KEY, - user_name VARCHAR(255) NOT NULL, - rule_id VARCHAR(255) NOT NULL, + user_name VARCHAR(150) NOT NULL, + rule_id VARCHAR(150) NOT NULL, enabled TINYINT, UNIQUE(user_name, rule_id) ) ; diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 336cd563d..124c9a9bd 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS event_forward_extremities( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, UNIQUE (event_id, room_id) ) ; @@ -24,8 +24,8 @@ CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_backward_extremities( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, UNIQUE (event_id, room_id) ) ; @@ -34,9 +34,9 @@ CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id CREATE TABLE IF NOT EXISTS event_edges( - event_id VARCHAR(255) NOT NULL, - prev_event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + prev_event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, is_state BOOL NOT NULL, UNIQUE (event_id, prev_event_id, room_id, is_state) ) ; @@ -46,7 +46,7 @@ CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); CREATE TABLE IF NOT EXISTS room_depth( - room_id VARCHAR(255) NOT NULL, + room_id VARCHAR(150) NOT NULL, min_depth INTEGER NOT NULL, UNIQUE (room_id) ) ; @@ -55,8 +55,8 @@ CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( - event_id VARCHAR(255) NOT NULL, - destination VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + destination VARCHAR(150) NOT NULL, delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ) ; @@ -65,10 +65,10 @@ CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id) CREATE TABLE IF NOT EXISTS state_forward_extremities( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, - type VARCHAR(255) NOT NULL, - state_key VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, UNIQUE (event_id, room_id) ) ; @@ -79,9 +79,9 @@ CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_auth( - event_id VARCHAR(255) NOT NULL, - auth_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + auth_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, UNIQUE (event_id, auth_id, room_id) ) ; diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index 11e611598..30e3f71c5 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS event_content_hashes ( - event_id VARCHAR(255), - algorithm VARCHAR(255), + event_id VARCHAR(150), + algorithm VARCHAR(150), hash BLOB, UNIQUE (event_id, algorithm) ) ; @@ -24,8 +24,8 @@ CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event CREATE TABLE IF NOT EXISTS event_reference_hashes ( - event_id VARCHAR(255), - algorithm VARCHAR(255), + event_id VARCHAR(150), + algorithm VARCHAR(150), hash BLOB, UNIQUE (event_id, algorithm) ) ; @@ -34,9 +34,9 @@ CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(e CREATE TABLE IF NOT EXISTS event_signatures ( - event_id VARCHAR(255), - signature_name VARCHAR(255), - key_id VARCHAR(255), + event_id VARCHAR(150), + signature_name VARCHAR(150), + key_id VARCHAR(150), signature BLOB, UNIQUE (event_id, signature_name, key_id) ) ; @@ -45,9 +45,9 @@ CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); CREATE TABLE IF NOT EXISTS event_edge_hashes( - event_id VARCHAR(255), - prev_event_id VARCHAR(255), - algorithm VARCHAR(255), + event_id VARCHAR(150), + prev_event_id VARCHAR(150), + algorithm VARCHAR(150), hash BLOB, UNIQUE (event_id, prev_event_id, algorithm) ) ; diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index a0fb33762..7cb8f802e 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -16,9 +16,9 @@ CREATE TABLE IF NOT EXISTS events( stream_ordering BIGINT PRIMARY KEY, topological_ordering BIGINT NOT NULL, - event_id VARCHAR(255) NOT NULL, - type VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, content BLOB NOT NULL, unrecognized_keys BLOB, processed BOOL NOT NULL, @@ -33,8 +33,8 @@ CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, internal_metadata BLOB NOT NULL, json BLOB NOT NULL, UNIQUE (event_id) @@ -44,11 +44,11 @@ CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); CREATE TABLE IF NOT EXISTS state_events( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, - type VARCHAR(255) NOT NULL, - state_key VARCHAR(255) NOT NULL, - prev_state VARCHAR(255), + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, + prev_state VARCHAR(150), UNIQUE (event_id) ) ; @@ -58,10 +58,10 @@ CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); CREATE TABLE IF NOT EXISTS current_state_events( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, - type VARCHAR(255) NOT NULL, - state_key VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, UNIQUE (event_id), UNIQUE (room_id, type, state_key) ) ; @@ -71,11 +71,11 @@ CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (ty CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); CREATE TABLE IF NOT EXISTS room_memberships( - event_id VARCHAR(255) NOT NULL, - user_id VARCHAR(255) NOT NULL, - sender VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, - membership VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + user_id VARCHAR(150) NOT NULL, + sender VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + membership VARCHAR(150) NOT NULL, UNIQUE (event_id) ) ; @@ -83,41 +83,41 @@ CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); CREATE TABLE IF NOT EXISTS feedback( - event_id VARCHAR(255) NOT NULL, - feedback_type VARCHAR(255), - target_event_id VARCHAR(255), - sender VARCHAR(255), - room_id VARCHAR(255), + event_id VARCHAR(150) NOT NULL, + feedback_type VARCHAR(150), + target_event_id VARCHAR(150), + sender VARCHAR(150), + room_id VARCHAR(150), UNIQUE (event_id) ) ; CREATE TABLE IF NOT EXISTS topics( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, - topic VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + topic VARCHAR(150) NOT NULL, UNIQUE (event_id) ) ; CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, - name VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + name VARCHAR(150) NOT NULL, UNIQUE (event_id) ); CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( - room_id VARCHAR(255) PRIMARY KEY NOT NULL, + room_id VARCHAR(150) PRIMARY KEY NOT NULL, is_public BOOL, - creator VARCHAR(255) + creator VARCHAR(150) ) ; CREATE TABLE IF NOT EXISTS room_hosts( - room_id VARCHAR(255) NOT NULL, - host VARCHAR(255) NOT NULL, + room_id VARCHAR(150) NOT NULL, + host VARCHAR(150) NOT NULL, UNIQUE (room_id, host) ) ; diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index a785cdb4c..062ca53fe 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -13,18 +13,18 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS server_tls_certificates( - server_name VARCHAR(255), -- Server name. - fingerprint VARCHAR(255), -- Certificate fingerprint. - from_server VARCHAR(255), -- Which key server the certificate was fetched from. + server_name VARCHAR(150), -- Server name. + fingerprint VARCHAR(150), -- Certificate fingerprint. + from_server VARCHAR(150), -- Which key server the certificate was fetched from. ts_added_ms BIGINT, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ) ; CREATE TABLE IF NOT EXISTS server_signature_keys( - server_name VARCHAR(255), -- Server name. - key_id VARCHAR(255), -- Key version. - from_server VARCHAR(255), -- Which key server the key was fetched form. + server_name VARCHAR(150), -- Server name. + key_id VARCHAR(150), -- Key version. + from_server VARCHAR(150), -- Which key server the key was fetched form. ts_added_ms BIGINT, -- When the key was added. verify_key BLOB, -- NACL verification key. UNIQUE (server_name, key_id) diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index 27fe297af..c8c5f1d2f 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -14,21 +14,21 @@ */ CREATE TABLE IF NOT EXISTS local_media_repository ( - media_id VARCHAR(255), -- The id used to refer to the media. - media_type VARCHAR(255), -- The MIME-type of the media. + media_id VARCHAR(150), -- The id used to refer to the media. + media_type VARCHAR(150), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. created_ts BIGINT, -- When the content was uploaded in ms. - upload_name VARCHAR(255), -- The name the media was uploaded with. - user_id VARCHAR(255), -- The user who uploaded the file. + upload_name VARCHAR(150), -- The name the media was uploaded with. + user_id VARCHAR(150), -- The user who uploaded the file. UNIQUE (media_id) ) ; CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( - media_id VARCHAR(255), -- The id used to refer to the media. + media_id VARCHAR(150), -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_type VARCHAR(255), -- The MIME-type of the thumbnail. - thumbnail_method VARCHAR(255), -- The method used to make the thumbnail. + thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. + thumbnail_method VARCHAR(150), -- The method used to make the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type @@ -39,25 +39,25 @@ CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); CREATE TABLE IF NOT EXISTS remote_media_cache ( - media_origin VARCHAR(255), -- The remote HS the media came from. - media_id VARCHAR(255), -- The id used to refer to the media on that server. - media_type VARCHAR(255), -- The MIME-type of the media. + media_origin VARCHAR(150), -- The remote HS the media came from. + media_id VARCHAR(150), -- The id used to refer to the media on that server. + media_type VARCHAR(150), -- The MIME-type of the media. created_ts BIGINT, -- When the content was uploaded in ms. - upload_name VARCHAR(255), -- The name the media was uploaded with. + upload_name VARCHAR(150), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. - filesystem_id VARCHAR(255), -- The name used to store the media on disk. + filesystem_id VARCHAR(150), -- The name used to store the media on disk. UNIQUE (media_origin, media_id) ) ; CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( - media_origin VARCHAR(255), -- The remote HS the media came from. - media_id VARCHAR(255), -- The id used to refer to the media. + media_origin VARCHAR(150), -- The remote HS the media came from. + media_id VARCHAR(150), -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_method VARCHAR(255), -- The method used to make the thumbnail - thumbnail_type VARCHAR(255), -- The MIME-type of the thumbnail. + thumbnail_method VARCHAR(150), -- The method used to make the thumbnail + thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - filesystem_id VARCHAR(255), -- The name used to store the media on disk. + filesystem_id VARCHAR(150), -- The name used to store the media on disk. UNIQUE ( media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index b48b110ae..273e61281 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -13,9 +13,9 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS presence( - user_id VARCHAR(255) NOT NULL, + user_id VARCHAR(150) NOT NULL, state VARCHAR(20), - status_msg VARCHAR(255), + status_msg VARCHAR(150), mtime BIGINT, -- miliseconds since last state change UNIQUE(user_id) ) ; @@ -23,16 +23,14 @@ CREATE TABLE IF NOT EXISTS presence( -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state CREATE TABLE IF NOT EXISTS presence_allow_inbound( - observed_user_id VARCHAR(255) NOT NULL, - observer_user_id VARCHAR(255), -- a UserID, - UNIQUE(observed_user_id) + observed_user_id VARCHAR(150) NOT NULL, + observer_user_id VARCHAR(150) -- a UserID, ) ; -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? CREATE TABLE IF NOT EXISTS presence_list( - user_id VARCHAR(255) NOT NULL, - observed_user_id VARCHAR(255), -- a UserID, - accepted BOOLEAN, - UNIQUE(user_id) + user_id VARCHAR(150) NOT NULL, + observed_user_id VARCHAR(150), -- a UserID, + accepted BOOLEAN ) ; diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index 92da48f97..023060a54 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS profiles( - user_id VARCHAR(255) NOT NULL, - displayname VARCHAR(255), - avatar_url VARCHAR(255), + user_id VARCHAR(150) NOT NULL, + displayname VARCHAR(150), + avatar_url VARCHAR(150), UNIQUE(user_id) ) ; diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index 9b52a2012..5c23188d6 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS redactions ( - event_id VARCHAR(255) NOT NULL, - redacts VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + redacts VARCHAR(150) NOT NULL, UNIQUE (event_id) ) ; diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index 220df8757..63fe0f5c6 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -14,12 +14,12 @@ */ CREATE TABLE IF NOT EXISTS room_aliases( - room_alias VARBINARY(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + room_alias VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, UNIQUE (room_alias) ) ; CREATE TABLE IF NOT EXISTS room_alias_servers( - room_alias VARBINARY(255) NOT NULL, - server VARCHAR(255) NOT NULL + room_alias VARCHAR(150) NOT NULL, + server VARCHAR(150) NOT NULL ) ; diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index 40584a325..acfb76439 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -15,21 +15,21 @@ CREATE TABLE IF NOT EXISTS state_groups( id VARCHAR(20) PRIMARY KEY, - room_id VARCHAR(255) NOT NULL, - event_id VARCHAR(255) NOT NULL + room_id VARCHAR(150) NOT NULL, + event_id VARCHAR(150) NOT NULL ) ; CREATE TABLE IF NOT EXISTS state_groups_state( state_group VARCHAR(20) NOT NULL, - room_id VARCHAR(255) NOT NULL, - type VARCHAR(255) NOT NULL, - state_key VARCHAR(255) NOT NULL, - event_id VARCHAR(255) NOT NULL + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, + event_id VARCHAR(150) NOT NULL ) ; CREATE TABLE IF NOT EXISTS event_to_state_groups( - event_id VARCHAR(255) NOT NULL, - state_group VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + state_group VARCHAR(150) NOT NULL, UNIQUE (event_id) ) ; diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index d33bdfb30..43541661c 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -14,8 +14,8 @@ */ -- Stores what transaction ids we have received and what our response was CREATE TABLE IF NOT EXISTS received_transactions( - transaction_id VARCHAR(255), - origin VARCHAR(255), + transaction_id VARCHAR(150), + origin VARCHAR(150), ts BIGINT, response_code INTEGER, response_json BLOB, @@ -30,8 +30,8 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering - transaction_id VARCHAR(255), - destination VARCHAR(255), + transaction_id VARCHAR(150), + destination VARCHAR(150), response_code INTEGER DEFAULT 0, response_json BLOB, ts BIGINT @@ -47,9 +47,9 @@ CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_c -- For sent transactions only. CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( transaction_id INTEGER, - destination VARCHAR(255), - pdu_id VARCHAR(255), - pdu_origin VARCHAR(255), + destination VARCHAR(150), + pdu_id VARCHAR(150), + pdu_origin VARCHAR(150), UNIQUE (transaction_id, destination) ) ; @@ -57,7 +57,7 @@ CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(d -- To track destination health CREATE TABLE IF NOT EXISTS destinations( - destination VARCHAR(255) PRIMARY KEY, + destination VARCHAR(150) PRIMARY KEY, retry_last_ts BIGINT, retry_interval INTEGER ) ; diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 28909f580..0271de352 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS users( - name VARCHAR(255), - password_hash VARCHAR(255), + name VARCHAR(150), + password_hash VARCHAR(150), creation_ts BIGINT, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) @@ -22,19 +22,19 @@ CREATE TABLE IF NOT EXISTS users( CREATE TABLE IF NOT EXISTS access_tokens( id BIGINT PRIMARY KEY, - user_id VARCHAR(255) NOT NULL, - device_id VARCHAR(255), - token VARCHAR(255) NOT NULL, + user_id VARCHAR(150) NOT NULL, + device_id VARCHAR(150), + token VARCHAR(150) NOT NULL, last_used BIGINT, UNIQUE(token) ) ; CREATE TABLE IF NOT EXISTS user_ips ( - user VARCHAR(255) NOT NULL, - access_token VARCHAR(255) NOT NULL, - device_id VARCHAR(255), - ip VARCHAR(255) NOT NULL, - user_agent VARCHAR(255) NOT NULL, + user VARCHAR(150) NOT NULL, + access_token VARCHAR(150) NOT NULL, + device_id VARCHAR(150), + ip VARCHAR(150) NOT NULL, + user_agent VARCHAR(150) NOT NULL, last_seen BIGINT NOT NULL, UNIQUE (user, access_token, ip, user_agent) ) ; diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql index 28762861e..e7fa6fe56 100644 --- a/synapse/storage/schema/schema_version.sql +++ b/synapse/storage/schema/schema_version.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS schema_version( CREATE TABLE IF NOT EXISTS applied_schema_deltas( `version` INTEGER NOT NULL, - `file` VARCHAR(255) NOT NULL, + `file` VARCHAR(150) NOT NULL, UNIQUE(version, file) ); From d0e444a648ec0e4168673665ff37d09a1c2c2292 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:04:02 +0100 Subject: [PATCH 024/104] Explicitly name the __main__ module logger --- synapse/app/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 9d7a58080..95190c1d7 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -64,7 +64,7 @@ import sqlite3 import yaml -logger = logging.getLogger(__name__) +logger = logging.getLogger("synapse.app.homeserver") class SynapseHomeServer(HomeServer): From 304111afd0fdda6a0c58d81238a1bbfa7f318208 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:05:36 +0100 Subject: [PATCH 025/104] Don't use AUTOINCREMENT, use an in memory version --- synapse/storage/_base.py | 54 +++++++++++++++++++++++++++++---- synapse/storage/events.py | 13 ++++---- synapse/storage/registration.py | 10 ++++-- synapse/storage/state.py | 6 ++-- synapse/storage/transactions.py | 4 ++- 5 files changed, 68 insertions(+), 19 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index de4f66197..9f63f0708 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -163,8 +163,8 @@ class LoggingTransaction(object): return self.txn.execute( sql, *args, **kwargs ) - except: - logger.exception("[SQL FAIL] {%s}", self.name) + except Exception as e: + logger.debug("[SQL FAIL] {%s} %s", self.name, e) raise finally: msecs = (time.time() * 1000) - start @@ -209,6 +209,46 @@ class PerformanceCounters(object): return top_n_counters +class IdGenerator(object): + def __init__(self, table, column, store): + self.table = table + self.column = column + self.store = store + self._lock = threading.Lock() + self._next_id = None + + @defer.inlineCallbacks + def get_next(self): + with self._lock: + if not self._next_id: + res = yield self.store._execute_and_decode( + "IdGenerator_%s" % (self.table,), + "SELECT MAX(%s) as mx FROM %s" % (self.column, self.table,) + ) + + self._next_id = (res and res[0] and res[0]["mx"]) or 1 + + i = self._next_id + self._next_id += 1 + defer.returnValue(i) + + def get_next_txn(self, txn): + with self._lock: + if self._next_id: + i = self._next_id + self._next_id += 1 + return i + else: + txn.execute( + "SELECT MAX(%s) FROM %s" % (self.column, self.table,) + ) + + val, = txn.fetchone() + self._next_id = val or 2 + + return 1 + + class SQLBaseStore(object): _TXN_ID = 0 @@ -234,8 +274,10 @@ class SQLBaseStore(object): # Pretend the getEventCache is just another named cache caches_by_name["*getEvent*"] = self._get_event_cache - self._next_stream_id_lock = threading.Lock() - self._next_stream_id = int(hs.get_clock().time_msec()) * 1000 + self._stream_id_gen = IdGenerator("events", "stream_ordering", self) + self._transaction_id_gen = IdGenerator("sent_transactions", "id", self) + self._state_groups_id_gen = IdGenerator("state_groups", "id", self) + self._access_tokens_id_gen = IdGenerator("access_tokens", "id", self) def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -292,8 +334,8 @@ class SQLBaseStore(object): LoggingTransaction(txn, name, self.database_engine), *args, **kwargs ) - except: - logger.exception("[TXN FAIL] {%s}", name) + except Exception as e: + logger.debug("[TXN FAIL] {%s}", name, e) raise finally: end = time.time() * 1000 diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 69f598967..514feebcb 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -96,12 +96,16 @@ class EventsStore(SQLBaseStore): # Remove the any existing cache entries for the event_id self._get_event_cache.pop(event.event_id) + if stream_ordering is None: + stream_ordering = self._stream_id_gen.get_next_txn(txn) + # We purposefully do this first since if we include a `current_state` # key, we *want* to update the `current_state_events` table if current_state: - txn.execute( - "DELETE FROM current_state_events WHERE room_id = ?", - (event.room_id,) + self._simple_delete_txn( + txn, + table="current_state_events", + keyvalues={"room_id": event.room_id}, ) for s in current_state: @@ -240,9 +244,6 @@ class EventsStore(SQLBaseStore): "depth": event.depth, } - if stream_ordering is None: - stream_ordering = self.get_next_stream_id() - unrec = { k: v for k, v in event.get_dict().items() diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 0c785ec98..b62b4a341 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -39,10 +39,12 @@ class RegistrationStore(SQLBaseStore): Raises: StoreError if there was a problem adding this. """ - yield self._simple_insert( + next_id = yield self._access_tokens_id_gen.get_next() + + self._simple_insert( "access_tokens", { - "id": self.get_next_stream_id(), + "id": next_id, "user_id": user_id, "token": token }, @@ -68,6 +70,8 @@ class RegistrationStore(SQLBaseStore): def _register(self, txn, user_id, token, password_hash): now = int(self.clock.time()) + next_id = self._access_tokens_id_gen.get_next_txn(txn) + try: txn.execute("INSERT INTO users(name, password_hash, creation_ts) " "VALUES (?,?,?)", @@ -82,7 +86,7 @@ class RegistrationStore(SQLBaseStore): txn.execute( "INSERT INTO access_tokens(id, user_id, token)" " VALUES (?,?,?)", - (self.get_next_stream_id(), user_id, token,) + (next_id, user_id, token,) ) @defer.inlineCallbacks diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 65ea9c4d8..3e55cb81b 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -93,12 +93,12 @@ class StateStore(SQLBaseStore): state_group = context.state_group if not state_group: - group = _make_group_id(self._clock) - state_group = self._simple_insert_txn( + state_group = _make_group_id(self._clock) + self._simple_insert_txn( txn, table="state_groups", values={ - "id": group, + "id": state_group, "room_id": event.room_id, "event_id": event.event_id, }, diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index e3e484fb2..9594fe1f2 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -123,6 +123,8 @@ class TransactionStore(SQLBaseStore): def _prep_send_transaction(self, txn, transaction_id, destination, origin_server_ts): + next_id = self._transaction_id_gen.get_next_txn(txn) + # First we find out what the prev_txns should be. # Since we know that we are only sending one transaction at a time, # we can simply take the last one. @@ -143,7 +145,7 @@ class TransactionStore(SQLBaseStore): txn, table=SentTransactions.table_name, values={ - "id": self.get_next_stream_id(), + "id": next_id, "transaction_id": transaction_id, "destination": destination, "ts": origin_server_ts, From c8d3f6486da29b1e0e36b447948b8971bdf05fd3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:06:01 +0100 Subject: [PATCH 026/104] Implement or_ignore flag on inserts --- synapse/storage/_base.py | 13 +++++++++---- synapse/storage/presence.py | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 9f63f0708..a0c1718c2 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -393,7 +393,8 @@ class SQLBaseStore(object): # "Simple" SQL API methods that operate on a single table with no JOINs, # no complex WHERE clauses, just a dict of values for columns. - def _simple_insert(self, table, values, desc="_simple_insert"): + def _simple_insert(self, table, values, or_ignore=False, + desc="_simple_insert"): """Executes an INSERT query on the named table. Args: @@ -403,10 +404,11 @@ class SQLBaseStore(object): return self.runInteraction( desc, self._simple_insert_txn, table, values, + or_ignore=or_ignore ) @log_function - def _simple_insert_txn(self, txn, table, values): + def _simple_insert_txn(self, txn, table, values, or_ignore=False): sql = "INSERT INTO %s (%s) VALUES(%s)" % ( table, ", ".join(k for k in values), @@ -418,8 +420,11 @@ class SQLBaseStore(object): sql, values.values(), ) - txn.execute(sql, values.values()) - return txn.lastrowid + try: + txn.execute(sql, values.values()) + except self.database_engine.module.IntegrityError: + if not or_ignore: + raise def _simple_upsert(self, table, keyvalues, values, desc="_simple_upsert"): """ diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index 87fba5543..22ec94bc1 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -57,6 +57,7 @@ class PresenceStore(SQLBaseStore): values={"observed_user_id": observed_localpart, "observer_user_id": observer_userid}, desc="allow_presence_visible", + or_ignore=True, ) def disallow_presence_visible(self, observed_localpart, observer_userid): From 0af5f5efaf0b24187514cf78d7982ef9b85a208c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:08:35 +0100 Subject: [PATCH 027/104] Don't use multiple UNIQUE constraints; it will cause deadlocks --- synapse/storage/_base.py | 4 ++-- synapse/storage/events.py | 14 +++++++------- .../storage/schema/full_schemas/11/presence.sql | 10 ++++++---- synapse/storage/schema/full_schemas/11/users.sql | 4 ++-- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index a0c1718c2..4ac61be89 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -837,11 +837,11 @@ class SQLBaseStore(object): return curr_time logger.debug("Got js: %r", js) - d = json.loads(js) + d = json.loads(str(js).decode("utf8")) start_time = update_counter("decode_json", start_time) logger.debug("Got internal_metadata: %r", internal_metadata) - internal_metadata = json.loads(internal_metadata) + internal_metadata = json.loads(str(internal_metadata).decode("utf8")) start_time = update_counter("decode_internal", start_time) ev = FrozenEvent(d, internal_metadata_dict=internal_metadata) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 514feebcb..3b3416716 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -179,7 +179,7 @@ class EventsStore(SQLBaseStore): ) txn.execute( sql, - (metadata_json.decode("UTF-8"), event.event_id,) + (buffer(metadata_json), event.event_id,) ) sql = ( @@ -224,14 +224,14 @@ class EventsStore(SQLBaseStore): values={ "event_id": event.event_id, "room_id": event.room_id, - "internal_metadata": metadata_json.decode("UTF-8"), - "json": encode_canonical_json(event_dict).decode("UTF-8"), + "internal_metadata": buffer(metadata_json), + "json": buffer(encode_canonical_json(event_dict)), }, ) - content = encode_canonical_json( + content = buffer(encode_canonical_json( event.content - ).decode("UTF-8") + )) vals = { "topological_ordering": event.depth, @@ -256,9 +256,9 @@ class EventsStore(SQLBaseStore): ] } - vals["unrecognized_keys"] = encode_canonical_json( + vals["unrecognized_keys"] = buffer(encode_canonical_json( unrec - ).decode("UTF-8") + )) sql = ( "INSERT INTO events" diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index 273e61281..00d803a5c 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -17,20 +17,22 @@ CREATE TABLE IF NOT EXISTS presence( state VARCHAR(20), status_msg VARCHAR(150), mtime BIGINT, -- miliseconds since last state change - UNIQUE(user_id) + UNIQUE (user_id) ) ; -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state CREATE TABLE IF NOT EXISTS presence_allow_inbound( observed_user_id VARCHAR(150) NOT NULL, - observer_user_id VARCHAR(150) -- a UserID, + observer_user_id VARCHAR(150) NOT NULL, -- a UserID, + UNIQUE (observed_user_id, observer_user_id) ) ; -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? CREATE TABLE IF NOT EXISTS presence_list( user_id VARCHAR(150) NOT NULL, - observed_user_id VARCHAR(150), -- a UserID, - accepted BOOLEAN + observed_user_id VARCHAR(150) NOT NULL, -- a UserID, + accepted BOOLEAN NOT NULL, + UNIQUE (user_id, observed_user_id) ) ; diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 0271de352..ba0f42d45 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -35,8 +35,8 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT NOT NULL, - UNIQUE (user, access_token, ip, user_agent) + last_seen BIGINT NOT NULL ) ; CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); +CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); From 8574bf62dc27ec3c2059c57a32c955e85d7e51df Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:09:36 +0100 Subject: [PATCH 028/104] Add index to presence table --- synapse/storage/schema/delta/15/presence_indices.sql | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 synapse/storage/schema/delta/15/presence_indices.sql diff --git a/synapse/storage/schema/delta/15/presence_indices.sql b/synapse/storage/schema/delta/15/presence_indices.sql new file mode 100644 index 000000000..6b8d0f1ca --- /dev/null +++ b/synapse/storage/schema/delta/15/presence_indices.sql @@ -0,0 +1,2 @@ + +CREATE INDEX IF NOT EXISTS presence_list_user_id ON presence_list (user_id); From 05a35d62b6c57d144cb0ca6b3c1071380d69b40c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:10:15 +0100 Subject: [PATCH 029/104] Bump database version --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index f8053484c..9b30cd6ad 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -51,7 +51,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 14 +SCHEMA_VERSION = 15 dir_path = os.path.abspath(os.path.dirname(__file__)) From 6bc9edd8b22309796470950a855fc43b88171bea Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:13:58 +0100 Subject: [PATCH 030/104] Fix prepare_sqlite3_database's convert_param_style --- synapse/storage/__init__.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 9b30cd6ad..e16414d73 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -442,8 +442,6 @@ def prepare_sqlite3_database(db_conn): new. This only affects sqlite databases since they were the only ones supported at the time. """ - import sqlite3 - with db_conn: schema_path = os.path.join( dir_path, "schema", "schema_version.sql", @@ -462,10 +460,7 @@ def prepare_sqlite3_database(db_conn): if row and row[0]: db_conn.execute( - _convert_param_style( - "REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)", - sqlite3 - ), + "REPLACE INTO schema_version (version, upgraded)" + " VALUES (?,?)", (row[0], False) ) From 0bfa78b39bf95ee24e78166c9545f59b34fd1d81 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:16:05 +0100 Subject: [PATCH 031/104] PEP8 --- synapse/app/homeserver.py | 7 ++----- synapse/storage/__init__.py | 4 ---- synapse/storage/registration.py | 1 - synapse/storage/roommember.py | 1 - synapse/storage/state.py | 1 - synapse/storage/transactions.py | 1 - 6 files changed, 2 insertions(+), 13 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 95190c1d7..f7c724c4b 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -17,9 +17,7 @@ import sys sys.dont_write_bytecode = True -from synapse.storage import ( - prepare_database, UpgradeDatabaseException, -) +from synapse.storage import UpgradeDatabaseException from synapse.storage.engines import create_engine from synapse.server import HomeServer @@ -60,7 +58,6 @@ import os import re import resource import subprocess -import sqlite3 import yaml @@ -322,7 +319,7 @@ def change_resource_limit(soft_file_no): resource.setrlimit(resource.RLIMIT_NOFILE, (soft_file_no, hard)) logger.info("Set file limit to: %d", soft_file_no) - except ( ValueError, resource.error) as e: + except (ValueError, resource.error) as e: logger.warn("Failed to set file limit: %s", e) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index e16414d73..231ec8169 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer - from .appservice import ApplicationServiceStore from .directory import DirectoryStore from .events import EventsStore @@ -43,8 +41,6 @@ import logging import os import re -import threading - logger = logging.getLogger(__name__) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index b62b4a341..0f9d898e5 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -105,7 +105,6 @@ class RegistrationStore(SQLBaseStore): defer.returnValue(user_info) - @cached() # TODO(paul): Currently there's no code to invalidate this cache. That # means if/when we ever add internal ways to invalidate access tokens or diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index df707f812..8ea5756d6 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -40,7 +40,6 @@ class RoomMemberStore(SQLBaseStore): """ try: target_user_id = event.state_key - domain = UserID.from_string(target_user_id).domain except: logger.exception( "Failed to parse target_user_id=%s", target_user_id diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 3e55cb81b..4994bacd6 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -159,4 +159,3 @@ class StateStore(SQLBaseStore): def _make_group_id(clock): return str(int(clock.time_msec())) + random_string(5) - diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 9594fe1f2..4c3dc5866 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -375,4 +375,3 @@ class DestinationsTable(object): "retry_last_ts", "retry_interval", ] - From 49d6aa1394bba42d52cae4013f7103d682756b28 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 15:28:37 +0100 Subject: [PATCH 032/104] Retry on deadlock --- synapse/storage/_base.py | 16 +++++++++++++--- synapse/storage/engines/maria.py | 5 +++++ synapse/storage/engines/sqlite3.py | 3 +++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 4ac61be89..c15cec0c7 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -160,9 +160,19 @@ class LoggingTransaction(object): start = time.time() * 1000 try: - return self.txn.execute( - sql, *args, **kwargs - ) + i = 0 + N = 5 + while True: + try: + return self.txn.execute( + sql, *args, **kwargs + ) + except self.database_engine.module.DatabaseError as e: + if self.database_engine.is_deadlock(e) and i < N: + i += 1 + logger.warn("[SQL DEADLOCK] {%s}", self.name) + continue + raise except Exception as e: logger.debug("[SQL FAIL] {%s} %s", self.name, e) raise diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py index 7fcb706a6..a279bfaf4 100644 --- a/synapse/storage/engines/maria.py +++ b/synapse/storage/engines/maria.py @@ -40,3 +40,8 @@ class MariaEngine(object): ) db_conn.commit() prepare_database(db_conn, self) + + def is_deadlock(self, error): + if isinstance(error, self.module.InternalError): + return error.sqlstate == 40001 and error.errno == 1213 + return False diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index e802b5d5f..72c11df46 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -32,3 +32,6 @@ class Sqlite3Engine(object): def prepare_database(self, db_conn): prepare_sqlite3_database(db_conn) prepare_database(db_conn, self) + + def is_deadlock(self, error): + return False From 24d8134ac135cbbd1f49b8cefacad431ed8f9d71 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Apr 2015 13:10:54 +0100 Subject: [PATCH 033/104] Fix maria engine to correctly recognize deadlocks --- synapse/storage/engines/maria.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py index a279bfaf4..230b32858 100644 --- a/synapse/storage/engines/maria.py +++ b/synapse/storage/engines/maria.py @@ -42,6 +42,6 @@ class MariaEngine(object): prepare_database(db_conn, self) def is_deadlock(self, error): - if isinstance(error, self.module.InternalError): - return error.sqlstate == 40001 and error.errno == 1213 + if isinstance(error, self.module.DatabaseError): + return error.sqlstate == "40001" and error.errno == 1213 return False From 9a05795619a4472460a791617efaee4ce1af110e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Apr 2015 13:11:28 +0100 Subject: [PATCH 034/104] Retry transaction, not SQL query --- synapse/storage/_base.py | 46 ++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 20fc1d0bb..fee713eb2 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -182,22 +182,12 @@ class LoggingTransaction(object): start = time.time() * 1000 try: - i = 0 - N = 5 - while True: - try: - return self.txn.execute( - sql, *args, **kwargs - ) - except self.database_engine.module.DatabaseError as e: - if self.database_engine.is_deadlock(e) and i < N: - i += 1 - logger.warn("[SQL DEADLOCK] {%s}", self.name) - continue - raise + return self.txn.execute( + sql, *args, **kwargs + ) except Exception as e: - logger.debug("[SQL FAIL] {%s} %s", self.name, e) - raise + logger.debug("[SQL FAIL] {%s} %s", self.name, e) + raise finally: msecs = (time.time() * 1000) - start sql_logger.debug("[SQL time] {%s} %f", self.name, msecs) @@ -347,7 +337,7 @@ class SQLBaseStore(object): start_time = time.time() * 1000 - def inner_func(txn, *args, **kwargs): + def inner_func(conn, *args, **kwargs): with LoggingContext("runInteraction") as context: current_context.copy_to(context) start = time.time() * 1000 @@ -362,10 +352,24 @@ class SQLBaseStore(object): sql_scheduling_timer.inc_by(time.time() * 1000 - start_time) transaction_logger.debug("[TXN START] {%s}", name) try: - return func( - LoggingTransaction(txn, name, self.database_engine), - *args, **kwargs - ) + i = 0 + N = 5 + while True: + try: + txn = conn.cursor() + return func( + LoggingTransaction(txn, name, self.database_engine), + *args, **kwargs + ) + except self.database_engine.module.DatabaseError as e: + logger.warn("[TXN DEADLOCK] {%s} %r, %r", name, e.errno, e.sqlstate) + if self.database_engine.is_deadlock(e): + logger.warn("[TXN DEADLOCK] {%s} %d/%d", name, i, N) + if i < N: + i += 1 + conn.rollback() + continue + raise except Exception as e: logger.debug("[TXN FAIL] {%s}", name, e) raise @@ -380,7 +384,7 @@ class SQLBaseStore(object): sql_txn_timer.inc_by(duration, desc) with PreserveLoggingContext(): - result = yield self._db_pool.runInteraction( + result = yield self._db_pool.runWithConnection( inner_func, *args, **kwargs ) defer.returnValue(result) From 279a547a8b7f7e9b9e8f4ed1ae25686824198107 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Apr 2015 16:53:48 +0100 Subject: [PATCH 035/104] Use generic db exceptions rather than sqlite3 specific ones --- synapse/storage/directory.py | 4 +--- synapse/storage/registration.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index cfb200570..2b2bdf861 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -21,8 +21,6 @@ from twisted.internet import defer from collections import namedtuple -import sqlite3 - RoomAliasMapping = namedtuple( "RoomAliasMapping", @@ -91,7 +89,7 @@ class DirectoryStore(SQLBaseStore): }, desc="create_room_alias_association", ) - except sqlite3.IntegrityError: + except self.database_engine.module.IntegrityError: raise SynapseError( 409, "Room alias %s already exists" % room_alias.to_string() ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 0f9d898e5..f7d829128 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -15,8 +15,6 @@ from twisted.internet import defer -from sqlite3 import IntegrityError - from synapse.api.errors import StoreError, Codes from ._base import SQLBaseStore, cached @@ -76,7 +74,7 @@ class RegistrationStore(SQLBaseStore): txn.execute("INSERT INTO users(name, password_hash, creation_ts) " "VALUES (?,?,?)", [user_id, password_hash, now]) - except IntegrityError: + except self.database_engine.module.IntegrityError: raise StoreError( 400, "User ID already taken.", errcode=Codes.USER_IN_USE ) From 22d7a593062eb06081d0cd1dc71e4fc79fc1cb85 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Apr 2015 16:57:14 +0100 Subject: [PATCH 036/104] Fix tests after commit 9a0579 --- tests/storage/test_base.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 5c17d3014..a64d2b821 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -33,12 +33,18 @@ class SQLBaseStoreTestCase(unittest.TestCase): def setUp(self): self.db_pool = Mock(spec=["runInteraction"]) self.mock_txn = Mock() + self.mock_conn = Mock(spec_set=["cursor"]) + self.mock_conn.cursor.return_value = self.mock_txn # Our fake runInteraction just runs synchronously inline def runInteraction(func, *args, **kwargs): return defer.succeed(func(self.mock_txn, *args, **kwargs)) self.db_pool.runInteraction = runInteraction + def runWithConnection(func, *args, **kwargs): + return defer.succeed(func(self.mock_conn, *args, **kwargs)) + self.db_pool.runWithConnection = runWithConnection + config = Mock() config.event_cache_size = 1 hs = HomeServer( From 8ad0f4912ed72daced74ae4d1c939ebdbc517476 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Apr 2015 11:41:36 +0100 Subject: [PATCH 037/104] Stream ordering and out of order insertions. Handle the fact that events can be persisted out of order, and so to get the "current max" stream token becomes non trivial - as we need to make sure that *all* stream tokens less than the current max have also successfully been persisted. --- synapse/storage/_base.py | 46 +--------- synapse/storage/events.py | 9 +- synapse/storage/stream.py | 23 +---- synapse/storage/util/__init__.py | 14 +++ synapse/storage/util/id_generators.py | 126 ++++++++++++++++++++++++++ 5 files changed, 153 insertions(+), 65 deletions(-) create mode 100644 synapse/storage/util/__init__.py create mode 100644 synapse/storage/util/id_generators.py diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 23289bbdd..badf9a5f4 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -22,6 +22,8 @@ from synapse.util.logcontext import PreserveLoggingContext, LoggingContext from synapse.util.lrucache import LruCache import synapse.metrics +from util.id_generators import IdGenerator, StreamIdGenerator + from twisted.internet import defer from collections import namedtuple, OrderedDict @@ -29,7 +31,6 @@ import functools import simplejson as json import sys import time -import threading logger = logging.getLogger(__name__) @@ -232,46 +233,6 @@ class PerformanceCounters(object): return top_n_counters -class IdGenerator(object): - def __init__(self, table, column, store): - self.table = table - self.column = column - self.store = store - self._lock = threading.Lock() - self._next_id = None - - @defer.inlineCallbacks - def get_next(self): - with self._lock: - if not self._next_id: - res = yield self.store._execute_and_decode( - "IdGenerator_%s" % (self.table,), - "SELECT MAX(%s) as mx FROM %s" % (self.column, self.table,) - ) - - self._next_id = (res and res[0] and res[0]["mx"]) or 1 - - i = self._next_id - self._next_id += 1 - defer.returnValue(i) - - def get_next_txn(self, txn): - with self._lock: - if self._next_id: - i = self._next_id - self._next_id += 1 - return i - else: - txn.execute( - "SELECT MAX(%s) FROM %s" % (self.column, self.table,) - ) - - val, = txn.fetchone() - self._next_id = val or 2 - - return 1 - - class SQLBaseStore(object): _TXN_ID = 0 @@ -297,7 +258,7 @@ class SQLBaseStore(object): # Pretend the getEventCache is just another named cache caches_by_name["*getEvent*"] = self._get_event_cache - self._stream_id_gen = IdGenerator("events", "stream_ordering", self) + self._stream_id_gen = StreamIdGenerator() self._transaction_id_gen = IdGenerator("sent_transactions", "id", self) self._state_groups_id_gen = IdGenerator("state_groups", "id", self) self._access_tokens_id_gen = IdGenerator("access_tokens", "id", self) @@ -363,7 +324,6 @@ class SQLBaseStore(object): *args, **kwargs ) except self.database_engine.module.DatabaseError as e: - logger.warn("[TXN DEADLOCK] {%s} %r, %r", name, e.errno, e.sqlstate) if self.database_engine.is_deadlock(e): logger.warn("[TXN DEADLOCK] {%s} %d/%d", name, i, N) if i < N: diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 3b3416716..f066484c7 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -52,7 +52,6 @@ class EventsStore(SQLBaseStore): is_new_state=is_new_state, current_state=current_state, ) - self.get_room_events_max_id.invalidate() except _RollbackButIsFineException: pass @@ -97,7 +96,13 @@ class EventsStore(SQLBaseStore): self._get_event_cache.pop(event.event_id) if stream_ordering is None: - stream_ordering = self._stream_id_gen.get_next_txn(txn) + with self._stream_id_gen.get_next_txn(txn) as stream_ordering: + return self._persist_event_txn( + txn, event, context, backfilled, + stream_ordering=stream_ordering, + is_new_state=is_new_state, + current_state=current_state, + ) # We purposefully do this first since if we include a `current_state` # key, we *want* to update the `current_state_events` table diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index e6bb5a807..9925f04bf 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -413,12 +413,10 @@ class StreamStore(SQLBaseStore): "get_recent_events_for_room", get_recent_events_for_room_txn ) - @cached(num_args=0) + @defer.inlineCallbacks def get_room_events_max_id(self): - return self.runInteraction( - "get_room_events_max_id", - self._get_room_events_max_id_txn - ) + token = yield self._stream_id_gen.get_max_token(self) + defer.returnValue("s%d" % (token,)) @defer.inlineCallbacks def _get_min_token(self): @@ -433,21 +431,6 @@ class StreamStore(SQLBaseStore): defer.returnValue(self.min_token) - def _get_room_events_max_id_txn(self, txn): - txn.execute( - "SELECT MAX(stream_ordering) as m FROM events" - ) - - res = self.cursor_to_dict(txn) - - logger.debug("get_room_events_max_id: %s", res) - - if not res or not res[0] or not res[0]["m"]: - return "s0" - - key = res[0]["m"] - return "s%d" % (key,) - @staticmethod def _set_before_and_after(events, rows): for event, row in zip(events, rows): diff --git a/synapse/storage/util/__init__.py b/synapse/storage/util/__init__.py new file mode 100644 index 000000000..c488b10d3 --- /dev/null +++ b/synapse/storage/util/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py new file mode 100644 index 000000000..8f419323a --- /dev/null +++ b/synapse/storage/util/id_generators.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from collections import deque +import contextlib +import threading + + +class IdGenerator(object): + def __init__(self, table, column, store): + self.table = table + self.column = column + self.store = store + self._lock = threading.Lock() + self._next_id = None + + @defer.inlineCallbacks + def get_next(self): + with self._lock: + if not self._next_id: + res = yield self.store._execute_and_decode( + "IdGenerator_%s" % (self.table,), + "SELECT MAX(%s) as mx FROM %s" % (self.column, self.table,) + ) + + self._next_id = (res and res[0] and res[0]["mx"]) or 1 + + i = self._next_id + self._next_id += 1 + defer.returnValue(i) + + def get_next_txn(self, txn): + with self._lock: + if self._next_id: + i = self._next_id + self._next_id += 1 + return i + else: + txn.execute( + "SELECT MAX(%s) FROM %s" % (self.column, self.table,) + ) + + val, = txn.fetchone() + self._next_id = val or 2 + + return 1 + + +class StreamIdGenerator(object): + """Used to generate new stream ids when persisting events while keeping + track of which transactions have been completed. + + This allows us to get the "current" stream id, i.e. the stream id such that + all ids less than or equal to it have completed. This handles the fact that + persistence of events can complete out of order. + + Usage: + with stream_id_gen.get_next_txn(txn) as stream_id: + # ... persist event ... + """ + def __init__(self): + self._lock = threading.Lock() + + self._current_max = None + self._unfinished_ids = deque() + + def get_next_txn(self, txn): + """ + Usage: + with stream_id_gen.get_next_txn(txn) as stream_id: + # ... persist event ... + """ + with self._lock: + if not self._current_max: + self._compute_current_max(txn) + + self._current_max += 1 + next_id = self._current_max + + self._unfinished_ids.append(next_id) + + @contextlib.contextmanager + def manager(): + yield next_id + with self._lock: + self._unfinished_ids.remove(next_id) + + return manager() + + def get_max_token(self, store): + """Returns the maximum stream id such that all stream ids less than or + equal to it have been successfully persisted. + """ + with self._lock: + if self._unfinished_ids: + return self._unfinished_ids[0] - 1 + + if not self._current_max: + return store.runInteraction( + "_compute_current_max", + self._compute_current_max, + ) + + return self._current_max + + def _compute_current_max(self, txn): + txn.execute("SELECT MAX(stream_ordering) FROM events") + val, = txn.fetchone() + + self._current_max = int(val) if val else 1 + + return self._current_max From 9707acfc40547277144438fd8f077bd3bcbf7001 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Apr 2015 13:45:20 +0100 Subject: [PATCH 038/104] Remove spurious spaces --- synapse/storage/schema/delta/12/v12.sql | 8 ++++---- synapse/storage/schema/delta/13/v13.sql | 4 ++-- synapse/storage/schema/delta/14/v14.sql | 2 +- .../schema/full_schemas/11/event_edges.sql | 14 +++++++------- .../full_schemas/11/event_signatures.sql | 8 ++++---- synapse/storage/schema/full_schemas/11/im.sql | 18 +++++++++--------- .../storage/schema/full_schemas/11/keys.sql | 4 ++-- .../full_schemas/11/media_repository.sql | 8 ++++---- .../schema/full_schemas/11/presence.sql | 6 +++--- .../schema/full_schemas/11/profiles.sql | 2 +- .../schema/full_schemas/11/redactions.sql | 2 +- .../schema/full_schemas/11/room_aliases.sql | 4 ++-- .../storage/schema/full_schemas/11/state.sql | 6 +++--- .../schema/full_schemas/11/transactions.sql | 8 ++++---- .../storage/schema/full_schemas/11/users.sql | 6 +++--- 15 files changed, 50 insertions(+), 50 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index 717d289f7..ed8a2f3f1 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS rejections( reason VARCHAR(150) NOT NULL, last_check VARCHAR(150) NOT NULL, UNIQUE (event_id) -) ; +); -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( @@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS pushers ( last_success BIGINT, failing_since BIGINT, UNIQUE (app_id, pushkey) -) ; +); CREATE TABLE IF NOT EXISTS push_rules ( id BIGINT PRIMARY KEY, @@ -48,7 +48,7 @@ CREATE TABLE IF NOT EXISTS push_rules ( conditions VARCHAR(150) NOT NULL, actions VARCHAR(150) NOT NULL, UNIQUE(user_name, rule_id) -) ; +); CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); @@ -56,7 +56,7 @@ CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(150), filter_id BIGINT, filter_json BLOB -) ; +); CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( user_id, filter_id diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index f5275a59b..ef6551f3a 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS application_services( hs_token VARCHAR(150), sender VARCHAR(150), UNIQUE(token) -) ; +); CREATE TABLE IF NOT EXISTS application_services_regex( id BIGINT PRIMARY KEY, @@ -28,4 +28,4 @@ CREATE TABLE IF NOT EXISTS application_services_regex( namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(150), FOREIGN KEY(as_id) REFERENCES application_services(id) -) ; +); diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index 1d582cc62..f1208c731 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -4,6 +4,6 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( rule_id VARCHAR(150) NOT NULL, enabled TINYINT, UNIQUE(user_name, rule_id) -) ; +); CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 124c9a9bd..602c11411 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, UNIQUE (event_id, room_id) -) ; +); CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); @@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, UNIQUE (event_id, room_id) -) ; +); CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_edges( room_id VARCHAR(150) NOT NULL, is_state BOOL NOT NULL, UNIQUE (event_id, prev_event_id, room_id, is_state) -) ; +); CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); @@ -49,7 +49,7 @@ CREATE TABLE IF NOT EXISTS room_depth( room_id VARCHAR(150) NOT NULL, min_depth INTEGER NOT NULL, UNIQUE (room_id) -) ; +); CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); @@ -59,7 +59,7 @@ create TABLE IF NOT EXISTS event_destinations( destination VARCHAR(150) NOT NULL, delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) -) ; +); CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); @@ -70,7 +70,7 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities( type VARCHAR(150) NOT NULL, state_key VARCHAR(150) NOT NULL, UNIQUE (event_id, room_id) -) ; +); CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( room_id, type, state_key @@ -83,7 +83,7 @@ CREATE TABLE IF NOT EXISTS event_auth( auth_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, UNIQUE (event_id, auth_id, room_id) -) ; +); CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index 30e3f71c5..0c27bc1f3 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( algorithm VARCHAR(150), hash BLOB, UNIQUE (event_id, algorithm) -) ; +); CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); @@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS event_reference_hashes ( algorithm VARCHAR(150), hash BLOB, UNIQUE (event_id, algorithm) -) ; +); CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_signatures ( key_id VARCHAR(150), signature BLOB, UNIQUE (event_id, signature_name, key_id) -) ; +); CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); @@ -50,6 +50,6 @@ CREATE TABLE IF NOT EXISTS event_edge_hashes( algorithm VARCHAR(150), hash BLOB, UNIQUE (event_id, prev_event_id, algorithm) -) ; +); CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index 7cb8f802e..363e2c842 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS events( outlier BOOL NOT NULL, depth BIGINT DEFAULT 0 NOT NULL, UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); @@ -38,7 +38,7 @@ CREATE TABLE IF NOT EXISTS event_json( internal_metadata BLOB NOT NULL, json BLOB NOT NULL, UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); @@ -50,7 +50,7 @@ CREATE TABLE IF NOT EXISTS state_events( state_key VARCHAR(150) NOT NULL, prev_state VARCHAR(150), UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); @@ -64,7 +64,7 @@ CREATE TABLE IF NOT EXISTS current_state_events( state_key VARCHAR(150) NOT NULL, UNIQUE (event_id), UNIQUE (room_id, type, state_key) -) ; +); CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); @@ -77,7 +77,7 @@ CREATE TABLE IF NOT EXISTS room_memberships( room_id VARCHAR(150) NOT NULL, membership VARCHAR(150) NOT NULL, UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); @@ -89,14 +89,14 @@ CREATE TABLE IF NOT EXISTS feedback( sender VARCHAR(150), room_id VARCHAR(150), UNIQUE (event_id) -) ; +); CREATE TABLE IF NOT EXISTS topics( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, topic VARCHAR(150) NOT NULL, UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); @@ -113,12 +113,12 @@ CREATE TABLE IF NOT EXISTS rooms( room_id VARCHAR(150) PRIMARY KEY NOT NULL, is_public BOOL, creator VARCHAR(150) -) ; +); CREATE TABLE IF NOT EXISTS room_hosts( room_id VARCHAR(150) NOT NULL, host VARCHAR(150) NOT NULL, UNIQUE (room_id, host) -) ; +); CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index 062ca53fe..0e80bf761 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -19,7 +19,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( ts_added_ms BIGINT, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) -) ; +); CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(150), -- Server name. @@ -28,4 +28,4 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( ts_added_ms BIGINT, -- When the key was added. verify_key BLOB, -- NACL verification key. UNIQUE (server_name, key_id) -) ; +); diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index c8c5f1d2f..08c42722d 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( upload_name VARCHAR(150), -- The name the media was uploaded with. user_id VARCHAR(150), -- The user who uploaded the file. UNIQUE (media_id) -) ; +); CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( media_id VARCHAR(150), -- The id used to refer to the media. @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type ) -) ; +); CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache ( media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(150), -- The name used to store the media on disk. UNIQUE (media_origin, media_id) -) ; +); CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( media_origin VARCHAR(150), -- The remote HS the media came from. @@ -62,7 +62,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type ) -) ; +); CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id ON remote_media_cache_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index 00d803a5c..f3f86e563 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS presence( status_msg VARCHAR(150), mtime BIGINT, -- miliseconds since last state change UNIQUE (user_id) -) ; +); -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS presence_allow_inbound( observed_user_id VARCHAR(150) NOT NULL, observer_user_id VARCHAR(150) NOT NULL, -- a UserID, UNIQUE (observed_user_id, observer_user_id) -) ; +); -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? @@ -35,4 +35,4 @@ CREATE TABLE IF NOT EXISTS presence_list( observed_user_id VARCHAR(150) NOT NULL, -- a UserID, accepted BOOLEAN NOT NULL, UNIQUE (user_id, observed_user_id) -) ; +); diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index 023060a54..21c58a99b 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -17,4 +17,4 @@ CREATE TABLE IF NOT EXISTS profiles( displayname VARCHAR(150), avatar_url VARCHAR(150), UNIQUE(user_id) -) ; +); diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index 5c23188d6..b81451eab 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS redactions ( event_id VARCHAR(150) NOT NULL, redacts VARCHAR(150) NOT NULL, UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index 63fe0f5c6..952cae35b 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -17,9 +17,9 @@ CREATE TABLE IF NOT EXISTS room_aliases( room_alias VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, UNIQUE (room_alias) -) ; +); CREATE TABLE IF NOT EXISTS room_alias_servers( room_alias VARCHAR(150) NOT NULL, server VARCHAR(150) NOT NULL -) ; +); diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index acfb76439..737c3e35c 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS state_groups( id VARCHAR(20) PRIMARY KEY, room_id VARCHAR(150) NOT NULL, event_id VARCHAR(150) NOT NULL -) ; +); CREATE TABLE IF NOT EXISTS state_groups_state( state_group VARCHAR(20) NOT NULL, @@ -25,13 +25,13 @@ CREATE TABLE IF NOT EXISTS state_groups_state( type VARCHAR(150) NOT NULL, state_key VARCHAR(150) NOT NULL, event_id VARCHAR(150) NOT NULL -) ; +); CREATE TABLE IF NOT EXISTS event_to_state_groups( event_id VARCHAR(150) NOT NULL, state_group VARCHAR(150) NOT NULL, UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 43541661c..c908109b6 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( response_json BLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) -) ; +); CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( response_code INTEGER DEFAULT 0, response_json BLOB, ts BIGINT -) ; +); CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); @@ -51,7 +51,7 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( pdu_id VARCHAR(150), pdu_origin VARCHAR(150), UNIQUE (transaction_id, destination) -) ; +); CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); @@ -60,4 +60,4 @@ CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(150) PRIMARY KEY, retry_last_ts BIGINT, retry_interval INTEGER -) ; +); diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index ba0f42d45..32dfc5b18 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS users( creation_ts BIGINT, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) -) ; +); CREATE TABLE IF NOT EXISTS access_tokens( id BIGINT PRIMARY KEY, @@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS access_tokens( token VARCHAR(150) NOT NULL, last_used BIGINT, UNIQUE(token) -) ; +); CREATE TABLE IF NOT EXISTS user_ips ( user VARCHAR(150) NOT NULL, @@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS user_ips ( ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, last_seen BIGINT NOT NULL -) ; +); CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); From 2ded3446201f833a69cdb7cf269c65e5f9de1f27 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Apr 2015 13:46:06 +0100 Subject: [PATCH 039/104] Remove unused import --- synapse/storage/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 9925f04bf..57c2e4dfe 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -35,7 +35,7 @@ what sort order was used: from twisted.internet import defer -from ._base import SQLBaseStore, cached +from ._base import SQLBaseStore from synapse.api.constants import EventTypes from synapse.api.errors import SynapseError from synapse.util.logutils import log_function From a1665c50949aaa257f3ee57fa02d1934b101642e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 10:05:44 +0100 Subject: [PATCH 040/104] Revert non-trivial schema changes and move them to a new schema version. --- synapse/storage/schema/full_schemas/11/im.sql | 19 ++- .../full_schemas/11/media_repository.sql | 3 - .../schema/full_schemas/11/presence.sql | 9 +- .../schema/full_schemas/11/profiles.sql | 3 +- .../schema/full_schemas/11/room_aliases.sql | 3 +- .../schema/full_schemas/11/transactions.sql | 4 +- .../storage/schema/full_schemas/11/users.sql | 5 +- .../schema/full_schemas/16/event_edges.sql | 89 +++++++++++++ .../full_schemas/16/event_signatures.sql | 55 ++++++++ synapse/storage/schema/full_schemas/16/im.sql | 124 ++++++++++++++++++ .../storage/schema/full_schemas/16/keys.sql | 31 +++++ .../full_schemas/16/media_repository.sql | 68 ++++++++++ .../schema/full_schemas/16/presence.sql | 38 ++++++ .../schema/full_schemas/16/profiles.sql | 20 +++ .../schema/full_schemas/16/redactions.sql | 22 ++++ .../schema/full_schemas/16/room_aliases.sql | 25 ++++ .../storage/schema/full_schemas/16/state.sql | 40 ++++++ .../schema/full_schemas/16/transactions.sql | 63 +++++++++ .../storage/schema/full_schemas/16/users.sql | 42 ++++++ 19 files changed, 636 insertions(+), 27 deletions(-) create mode 100644 synapse/storage/schema/full_schemas/16/event_edges.sql create mode 100644 synapse/storage/schema/full_schemas/16/event_signatures.sql create mode 100644 synapse/storage/schema/full_schemas/16/im.sql create mode 100644 synapse/storage/schema/full_schemas/16/keys.sql create mode 100644 synapse/storage/schema/full_schemas/16/media_repository.sql create mode 100644 synapse/storage/schema/full_schemas/16/presence.sql create mode 100644 synapse/storage/schema/full_schemas/16/profiles.sql create mode 100644 synapse/storage/schema/full_schemas/16/redactions.sql create mode 100644 synapse/storage/schema/full_schemas/16/room_aliases.sql create mode 100644 synapse/storage/schema/full_schemas/16/state.sql create mode 100644 synapse/storage/schema/full_schemas/16/transactions.sql create mode 100644 synapse/storage/schema/full_schemas/16/users.sql diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index 363e2c842..eac9c6af4 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering BIGINT PRIMARY KEY, + stream_ordering BIGINT PRIMARY KEY AUTOINCREMENT, topological_ordering BIGINT NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, @@ -62,10 +62,10 @@ CREATE TABLE IF NOT EXISTS current_state_events( room_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, state_key VARCHAR(150) NOT NULL, - UNIQUE (event_id), UNIQUE (room_id, type, state_key) ); +CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id); CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); @@ -75,10 +75,10 @@ CREATE TABLE IF NOT EXISTS room_memberships( user_id VARCHAR(150) NOT NULL, sender VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - membership VARCHAR(150) NOT NULL, - UNIQUE (event_id) + membership VARCHAR(150) NOT NULL ); +CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id); CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); @@ -87,26 +87,25 @@ CREATE TABLE IF NOT EXISTS feedback( feedback_type VARCHAR(150), target_event_id VARCHAR(150), sender VARCHAR(150), - room_id VARCHAR(150), - UNIQUE (event_id) + room_id VARCHAR(150) ); CREATE TABLE IF NOT EXISTS topics( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - topic VARCHAR(150) NOT NULL, - UNIQUE (event_id) + topic VARCHAR(150) NOT NULL ); +CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id); CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - name VARCHAR(150) NOT NULL, - UNIQUE (event_id) + name VARCHAR(150) NOT NULL ); +CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id); CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index 08c42722d..c6ce6cf69 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -63,6 +63,3 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( thumbnail_type ) ); - -CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id - ON remote_media_cache_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index f3f86e563..fce324b89 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -16,16 +16,14 @@ CREATE TABLE IF NOT EXISTS presence( user_id VARCHAR(150) NOT NULL, state VARCHAR(20), status_msg VARCHAR(150), - mtime BIGINT, -- miliseconds since last state change - UNIQUE (user_id) + mtime BIGINT -- miliseconds since last state change ); -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state CREATE TABLE IF NOT EXISTS presence_allow_inbound( observed_user_id VARCHAR(150) NOT NULL, - observer_user_id VARCHAR(150) NOT NULL, -- a UserID, - UNIQUE (observed_user_id, observer_user_id) + observer_user_id VARCHAR(150) NOT NULL -- a UserID, ); -- For each of /my/ users (watcher), which possibly-remote users are they @@ -33,6 +31,5 @@ CREATE TABLE IF NOT EXISTS presence_allow_inbound( CREATE TABLE IF NOT EXISTS presence_list( user_id VARCHAR(150) NOT NULL, observed_user_id VARCHAR(150) NOT NULL, -- a UserID, - accepted BOOLEAN NOT NULL, - UNIQUE (user_id, observed_user_id) + accepted BOOLEAN NOT NULL ); diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index 21c58a99b..ffe75edf9 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -15,6 +15,5 @@ CREATE TABLE IF NOT EXISTS profiles( user_id VARCHAR(150) NOT NULL, displayname VARCHAR(150), - avatar_url VARCHAR(150), - UNIQUE(user_id) + avatar_url VARCHAR(150) ); diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index 952cae35b..622691322 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -15,8 +15,7 @@ CREATE TABLE IF NOT EXISTS room_aliases( room_alias VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - UNIQUE (room_alias) + room_id VARCHAR(150) NOT NULL ); CREATE TABLE IF NOT EXISTS room_alias_servers( diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index c908109b6..c3d46763a 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -49,10 +49,10 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( transaction_id INTEGER, destination VARCHAR(150), pdu_id VARCHAR(150), - pdu_origin VARCHAR(150), - UNIQUE (transaction_id, destination) + pdu_origin VARCHAR(150) ); +CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination) CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 32dfc5b18..94c91e529 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -13,6 +13,7 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS users( + id BIGINT PRIMARY KEY AUTOINCREMENT, name VARCHAR(150), password_hash VARCHAR(150), creation_ts BIGINT, @@ -35,8 +36,8 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT NOT NULL + last_seen BIGINT NOT NULL, + UNIQUE (user, access_token, ip, user_agent) ); CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); -CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); diff --git a/synapse/storage/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/full_schemas/16/event_edges.sql new file mode 100644 index 000000000..602c11411 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/event_edges.sql @@ -0,0 +1,89 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS event_forward_extremities( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + UNIQUE (event_id, room_id) +); + +CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); +CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); + + +CREATE TABLE IF NOT EXISTS event_backward_extremities( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + UNIQUE (event_id, room_id) +); + +CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); +CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); + + +CREATE TABLE IF NOT EXISTS event_edges( + event_id VARCHAR(150) NOT NULL, + prev_event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + is_state BOOL NOT NULL, + UNIQUE (event_id, prev_event_id, room_id, is_state) +); + +CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); +CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); + + +CREATE TABLE IF NOT EXISTS room_depth( + room_id VARCHAR(150) NOT NULL, + min_depth INTEGER NOT NULL, + UNIQUE (room_id) +); + +CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); + + +create TABLE IF NOT EXISTS event_destinations( + event_id VARCHAR(150) NOT NULL, + destination VARCHAR(150) NOT NULL, + delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered + UNIQUE (event_id, destination) +); + +CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); + + +CREATE TABLE IF NOT EXISTS state_forward_extremities( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, + UNIQUE (event_id, room_id) +); + +CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( + room_id, type, state_key +); +CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); + + +CREATE TABLE IF NOT EXISTS event_auth( + event_id VARCHAR(150) NOT NULL, + auth_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + UNIQUE (event_id, auth_id, room_id) +); + +CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); +CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/full_schemas/16/event_signatures.sql new file mode 100644 index 000000000..0c27bc1f3 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/event_signatures.sql @@ -0,0 +1,55 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS event_content_hashes ( + event_id VARCHAR(150), + algorithm VARCHAR(150), + hash BLOB, + UNIQUE (event_id, algorithm) +); + +CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); + + +CREATE TABLE IF NOT EXISTS event_reference_hashes ( + event_id VARCHAR(150), + algorithm VARCHAR(150), + hash BLOB, + UNIQUE (event_id, algorithm) +); + +CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); + + +CREATE TABLE IF NOT EXISTS event_signatures ( + event_id VARCHAR(150), + signature_name VARCHAR(150), + key_id VARCHAR(150), + signature BLOB, + UNIQUE (event_id, signature_name, key_id) +); + +CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); + + +CREATE TABLE IF NOT EXISTS event_edge_hashes( + event_id VARCHAR(150), + prev_event_id VARCHAR(150), + algorithm VARCHAR(150), + hash BLOB, + UNIQUE (event_id, prev_event_id, algorithm) +); + +CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql new file mode 100644 index 000000000..363e2c842 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -0,0 +1,124 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS events( + stream_ordering BIGINT PRIMARY KEY, + topological_ordering BIGINT NOT NULL, + event_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + content BLOB NOT NULL, + unrecognized_keys BLOB, + processed BOOL NOT NULL, + outlier BOOL NOT NULL, + depth BIGINT DEFAULT 0 NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); +CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); +CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); + + +CREATE TABLE IF NOT EXISTS event_json( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + internal_metadata BLOB NOT NULL, + json BLOB NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); + + +CREATE TABLE IF NOT EXISTS state_events( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, + prev_state VARCHAR(150), + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); +CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); +CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); + + +CREATE TABLE IF NOT EXISTS current_state_events( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, + UNIQUE (event_id), + UNIQUE (room_id, type, state_key) +); + +CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); +CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); +CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); + +CREATE TABLE IF NOT EXISTS room_memberships( + event_id VARCHAR(150) NOT NULL, + user_id VARCHAR(150) NOT NULL, + sender VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + membership VARCHAR(150) NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); +CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); + +CREATE TABLE IF NOT EXISTS feedback( + event_id VARCHAR(150) NOT NULL, + feedback_type VARCHAR(150), + target_event_id VARCHAR(150), + sender VARCHAR(150), + room_id VARCHAR(150), + UNIQUE (event_id) +); + +CREATE TABLE IF NOT EXISTS topics( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + topic VARCHAR(150) NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); + +CREATE TABLE IF NOT EXISTS room_names( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + name VARCHAR(150) NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); + +CREATE TABLE IF NOT EXISTS rooms( + room_id VARCHAR(150) PRIMARY KEY NOT NULL, + is_public BOOL, + creator VARCHAR(150) +); + +CREATE TABLE IF NOT EXISTS room_hosts( + room_id VARCHAR(150) NOT NULL, + host VARCHAR(150) NOT NULL, + UNIQUE (room_id, host) +); + +CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql new file mode 100644 index 000000000..0e80bf761 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/keys.sql @@ -0,0 +1,31 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS server_tls_certificates( + server_name VARCHAR(150), -- Server name. + fingerprint VARCHAR(150), -- Certificate fingerprint. + from_server VARCHAR(150), -- Which key server the certificate was fetched from. + ts_added_ms BIGINT, -- When the certifcate was added. + tls_certificate BLOB, -- DER encoded x509 certificate. + UNIQUE (server_name, fingerprint) +); + +CREATE TABLE IF NOT EXISTS server_signature_keys( + server_name VARCHAR(150), -- Server name. + key_id VARCHAR(150), -- Key version. + from_server VARCHAR(150), -- Which key server the key was fetched form. + ts_added_ms BIGINT, -- When the key was added. + verify_key BLOB, -- NACL verification key. + UNIQUE (server_name, key_id) +); diff --git a/synapse/storage/schema/full_schemas/16/media_repository.sql b/synapse/storage/schema/full_schemas/16/media_repository.sql new file mode 100644 index 000000000..08c42722d --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/media_repository.sql @@ -0,0 +1,68 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS local_media_repository ( + media_id VARCHAR(150), -- The id used to refer to the media. + media_type VARCHAR(150), -- The MIME-type of the media. + media_length INTEGER, -- Length of the media in bytes. + created_ts BIGINT, -- When the content was uploaded in ms. + upload_name VARCHAR(150), -- The name the media was uploaded with. + user_id VARCHAR(150), -- The user who uploaded the file. + UNIQUE (media_id) +); + +CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( + media_id VARCHAR(150), -- The id used to refer to the media. + thumbnail_width INTEGER, -- The width of the thumbnail in pixels. + thumbnail_height INTEGER, -- The height of the thumbnail in pixels. + thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. + thumbnail_method VARCHAR(150), -- The method used to make the thumbnail. + thumbnail_length INTEGER, -- The length of the thumbnail in bytes. + UNIQUE ( + media_id, thumbnail_width, thumbnail_height, thumbnail_type + ) +); + +CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id + ON local_media_repository_thumbnails (media_id); + +CREATE TABLE IF NOT EXISTS remote_media_cache ( + media_origin VARCHAR(150), -- The remote HS the media came from. + media_id VARCHAR(150), -- The id used to refer to the media on that server. + media_type VARCHAR(150), -- The MIME-type of the media. + created_ts BIGINT, -- When the content was uploaded in ms. + upload_name VARCHAR(150), -- The name the media was uploaded with. + media_length INTEGER, -- Length of the media in bytes. + filesystem_id VARCHAR(150), -- The name used to store the media on disk. + UNIQUE (media_origin, media_id) +); + +CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( + media_origin VARCHAR(150), -- The remote HS the media came from. + media_id VARCHAR(150), -- The id used to refer to the media. + thumbnail_width INTEGER, -- The width of the thumbnail in pixels. + thumbnail_height INTEGER, -- The height of the thumbnail in pixels. + thumbnail_method VARCHAR(150), -- The method used to make the thumbnail + thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. + thumbnail_length INTEGER, -- The length of the thumbnail in bytes. + filesystem_id VARCHAR(150), -- The name used to store the media on disk. + UNIQUE ( + media_origin, media_id, thumbnail_width, thumbnail_height, + thumbnail_type + ) +); + +CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id + ON remote_media_cache_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql new file mode 100644 index 000000000..f3f86e563 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/presence.sql @@ -0,0 +1,38 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS presence( + user_id VARCHAR(150) NOT NULL, + state VARCHAR(20), + status_msg VARCHAR(150), + mtime BIGINT, -- miliseconds since last state change + UNIQUE (user_id) +); + +-- For each of /my/ users which possibly-remote users are allowed to see their +-- presence state +CREATE TABLE IF NOT EXISTS presence_allow_inbound( + observed_user_id VARCHAR(150) NOT NULL, + observer_user_id VARCHAR(150) NOT NULL, -- a UserID, + UNIQUE (observed_user_id, observer_user_id) +); + +-- For each of /my/ users (watcher), which possibly-remote users are they +-- watching? +CREATE TABLE IF NOT EXISTS presence_list( + user_id VARCHAR(150) NOT NULL, + observed_user_id VARCHAR(150) NOT NULL, -- a UserID, + accepted BOOLEAN NOT NULL, + UNIQUE (user_id, observed_user_id) +); diff --git a/synapse/storage/schema/full_schemas/16/profiles.sql b/synapse/storage/schema/full_schemas/16/profiles.sql new file mode 100644 index 000000000..21c58a99b --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/profiles.sql @@ -0,0 +1,20 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS profiles( + user_id VARCHAR(150) NOT NULL, + displayname VARCHAR(150), + avatar_url VARCHAR(150), + UNIQUE(user_id) +); diff --git a/synapse/storage/schema/full_schemas/16/redactions.sql b/synapse/storage/schema/full_schemas/16/redactions.sql new file mode 100644 index 000000000..b81451eab --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/redactions.sql @@ -0,0 +1,22 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS redactions ( + event_id VARCHAR(150) NOT NULL, + redacts VARCHAR(150) NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); +CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/full_schemas/16/room_aliases.sql b/synapse/storage/schema/full_schemas/16/room_aliases.sql new file mode 100644 index 000000000..952cae35b --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/room_aliases.sql @@ -0,0 +1,25 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS room_aliases( + room_alias VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + UNIQUE (room_alias) +); + +CREATE TABLE IF NOT EXISTS room_alias_servers( + room_alias VARCHAR(150) NOT NULL, + server VARCHAR(150) NOT NULL +); diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql new file mode 100644 index 000000000..737c3e35c --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/state.sql @@ -0,0 +1,40 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS state_groups( + id VARCHAR(20) PRIMARY KEY, + room_id VARCHAR(150) NOT NULL, + event_id VARCHAR(150) NOT NULL +); + +CREATE TABLE IF NOT EXISTS state_groups_state( + state_group VARCHAR(20) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, + event_id VARCHAR(150) NOT NULL +); + +CREATE TABLE IF NOT EXISTS event_to_state_groups( + event_id VARCHAR(150) NOT NULL, + state_group VARCHAR(150) NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); + +CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(state_group); +CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(room_id, type, state_key); +CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id); \ No newline at end of file diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql new file mode 100644 index 000000000..c908109b6 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/transactions.sql @@ -0,0 +1,63 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +-- Stores what transaction ids we have received and what our response was +CREATE TABLE IF NOT EXISTS received_transactions( + transaction_id VARCHAR(150), + origin VARCHAR(150), + ts BIGINT, + response_code INTEGER, + response_json BLOB, + has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx + UNIQUE (transaction_id, origin) +); + +CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; + + +-- Stores what transactions we've sent, what their response was (if we got one) and whether we have +-- since referenced the transaction in another outgoing transaction +CREATE TABLE IF NOT EXISTS sent_transactions( + id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering + transaction_id VARCHAR(150), + destination VARCHAR(150), + response_code INTEGER DEFAULT 0, + response_json BLOB, + ts BIGINT +); + +CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); +CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); +-- So that we can do an efficient look up of all transactions that have yet to be successfully +-- sent. +CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code); + + +-- For sent transactions only. +CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( + transaction_id INTEGER, + destination VARCHAR(150), + pdu_id VARCHAR(150), + pdu_origin VARCHAR(150), + UNIQUE (transaction_id, destination) +); + +CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); + +-- To track destination health +CREATE TABLE IF NOT EXISTS destinations( + destination VARCHAR(150) PRIMARY KEY, + retry_last_ts BIGINT, + retry_interval INTEGER +); diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql new file mode 100644 index 000000000..32dfc5b18 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/users.sql @@ -0,0 +1,42 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS users( + name VARCHAR(150), + password_hash VARCHAR(150), + creation_ts BIGINT, + admin BOOL DEFAULT 0 NOT NULL, + UNIQUE(name) +); + +CREATE TABLE IF NOT EXISTS access_tokens( + id BIGINT PRIMARY KEY, + user_id VARCHAR(150) NOT NULL, + device_id VARCHAR(150), + token VARCHAR(150) NOT NULL, + last_used BIGINT, + UNIQUE(token) +); + +CREATE TABLE IF NOT EXISTS user_ips ( + user VARCHAR(150) NOT NULL, + access_token VARCHAR(150) NOT NULL, + device_id VARCHAR(150), + ip VARCHAR(150) NOT NULL, + user_agent VARCHAR(150) NOT NULL, + last_seen BIGINT NOT NULL +); + +CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); +CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); From e2722f58ee98e4af7b54dfa230f63520b1ba8558 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 10:16:29 +0100 Subject: [PATCH 041/104] Fix schema again --- synapse/storage/schema/full_schemas/11/transactions.sql | 4 ++-- synapse/storage/schema/full_schemas/11/users.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index c3d46763a..093bb275b 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -29,7 +29,7 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin -- Stores what transactions we've sent, what their response was (if we got one) and whether we have -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( - id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering + id BIGINT PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, @@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( pdu_origin VARCHAR(150) ); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination) +CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination); CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 94c91e529..e9a9fc94f 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS users( ); CREATE TABLE IF NOT EXISTS access_tokens( - id BIGINT PRIMARY KEY, + id BIGINT PRIMARY KEY AUTOINCREMENT, user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, From cda4a6f93f695cf35eea334ba17e8694aeeab58c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 10:19:50 +0100 Subject: [PATCH 042/104] Revert non-trivial changes to upgrade scripts --- synapse/storage/schema/delta/12/v12.sql | 4 ++-- synapse/storage/schema/delta/13/v13.sql | 4 ++-- synapse/storage/schema/delta/14/v14.sql | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index ed8a2f3f1..f0d7e74b8 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS rejections( -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( - id BIGINT PRIMARY KEY, + id BIGINT PRIMARY KEY AUTOINCREMENT, user_name VARCHAR(150) NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, @@ -40,7 +40,7 @@ CREATE TABLE IF NOT EXISTS pushers ( ); CREATE TABLE IF NOT EXISTS push_rules ( - id BIGINT PRIMARY KEY, + id BIGINT PRIMARY KEY AUTOINCREMENT, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, priority_class TINYINT NOT NULL, diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index ef6551f3a..202adc720 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS application_services( - id BIGINT PRIMARY KEY, + id BIGINT PRIMARY KEY AUTOINCREMENT, url VARCHAR(150), token VARCHAR(150), hs_token VARCHAR(150), @@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS application_services( ); CREATE TABLE IF NOT EXISTS application_services_regex( - id BIGINT PRIMARY KEY, + id BIGINT PRIMARY KEY AUTOINCREMENT, as_id BIGINT NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(150), diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index f1208c731..db091fcd5 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -1,5 +1,5 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( - id BIGINT PRIMARY KEY, + id BIGINT PRIMARY KEY AUTOINCREMENT, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, enabled TINYINT, From d5d4281647246f14630bebba1820057db7d81c87 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 10:59:46 +0100 Subject: [PATCH 043/104] Update full_schemas/16 to match delta files. Add delta/16 scripts --- .../delta/16/remote_media_cache_index.sql | 2 + .../schema/delta/16/unique_constraints.sql | 64 ++++++++++++++++ synapse/storage/schema/delta/16/users.sql | 56 ++++++++++++++ .../full_schemas/16/application_services.sql | 48 ++++++++++++ .../schema/full_schemas/16/presence.sql | 2 + .../storage/schema/full_schemas/16/push.sql | 73 +++++++++++++++++++ 6 files changed, 245 insertions(+) create mode 100644 synapse/storage/schema/delta/16/remote_media_cache_index.sql create mode 100644 synapse/storage/schema/delta/16/unique_constraints.sql create mode 100644 synapse/storage/schema/delta/16/users.sql create mode 100644 synapse/storage/schema/full_schemas/16/application_services.sql create mode 100644 synapse/storage/schema/full_schemas/16/push.sql diff --git a/synapse/storage/schema/delta/16/remote_media_cache_index.sql b/synapse/storage/schema/delta/16/remote_media_cache_index.sql new file mode 100644 index 000000000..7a15265cb --- /dev/null +++ b/synapse/storage/schema/delta/16/remote_media_cache_index.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id + ON remote_media_cache_thumbnails (media_id); \ No newline at end of file diff --git a/synapse/storage/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/delta/16/unique_constraints.sql new file mode 100644 index 000000000..b2b677e51 --- /dev/null +++ b/synapse/storage/schema/delta/16/unique_constraints.sql @@ -0,0 +1,64 @@ + +-- We can use SQLite features here, since mysql support was only added in v16 + +-- +DELETE FROM current_state_events WHERE rowid not in ( + SELECT MIN(rowid) FROM current_state_events GROUP BY event_id +); + +CREATE UNIQUE INDEX current_state_events_event_id ON current_state_events(event_id); + +-- +DELETE FROM room_memberships WHERE rowid not in ( + SELECT MIN(rowid) FROM room_memberships GROUP BY event_id +); + +CREATE UNIQUE INDEX room_memberships_event_id ON room_memberships(event_id); + +-- +DELETE FROM feedback WHERE rowid not in ( + SELECT MIN(rowid) FROM feedback GROUP BY event_id +); + +CREATE UNIQUE INDEX feedback_event_id ON feedback(event_id); + +-- +DELETE FROM topics WHERE rowid not in ( + SELECT MIN(rowid) FROM topics GROUP BY event_id +); + +CREATE UNIQUE INDEX topics_event_id ON topics(event_id); + +-- +DELETE FROM room_names WHERE rowid not in ( + SELECT MIN(rowid) FROM room_names GROUP BY event_id +); + +CREATE UNIQUE INDEX room_names_id ON room_names(event_id); + +-- +DELETE FROM presence WHERE rowid not in ( + SELECT MIN(rowid) FROM presence GROUP BY user_id +); + +CREATE UNIQUE INDEX presence_id ON presence(user_id); + +-- +DELETE FROM presence_allow_inbound WHERE rowid not in ( + SELECT MIN(rowid) FROM presence_allow_inbound + GROUP BY observed_user_id, observer_user_id +); + +CREATE UNIQUE INDEX presence_allow_inbound_observers ON presence_allow_inbound( + observed_user_id, observer_user_id +); + +-- +DELETE FROM presence_list WHERE rowid not in ( + SELECT MIN(rowid) FROM presence_list + GROUP BY user_id, observed_user_id +); + +CREATE UNIQUE INDEX presence_list_observers ON presence_list( + user_id, observed_user_id +); diff --git a/synapse/storage/schema/delta/16/users.sql b/synapse/storage/schema/delta/16/users.sql new file mode 100644 index 000000000..1bbd1c0a1 --- /dev/null +++ b/synapse/storage/schema/delta/16/users.sql @@ -0,0 +1,56 @@ +-- Convert `access_tokens`.user from rowids to user strings. +-- MUST BE DONE BEFORE REMOVING ID COLUMN FROM USERS TABLE BELOW +CREATE TABLE IF NOT EXISTS new_access_tokens( + id BIGINT PRIMARY KEY, + user_id VARCHAR(150) NOT NULL, + device_id VARCHAR(150), + token VARCHAR(150) NOT NULL, + last_used BIGINT, + UNIQUE(token) +); + +INSERT INTO new_access_tokens + SELECT a.id, u.name, a.device_id, a.token, a.last_used + FROM access_tokens as a + INNER JOIN users as u ON u.id = a.user_id; + +DROP TABLE access_tokens; + +ALTER TABLE new_access_tokens RENAME TO access_tokens; + +-- Remove ID column from `users` table +CREATE TABLE IF NOT EXISTS new_users( + name VARCHAR(150), + password_hash VARCHAR(150), + creation_ts BIGINT, + admin BOOL DEFAULT 0 NOT NULL, + UNIQUE(name) +); + +INSERT INTO new_users SELECT name, password_hash, creation_ts, admin FROM users; + +DROP TABLE users; + +ALTER TABLE new_users RENAME TO users; + + +-- Remove UNIQUE constraint from `user_ips` table +CREATE TABLE IF NOT EXISTS new_user_ips ( + user VARCHAR(150) NOT NULL, + access_token VARCHAR(150) NOT NULL, + device_id VARCHAR(150), + ip VARCHAR(150) NOT NULL, + user_agent VARCHAR(150) NOT NULL, + last_seen BIGINT NOT NULL +); + +INSERT INTO new_user_ips + SELECT user, access_token, device_id, ip, user_agent, last_seen FROM user_ips; + +DROP TABLE user_ips; + +ALTER TABLE new_user_ips RENAME TO user_ips; + +CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); +CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); + diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql new file mode 100644 index 000000000..7a5c2510d --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/application_services.sql @@ -0,0 +1,48 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS application_services( + id BIGINT PRIMARY KEY, + url VARCHAR(150), + token VARCHAR(150), + hs_token VARCHAR(150), + sender VARCHAR(150), + UNIQUE(token) +); + +CREATE TABLE IF NOT EXISTS application_services_regex( + id BIGINT PRIMARY KEY, + as_id BIGINT NOT NULL, + namespace INTEGER, /* enum[room_id|room_alias|user_id] */ + regex VARCHAR(150), + FOREIGN KEY(as_id) REFERENCES application_services(id) +); + +CREATE TABLE IF NOT EXISTS application_services_state( + as_id VARCHAR(150) PRIMARY KEY, + state VARCHAR(5), + last_txn INTEGER +); + +CREATE TABLE IF NOT EXISTS application_services_txns( + as_id VARCHAR(150) NOT NULL, + txn_id INTEGER NOT NULL, + event_ids BLOB NOT NULL, + UNIQUE(as_id, txn_id) +); + +CREATE INDEX IF NOT EXISTS application_services_txns_id ON application_services_txns ( + as_id +); diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql index f3f86e563..15a6ad3c7 100644 --- a/synapse/storage/schema/full_schemas/16/presence.sql +++ b/synapse/storage/schema/full_schemas/16/presence.sql @@ -36,3 +36,5 @@ CREATE TABLE IF NOT EXISTS presence_list( accepted BOOLEAN NOT NULL, UNIQUE (user_id, observed_user_id) ); + +CREATE INDEX IF NOT EXISTS presence_list_user_id ON presence_list (user_id); diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql new file mode 100644 index 000000000..4c8faa921 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/push.sql @@ -0,0 +1,73 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS rejections( + event_id VARCHAR(150) NOT NULL, + reason VARCHAR(150) NOT NULL, + last_check VARCHAR(150) NOT NULL, + UNIQUE (event_id) +); + +-- Push notification endpoints that users have configured +CREATE TABLE IF NOT EXISTS pushers ( + id BIGINT PRIMARY KEY, + user_name VARCHAR(150) NOT NULL, + profile_tag VARCHAR(32) NOT NULL, + kind VARCHAR(8) NOT NULL, + app_id VARCHAR(64) NOT NULL, + app_display_name VARCHAR(64) NOT NULL, + device_display_name VARCHAR(128) NOT NULL, + pushkey VARBINARY(512) NOT NULL, + ts BIGINT NOT NULL, + lang VARCHAR(8), + data BLOB, + last_token TEXT, + last_success BIGINT, + failing_since BIGINT, + UNIQUE (app_id, pushkey) +); + +CREATE TABLE IF NOT EXISTS push_rules ( + id BIGINT PRIMARY KEY, + user_name VARCHAR(150) NOT NULL, + rule_id VARCHAR(150) NOT NULL, + priority_class TINYINT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + conditions VARCHAR(150) NOT NULL, + actions VARCHAR(150) NOT NULL, + UNIQUE(user_name, rule_id) +); + +CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); + +CREATE TABLE IF NOT EXISTS user_filters( + user_id VARCHAR(150), + filter_id BIGINT, + filter_json BLOB +); + +CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( + user_id, filter_id +); + +CREATE TABLE IF NOT EXISTS push_rules_enable ( + id BIGINT PRIMARY KEY, + user_name VARCHAR(150) NOT NULL, + rule_id VARCHAR(150) NOT NULL, + enabled TINYINT, + UNIQUE(user_name, rule_id) +); + +CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name); From 0f12772e320291408c8c7b475a391556db4c6d2b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 11:16:09 +0100 Subject: [PATCH 044/104] SQLite wants INTEGER and not BIGINT for primary keys --- synapse/storage/schema/delta/12/v12.sql | 4 ++-- synapse/storage/schema/delta/13/v13.sql | 4 ++-- synapse/storage/schema/delta/14/v14.sql | 2 +- synapse/storage/schema/full_schemas/11/im.sql | 2 +- synapse/storage/schema/full_schemas/11/transactions.sql | 2 +- synapse/storage/schema/full_schemas/11/users.sql | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index f0d7e74b8..4107cba71 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS rejections( -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, user_name VARCHAR(150) NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, @@ -40,7 +40,7 @@ CREATE TABLE IF NOT EXISTS pushers ( ); CREATE TABLE IF NOT EXISTS push_rules ( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, priority_class TINYINT NOT NULL, diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index 202adc720..17a139b86 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS application_services( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, url VARCHAR(150), token VARCHAR(150), hs_token VARCHAR(150), @@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS application_services( ); CREATE TABLE IF NOT EXISTS application_services_regex( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, as_id BIGINT NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(150), diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index db091fcd5..8c47d4b0f 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -1,5 +1,5 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, enabled TINYINT, diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index eac9c6af4..91ae9b9ee 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering BIGINT PRIMARY KEY AUTOINCREMENT, + stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, topological_ordering BIGINT NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 093bb275b..a9bf301ef 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -29,7 +29,7 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin -- Stores what transactions we've sent, what their response was (if we got one) and whether we have -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( - id BIGINT PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering + id INTEGER PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index e9a9fc94f..ad5e403f9 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -13,7 +13,7 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS users( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(150), password_hash VARCHAR(150), creation_ts BIGINT, @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS users( ); CREATE TABLE IF NOT EXISTS access_tokens( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, From 7e863c51e6a6a5b6f757cb279f5d7308db1432d4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 11:22:04 +0100 Subject: [PATCH 045/104] Use unsigned bigint --- synapse/storage/schema/delta/12/v12.sql | 8 ++++---- synapse/storage/schema/delta/13/v13.sql | 2 +- synapse/storage/schema/delta/16/users.sql | 8 ++++---- .../storage/schema/full_schemas/11/event_edges.sql | 2 +- synapse/storage/schema/full_schemas/11/im.sql | 4 ++-- synapse/storage/schema/full_schemas/11/keys.sql | 4 ++-- .../schema/full_schemas/11/media_repository.sql | 4 ++-- .../storage/schema/full_schemas/11/presence.sql | 2 +- .../schema/full_schemas/11/transactions.sql | 6 +++--- synapse/storage/schema/full_schemas/11/users.sql | 6 +++--- .../full_schemas/16/application_services.sql | 6 +++--- .../storage/schema/full_schemas/16/event_edges.sql | 2 +- synapse/storage/schema/full_schemas/16/im.sql | 6 +++--- synapse/storage/schema/full_schemas/16/keys.sql | 4 ++-- .../schema/full_schemas/16/media_repository.sql | 4 ++-- .../storage/schema/full_schemas/16/presence.sql | 2 +- synapse/storage/schema/full_schemas/16/push.sql | 14 +++++++------- .../schema/full_schemas/16/transactions.sql | 8 ++++---- synapse/storage/schema/full_schemas/16/users.sql | 8 ++++---- 19 files changed, 50 insertions(+), 50 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index 4107cba71..1247fd175 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -30,12 +30,12 @@ CREATE TABLE IF NOT EXISTS pushers ( app_display_name VARCHAR(64) NOT NULL, device_display_name VARCHAR(128) NOT NULL, pushkey VARBINARY(512) NOT NULL, - ts BIGINT NOT NULL, + ts BIGINT UNSIGNED NOT NULL, lang VARCHAR(8), data BLOB, last_token TEXT, - last_success BIGINT, - failing_since BIGINT, + last_success BIGINT UNSIGNED, + failing_since BIGINT UNSIGNED, UNIQUE (app_id, pushkey) ); @@ -54,7 +54,7 @@ CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(150), - filter_id BIGINT, + filter_id BIGINT UNSIGNED, filter_json BLOB ); diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index 17a139b86..d1da2b48e 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS application_services( CREATE TABLE IF NOT EXISTS application_services_regex( id INTEGER PRIMARY KEY AUTOINCREMENT, - as_id BIGINT NOT NULL, + as_id BIGINT UNSIGNED NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(150), FOREIGN KEY(as_id) REFERENCES application_services(id) diff --git a/synapse/storage/schema/delta/16/users.sql b/synapse/storage/schema/delta/16/users.sql index 1bbd1c0a1..f04705cd7 100644 --- a/synapse/storage/schema/delta/16/users.sql +++ b/synapse/storage/schema/delta/16/users.sql @@ -1,11 +1,11 @@ -- Convert `access_tokens`.user from rowids to user strings. -- MUST BE DONE BEFORE REMOVING ID COLUMN FROM USERS TABLE BELOW CREATE TABLE IF NOT EXISTS new_access_tokens( - id BIGINT PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, - last_used BIGINT, + last_used BIGINT UNSIGNED, UNIQUE(token) ); @@ -22,7 +22,7 @@ ALTER TABLE new_access_tokens RENAME TO access_tokens; CREATE TABLE IF NOT EXISTS new_users( name VARCHAR(150), password_hash VARCHAR(150), - creation_ts BIGINT, + creation_ts BIGINT UNSIGNED, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) ); @@ -41,7 +41,7 @@ CREATE TABLE IF NOT EXISTS new_user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT NOT NULL + last_seen BIGINT UNSIGNED NOT NULL ); INSERT INTO new_user_ips diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 602c11411..bdb110909 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -57,7 +57,7 @@ CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( event_id VARCHAR(150) NOT NULL, destination VARCHAR(150) NOT NULL, - delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered + delivered_ts BIGINT UNSIGNED DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index 91ae9b9ee..e75093a44 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS events( stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, - topological_ordering BIGINT NOT NULL, + topological_ordering BIGINT UNSIGNED NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, @@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS events( unrecognized_keys BLOB, processed BOOL NOT NULL, outlier BOOL NOT NULL, - depth BIGINT DEFAULT 0 NOT NULL, + depth BIGINT UNSIGNED DEFAULT 0 NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index 0e80bf761..551650cf1 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( server_name VARCHAR(150), -- Server name. fingerprint VARCHAR(150), -- Certificate fingerprint. from_server VARCHAR(150), -- Which key server the certificate was fetched from. - ts_added_ms BIGINT, -- When the certifcate was added. + ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(150), -- Server name. key_id VARCHAR(150), -- Key version. from_server VARCHAR(150), -- Which key server the key was fetched form. - ts_added_ms BIGINT, -- When the key was added. + ts_added_ms BIGINT UNSIGNED, -- When the key was added. verify_key BLOB, -- NACL verification key. UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index c6ce6cf69..134e7fbce 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( media_id VARCHAR(150), -- The id used to refer to the media. media_type VARCHAR(150), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. - created_ts BIGINT, -- When the content was uploaded in ms. + created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. user_id VARCHAR(150), -- The user who uploaded the file. UNIQUE (media_id) @@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache ( media_origin VARCHAR(150), -- The remote HS the media came from. media_id VARCHAR(150), -- The id used to refer to the media on that server. media_type VARCHAR(150), -- The MIME-type of the media. - created_ts BIGINT, -- When the content was uploaded in ms. + created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(150), -- The name used to store the media on disk. diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index fce324b89..c617ebea7 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS presence( user_id VARCHAR(150) NOT NULL, state VARCHAR(20), status_msg VARCHAR(150), - mtime BIGINT -- miliseconds since last state change + mtime BIGINT UNSIGNED -- miliseconds since last state change ); -- For each of /my/ users which possibly-remote users are allowed to see their diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index a9bf301ef..67c79db3e 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( transaction_id VARCHAR(150), origin VARCHAR(150), - ts BIGINT, + ts BIGINT UNSIGNED, response_code INTEGER, response_json BLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx @@ -34,7 +34,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( destination VARCHAR(150), response_code INTEGER DEFAULT 0, response_json BLOB, - ts BIGINT + ts BIGINT UNSIGNED ); CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); @@ -58,6 +58,6 @@ CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(d -- To track destination health CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(150) PRIMARY KEY, - retry_last_ts BIGINT, + retry_last_ts BIGINT UNSIGNED, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index ad5e403f9..0ddfccd41 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS users( id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(150), password_hash VARCHAR(150), - creation_ts BIGINT, + creation_ts BIGINT UNSIGNED, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) ); @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS access_tokens( user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, - last_used BIGINT, + last_used BIGINT UNSIGNED, UNIQUE(token) ); @@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT NOT NULL, + last_seen BIGINT UNSIGNED NOT NULL, UNIQUE (user, access_token, ip, user_agent) ); diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql index 7a5c2510d..fbc3e08a1 100644 --- a/synapse/storage/schema/full_schemas/16/application_services.sql +++ b/synapse/storage/schema/full_schemas/16/application_services.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS application_services( - id BIGINT PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, url VARCHAR(150), token VARCHAR(150), hs_token VARCHAR(150), @@ -23,8 +23,8 @@ CREATE TABLE IF NOT EXISTS application_services( ); CREATE TABLE IF NOT EXISTS application_services_regex( - id BIGINT PRIMARY KEY, - as_id BIGINT NOT NULL, + id BIGINT UNSIGNED PRIMARY KEY, + as_id BIGINT UNSIGNED NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(150), FOREIGN KEY(as_id) REFERENCES application_services(id) diff --git a/synapse/storage/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/full_schemas/16/event_edges.sql index 602c11411..bdb110909 100644 --- a/synapse/storage/schema/full_schemas/16/event_edges.sql +++ b/synapse/storage/schema/full_schemas/16/event_edges.sql @@ -57,7 +57,7 @@ CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( event_id VARCHAR(150) NOT NULL, destination VARCHAR(150) NOT NULL, - delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered + delivered_ts BIGINT UNSIGNED DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index 363e2c842..81dae2a91 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering BIGINT PRIMARY KEY, - topological_ordering BIGINT NOT NULL, + stream_ordering BIGINT UNSIGNED PRIMARY KEY, + topological_ordering BIGINT UNSIGNED NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, @@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS events( unrecognized_keys BLOB, processed BOOL NOT NULL, outlier BOOL NOT NULL, - depth BIGINT DEFAULT 0 NOT NULL, + depth BIGINT UNSIGNED DEFAULT 0 NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql index 0e80bf761..551650cf1 100644 --- a/synapse/storage/schema/full_schemas/16/keys.sql +++ b/synapse/storage/schema/full_schemas/16/keys.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( server_name VARCHAR(150), -- Server name. fingerprint VARCHAR(150), -- Certificate fingerprint. from_server VARCHAR(150), -- Which key server the certificate was fetched from. - ts_added_ms BIGINT, -- When the certifcate was added. + ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(150), -- Server name. key_id VARCHAR(150), -- Key version. from_server VARCHAR(150), -- Which key server the key was fetched form. - ts_added_ms BIGINT, -- When the key was added. + ts_added_ms BIGINT UNSIGNED, -- When the key was added. verify_key BLOB, -- NACL verification key. UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/16/media_repository.sql b/synapse/storage/schema/full_schemas/16/media_repository.sql index 08c42722d..014bce4ae 100644 --- a/synapse/storage/schema/full_schemas/16/media_repository.sql +++ b/synapse/storage/schema/full_schemas/16/media_repository.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( media_id VARCHAR(150), -- The id used to refer to the media. media_type VARCHAR(150), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. - created_ts BIGINT, -- When the content was uploaded in ms. + created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. user_id VARCHAR(150), -- The user who uploaded the file. UNIQUE (media_id) @@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache ( media_origin VARCHAR(150), -- The remote HS the media came from. media_id VARCHAR(150), -- The id used to refer to the media on that server. media_type VARCHAR(150), -- The MIME-type of the media. - created_ts BIGINT, -- When the content was uploaded in ms. + created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(150), -- The name used to store the media on disk. diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql index 15a6ad3c7..fbe5b0af6 100644 --- a/synapse/storage/schema/full_schemas/16/presence.sql +++ b/synapse/storage/schema/full_schemas/16/presence.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS presence( user_id VARCHAR(150) NOT NULL, state VARCHAR(20), status_msg VARCHAR(150), - mtime BIGINT, -- miliseconds since last state change + mtime BIGINT UNSIGNED, -- miliseconds since last state change UNIQUE (user_id) ); diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql index 4c8faa921..99175b963 100644 --- a/synapse/storage/schema/full_schemas/16/push.sql +++ b/synapse/storage/schema/full_schemas/16/push.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS rejections( -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( - id BIGINT PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, user_name VARCHAR(150) NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, @@ -30,17 +30,17 @@ CREATE TABLE IF NOT EXISTS pushers ( app_display_name VARCHAR(64) NOT NULL, device_display_name VARCHAR(128) NOT NULL, pushkey VARBINARY(512) NOT NULL, - ts BIGINT NOT NULL, + ts BIGINT UNSIGNED NOT NULL, lang VARCHAR(8), data BLOB, last_token TEXT, - last_success BIGINT, - failing_since BIGINT, + last_success BIGINT UNSIGNED, + failing_since BIGINT UNSIGNED, UNIQUE (app_id, pushkey) ); CREATE TABLE IF NOT EXISTS push_rules ( - id BIGINT PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, priority_class TINYINT NOT NULL, @@ -54,7 +54,7 @@ CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(150), - filter_id BIGINT, + filter_id BIGINT UNSIGNED, filter_json BLOB ); @@ -63,7 +63,7 @@ CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( ); CREATE TABLE IF NOT EXISTS push_rules_enable ( - id BIGINT PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, enabled TINYINT, diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql index c908109b6..e48eb4414 100644 --- a/synapse/storage/schema/full_schemas/16/transactions.sql +++ b/synapse/storage/schema/full_schemas/16/transactions.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( transaction_id VARCHAR(150), origin VARCHAR(150), - ts BIGINT, + ts BIGINT UNSIGNED, response_code INTEGER, response_json BLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx @@ -29,12 +29,12 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin -- Stores what transactions we've sent, what their response was (if we got one) and whether we have -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( - id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering + id BIGINT UNSIGNED PRIMARY KEY, -- This is used to apply insertion ordering transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, response_json BLOB, - ts BIGINT + ts BIGINT UNSIGNED ); CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); @@ -58,6 +58,6 @@ CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(d -- To track destination health CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(150) PRIMARY KEY, - retry_last_ts BIGINT, + retry_last_ts BIGINT UNSIGNED, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql index 32dfc5b18..d0011c04b 100644 --- a/synapse/storage/schema/full_schemas/16/users.sql +++ b/synapse/storage/schema/full_schemas/16/users.sql @@ -15,17 +15,17 @@ CREATE TABLE IF NOT EXISTS users( name VARCHAR(150), password_hash VARCHAR(150), - creation_ts BIGINT, + creation_ts BIGINT UNSIGNED, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) ); CREATE TABLE IF NOT EXISTS access_tokens( - id BIGINT PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, - last_used BIGINT, + last_used BIGINT UNSIGNED, UNIQUE(token) ); @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT NOT NULL + last_seen BIGINT UNSIGNED NOT NULL ); CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); From ce797ad3738afdef681bdca48e9cabe0baa7517b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 11:22:30 +0100 Subject: [PATCH 046/104] Bump schema version --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 87db382fb..b46cafd25 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -51,7 +51,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 15 +SCHEMA_VERSION = 16 dir_path = os.path.abspath(os.path.dirname(__file__)) From 7ed2ec3061b12ad79f3bd07da8758c5e41af8e5e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 13:41:54 +0100 Subject: [PATCH 047/104] Handle the fact that in sqlite binary data might be stored as unicode or bytes --- synapse/storage/_base.py | 7 +++++-- synapse/storage/engines/maria.py | 5 ++++- synapse/storage/engines/sqlite3.py | 7 +++++++ synapse/storage/profile.py | 2 +- synapse/storage/registration.py | 4 +++- 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index c1bf98cdc..e30514cd5 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -812,6 +812,9 @@ class SQLBaseStore(object): internal_metadata, js, redacted, rejected_reason = res + internal_metadata = self.database_engine.load_unicode(internal_metadata) + js = self.database_engine.load_unicode(js) + start_time = update_counter("select_event", start_time) result = self._get_event_from_row_txn( @@ -839,11 +842,11 @@ class SQLBaseStore(object): return curr_time logger.debug("Got js: %r", js) - d = json.loads(str(js).decode("utf8")) + d = json.loads(js) start_time = update_counter("decode_json", start_time) logger.debug("Got internal_metadata: %r", internal_metadata) - internal_metadata = json.loads(str(internal_metadata).decode("utf8")) + internal_metadata = json.loads(internal_metadata) start_time = update_counter("decode_internal", start_time) ev = FrozenEvent( diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py index 230b32858..c93437693 100644 --- a/synapse/storage/engines/maria.py +++ b/synapse/storage/engines/maria.py @@ -27,7 +27,7 @@ class MariaEngine(object): def encode_parameter(self, param): if isinstance(param, types.BufferType): - return str(param) + return bytes(param) return param def on_new_connection(self, db_conn): @@ -45,3 +45,6 @@ class MariaEngine(object): if isinstance(error, self.module.DatabaseError): return error.sqlstate == "40001" and error.errno == 1213 return False + + def load_unicode(self, v): + return bytes(v).decode("UTF8") diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 72c11df46..389df35eb 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -15,6 +15,8 @@ from synapse.storage import prepare_database, prepare_sqlite3_database +import types + class Sqlite3Engine(object): def __init__(self, database_module): @@ -35,3 +37,8 @@ class Sqlite3Engine(object): def is_deadlock(self, error): return False + + def load_unicode(self, v): + if isinstance(v, types.UnicodeType): + return v + return bytes(v).decode("UTF8") diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py index 09778045b..e33963d0b 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py @@ -36,7 +36,7 @@ class ProfileStore(SQLBaseStore): ) if name: - name = name.decode("utf8") + name = self.database_engine.load_unicode(name) defer.returnValue(name) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index f7d829128..86fd2600f 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -99,7 +99,9 @@ class RegistrationStore(SQLBaseStore): ) if user_info: - user_info["password_hash"] = user_info["password_hash"].decode("utf8") + user_info["password_hash"] = self.database_engine.load_unicode( + user_info["password_hash"] + ) defer.returnValue(user_info) From 4d5b0986261db7e9a4389c6fe2eb4808e5946e91 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 18:45:09 +0100 Subject: [PATCH 048/104] Use LONGBLOB and TEXT for arbitary length rows --- synapse/storage/schema/delta/12/v12.sql | 4 ++-- synapse/storage/schema/delta/15/appservice_txns.sql | 2 +- .../schema/full_schemas/11/event_signatures.sql | 8 ++++---- synapse/storage/schema/full_schemas/11/im.sql | 12 ++++++------ synapse/storage/schema/full_schemas/11/keys.sql | 4 ++-- .../storage/schema/full_schemas/11/transactions.sql | 4 ++-- .../schema/full_schemas/16/application_services.sql | 2 +- .../schema/full_schemas/16/event_signatures.sql | 8 ++++---- synapse/storage/schema/full_schemas/16/im.sql | 12 ++++++------ synapse/storage/schema/full_schemas/16/keys.sql | 4 ++-- synapse/storage/schema/full_schemas/16/push.sql | 4 ++-- .../storage/schema/full_schemas/16/transactions.sql | 4 ++-- 12 files changed, 34 insertions(+), 34 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index 1247fd175..a246943f5 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -32,7 +32,7 @@ CREATE TABLE IF NOT EXISTS pushers ( pushkey VARBINARY(512) NOT NULL, ts BIGINT UNSIGNED NOT NULL, lang VARCHAR(8), - data BLOB, + data LONGBLOB, last_token TEXT, last_success BIGINT UNSIGNED, failing_since BIGINT UNSIGNED, @@ -55,7 +55,7 @@ CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(150), filter_id BIGINT UNSIGNED, - filter_json BLOB + filter_json LONGBLOB ); CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( diff --git a/synapse/storage/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/delta/15/appservice_txns.sql index 2f4c3eae5..ddea8fc69 100644 --- a/synapse/storage/schema/delta/15/appservice_txns.sql +++ b/synapse/storage/schema/delta/15/appservice_txns.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS application_services_state( CREATE TABLE IF NOT EXISTS application_services_txns( as_id VARCHAR(150) NOT NULL, txn_id INTEGER NOT NULL, - event_ids BLOB NOT NULL, + event_ids LONGBLOB NOT NULL, UNIQUE(as_id, txn_id) ); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index 0c27bc1f3..09886f607 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash BLOB, + hash LONGBLOB, UNIQUE (event_id, algorithm) ); @@ -26,7 +26,7 @@ CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event CREATE TABLE IF NOT EXISTS event_reference_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash BLOB, + hash LONGBLOB, UNIQUE (event_id, algorithm) ); @@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS event_signatures ( event_id VARCHAR(150), signature_name VARCHAR(150), key_id VARCHAR(150), - signature BLOB, + signature LONGBLOB, UNIQUE (event_id, signature_name, key_id) ); @@ -48,7 +48,7 @@ CREATE TABLE IF NOT EXISTS event_edge_hashes( event_id VARCHAR(150), prev_event_id VARCHAR(150), algorithm VARCHAR(150), - hash BLOB, + hash LONGBLOB, UNIQUE (event_id, prev_event_id, algorithm) ); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index e75093a44..9c47f5174 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS events( event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - content BLOB NOT NULL, - unrecognized_keys BLOB, + content LONGBLOB NOT NULL, + unrecognized_keys LONGBLOB, processed BOOL NOT NULL, outlier BOOL NOT NULL, depth BIGINT UNSIGNED DEFAULT 0 NOT NULL, @@ -35,8 +35,8 @@ CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - internal_metadata BLOB NOT NULL, - json BLOB NOT NULL, + internal_metadata LONGBLOB NOT NULL, + json LONGBLOB NOT NULL, UNIQUE (event_id) ); @@ -93,7 +93,7 @@ CREATE TABLE IF NOT EXISTS feedback( CREATE TABLE IF NOT EXISTS topics( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - topic VARCHAR(150) NOT NULL + topic TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id); @@ -102,7 +102,7 @@ CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - name VARCHAR(150) NOT NULL + name TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index 551650cf1..35f141c28 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( fingerprint VARCHAR(150), -- Certificate fingerprint. from_server VARCHAR(150), -- Which key server the certificate was fetched from. ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added. - tls_certificate BLOB, -- DER encoded x509 certificate. + tls_certificate LONGBLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); @@ -26,6 +26,6 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( key_id VARCHAR(150), -- Key version. from_server VARCHAR(150), -- Which key server the key was fetched form. ts_added_ms BIGINT UNSIGNED, -- When the key was added. - verify_key BLOB, -- NACL verification key. + verify_key LONGBLOB, -- NACL verification key. UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 67c79db3e..c2fab10aa 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( origin VARCHAR(150), ts BIGINT UNSIGNED, response_code INTEGER, - response_json BLOB, + response_json LONGBLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) ); @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, - response_json BLOB, + response_json LONGBLOB, ts BIGINT UNSIGNED ); diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql index fbc3e08a1..bc709df92 100644 --- a/synapse/storage/schema/full_schemas/16/application_services.sql +++ b/synapse/storage/schema/full_schemas/16/application_services.sql @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS application_services_state( CREATE TABLE IF NOT EXISTS application_services_txns( as_id VARCHAR(150) NOT NULL, txn_id INTEGER NOT NULL, - event_ids BLOB NOT NULL, + event_ids LONGBLOB NOT NULL, UNIQUE(as_id, txn_id) ); diff --git a/synapse/storage/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/full_schemas/16/event_signatures.sql index 0c27bc1f3..09886f607 100644 --- a/synapse/storage/schema/full_schemas/16/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/16/event_signatures.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash BLOB, + hash LONGBLOB, UNIQUE (event_id, algorithm) ); @@ -26,7 +26,7 @@ CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event CREATE TABLE IF NOT EXISTS event_reference_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash BLOB, + hash LONGBLOB, UNIQUE (event_id, algorithm) ); @@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS event_signatures ( event_id VARCHAR(150), signature_name VARCHAR(150), key_id VARCHAR(150), - signature BLOB, + signature LONGBLOB, UNIQUE (event_id, signature_name, key_id) ); @@ -48,7 +48,7 @@ CREATE TABLE IF NOT EXISTS event_edge_hashes( event_id VARCHAR(150), prev_event_id VARCHAR(150), algorithm VARCHAR(150), - hash BLOB, + hash LONGBLOB, UNIQUE (event_id, prev_event_id, algorithm) ); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index 81dae2a91..19f0f3414 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS events( event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - content BLOB NOT NULL, - unrecognized_keys BLOB, + content LONGBLOB NOT NULL, + unrecognized_keys LONGBLOB, processed BOOL NOT NULL, outlier BOOL NOT NULL, depth BIGINT UNSIGNED DEFAULT 0 NOT NULL, @@ -35,8 +35,8 @@ CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - internal_metadata BLOB NOT NULL, - json BLOB NOT NULL, + internal_metadata LONGBLOB NOT NULL, + json LONGBLOB NOT NULL, UNIQUE (event_id) ); @@ -94,7 +94,7 @@ CREATE TABLE IF NOT EXISTS feedback( CREATE TABLE IF NOT EXISTS topics( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - topic VARCHAR(150) NOT NULL, + topic TEXT NOT NULL, UNIQUE (event_id) ); @@ -103,7 +103,7 @@ CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - name VARCHAR(150) NOT NULL, + name TEXT NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql index 551650cf1..35f141c28 100644 --- a/synapse/storage/schema/full_schemas/16/keys.sql +++ b/synapse/storage/schema/full_schemas/16/keys.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( fingerprint VARCHAR(150), -- Certificate fingerprint. from_server VARCHAR(150), -- Which key server the certificate was fetched from. ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added. - tls_certificate BLOB, -- DER encoded x509 certificate. + tls_certificate LONGBLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); @@ -26,6 +26,6 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( key_id VARCHAR(150), -- Key version. from_server VARCHAR(150), -- Which key server the key was fetched form. ts_added_ms BIGINT UNSIGNED, -- When the key was added. - verify_key BLOB, -- NACL verification key. + verify_key LONGBLOB, -- NACL verification key. UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql index 99175b963..33300736f 100644 --- a/synapse/storage/schema/full_schemas/16/push.sql +++ b/synapse/storage/schema/full_schemas/16/push.sql @@ -32,7 +32,7 @@ CREATE TABLE IF NOT EXISTS pushers ( pushkey VARBINARY(512) NOT NULL, ts BIGINT UNSIGNED NOT NULL, lang VARCHAR(8), - data BLOB, + data LONGBLOB, last_token TEXT, last_success BIGINT UNSIGNED, failing_since BIGINT UNSIGNED, @@ -55,7 +55,7 @@ CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(150), filter_id BIGINT UNSIGNED, - filter_json BLOB + filter_json LONGBLOB ); CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql index e48eb4414..f381e6760 100644 --- a/synapse/storage/schema/full_schemas/16/transactions.sql +++ b/synapse/storage/schema/full_schemas/16/transactions.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( origin VARCHAR(150), ts BIGINT UNSIGNED, response_code INTEGER, - response_json BLOB, + response_json LONGBLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) ); @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, - response_json BLOB, + response_json LONGBLOB, ts BIGINT UNSIGNED ); From 4103b1c4709b374785ffb7eca75c8f338218bc17 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 18:46:09 +0100 Subject: [PATCH 049/104] DROP indexes before recreating them --- synapse/storage/schema/delta/16/unique_constraints.sql | 8 ++++++++ synapse/storage/schema/full_schemas/16/state.sql | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/synapse/storage/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/delta/16/unique_constraints.sql index b2b677e51..f9fbb6b44 100644 --- a/synapse/storage/schema/delta/16/unique_constraints.sql +++ b/synapse/storage/schema/delta/16/unique_constraints.sql @@ -6,6 +6,7 @@ DELETE FROM current_state_events WHERE rowid not in ( SELECT MIN(rowid) FROM current_state_events GROUP BY event_id ); +DROP INDEX IF EXISTS current_state_events_event_id; CREATE UNIQUE INDEX current_state_events_event_id ON current_state_events(event_id); -- @@ -13,6 +14,7 @@ DELETE FROM room_memberships WHERE rowid not in ( SELECT MIN(rowid) FROM room_memberships GROUP BY event_id ); +DROP INDEX IF EXISTS room_memberships_event_id; CREATE UNIQUE INDEX room_memberships_event_id ON room_memberships(event_id); -- @@ -20,6 +22,7 @@ DELETE FROM feedback WHERE rowid not in ( SELECT MIN(rowid) FROM feedback GROUP BY event_id ); +DROP INDEX IF EXISTS feedback_event_id; CREATE UNIQUE INDEX feedback_event_id ON feedback(event_id); -- @@ -27,6 +30,7 @@ DELETE FROM topics WHERE rowid not in ( SELECT MIN(rowid) FROM topics GROUP BY event_id ); +DROP INDEX IF EXISTS topics_event_id; CREATE UNIQUE INDEX topics_event_id ON topics(event_id); -- @@ -34,6 +38,7 @@ DELETE FROM room_names WHERE rowid not in ( SELECT MIN(rowid) FROM room_names GROUP BY event_id ); +DROP INDEX IF EXISTS room_names_id; CREATE UNIQUE INDEX room_names_id ON room_names(event_id); -- @@ -41,6 +46,7 @@ DELETE FROM presence WHERE rowid not in ( SELECT MIN(rowid) FROM presence GROUP BY user_id ); +DROP INDEX IF EXISTS presence_id; CREATE UNIQUE INDEX presence_id ON presence(user_id); -- @@ -49,6 +55,7 @@ DELETE FROM presence_allow_inbound WHERE rowid not in ( GROUP BY observed_user_id, observer_user_id ); +DROP INDEX IF EXISTS presence_allow_inbound_observers; CREATE UNIQUE INDEX presence_allow_inbound_observers ON presence_allow_inbound( observed_user_id, observer_user_id ); @@ -59,6 +66,7 @@ DELETE FROM presence_list WHERE rowid not in ( GROUP BY user_id, observed_user_id ); +DROP INDEX IF EXISTS presence_list_observers; CREATE UNIQUE INDEX presence_list_observers ON presence_list( user_id, observed_user_id ); diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql index 737c3e35c..8d6b94a61 100644 --- a/synapse/storage/schema/full_schemas/16/state.sql +++ b/synapse/storage/schema/full_schemas/16/state.sql @@ -37,4 +37,4 @@ CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(state_group); CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(room_id, type, state_key); -CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id); \ No newline at end of file +CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id); From c5365dee564ffcf46300d1f979212e0ebaf98692 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 18:46:33 +0100 Subject: [PATCH 050/104] Use case sensitive collations --- synapse/app/homeserver.py | 3 +-- synapse/storage/engines/maria.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index fbc9a43d6..a47e548d6 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -363,7 +363,6 @@ def setup(config_options): db_config = { k: v for k, v in db_config.items() - if not k.startswith("cp_") } name = db_config.get("name", None) @@ -372,7 +371,7 @@ def setup(config_options): "sql_mode": "TRADITIONAL", "charset": "utf8mb4", "use_unicode": True, - "collation": "utf8mb4_general_ci", + "collation": "utf8mb4_bin", }) elif name == "sqlite3": db_config.setdefault("args", {}).update({ diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py index c93437693..90165f684 100644 --- a/synapse/storage/engines/maria.py +++ b/synapse/storage/engines/maria.py @@ -36,7 +36,7 @@ class MariaEngine(object): def prepare_database(self, db_conn): cur = db_conn.cursor() cur.execute( - "ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci" + "ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_bin" ) db_conn.commit() prepare_database(db_conn, self) From 93937b2b313e5b1020d21f3f2cad404ecff659c2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 18:47:03 +0100 Subject: [PATCH 051/104] Remove duplicate rows --- synapse/storage/schema/delta/16/remove_duplicates.sql | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 synapse/storage/schema/delta/16/remove_duplicates.sql diff --git a/synapse/storage/schema/delta/16/remove_duplicates.sql b/synapse/storage/schema/delta/16/remove_duplicates.sql new file mode 100644 index 000000000..65c97b5e2 --- /dev/null +++ b/synapse/storage/schema/delta/16/remove_duplicates.sql @@ -0,0 +1,9 @@ + + +DELETE FROM event_to_state_groups WHERE state_group not in ( + SELECT MAX(state_group) FROM event_to_state_groups GROUP BY event_id +); + +DELETE FROM event_to_state_groups WHERE rowid not in ( + SELECT MIN(rowid) FROM event_to_state_groups GROUP BY event_id +); From 1bede478430c7a008ddaae195a81eca4ea155630 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 18:47:20 +0100 Subject: [PATCH 052/104] Add beginnings of migration sqlite->maria db script --- scripts/port_to_maria.py | 271 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 271 insertions(+) create mode 100644 scripts/port_to_maria.py diff --git a/scripts/port_to_maria.py b/scripts/port_to_maria.py new file mode 100644 index 000000000..6e0adf703 --- /dev/null +++ b/scripts/port_to_maria.py @@ -0,0 +1,271 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer, reactor +from twisted.enterprise import adbapi + +from synapse.storage._base import LoggingTransaction, SQLBaseStore +from synapse.storage.engines import create_engine + +import argparse +import itertools +import logging +import yaml + + +logger = logging.getLogger("port_to_maria") + + +BINARY_COLUMNS = { + "event_content_hashes": ["hash"], + "event_reference_hashes": ["hash"], + "event_signatures": ["signature"], + "event_edge_hashes": ["hash"], + "events": ["content", "unrecognized_keys"], + "event_json": ["internal_metadata", "json"], + "application_services_txns": ["event_ids"], + "received_transactions": ["response_json"], + "sent_transactions": ["response_json"], + "server_tls_certificates": ["tls_certificate"], + "server_signature_keys": ["verify_key"], + "pushers": ["pushkey", "data"], + "user_filters": ["filter_json"], +} + +UNICODE_COLUMNS = { + "events": ["content", "unrecognized_keys"], + "event_json": ["internal_metadata", "json"], + "users": ["password_hash"], +} + + +class Store(object): + def __init__(self, db_pool, engine): + self.db_pool = db_pool + self.engine = engine + + _simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"] + _simple_insert = SQLBaseStore.__dict__["_simple_insert"] + + _simple_select_onecol_txn = SQLBaseStore.__dict__["_simple_select_onecol_txn"] + _simple_select_onecol = SQLBaseStore.__dict__["_simple_select_onecol"] + + _execute_and_decode = SQLBaseStore.__dict__["_execute_and_decode"] + + def runInteraction(self, desc, func, *args, **kwargs): + def r(conn): + try: + i = 0 + N = 5 + while True: + try: + txn = conn.cursor() + return func( + LoggingTransaction(txn, desc, self.engine), + *args, **kwargs + ) + except self.engine.module.DatabaseError as e: + if self.engine.is_deadlock(e): + logger.warn("[TXN DEADLOCK] {%s} %d/%d", desc, i, N) + if i < N: + i += 1 + conn.rollback() + continue + raise + except Exception as e: + logger.debug("[TXN FAIL] {%s}", desc, e) + raise + + return self.db_pool.runWithConnection(r) + + def insert_many(self, table, headers, rows): + sql = "INSERT INTO %s (%s) VALUES (%s)" % ( + table, + ", ".join(k for k in headers), + ", ".join("%s" for _ in headers) + ) + + def t(txn): + try: + txn.executemany(sql, rows) + except: + logger.exception( + "Failed to insert: %s", + table, + ) + raise + + return self.runInteraction("insert_many", t) + + +def chunks(n): + for i in itertools.count(0, n): + yield range(i, i+n) + + +@defer.inlineCallbacks +def handle_table(table, sqlite_store, mysql_store): + N = 1000 + + select = "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,) + + uni_col_names = UNICODE_COLUMNS.get(table, []) + + def conv_uni(c): + return sqlite_store.engine.load_unicode(c) + + next_chunk = 0 + while True: + def r(txn): + txn.execute(select, (next_chunk, N,)) + rows = txn.fetchall() + headers = [column[0] for column in txn.description] + + return headers, rows + + headers, rows = yield sqlite_store.runInteraction("select", r) + + logger.info("Got %d rows for %s", len(rows), table) + + if rows: + uni_cols = [i for i, h in enumerate(headers) if h in uni_col_names] + next_chunk = rows[-1][0] + 1 + + for i, row in enumerate(rows): + rows[i] = tuple( + mysql_store.engine.encode_parameter( + conv_uni(col) if j in uni_cols else col + ) + for j, col in enumerate(row) + if j > 0 + ) + + yield mysql_store.insert_many(table, headers[1:], rows) + else: + return + + +def setup_db(db_config, database_engine): + db_conn = database_engine.module.connect( + **{ + k: v for k, v in db_config.get("args", {}).items() + if not k.startswith("cp_") + } + ) + + database_engine.prepare_database(db_conn) + + db_conn.commit() + + +@defer.inlineCallbacks +def main(sqlite_config, mysql_config): + try: + sqlite_db_pool = adbapi.ConnectionPool( + sqlite_config["name"], + **sqlite_config["args"] + ) + + mysql_db_pool = adbapi.ConnectionPool( + mysql_config["name"], + **mysql_config["args"] + ) + + sqlite_engine = create_engine("sqlite3") + mysql_engine = create_engine("mysql.connector") + + sqlite_store = Store(sqlite_db_pool, sqlite_engine) + mysql_store = Store(mysql_db_pool, mysql_engine) + + # Step 1. Set up mysql database. + logger.info("Preparing sqlite database...") + setup_db(sqlite_config, sqlite_engine) + + logger.info("Preparing mysql database...") + setup_db(mysql_config, mysql_engine) + + # Step 2. Get tables. + logger.info("Fetching tables...") + tables = yield sqlite_store._simple_select_onecol( + table="sqlite_master", + keyvalues={ + "type": "table", + }, + retcol="name", + ) + + logger.info("Found %d tables", len(tables)) + + # Process tables. + yield defer.gatherResults( + [ + handle_table(table, sqlite_store, mysql_store) + for table in tables + if table not in ["schema_version", "applied_schema_deltas"] + and not table.startswith("sqlite_") + ], + consumeErrors=True, + ) + + # for table in ["current_state_events"]: # tables: + # if table not in ["schema_version", "applied_schema_deltas"]: + # if not table.startswith("sqlite_"): + # yield handle_table(table, sqlite_store, mysql_store) + except: + logger.exception("") + finally: + reactor.stop() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--sqlite-database") + parser.add_argument( + "--mysql-config", type=argparse.FileType('r'), + ) + + args = parser.parse_args() + logging.basicConfig(level=logging.INFO) + + sqlite_config = { + "name": "sqlite3", + "args": { + "database": args.sqlite_database, + "cp_min": 1, + "cp_max": 1, + "check_same_thread": False, + }, + } + + mysql_config = yaml.safe_load(args.mysql_config) + mysql_config["args"].update({ + "sql_mode": "TRADITIONAL", + "charset": "utf8mb4", + "use_unicode": True, + "collation": "utf8mb4_bin", + }) + + import codecs + codecs.register( + lambda name: codecs.lookup('utf8') if name == "utf8mb4" else None + ) + + reactor.callWhenRunning( + main, + sqlite_config=sqlite_config, + mysql_config=mysql_config, + ) + + reactor.run() From 90bcb869579fe9c33c223851fece8bd3eeabf942 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 13 Apr 2015 13:45:04 +0100 Subject: [PATCH 053/104] Support running porting script multiple times --- scripts/port_to_maria.py | 108 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 100 insertions(+), 8 deletions(-) diff --git a/scripts/port_to_maria.py b/scripts/port_to_maria.py index 6e0adf703..b14cca891 100644 --- a/scripts/port_to_maria.py +++ b/scripts/port_to_maria.py @@ -51,16 +51,52 @@ UNICODE_COLUMNS = { } +APPEND_ONLY_TABLES = [ + "event_content_hashes", + "event_reference_hashes", + "event_signatures", + "event_edge_hashes", + "events", + "event_json", + "state_events", + "room_memberships", + "feedback", + "topics", + "room_names", + "rooms", + "local_media_repository", + "local_media_repository_thumbnails", + "remote_media_cache", + "remote_media_cache_thumbnails", + "redactions", + "event_edges", + "event_auth", + "received_transactions", + "sent_transactions", + "transaction_id_to_pdu", + "users", + "state_groups", + "state_groups_state", + "event_to_state_groups", + "rejections", +] + + class Store(object): def __init__(self, db_pool, engine): self.db_pool = db_pool - self.engine = engine + self.database_engine = engine _simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"] _simple_insert = SQLBaseStore.__dict__["_simple_insert"] _simple_select_onecol_txn = SQLBaseStore.__dict__["_simple_select_onecol_txn"] _simple_select_onecol = SQLBaseStore.__dict__["_simple_select_onecol"] + _simple_select_one_onecol = SQLBaseStore.__dict__["_simple_select_one_onecol"] + _simple_select_one_onecol_txn = SQLBaseStore.__dict__["_simple_select_one_onecol_txn"] + + _simple_update_one = SQLBaseStore.__dict__["_simple_update_one"] + _simple_update_one_txn = SQLBaseStore.__dict__["_simple_update_one_txn"] _execute_and_decode = SQLBaseStore.__dict__["_execute_and_decode"] @@ -73,11 +109,11 @@ class Store(object): try: txn = conn.cursor() return func( - LoggingTransaction(txn, desc, self.engine), + LoggingTransaction(txn, desc, self.database_engine), *args, **kwargs ) - except self.engine.module.DatabaseError as e: - if self.engine.is_deadlock(e): + except self.database_engine.module.DatabaseError as e: + if self.database_engine.is_deadlock(e): logger.warn("[TXN DEADLOCK] {%s} %d/%d", desc, i, N) if i < N: i += 1 @@ -117,16 +153,50 @@ def chunks(n): @defer.inlineCallbacks def handle_table(table, sqlite_store, mysql_store): - N = 1000 + if table in APPEND_ONLY_TABLES: + # It's safe to just carry on inserting. + next_chunk = yield mysql_store._simple_select_one_onecol( + table="port_from_sqlite3", + keyvalues={"table_name": table}, + retcol="rowid", + allow_none=True, + ) + + if next_chunk is None: + yield mysql_store._simple_insert( + table="port_from_sqlite3", + values={"table_name": table, "rowid": 0} + ) + + next_chunk = 0 + else: + def delete_all(txn): + txn.execute( + "DELETE FROM port_from_sqlite3 WHERE table_name = %s", + (table,) + ) + txn.execute("DELETE FROM %s" % (table,)) + mysql_store._simple_insert_txn( + txn, + table="port_from_sqlite3", + values={"table_name": table, "rowid": 0} + ) + + yield mysql_store.runInteraction( + "delete_non_append_only", delete_all + ) + + next_chunk = 0 + + N = 5000 select = "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,) uni_col_names = UNICODE_COLUMNS.get(table, []) def conv_uni(c): - return sqlite_store.engine.load_unicode(c) + return sqlite_store.database_engine.load_unicode(c) - next_chunk = 0 while True: def r(txn): txn.execute(select, (next_chunk, N,)) @@ -145,7 +215,7 @@ def handle_table(table, sqlite_store, mysql_store): for i, row in enumerate(rows): rows[i] = tuple( - mysql_store.engine.encode_parameter( + mysql_store.database_engine.encode_parameter( conv_uni(col) if j in uni_cols else col ) for j, col in enumerate(row) @@ -153,6 +223,12 @@ def handle_table(table, sqlite_store, mysql_store): ) yield mysql_store.insert_many(table, headers[1:], rows) + + yield mysql_store._simple_update_one( + table="port_from_sqlite3", + keyvalues={"table_name": table}, + updatevalues={"rowid": next_chunk}, + ) else: return @@ -208,6 +284,22 @@ def main(sqlite_config, mysql_config): logger.info("Found %d tables", len(tables)) + def create_port_table(txn): + try: + txn.execute( + "CREATE TABLE port_from_sqlite3 (" + " `table_name` varchar(100) NOT NULL UNIQUE," + " `rowid` bigint unsigned NOT NULL" + ")" + ) + except mysql_engine.module.DatabaseError as e: + if e.errno != mysql_engine.module.errorcode.ER_TABLE_EXISTS_ERROR: + raise + + yield mysql_store.runInteraction( + "create_port_table", create_port_table + ) + # Process tables. yield defer.gatherResults( [ From 86fc9b617c5eb91186a8466370ddd97f3acd34a2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 13 Apr 2015 17:03:49 +0100 Subject: [PATCH 054/104] For backwards compat, make state_groups.id have a type of int, not varchar --- synapse/storage/schema/full_schemas/16/state.sql | 2 +- synapse/storage/state.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql index 8d6b94a61..8c5161039 100644 --- a/synapse/storage/schema/full_schemas/16/state.sql +++ b/synapse/storage/schema/full_schemas/16/state.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS state_groups( - id VARCHAR(20) PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, room_id VARCHAR(150) NOT NULL, event_id VARCHAR(150) NOT NULL ); diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 4994bacd6..553ba9dd1 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -93,7 +93,7 @@ class StateStore(SQLBaseStore): state_group = context.state_group if not state_group: - state_group = _make_group_id(self._clock) + state_group = self._state_groups_id_gen.get_next_txn(txn) self._simple_insert_txn( txn, table="state_groups", From 3c741682e59a41fcc45a5b9a370c7f268be7729e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Apr 2015 09:54:44 +0100 Subject: [PATCH 055/104] Correctly increment the _next_id initially --- synapse/storage/util/id_generators.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index 8f419323a..2e2a40898 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -55,9 +55,11 @@ class IdGenerator(object): ) val, = txn.fetchone() - self._next_id = val or 2 + cur = val or 0 + cur += 1 + self._next_id = cur + 1 - return 1 + return cur class StreamIdGenerator(object): From 58d83399663a080c123d2f112b4f4d84accbc638 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Apr 2015 13:53:20 +0100 Subject: [PATCH 056/104] Add support for postgres instead of mysql. Change sql accourdingly. blob + varbinary -> bytea. No support for UNSIGNED or CREATE INDEX IF NOT EXISTS. --- synapse/app/homeserver.py | 2 + synapse/storage/__init__.py | 15 +++---- synapse/storage/_base.py | 2 +- synapse/storage/engines/__init__.py | 2 + synapse/storage/event_federation.py | 10 ++--- synapse/storage/events.py | 4 +- synapse/storage/room.py | 34 +++++++++----- .../full_schemas/16/application_services.sql | 10 ++--- .../schema/full_schemas/16/event_edges.sql | 26 +++++------ .../full_schemas/16/event_signatures.sql | 16 +++---- synapse/storage/schema/full_schemas/16/im.sql | 44 +++++++++---------- .../storage/schema/full_schemas/16/keys.sql | 8 ++-- .../full_schemas/16/media_repository.sql | 8 ++-- .../schema/full_schemas/16/presence.sql | 4 +- .../storage/schema/full_schemas/16/push.sql | 30 ++++++------- .../schema/full_schemas/16/redactions.sql | 4 +- .../storage/schema/full_schemas/16/state.sql | 10 ++--- .../schema/full_schemas/16/transactions.sql | 24 +++++----- .../storage/schema/full_schemas/16/users.sql | 10 ++--- synapse/storage/schema/schema_version.sql | 14 +++--- synapse/storage/stream.py | 16 +++---- 21 files changed, 153 insertions(+), 140 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index a47e548d6..033011e1d 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -373,6 +373,8 @@ def setup(config_options): "use_unicode": True, "collation": "utf8mb4_bin", }) + elif name == "psycopg2": + pass elif name == "sqlite3": db_config.setdefault("args", {}).update({ "cp_min": 1, diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index b46cafd25..272420194 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -236,7 +236,7 @@ def _setup_new_database(cur, database_engine): cur.execute( database_engine.convert_param_style( - "REPLACE INTO schema_version (version, upgraded)" + "INSERT INTO schema_version (version, upgraded)" " VALUES (?,?)" ), (max_current_ver, False,) @@ -432,14 +432,11 @@ def executescript(txn, schema_path): def _get_or_create_schema_state(txn, database_engine): - try: - # Bluntly try creating the schema_version tables. - schema_path = os.path.join( - dir_path, "schema", "schema_version.sql", - ) - executescript(txn, schema_path) - except: - pass + # Bluntly try creating the schema_version tables. + schema_path = os.path.join( + dir_path, "schema", "schema_version.sql", + ) + executescript(txn, schema_path) txn.execute("SELECT version, upgraded FROM schema_version") row = txn.fetchone() diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index e30514cd5..fa5199104 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -330,7 +330,7 @@ class SQLBaseStore(object): continue raise except Exception as e: - logger.debug("[TXN FAIL] {%s}", name, e) + logger.debug("[TXN FAIL] {%s} %s", name, e) raise finally: end = time.time() * 1000 diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index 29702be92..548d4e1b4 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -14,6 +14,7 @@ # limitations under the License. from .maria import MariaEngine +from .postgres import PostgresEngine from .sqlite3 import Sqlite3Engine import importlib @@ -22,6 +23,7 @@ import importlib SUPPORTED_MODULE = { "sqlite3": Sqlite3Engine, "mysql.connector": MariaEngine, + "psycopg2": PostgresEngine, } diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 79ad5ddc9..54a3c9d80 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -153,7 +153,7 @@ class EventFederationStore(SQLBaseStore): results = self._get_prev_events_and_state( txn, event_id, - is_state=1, + is_state=True, ) return [(e_id, h, ) for e_id, h, _ in results] @@ -164,7 +164,7 @@ class EventFederationStore(SQLBaseStore): } if is_state is not None: - keyvalues["is_state"] = is_state + keyvalues["is_state"] = bool(is_state) res = self._simple_select_list_txn( txn, @@ -259,7 +259,7 @@ class EventFederationStore(SQLBaseStore): "event_id": event_id, "prev_event_id": e_id, "room_id": room_id, - "is_state": 0, + "is_state": False, }, ) @@ -397,7 +397,7 @@ class EventFederationStore(SQLBaseStore): query = ( "SELECT prev_event_id FROM event_edges " - "WHERE room_id = ? AND event_id = ? AND is_state = 0 " + "WHERE room_id = ? AND event_id = ? AND is_state = ? " "LIMIT ?" ) @@ -406,7 +406,7 @@ class EventFederationStore(SQLBaseStore): for event_id in front: txn.execute( query, - (room_id, event_id, limit - len(event_results)) + (room_id, event_id, False, limit - len(event_results)) ) for e_id, in txn.fetchall(): diff --git a/synapse/storage/events.py b/synapse/storage/events.py index a2e87c27c..9fe2effb4 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -188,12 +188,12 @@ class EventsStore(SQLBaseStore): ) sql = ( - "UPDATE events SET outlier = 0" + "UPDATE events SET outlier = ?" " WHERE event_id = ?" ) txn.execute( sql, - (event.event_id,) + (False, event.event_id,) ) return diff --git a/synapse/storage/room.py b/synapse/storage/room.py index a1a76280f..48ebb3305 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -105,14 +105,12 @@ class RoomStore(SQLBaseStore): # We use non printing ascii character US (\x1F) as a separator sql = ( - "SELECT r.room_id, n.name, t.topic, " - "group_concat(a.room_alias, '\x1F') " - "FROM rooms AS r " - "LEFT JOIN (%(topic)s) AS t ON t.room_id = r.room_id " - "LEFT JOIN (%(name)s) AS n ON n.room_id = r.room_id " - "INNER JOIN room_aliases AS a ON a.room_id = r.room_id " - "WHERE r.is_public = ? " - "GROUP BY r.room_id " + "SELECT r.room_id, max(n.name), max(t.topic)" + " FROM rooms AS r" + " LEFT JOIN (%(topic)s) AS t ON t.room_id = r.room_id" + " LEFT JOIN (%(name)s) AS n ON n.room_id = r.room_id" + " WHERE r.is_public = ?" + " GROUP BY r.room_id" ) % { "topic": topic_subquery, "name": name_subquery, @@ -120,7 +118,22 @@ class RoomStore(SQLBaseStore): txn.execute(sql, (is_public,)) - return txn.fetchall() + rows = txn.fetchall() + + for i, row in enumerate(rows): + room_id = row[0] + aliases = self._simple_select_onecol_txn( + txn, + table="room_aliases", + keyvalues={ + "room_id": room_id + }, + retcol="room_alias", + ) + + rows[i] = list(row) + [aliases] + + return rows rows = yield self.runInteraction( "get_rooms", f @@ -131,9 +144,10 @@ class RoomStore(SQLBaseStore): "room_id": r[0], "name": r[1], "topic": r[2], - "aliases": r[3].split("\x1F"), + "aliases": r[3], } for r in rows + if r[3] # We only return rooms that have at least one alias. ] defer.returnValue(ret) diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql index bc709df92..f08c5bcf7 100644 --- a/synapse/storage/schema/full_schemas/16/application_services.sql +++ b/synapse/storage/schema/full_schemas/16/application_services.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS application_services( - id BIGINT UNSIGNED PRIMARY KEY, + id BIGINT PRIMARY KEY, url VARCHAR(150), token VARCHAR(150), hs_token VARCHAR(150), @@ -23,8 +23,8 @@ CREATE TABLE IF NOT EXISTS application_services( ); CREATE TABLE IF NOT EXISTS application_services_regex( - id BIGINT UNSIGNED PRIMARY KEY, - as_id BIGINT UNSIGNED NOT NULL, + id BIGINT PRIMARY KEY, + as_id BIGINT NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(150), FOREIGN KEY(as_id) REFERENCES application_services(id) @@ -39,10 +39,10 @@ CREATE TABLE IF NOT EXISTS application_services_state( CREATE TABLE IF NOT EXISTS application_services_txns( as_id VARCHAR(150) NOT NULL, txn_id INTEGER NOT NULL, - event_ids LONGBLOB NOT NULL, + event_ids bytea NOT NULL, UNIQUE(as_id, txn_id) ); -CREATE INDEX IF NOT EXISTS application_services_txns_id ON application_services_txns ( +CREATE INDEX application_services_txns_id ON application_services_txns ( as_id ); diff --git a/synapse/storage/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/full_schemas/16/event_edges.sql index bdb110909..05d0874f0 100644 --- a/synapse/storage/schema/full_schemas/16/event_edges.sql +++ b/synapse/storage/schema/full_schemas/16/event_edges.sql @@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities( UNIQUE (event_id, room_id) ); -CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); -CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); +CREATE INDEX ev_extrem_room ON event_forward_extremities(room_id); +CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_backward_extremities( @@ -29,8 +29,8 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities( UNIQUE (event_id, room_id) ); -CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); -CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); +CREATE INDEX ev_b_extrem_room ON event_backward_extremities(room_id); +CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_edges( @@ -41,8 +41,8 @@ CREATE TABLE IF NOT EXISTS event_edges( UNIQUE (event_id, prev_event_id, room_id, is_state) ); -CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); -CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); +CREATE INDEX ev_edges_id ON event_edges(event_id); +CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id); CREATE TABLE IF NOT EXISTS room_depth( @@ -51,17 +51,17 @@ CREATE TABLE IF NOT EXISTS room_depth( UNIQUE (room_id) ); -CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); +CREATE INDEX room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( event_id VARCHAR(150) NOT NULL, destination VARCHAR(150) NOT NULL, - delivered_ts BIGINT UNSIGNED DEFAULT 0, -- or 0 if not delivered + delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ); -CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); +CREATE INDEX event_destinations_id ON event_destinations(event_id); CREATE TABLE IF NOT EXISTS state_forward_extremities( @@ -72,10 +72,10 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities( UNIQUE (event_id, room_id) ); -CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( +CREATE INDEX st_extrem_keys ON state_forward_extremities( room_id, type, state_key ); -CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); +CREATE INDEX st_extrem_id ON state_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_auth( @@ -85,5 +85,5 @@ CREATE TABLE IF NOT EXISTS event_auth( UNIQUE (event_id, auth_id, room_id) ); -CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); -CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); +CREATE INDEX evauth_edges_id ON event_auth(event_id); +CREATE INDEX evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/full_schemas/16/event_signatures.sql index 09886f607..429182736 100644 --- a/synapse/storage/schema/full_schemas/16/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/16/event_signatures.sql @@ -16,40 +16,40 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash LONGBLOB, + hash bytea, UNIQUE (event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); +CREATE INDEX event_content_hashes_id ON event_content_hashes(event_id); CREATE TABLE IF NOT EXISTS event_reference_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash LONGBLOB, + hash bytea, UNIQUE (event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); +CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id); CREATE TABLE IF NOT EXISTS event_signatures ( event_id VARCHAR(150), signature_name VARCHAR(150), key_id VARCHAR(150), - signature LONGBLOB, + signature bytea, UNIQUE (event_id, signature_name, key_id) ); -CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); +CREATE INDEX event_signatures_id ON event_signatures(event_id); CREATE TABLE IF NOT EXISTS event_edge_hashes( event_id VARCHAR(150), prev_event_id VARCHAR(150), algorithm VARCHAR(150), - hash LONGBLOB, + hash bytea, UNIQUE (event_id, prev_event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); +CREATE INDEX event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index 19f0f3414..a661fc160 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -14,33 +14,33 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering BIGINT UNSIGNED PRIMARY KEY, - topological_ordering BIGINT UNSIGNED NOT NULL, + stream_ordering BIGINT PRIMARY KEY, + topological_ordering BIGINT NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - content LONGBLOB NOT NULL, - unrecognized_keys LONGBLOB, + content bytea NOT NULL, + unrecognized_keys bytea, processed BOOL NOT NULL, outlier BOOL NOT NULL, - depth BIGINT UNSIGNED DEFAULT 0 NOT NULL, + depth BIGINT DEFAULT 0 NOT NULL, UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); -CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); -CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); +CREATE INDEX events_stream_ordering ON events (stream_ordering); +CREATE INDEX events_topological_ordering ON events (topological_ordering); +CREATE INDEX events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - internal_metadata LONGBLOB NOT NULL, - json LONGBLOB NOT NULL, + internal_metadata bytea NOT NULL, + json bytea NOT NULL, UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); +CREATE INDEX event_json_room_id ON event_json(room_id); CREATE TABLE IF NOT EXISTS state_events( @@ -52,9 +52,9 @@ CREATE TABLE IF NOT EXISTS state_events( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); -CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); -CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); +CREATE INDEX state_events_room_id ON state_events (room_id); +CREATE INDEX state_events_type ON state_events (type); +CREATE INDEX state_events_state_key ON state_events (state_key); CREATE TABLE IF NOT EXISTS current_state_events( @@ -66,9 +66,9 @@ CREATE TABLE IF NOT EXISTS current_state_events( UNIQUE (room_id, type, state_key) ); -CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); -CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); -CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); +CREATE INDEX current_state_events_room_id ON current_state_events (room_id); +CREATE INDEX current_state_events_type ON current_state_events (type); +CREATE INDEX current_state_events_state_key ON current_state_events (state_key); CREATE TABLE IF NOT EXISTS room_memberships( event_id VARCHAR(150) NOT NULL, @@ -79,8 +79,8 @@ CREATE TABLE IF NOT EXISTS room_memberships( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); -CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); +CREATE INDEX room_memberships_room_id ON room_memberships (room_id); +CREATE INDEX room_memberships_user_id ON room_memberships (user_id); CREATE TABLE IF NOT EXISTS feedback( event_id VARCHAR(150) NOT NULL, @@ -98,7 +98,7 @@ CREATE TABLE IF NOT EXISTS topics( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); +CREATE INDEX topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( event_id VARCHAR(150) NOT NULL, @@ -107,7 +107,7 @@ CREATE TABLE IF NOT EXISTS room_names( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); +CREATE INDEX room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( room_id VARCHAR(150) PRIMARY KEY NOT NULL, @@ -121,4 +121,4 @@ CREATE TABLE IF NOT EXISTS room_hosts( UNIQUE (room_id, host) ); -CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); +CREATE INDEX room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql index 35f141c28..459b51042 100644 --- a/synapse/storage/schema/full_schemas/16/keys.sql +++ b/synapse/storage/schema/full_schemas/16/keys.sql @@ -16,8 +16,8 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( server_name VARCHAR(150), -- Server name. fingerprint VARCHAR(150), -- Certificate fingerprint. from_server VARCHAR(150), -- Which key server the certificate was fetched from. - ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added. - tls_certificate LONGBLOB, -- DER encoded x509 certificate. + ts_added_ms BIGINT, -- When the certifcate was added. + tls_certificate bytea, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(150), -- Server name. key_id VARCHAR(150), -- Key version. from_server VARCHAR(150), -- Which key server the key was fetched form. - ts_added_ms BIGINT UNSIGNED, -- When the key was added. - verify_key LONGBLOB, -- NACL verification key. + ts_added_ms BIGINT, -- When the key was added. + verify_key bytea, -- NACL verification key. UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/16/media_repository.sql b/synapse/storage/schema/full_schemas/16/media_repository.sql index 014bce4ae..0e819fca3 100644 --- a/synapse/storage/schema/full_schemas/16/media_repository.sql +++ b/synapse/storage/schema/full_schemas/16/media_repository.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( media_id VARCHAR(150), -- The id used to refer to the media. media_type VARCHAR(150), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. - created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. + created_ts BIGINT, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. user_id VARCHAR(150), -- The user who uploaded the file. UNIQUE (media_id) @@ -35,14 +35,14 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( ) ); -CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id +CREATE INDEX local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); CREATE TABLE IF NOT EXISTS remote_media_cache ( media_origin VARCHAR(150), -- The remote HS the media came from. media_id VARCHAR(150), -- The id used to refer to the media on that server. media_type VARCHAR(150), -- The MIME-type of the media. - created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. + created_ts BIGINT, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(150), -- The name used to store the media on disk. @@ -64,5 +64,5 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( ) ); -CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id +CREATE INDEX remote_media_cache_thumbnails_media_id ON remote_media_cache_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql index fbe5b0af6..9c41be296 100644 --- a/synapse/storage/schema/full_schemas/16/presence.sql +++ b/synapse/storage/schema/full_schemas/16/presence.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS presence( user_id VARCHAR(150) NOT NULL, state VARCHAR(20), status_msg VARCHAR(150), - mtime BIGINT UNSIGNED, -- miliseconds since last state change + mtime BIGINT, -- miliseconds since last state change UNIQUE (user_id) ); @@ -37,4 +37,4 @@ CREATE TABLE IF NOT EXISTS presence_list( UNIQUE (user_id, observed_user_id) ); -CREATE INDEX IF NOT EXISTS presence_list_user_id ON presence_list (user_id); +CREATE INDEX presence_list_user_id ON presence_list (user_id); diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql index 33300736f..5c0c7bc20 100644 --- a/synapse/storage/schema/full_schemas/16/push.sql +++ b/synapse/storage/schema/full_schemas/16/push.sql @@ -22,52 +22,52 @@ CREATE TABLE IF NOT EXISTS rejections( -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( - id BIGINT UNSIGNED PRIMARY KEY, + id BIGINT PRIMARY KEY, user_name VARCHAR(150) NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, app_id VARCHAR(64) NOT NULL, app_display_name VARCHAR(64) NOT NULL, device_display_name VARCHAR(128) NOT NULL, - pushkey VARBINARY(512) NOT NULL, - ts BIGINT UNSIGNED NOT NULL, + pushkey bytea NOT NULL, + ts BIGINT NOT NULL, lang VARCHAR(8), - data LONGBLOB, + data bytea, last_token TEXT, - last_success BIGINT UNSIGNED, - failing_since BIGINT UNSIGNED, + last_success BIGINT, + failing_since BIGINT, UNIQUE (app_id, pushkey) ); CREATE TABLE IF NOT EXISTS push_rules ( - id BIGINT UNSIGNED PRIMARY KEY, + id BIGINT PRIMARY KEY, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, - priority_class TINYINT NOT NULL, + priority_class SMALLINT NOT NULL, priority INTEGER NOT NULL DEFAULT 0, conditions VARCHAR(150) NOT NULL, actions VARCHAR(150) NOT NULL, UNIQUE(user_name, rule_id) ); -CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); +CREATE INDEX push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(150), - filter_id BIGINT UNSIGNED, - filter_json LONGBLOB + filter_id BIGINT, + filter_json bytea ); -CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( +CREATE INDEX user_filters_by_user_id_filter_id ON user_filters( user_id, filter_id ); CREATE TABLE IF NOT EXISTS push_rules_enable ( - id BIGINT UNSIGNED PRIMARY KEY, + id BIGINT PRIMARY KEY, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, - enabled TINYINT, + enabled SMALLINT, UNIQUE(user_name, rule_id) ); -CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name); +CREATE INDEX push_rules_enable_user_name on push_rules_enable (user_name); diff --git a/synapse/storage/schema/full_schemas/16/redactions.sql b/synapse/storage/schema/full_schemas/16/redactions.sql index b81451eab..492fd2203 100644 --- a/synapse/storage/schema/full_schemas/16/redactions.sql +++ b/synapse/storage/schema/full_schemas/16/redactions.sql @@ -18,5 +18,5 @@ CREATE TABLE IF NOT EXISTS redactions ( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); -CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); +CREATE INDEX redactions_event_id ON redactions (event_id); +CREATE INDEX redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql index 8c5161039..3c54595e6 100644 --- a/synapse/storage/schema/full_schemas/16/state.sql +++ b/synapse/storage/schema/full_schemas/16/state.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS state_groups( - id BIGINT UNSIGNED PRIMARY KEY, + id BIGINT PRIMARY KEY, room_id VARCHAR(150) NOT NULL, event_id VARCHAR(150) NOT NULL ); @@ -33,8 +33,8 @@ CREATE TABLE IF NOT EXISTS event_to_state_groups( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); +CREATE INDEX state_groups_id ON state_groups(id); -CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(state_group); -CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(room_id, type, state_key); -CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id); +CREATE INDEX state_groups_state_id ON state_groups_state(state_group); +CREATE INDEX state_groups_state_tuple ON state_groups_state(room_id, type, state_key); +CREATE INDEX event_to_state_groups_id ON event_to_state_groups(event_id); diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql index f381e6760..bc6406493 100644 --- a/synapse/storage/schema/full_schemas/16/transactions.sql +++ b/synapse/storage/schema/full_schemas/16/transactions.sql @@ -16,32 +16,32 @@ CREATE TABLE IF NOT EXISTS received_transactions( transaction_id VARCHAR(150), origin VARCHAR(150), - ts BIGINT UNSIGNED, + ts BIGINT, response_code INTEGER, - response_json LONGBLOB, - has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx + response_json bytea, + has_been_referenced smallint default 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) ); -CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; +CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; -- Stores what transactions we've sent, what their response was (if we got one) and whether we have -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( - id BIGINT UNSIGNED PRIMARY KEY, -- This is used to apply insertion ordering + id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, - response_json LONGBLOB, - ts BIGINT UNSIGNED + response_json bytea, + ts BIGINT ); -CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); -CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); +CREATE INDEX sent_transaction_dest ON sent_transactions(destination); +CREATE INDEX sent_transaction_txn_id ON sent_transactions(transaction_id); -- So that we can do an efficient look up of all transactions that have yet to be successfully -- sent. -CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code); +CREATE INDEX sent_transaction_sent ON sent_transactions(response_code); -- For sent transactions only. @@ -53,11 +53,11 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( UNIQUE (transaction_id, destination) ); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); +CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(150) PRIMARY KEY, - retry_last_ts BIGINT UNSIGNED, + retry_last_ts BIGINT, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql index d0011c04b..267284d07 100644 --- a/synapse/storage/schema/full_schemas/16/users.sql +++ b/synapse/storage/schema/full_schemas/16/users.sql @@ -15,17 +15,17 @@ CREATE TABLE IF NOT EXISTS users( name VARCHAR(150), password_hash VARCHAR(150), - creation_ts BIGINT UNSIGNED, - admin BOOL DEFAULT 0 NOT NULL, + creation_ts BIGINT, + admin SMALLINT DEFAULT 0 NOT NULL, UNIQUE(name) ); CREATE TABLE IF NOT EXISTS access_tokens( - id BIGINT UNSIGNED PRIMARY KEY, + id BIGINT PRIMARY KEY, user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, - last_used BIGINT UNSIGNED, + last_used BIGINT, UNIQUE(token) ); @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT UNSIGNED NOT NULL + last_seen BIGINT NOT NULL ); CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql index e7fa6fe56..d9494611e 100644 --- a/synapse/storage/schema/schema_version.sql +++ b/synapse/storage/schema/schema_version.sql @@ -14,16 +14,14 @@ */ CREATE TABLE IF NOT EXISTS schema_version( - `Lock` CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row. - `version` INTEGER NOT NULL, - `upgraded` BOOL NOT NULL, -- Whether we reached this version from an upgrade or an initial schema. - CHECK (`Lock`='X') + Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row. + version INTEGER NOT NULL, + upgraded BOOL NOT NULL, -- Whether we reached this version from an upgrade or an initial schema. + CHECK (Lock='X') ); CREATE TABLE IF NOT EXISTS applied_schema_deltas( - `version` INTEGER NOT NULL, - `file` VARCHAR(150) NOT NULL, + version INTEGER NOT NULL, + file VARCHAR(150) NOT NULL, UNIQUE(version, file) ); - -CREATE INDEX IF NOT EXISTS schema_deltas_ver ON applied_schema_deltas(version); diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 57c2e4dfe..df6de7cbc 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -240,7 +240,7 @@ class StreamStore(SQLBaseStore): sql = ( "SELECT e.event_id, e.stream_ordering FROM events AS e WHERE " - "(e.outlier = 0 AND (room_id IN (%(current)s)) OR " + "(e.outlier = ? AND (room_id IN (%(current)s)) OR " "(event_id IN (%(invites)s))) " "AND e.stream_ordering > ? AND e.stream_ordering <= ? " "ORDER BY stream_ordering ASC LIMIT %(limit)d " @@ -251,7 +251,7 @@ class StreamStore(SQLBaseStore): } def f(txn): - txn.execute(sql, (user_id, user_id, from_id.stream, to_id.stream,)) + txn.execute(sql, (False, user_id, user_id, from_id.stream, to_id.stream,)) rows = self.cursor_to_dict(txn) @@ -283,7 +283,7 @@ class StreamStore(SQLBaseStore): # Tokens really represent positions between elements, but we use # the convention of pointing to the event before the gap. Hence # we have a bit of asymmetry when it comes to equalities. - args = [room_id] + args = [False, room_id] if direction == 'b': order = "DESC" bounds = _StreamToken.parse(from_key).upper_bound() @@ -307,7 +307,7 @@ class StreamStore(SQLBaseStore): sql = ( "SELECT * FROM events" - " WHERE outlier = 0 AND room_id = ? AND %(bounds)s" + " WHERE outlier = ? AND room_id = ? AND %(bounds)s" " ORDER BY topological_ordering %(order)s," " stream_ordering %(order)s %(limit)s" ) % { @@ -358,7 +358,7 @@ class StreamStore(SQLBaseStore): sql = ( "SELECT stream_ordering, topological_ordering, event_id" " FROM events" - " WHERE room_id = ? AND stream_ordering <= ? AND outlier = 0" + " WHERE room_id = ? AND stream_ordering <= ? AND outlier = ?" " ORDER BY topological_ordering DESC, stream_ordering DESC" " LIMIT ?" ) @@ -368,17 +368,17 @@ class StreamStore(SQLBaseStore): "SELECT stream_ordering, topological_ordering, event_id" " FROM events" " WHERE room_id = ? AND stream_ordering > ?" - " AND stream_ordering <= ? AND outlier = 0" + " AND stream_ordering <= ? AND outlier = ?" " ORDER BY topological_ordering DESC, stream_ordering DESC" " LIMIT ?" ) def get_recent_events_for_room_txn(txn): if from_token is None: - txn.execute(sql, (room_id, end_token.stream, limit,)) + txn.execute(sql, (room_id, end_token.stream, False, limit,)) else: txn.execute(sql, ( - room_id, from_token.stream, end_token.stream, limit + room_id, from_token.stream, end_token.stream, False, limit )) rows = self.cursor_to_dict(txn) From 5a95cd444297dfc2aab98259d6fc6a07bed837b9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Apr 2015 13:54:09 +0100 Subject: [PATCH 057/104] Rename user_ips.user -> user_id --- synapse/storage/__init__.py | 4 ++-- synapse/storage/schema/delta/16/users.sql | 6 +++--- synapse/storage/schema/full_schemas/16/users.sql | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 272420194..995114e40 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -107,7 +107,7 @@ class DataStore(RoomMemberStore, RoomStore, yield self._simple_upsert( "user_ips", keyvalues={ - "user": user.to_string(), + "user_id": user.to_string(), "access_token": access_token, "ip": ip, "user_agent": user_agent, @@ -122,7 +122,7 @@ class DataStore(RoomMemberStore, RoomStore, def get_user_ip_and_agents(self, user): return self._simple_select_list( table="user_ips", - keyvalues={"user": user.to_string()}, + keyvalues={"user_id": user.to_string()}, retcols=[ "device_id", "access_token", "ip", "user_agent", "last_seen" ], diff --git a/synapse/storage/schema/delta/16/users.sql b/synapse/storage/schema/delta/16/users.sql index f04705cd7..db27bdca0 100644 --- a/synapse/storage/schema/delta/16/users.sql +++ b/synapse/storage/schema/delta/16/users.sql @@ -36,7 +36,7 @@ ALTER TABLE new_users RENAME TO users; -- Remove UNIQUE constraint from `user_ips` table CREATE TABLE IF NOT EXISTS new_user_ips ( - user VARCHAR(150) NOT NULL, + user_id VARCHAR(150) NOT NULL, access_token VARCHAR(150) NOT NULL, device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, @@ -51,6 +51,6 @@ DROP TABLE user_ips; ALTER TABLE new_user_ips RENAME TO user_ips; -CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); -CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); +CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user_id); +CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user_id, access_token, ip); diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql index 267284d07..006b249fc 100644 --- a/synapse/storage/schema/full_schemas/16/users.sql +++ b/synapse/storage/schema/full_schemas/16/users.sql @@ -30,7 +30,7 @@ CREATE TABLE IF NOT EXISTS access_tokens( ); CREATE TABLE IF NOT EXISTS user_ips ( - user VARCHAR(150) NOT NULL, + user_id VARCHAR(150) NOT NULL, access_token VARCHAR(150) NOT NULL, device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, @@ -38,5 +38,5 @@ CREATE TABLE IF NOT EXISTS user_ips ( last_seen BIGINT NOT NULL ); -CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); -CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); +CREATE INDEX user_ips_user ON user_ips(user_id); +CREATE INDEX user_ips_user_ip ON user_ips(user_id, access_token, ip); From 127fad17dda0ba967d7e334e972cbf95bb689bf7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Apr 2015 14:50:29 +0100 Subject: [PATCH 058/104] Add postgres database engine --- synapse/storage/engines/postgres.py | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 synapse/storage/engines/postgres.py diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py new file mode 100644 index 000000000..9910de5dc --- /dev/null +++ b/synapse/storage/engines/postgres.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.storage import prepare_database + + +class PostgresEngine(object): + def __init__(self, database_module): + self.module = database_module + self.module.extensions.register_type(self.module.extensions.UNICODE) + + def convert_param_style(self, sql): + return sql.replace("?", "%s") + + def encode_parameter(self, param): + return param + + def on_new_connection(self, db_conn): + db_conn.set_isolation_level( + self.module.extensions.ISOLATION_LEVEL_REPEATABLE_READ + ) + + def prepare_database(self, db_conn): + prepare_database(db_conn, self) + + def is_deadlock(self, error): + if isinstance(error, self.module.DatabaseError): + return error.pgcode == "40P01" + return False + + def load_unicode(self, v): + return bytes(v).decode("UTF8") From 63677d1f4752fd928fb2b6f1e97c819896bc5323 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 10:23:24 +0100 Subject: [PATCH 059/104] Change port script to work with postgres --- scripts/port_to_maria.py | 114 +++++++++++++++++++++++---------------- 1 file changed, 67 insertions(+), 47 deletions(-) diff --git a/scripts/port_to_maria.py b/scripts/port_to_maria.py index b14cca891..0d7ba9235 100644 --- a/scripts/port_to_maria.py +++ b/scripts/port_to_maria.py @@ -22,6 +22,7 @@ from synapse.storage.engines import create_engine import argparse import itertools import logging +import types import yaml @@ -51,6 +52,14 @@ UNICODE_COLUMNS = { } +BOOLEAN_COLUMNS = { + "events": ["processed", "outlier"], + "rooms": ["is_public"], + "event_edges": ["is_state"], + "presence_list": ["accepted"], +} + + APPEND_ONLY_TABLES = [ "event_content_hashes", "event_reference_hashes", @@ -126,24 +135,22 @@ class Store(object): return self.db_pool.runWithConnection(r) - def insert_many(self, table, headers, rows): + def insert_many_txn(self, txn, table, headers, rows): sql = "INSERT INTO %s (%s) VALUES (%s)" % ( table, ", ".join(k for k in headers), ", ".join("%s" for _ in headers) ) - def t(txn): - try: - txn.executemany(sql, rows) - except: - logger.exception( - "Failed to insert: %s", - table, - ) - raise + try: + txn.executemany(sql, rows) + except: + logger.exception( + "Failed to insert: %s", + table, + ) + raise - return self.runInteraction("insert_many", t) def chunks(n): @@ -175,7 +182,7 @@ def handle_table(table, sqlite_store, mysql_store): "DELETE FROM port_from_sqlite3 WHERE table_name = %s", (table,) ) - txn.execute("DELETE FROM %s" % (table,)) + txn.execute("TRUNCATE %s CASCADE" % (table,)) mysql_store._simple_insert_txn( txn, table="port_from_sqlite3", @@ -188,14 +195,15 @@ def handle_table(table, sqlite_store, mysql_store): next_chunk = 0 + logger.info("next_chunk for %s: %d", table, next_chunk) + N = 5000 select = "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,) uni_col_names = UNICODE_COLUMNS.get(table, []) - - def conv_uni(c): - return sqlite_store.database_engine.load_unicode(c) + bool_col_names = BOOLEAN_COLUMNS.get(table, []) + bin_col_names = BINARY_COLUMNS.get(table, []) while True: def r(txn): @@ -211,24 +219,42 @@ def handle_table(table, sqlite_store, mysql_store): if rows: uni_cols = [i for i, h in enumerate(headers) if h in uni_col_names] + bool_cols = [i for i, h in enumerate(headers) if h in bool_col_names] + bin_cols = [i for i, h in enumerate(headers) if h in bin_col_names] next_chunk = rows[-1][0] + 1 + def conv(j, col): + if j in uni_cols: + col = sqlite_store.database_engine.load_unicode(col) + if j in bool_cols: + return bool(col) + + if j in bin_cols: + if isinstance(col, types.UnicodeType): + col = buffer(col.encode("utf8")) + + return col + for i, row in enumerate(rows): rows[i] = tuple( mysql_store.database_engine.encode_parameter( - conv_uni(col) if j in uni_cols else col + conv(j, col) ) for j, col in enumerate(row) if j > 0 ) - yield mysql_store.insert_many(table, headers[1:], rows) + def ins(txn): + mysql_store.insert_many_txn(txn, table, headers[1:], rows) - yield mysql_store._simple_update_one( - table="port_from_sqlite3", - keyvalues={"table_name": table}, - updatevalues={"rowid": next_chunk}, - ) + mysql_store._simple_update_one_txn( + txn, + table="port_from_sqlite3", + keyvalues={"table_name": table}, + updatevalues={"rowid": next_chunk}, + ) + + yield mysql_store.runInteraction("insert_many", ins) else: return @@ -260,7 +286,7 @@ def main(sqlite_config, mysql_config): ) sqlite_engine = create_engine("sqlite3") - mysql_engine = create_engine("mysql.connector") + mysql_engine = create_engine("psycopg2") sqlite_store = Store(sqlite_db_pool, sqlite_engine) mysql_store = Store(mysql_db_pool, mysql_engine) @@ -285,20 +311,19 @@ def main(sqlite_config, mysql_config): logger.info("Found %d tables", len(tables)) def create_port_table(txn): - try: - txn.execute( - "CREATE TABLE port_from_sqlite3 (" - " `table_name` varchar(100) NOT NULL UNIQUE," - " `rowid` bigint unsigned NOT NULL" - ")" - ) - except mysql_engine.module.DatabaseError as e: - if e.errno != mysql_engine.module.errorcode.ER_TABLE_EXISTS_ERROR: - raise + txn.execute( + "CREATE TABLE port_from_sqlite3 (" + " table_name varchar(100) NOT NULL UNIQUE," + " rowid bigint NOT NULL" + ")" + ) - yield mysql_store.runInteraction( - "create_port_table", create_port_table - ) + try: + yield mysql_store.runInteraction( + "create_port_table", create_port_table + ) + except Exception as e: + logger.info("Failed to create port table: %s", e) # Process tables. yield defer.gatherResults( @@ -342,17 +367,12 @@ if __name__ == "__main__": } mysql_config = yaml.safe_load(args.mysql_config) - mysql_config["args"].update({ - "sql_mode": "TRADITIONAL", - "charset": "utf8mb4", - "use_unicode": True, - "collation": "utf8mb4_bin", - }) - - import codecs - codecs.register( - lambda name: codecs.lookup('utf8') if name == "utf8mb4" else None - ) + # mysql_config["args"].update({ + # "sql_mode": "TRADITIONAL", + # "charset": "utf8mb4", + # "use_unicode": True, + # "collation": "utf8mb4_bin", + # }) reactor.callWhenRunning( main, From c756dfeb143d61410303dbd51e17303d0b08e7d4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 10:23:42 +0100 Subject: [PATCH 060/104] Correctly identify deadlocks --- synapse/storage/engines/postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 9910de5dc..457c1f70a 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -37,7 +37,7 @@ class PostgresEngine(object): def is_deadlock(self, error): if isinstance(error, self.module.DatabaseError): - return error.pgcode == "40P01" + return error.pgcode in ["40001", "40P01"] return False def load_unicode(self, v): From f9e12f79ca48bc87b395537438eaa74fa6d96745 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 10:24:07 +0100 Subject: [PATCH 061/104] Add missing yield in storage func --- synapse/storage/registration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 86fd2600f..8a63fe469 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -39,7 +39,7 @@ class RegistrationStore(SQLBaseStore): """ next_id = yield self._access_tokens_id_gen.get_next() - self._simple_insert( + yield self._simple_insert( "access_tokens", { "id": next_id, From ded41289655d083194061678c24d3e5551e64a7d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 10:24:24 +0100 Subject: [PATCH 062/104] Use True for True rather than 1 --- synapse/storage/events.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 9fe2effb4..0373d152b 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -269,13 +269,15 @@ class EventsStore(SQLBaseStore): "INSERT INTO events" " (stream_ordering, topological_ordering, event_id, type," " room_id, content, processed, outlier, depth)" - " VALUES (%s,?,?,?,?,?,?,?,?)" - ) % (stream_ordering,) + " VALUES (?,?,?,?,?,?,?,?,?)" + ) txn.execute( sql, - (event.depth, event.event_id, event.type, event.room_id, - content, True, outlier, event.depth) + ( + stream_ordering, event.depth, event.event_id, event.type, + event.room_id, content, True, outlier, event.depth + ) ) if context.rejected: @@ -321,7 +323,7 @@ class EventsStore(SQLBaseStore): "event_id": event.event_id, "prev_event_id": e_id, "room_id": event.room_id, - "is_state": 1, + "is_state": True, }, ) From a971fa9d584b35c35b675b65ed86faed77b46cf7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 10:25:43 +0100 Subject: [PATCH 063/104] Use try..finally in contextlib.contextmanager --- synapse/storage/util/id_generators.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index 2e2a40898..e5dec1c94 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -97,9 +97,11 @@ class StreamIdGenerator(object): @contextlib.contextmanager def manager(): - yield next_id - with self._lock: - self._unfinished_ids.remove(next_id) + try: + yield next_id + finally: + with self._lock: + self._unfinished_ids.remove(next_id) return manager() From cce5d057d34ac6fc953d1987bbce3bce2ce42bd8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 11:43:25 +0100 Subject: [PATCH 064/104] Add index on events (topological_ordering, stream_ordering) to help with ORDER BY clauses --- synapse/storage/schema/delta/16/events_order_index.sql | 1 + synapse/storage/schema/full_schemas/16/im.sql | 1 + 2 files changed, 2 insertions(+) create mode 100644 synapse/storage/schema/delta/16/events_order_index.sql diff --git a/synapse/storage/schema/delta/16/events_order_index.sql b/synapse/storage/schema/delta/16/events_order_index.sql new file mode 100644 index 000000000..c01541466 --- /dev/null +++ b/synapse/storage/schema/delta/16/events_order_index.sql @@ -0,0 +1 @@ +CREATE INDEX events_order ON events (topological_ordering, stream_ordering); \ No newline at end of file diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index a661fc160..b8faed506 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -29,6 +29,7 @@ CREATE TABLE IF NOT EXISTS events( CREATE INDEX events_stream_ordering ON events (stream_ordering); CREATE INDEX events_topological_ordering ON events (topological_ordering); +CREATE INDEX events_order ON events (topological_ordering, stream_ordering); CREATE INDEX events_room_id ON events (room_id); From 25d2914fbaa5c3670464af5f9a2430a277fdda1d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 14:09:35 +0100 Subject: [PATCH 065/104] Turn off persistance of sent_transactions.response_json --- synapse/storage/transactions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 4c3dc5866..7e3add528 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -184,7 +184,7 @@ class TransactionStore(SQLBaseStore): }, updatevalues={ "response_code": code, - "response_json": response_json, + "response_json": None, # For now, don't persist response_json } ) From 1ec6fa98c93114f1799a26100a970273e97576bb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 14:17:16 +0100 Subject: [PATCH 066/104] Parellelize initial sync --- synapse/handlers/message.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 7b9685be7..9c2ef9429 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -274,7 +274,8 @@ class MessageHandler(BaseHandler): if limit is None: limit = 10 - for event in room_list: + @defer.inlineCallbacks + def handle_room(event): d = { "room_id": event.room_id, "membership": event.membership, @@ -290,7 +291,7 @@ class MessageHandler(BaseHandler): rooms_ret.append(d) if event.membership != Membership.JOIN: - continue + return try: messages, token = yield self.store.get_recent_events_for_room( event.room_id, @@ -321,6 +322,11 @@ class MessageHandler(BaseHandler): except: logger.exception("Failed to get snapshot") + yield defer.gatherResults( + [handle_room(e) for e in room_list], + consumeErrors=True + ) + ret = { "rooms": rooms_ret, "presence": presence, From bc6cef823f20f3d14c25ee5409fdea44e518087e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 14:21:59 +0100 Subject: [PATCH 067/104] Do more parellelization for initialSync --- synapse/handlers/message.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 9c2ef9429..9667bb867 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -293,10 +293,17 @@ class MessageHandler(BaseHandler): if event.membership != Membership.JOIN: return try: - messages, token = yield self.store.get_recent_events_for_room( - event.room_id, - limit=limit, - end_token=now_token.room_key, + (messages, token), current_state = yield defer.gatherResults( + [ + self.store.get_recent_events_for_room( + event.room_id, + limit=limit, + end_token=now_token.room_key, + ), + self.state_handler.get_current_state( + event.room_id + ), + ] ) start_token = now_token.copy_and_replace("room_key", token[0]) @@ -312,9 +319,6 @@ class MessageHandler(BaseHandler): "end": end_token.to_string(), } - current_state = yield self.state_handler.get_current_state( - event.room_id - ) d["state"] = [ serialize_event(c, time_now, as_client_event) for c in current_state.values() From 4af32a2817f4a71ee960dc4000299de80bb1efb9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 14:51:21 +0100 Subject: [PATCH 068/104] Postgres does not allow you to continue using a cursor after a DB exception has been raised, so move _simple_insert or_ignore flag out of transaction --- synapse/storage/_base.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index fa5199104..c9677710b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -388,6 +388,7 @@ class SQLBaseStore(object): # "Simple" SQL API methods that operate on a single table with no JOINs, # no complex WHERE clauses, just a dict of values for columns. + @defer.inlineCallbacks def _simple_insert(self, table, values, or_ignore=False, desc="_simple_insert"): """Executes an INSERT query on the named table. @@ -396,14 +397,20 @@ class SQLBaseStore(object): table : string giving the table name values : dict of new column names and values for them """ - return self.runInteraction( - desc, - self._simple_insert_txn, table, values, - or_ignore=or_ignore - ) + try: + yield self.runInteraction( + desc, + self._simple_insert_txn, table, values, + or_ignore=or_ignore + ) + except self.database_engine.module.IntegrityError: + # We have to do or_ignore flag at this layer, since we can't reuse + # a cursor after we receive an error from the db. + if not or_ignore: + raise @log_function - def _simple_insert_txn(self, txn, table, values, or_ignore=False): + def _simple_insert_txn(self, txn, table, values): sql = "INSERT INTO %s (%s) VALUES(%s)" % ( table, ", ".join(k for k in values), @@ -415,11 +422,7 @@ class SQLBaseStore(object): sql, values.values(), ) - try: - txn.execute(sql, values.values()) - except self.database_engine.module.IntegrityError: - if not or_ignore: - raise + txn.execute(sql, values.values()) def _simple_upsert(self, table, keyvalues, values, desc="_simple_upsert"): """ From e845434028ff52f25c5b967c2d1dffcb716ed31e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 15:05:45 +0100 Subject: [PATCH 069/104] Remove run_on_reactor()s --- synapse/handlers/_base.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 48816a242..d0c027de3 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -58,8 +58,6 @@ class BaseHandler(object): @defer.inlineCallbacks def _create_new_client_event(self, builder): - yield run_on_reactor() - latest_ret = yield self.store.get_latest_events_in_room( builder.room_id, ) @@ -101,8 +99,6 @@ class BaseHandler(object): @defer.inlineCallbacks def handle_new_client_event(self, event, context, extra_destinations=[], extra_users=[], suppress_auth=False): - yield run_on_reactor() - # We now need to go and hit out to wherever we need to hit out to. if not suppress_auth: From d5ff9effcf59b55009eb8226d5f1bbac5a95bdff Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 15:05:57 +0100 Subject: [PATCH 070/104] Don't wait on federation_handler.handle_new_event --- synapse/handlers/_base.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index d0c027de3..e4471dd9b 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -139,7 +139,9 @@ class BaseHandler(object): ) # Don't block waiting on waking up all the listeners. - d = self.notifier.on_new_room_event(event, extra_users=extra_users) + notify_d = self.notifier.on_new_room_event( + event, extra_users=extra_users + ) def log_failure(f): logger.warn( @@ -147,8 +149,10 @@ class BaseHandler(object): event.event_id, f.value ) - d.addErrback(log_failure) + notify_d.addErrback(log_failure) - yield federation_handler.handle_new_event( + fed_d = federation_handler.handle_new_event( event, destinations=destinations, ) + + fed_d.addErrback(log_failure) From e19f794fee8fe32215a3dcd49ac37ed7b17a4608 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 15:12:57 +0100 Subject: [PATCH 071/104] Change from exception to warn --- synapse/notifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index d750a6fcf..ea854482b 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -98,7 +98,7 @@ class _NotificationListener(object): try: notifier.clock.cancel_call_later(self.timer) except: - logger.exception("Failed to cancel notifier timer") + logger.warn("Failed to cancel notifier timer") class Notifier(object): From a5c72780e6fdf12c003d059f685632541932e3ed Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 15:13:22 +0100 Subject: [PATCH 072/104] Don't pass in removed flag --- synapse/storage/_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index c9677710b..ede25c43b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -401,7 +401,6 @@ class SQLBaseStore(object): yield self.runInteraction( desc, self._simple_insert_txn, table, values, - or_ignore=or_ignore ) except self.database_engine.module.IntegrityError: # We have to do or_ignore flag at this layer, since we can't reuse From 806f380a8b448ce03671f1ce07f7dfcbdc5261e1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 16:07:59 +0100 Subject: [PATCH 073/104] Make LruCache thread safe, as its used for event cache --- synapse/util/lrucache.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/synapse/util/lrucache.py b/synapse/util/lrucache.py index 2f7b615f7..96163c90f 100644 --- a/synapse/util/lrucache.py +++ b/synapse/util/lrucache.py @@ -14,6 +14,10 @@ # limitations under the License. +from functools import wraps +import threading + + class LruCache(object): """Least-recently-used cache.""" # TODO(mjark) Add mutex for linked list for thread safety. @@ -24,6 +28,16 @@ class LruCache(object): PREV, NEXT, KEY, VALUE = 0, 1, 2, 3 + lock = threading.Lock() + + def synchronized(f): + @wraps(f) + def inner(*args, **kwargs): + with lock: + return f(*args, **kwargs) + + return inner + def add_node(key, value): prev_node = list_root next_node = prev_node[NEXT] @@ -51,6 +65,7 @@ class LruCache(object): next_node[PREV] = prev_node cache.pop(node[KEY], None) + @synchronized def cache_get(key, default=None): node = cache.get(key, None) if node is not None: @@ -59,6 +74,7 @@ class LruCache(object): else: return default + @synchronized def cache_set(key, value): node = cache.get(key, None) if node is not None: @@ -69,6 +85,7 @@ class LruCache(object): if len(cache) > max_size: delete_node(list_root[PREV]) + @synchronized def cache_set_default(key, value): node = cache.get(key, None) if node is not None: @@ -79,6 +96,7 @@ class LruCache(object): delete_node(list_root[PREV]) return value + @synchronized def cache_pop(key, default=None): node = cache.get(key, None) if node: @@ -87,9 +105,11 @@ class LruCache(object): else: return default + @synchronized def cache_len(): return len(cache) + @synchronized def cache_contains(key): return key in cache From ed26e4012bb68cbde33f020b2756efe2bc87455a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 16:24:14 +0100 Subject: [PATCH 074/104] pushers table requires a unique id. --- synapse/storage/_base.py | 10 +++++++--- synapse/storage/pusher.py | 4 ++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index ede25c43b..43a923d91 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -260,6 +260,7 @@ class SQLBaseStore(object): self._transaction_id_gen = IdGenerator("sent_transactions", "id", self) self._state_groups_id_gen = IdGenerator("state_groups", "id", self) self._access_tokens_id_gen = IdGenerator("access_tokens", "id", self) + self._pushers_id_gen = IdGenerator("pushers", "id", self) def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -423,20 +424,22 @@ class SQLBaseStore(object): txn.execute(sql, values.values()) - def _simple_upsert(self, table, keyvalues, values, desc="_simple_upsert"): + def _simple_upsert(self, table, keyvalues, values, + insertion_values={}, desc="_simple_upsert"): """ Args: table (str): The table to upsert into keyvalues (dict): The unique key tables and their new values values (dict): The nonunique columns and their new values + insertion_values (dict): key/values to use when inserting Returns: A deferred """ return self.runInteraction( desc, - self._simple_upsert_txn, table, keyvalues, values + self._simple_upsert_txn, table, keyvalues, values, insertion_values, ) - def _simple_upsert_txn(self, txn, table, keyvalues, values): + def _simple_upsert_txn(self, txn, table, keyvalues, values, insertion_values={}): # Try to update sql = "UPDATE %s SET %s WHERE %s" % ( table, @@ -455,6 +458,7 @@ class SQLBaseStore(object): allvalues = {} allvalues.update(keyvalues) allvalues.update(values) + allvalues.update(insertion_values) sql = "INSERT INTO %s (%s) VALUES (%s)" % ( table, diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 000502b4f..e4135ba92 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -99,6 +99,7 @@ class PusherStore(SQLBaseStore): app_display_name, device_display_name, pushkey, pushkey_ts, lang, data): try: + next_id = self._pushers_id_gen.get_next() yield self._simple_upsert( PushersTable.table_name, dict( @@ -115,6 +116,9 @@ class PusherStore(SQLBaseStore): lang=lang, data=data ), + insertion_values=dict( + id=next_id, + ), desc="add_pusher", ) except Exception as e: From 6e91f14d09cf64100dc8fecc83c333f59fe8e77a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 16:25:07 +0100 Subject: [PATCH 075/104] Add missing yield --- synapse/storage/pusher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index e4135ba92..80c3ba154 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -99,7 +99,7 @@ class PusherStore(SQLBaseStore): app_display_name, device_display_name, pushkey, pushkey_ts, lang, data): try: - next_id = self._pushers_id_gen.get_next() + next_id = yield self._pushers_id_gen.get_next() yield self._simple_upsert( PushersTable.table_name, dict( From 5b31afcbd1c2016d03a29169e3fdc22a1adf7f52 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 16:27:04 +0100 Subject: [PATCH 076/104] Remove debug logging --- synapse/storage/_base.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 43a923d91..5ec1d2613 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -847,11 +847,9 @@ class SQLBaseStore(object): sql_getevents_timer.inc_by(curr_time - last_time, desc) return curr_time - logger.debug("Got js: %r", js) d = json.loads(js) start_time = update_counter("decode_json", start_time) - logger.debug("Got internal_metadata: %r", internal_metadata) internal_metadata = json.loads(internal_metadata) start_time = update_counter("decode_internal", start_time) From cf04cedf216a21f36a476703470b7ffe1a727967 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 16:53:47 +0100 Subject: [PATCH 077/104] Change full_schemas/11 to work with postgres --- .../schema/delta/16/events_order_index.sql | 5 +- .../schema/full_schemas/11/event_edges.sql | 26 +++++----- .../full_schemas/11/event_signatures.sql | 16 +++--- synapse/storage/schema/full_schemas/11/im.sql | 50 +++++++++---------- .../storage/schema/full_schemas/11/keys.sql | 8 +-- .../full_schemas/11/media_repository.sql | 6 +-- .../schema/full_schemas/11/presence.sql | 2 +- .../schema/full_schemas/11/redactions.sql | 4 +- .../storage/schema/full_schemas/11/state.sql | 14 +++--- .../schema/full_schemas/11/transactions.sql | 24 ++++----- .../storage/schema/full_schemas/11/users.sql | 10 ++-- synapse/storage/schema/full_schemas/16/im.sql | 5 +- .../storage/schema/full_schemas/16/state.sql | 4 +- 13 files changed, 90 insertions(+), 84 deletions(-) diff --git a/synapse/storage/schema/delta/16/events_order_index.sql b/synapse/storage/schema/delta/16/events_order_index.sql index c01541466..a48f21517 100644 --- a/synapse/storage/schema/delta/16/events_order_index.sql +++ b/synapse/storage/schema/delta/16/events_order_index.sql @@ -1 +1,4 @@ -CREATE INDEX events_order ON events (topological_ordering, stream_ordering); \ No newline at end of file +CREATE INDEX events_order ON events (topological_ordering, stream_ordering); +CREATE INDEX events_order_room ON events ( + room_id, topological_ordering, stream_ordering +); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index bdb110909..05d0874f0 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities( UNIQUE (event_id, room_id) ); -CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); -CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); +CREATE INDEX ev_extrem_room ON event_forward_extremities(room_id); +CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_backward_extremities( @@ -29,8 +29,8 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities( UNIQUE (event_id, room_id) ); -CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); -CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); +CREATE INDEX ev_b_extrem_room ON event_backward_extremities(room_id); +CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_edges( @@ -41,8 +41,8 @@ CREATE TABLE IF NOT EXISTS event_edges( UNIQUE (event_id, prev_event_id, room_id, is_state) ); -CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); -CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); +CREATE INDEX ev_edges_id ON event_edges(event_id); +CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id); CREATE TABLE IF NOT EXISTS room_depth( @@ -51,17 +51,17 @@ CREATE TABLE IF NOT EXISTS room_depth( UNIQUE (room_id) ); -CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); +CREATE INDEX room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( event_id VARCHAR(150) NOT NULL, destination VARCHAR(150) NOT NULL, - delivered_ts BIGINT UNSIGNED DEFAULT 0, -- or 0 if not delivered + delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ); -CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); +CREATE INDEX event_destinations_id ON event_destinations(event_id); CREATE TABLE IF NOT EXISTS state_forward_extremities( @@ -72,10 +72,10 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities( UNIQUE (event_id, room_id) ); -CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( +CREATE INDEX st_extrem_keys ON state_forward_extremities( room_id, type, state_key ); -CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); +CREATE INDEX st_extrem_id ON state_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_auth( @@ -85,5 +85,5 @@ CREATE TABLE IF NOT EXISTS event_auth( UNIQUE (event_id, auth_id, room_id) ); -CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); -CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); +CREATE INDEX evauth_edges_id ON event_auth(event_id); +CREATE INDEX evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index 09886f607..429182736 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -16,40 +16,40 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash LONGBLOB, + hash bytea, UNIQUE (event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); +CREATE INDEX event_content_hashes_id ON event_content_hashes(event_id); CREATE TABLE IF NOT EXISTS event_reference_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash LONGBLOB, + hash bytea, UNIQUE (event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); +CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id); CREATE TABLE IF NOT EXISTS event_signatures ( event_id VARCHAR(150), signature_name VARCHAR(150), key_id VARCHAR(150), - signature LONGBLOB, + signature bytea, UNIQUE (event_id, signature_name, key_id) ); -CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); +CREATE INDEX event_signatures_id ON event_signatures(event_id); CREATE TABLE IF NOT EXISTS event_edge_hashes( event_id VARCHAR(150), prev_event_id VARCHAR(150), algorithm VARCHAR(150), - hash LONGBLOB, + hash bytea, UNIQUE (event_id, prev_event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); +CREATE INDEX event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index 9c47f5174..e9e09214d 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -15,32 +15,32 @@ CREATE TABLE IF NOT EXISTS events( stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, - topological_ordering BIGINT UNSIGNED NOT NULL, + topological_ordering BIGINT NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - content LONGBLOB NOT NULL, - unrecognized_keys LONGBLOB, + content bytea NOT NULL, + unrecognized_keys bytea, processed BOOL NOT NULL, outlier BOOL NOT NULL, - depth BIGINT UNSIGNED DEFAULT 0 NOT NULL, + depth BIGINT DEFAULT 0 NOT NULL, UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); -CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); -CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); +CREATE INDEX events_stream_ordering ON events (stream_ordering); +CREATE INDEX events_topological_ordering ON events (topological_ordering); +CREATE INDEX events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - internal_metadata LONGBLOB NOT NULL, - json LONGBLOB NOT NULL, + internal_metadata bytea NOT NULL, + json bytea NOT NULL, UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); +CREATE INDEX event_json_room_id ON event_json(room_id); CREATE TABLE IF NOT EXISTS state_events( @@ -52,9 +52,9 @@ CREATE TABLE IF NOT EXISTS state_events( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); -CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); -CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); +CREATE INDEX state_events_room_id ON state_events (room_id); +CREATE INDEX state_events_type ON state_events (type); +CREATE INDEX state_events_state_key ON state_events (state_key); CREATE TABLE IF NOT EXISTS current_state_events( @@ -65,10 +65,10 @@ CREATE TABLE IF NOT EXISTS current_state_events( UNIQUE (room_id, type, state_key) ); -CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id); -CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); -CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); -CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); +CREATE INDEX curr_events_event_id ON current_state_events (event_id); +CREATE INDEX current_state_events_room_id ON current_state_events (room_id); +CREATE INDEX current_state_events_type ON current_state_events (type); +CREATE INDEX current_state_events_state_key ON current_state_events (state_key); CREATE TABLE IF NOT EXISTS room_memberships( event_id VARCHAR(150) NOT NULL, @@ -78,9 +78,9 @@ CREATE TABLE IF NOT EXISTS room_memberships( membership VARCHAR(150) NOT NULL ); -CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id); -CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); -CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); +CREATE INDEX room_memberships_event_id ON room_memberships (event_id); +CREATE INDEX room_memberships_room_id ON room_memberships (room_id); +CREATE INDEX room_memberships_user_id ON room_memberships (user_id); CREATE TABLE IF NOT EXISTS feedback( event_id VARCHAR(150) NOT NULL, @@ -96,8 +96,8 @@ CREATE TABLE IF NOT EXISTS topics( topic TEXT NOT NULL ); -CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id); -CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); +CREATE INDEX topics_event_id ON topics(event_id); +CREATE INDEX topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( event_id VARCHAR(150) NOT NULL, @@ -105,8 +105,8 @@ CREATE TABLE IF NOT EXISTS room_names( name TEXT NOT NULL ); -CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id); -CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); +CREATE INDEX room_names_event_id ON room_names(event_id); +CREATE INDEX room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( room_id VARCHAR(150) PRIMARY KEY NOT NULL, @@ -120,4 +120,4 @@ CREATE TABLE IF NOT EXISTS room_hosts( UNIQUE (room_id, host) ); -CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); +CREATE INDEX room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index 35f141c28..459b51042 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -16,8 +16,8 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( server_name VARCHAR(150), -- Server name. fingerprint VARCHAR(150), -- Certificate fingerprint. from_server VARCHAR(150), -- Which key server the certificate was fetched from. - ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added. - tls_certificate LONGBLOB, -- DER encoded x509 certificate. + ts_added_ms BIGINT, -- When the certifcate was added. + tls_certificate bytea, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(150), -- Server name. key_id VARCHAR(150), -- Key version. from_server VARCHAR(150), -- Which key server the key was fetched form. - ts_added_ms BIGINT UNSIGNED, -- When the key was added. - verify_key LONGBLOB, -- NACL verification key. + ts_added_ms BIGINT, -- When the key was added. + verify_key bytea, -- NACL verification key. UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index 134e7fbce..6e0ee0db4 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( media_id VARCHAR(150), -- The id used to refer to the media. media_type VARCHAR(150), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. - created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. + created_ts BIGINT, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. user_id VARCHAR(150), -- The user who uploaded the file. UNIQUE (media_id) @@ -35,14 +35,14 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( ) ); -CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id +CREATE INDEX local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); CREATE TABLE IF NOT EXISTS remote_media_cache ( media_origin VARCHAR(150), -- The remote HS the media came from. media_id VARCHAR(150), -- The id used to refer to the media on that server. media_type VARCHAR(150), -- The MIME-type of the media. - created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. + created_ts BIGINT, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(150), -- The name used to store the media on disk. diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index c617ebea7..fce324b89 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS presence( user_id VARCHAR(150) NOT NULL, state VARCHAR(20), status_msg VARCHAR(150), - mtime BIGINT UNSIGNED -- miliseconds since last state change + mtime BIGINT -- miliseconds since last state change ); -- For each of /my/ users which possibly-remote users are allowed to see their diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index b81451eab..492fd2203 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -18,5 +18,5 @@ CREATE TABLE IF NOT EXISTS redactions ( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); -CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); +CREATE INDEX redactions_event_id ON redactions (event_id); +CREATE INDEX redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index 737c3e35c..62c20819f 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -14,13 +14,13 @@ */ CREATE TABLE IF NOT EXISTS state_groups( - id VARCHAR(20) PRIMARY KEY, + id INTEGER PRIMARY KEY, room_id VARCHAR(150) NOT NULL, event_id VARCHAR(150) NOT NULL ); CREATE TABLE IF NOT EXISTS state_groups_state( - state_group VARCHAR(20) NOT NULL, + state_group INTEGER NOT NULL, room_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, state_key VARCHAR(150) NOT NULL, @@ -29,12 +29,12 @@ CREATE TABLE IF NOT EXISTS state_groups_state( CREATE TABLE IF NOT EXISTS event_to_state_groups( event_id VARCHAR(150) NOT NULL, - state_group VARCHAR(150) NOT NULL, + state_group INTEGER NOT NULL, UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); +CREATE INDEX state_groups_id ON state_groups(id); -CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(state_group); -CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(room_id, type, state_key); -CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id); \ No newline at end of file +CREATE INDEX state_groups_state_id ON state_groups_state(state_group); +CREATE INDEX state_groups_state_tuple ON state_groups_state(room_id, type, state_key); +CREATE INDEX event_to_state_groups_id ON event_to_state_groups(event_id); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index c2fab10aa..f01d9eba3 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -16,14 +16,14 @@ CREATE TABLE IF NOT EXISTS received_transactions( transaction_id VARCHAR(150), origin VARCHAR(150), - ts BIGINT UNSIGNED, + ts BIGINT, response_code INTEGER, - response_json LONGBLOB, - has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx + response_json bytea, + has_been_referenced SMALLINT DEFAULT 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) ); -CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; +CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; -- Stores what transactions we've sent, what their response was (if we got one) and whether we have @@ -33,15 +33,15 @@ CREATE TABLE IF NOT EXISTS sent_transactions( transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, - response_json LONGBLOB, - ts BIGINT UNSIGNED + response_json bytea, + ts BIGINT ); -CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); -CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); +CREATE INDEX sent_transaction_dest ON sent_transactions(destination); +CREATE INDEX sent_transaction_txn_id ON sent_transactions(transaction_id); -- So that we can do an efficient look up of all transactions that have yet to be successfully -- sent. -CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code); +CREATE INDEX sent_transaction_sent ON sent_transactions(response_code); -- For sent transactions only. @@ -52,12 +52,12 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( pdu_origin VARCHAR(150) ); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); +CREATE INDEX transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination); +CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(150) PRIMARY KEY, - retry_last_ts BIGINT UNSIGNED, + retry_last_ts BIGINT, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 0ddfccd41..2edc13c3b 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -16,8 +16,8 @@ CREATE TABLE IF NOT EXISTS users( id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(150), password_hash VARCHAR(150), - creation_ts BIGINT UNSIGNED, - admin BOOL DEFAULT 0 NOT NULL, + creation_ts BIGINT, + admin SMALLINT DEFAULT 0 NOT NULL, UNIQUE(name) ); @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS access_tokens( user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, - last_used BIGINT UNSIGNED, + last_used BIGINT, UNIQUE(token) ); @@ -36,8 +36,8 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT UNSIGNED NOT NULL, + last_seen BIGINT NOT NULL, UNIQUE (user, access_token, ip, user_agent) ); -CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); +CREATE INDEX user_ips_user ON user_ips(user); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index b8faed506..17e4c949b 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering BIGINT PRIMARY KEY, + stream_ordering INTEGER PRIMARY KEY, topological_ordering BIGINT NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, @@ -31,6 +31,9 @@ CREATE INDEX events_stream_ordering ON events (stream_ordering); CREATE INDEX events_topological_ordering ON events (topological_ordering); CREATE INDEX events_order ON events (topological_ordering, stream_ordering); CREATE INDEX events_room_id ON events (room_id); +CREATE INDEX events_order_room ON events ( + room_id, topological_ordering, stream_ordering +); CREATE TABLE IF NOT EXISTS event_json( diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql index 3c54595e6..b0cd5ee75 100644 --- a/synapse/storage/schema/full_schemas/16/state.sql +++ b/synapse/storage/schema/full_schemas/16/state.sql @@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS state_groups( ); CREATE TABLE IF NOT EXISTS state_groups_state( - state_group VARCHAR(20) NOT NULL, + state_group BIGINT NOT NULL, room_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, state_key VARCHAR(150) NOT NULL, @@ -29,7 +29,7 @@ CREATE TABLE IF NOT EXISTS state_groups_state( CREATE TABLE IF NOT EXISTS event_to_state_groups( event_id VARCHAR(150) NOT NULL, - state_group VARCHAR(150) NOT NULL, + state_group BIGINT NOT NULL, UNIQUE (event_id) ); From ffad75bd6284873c27efb2cfdfdcf9f909eb9db3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 17:00:50 +0100 Subject: [PATCH 078/104] Remove mysql/maria support --- ...ria.py => port_from_sqlite_to_postgres.py} | 58 ++++++++----------- synapse/app/homeserver.py | 9 +-- synapse/storage/engines/__init__.py | 2 - synapse/storage/engines/maria.py | 50 ---------------- .../schema/delta/16/unique_constraints.sql | 2 +- 5 files changed, 26 insertions(+), 95 deletions(-) rename scripts/{port_to_maria.py => port_from_sqlite_to_postgres.py} (85%) delete mode 100644 synapse/storage/engines/maria.py diff --git a/scripts/port_to_maria.py b/scripts/port_from_sqlite_to_postgres.py similarity index 85% rename from scripts/port_to_maria.py rename to scripts/port_from_sqlite_to_postgres.py index 0d7ba9235..4b3fd9e52 100644 --- a/scripts/port_to_maria.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -26,7 +26,7 @@ import types import yaml -logger = logging.getLogger("port_to_maria") +logger = logging.getLogger("port_from_sqlite_to_postgres") BINARY_COLUMNS = { @@ -159,10 +159,10 @@ def chunks(n): @defer.inlineCallbacks -def handle_table(table, sqlite_store, mysql_store): +def handle_table(table, sqlite_store, postgres_store): if table in APPEND_ONLY_TABLES: # It's safe to just carry on inserting. - next_chunk = yield mysql_store._simple_select_one_onecol( + next_chunk = yield postgres_store._simple_select_one_onecol( table="port_from_sqlite3", keyvalues={"table_name": table}, retcol="rowid", @@ -170,7 +170,7 @@ def handle_table(table, sqlite_store, mysql_store): ) if next_chunk is None: - yield mysql_store._simple_insert( + yield postgres_store._simple_insert( table="port_from_sqlite3", values={"table_name": table, "rowid": 0} ) @@ -183,13 +183,13 @@ def handle_table(table, sqlite_store, mysql_store): (table,) ) txn.execute("TRUNCATE %s CASCADE" % (table,)) - mysql_store._simple_insert_txn( + postgres_store._simple_insert_txn( txn, table="port_from_sqlite3", values={"table_name": table, "rowid": 0} ) - yield mysql_store.runInteraction( + yield postgres_store.runInteraction( "delete_non_append_only", delete_all ) @@ -237,7 +237,7 @@ def handle_table(table, sqlite_store, mysql_store): for i, row in enumerate(rows): rows[i] = tuple( - mysql_store.database_engine.encode_parameter( + postgres_store.database_engine.encode_parameter( conv(j, col) ) for j, col in enumerate(row) @@ -245,16 +245,16 @@ def handle_table(table, sqlite_store, mysql_store): ) def ins(txn): - mysql_store.insert_many_txn(txn, table, headers[1:], rows) + postgres_store.insert_many_txn(txn, table, headers[1:], rows) - mysql_store._simple_update_one_txn( + postgres_store._simple_update_one_txn( txn, table="port_from_sqlite3", keyvalues={"table_name": table}, updatevalues={"rowid": next_chunk}, ) - yield mysql_store.runInteraction("insert_many", ins) + yield postgres_store.runInteraction("insert_many", ins) else: return @@ -273,30 +273,30 @@ def setup_db(db_config, database_engine): @defer.inlineCallbacks -def main(sqlite_config, mysql_config): +def main(sqlite_config, postgress_config): try: sqlite_db_pool = adbapi.ConnectionPool( sqlite_config["name"], **sqlite_config["args"] ) - mysql_db_pool = adbapi.ConnectionPool( - mysql_config["name"], - **mysql_config["args"] + postgres_db_pool = adbapi.ConnectionPool( + postgress_config["name"], + **postgress_config["args"] ) sqlite_engine = create_engine("sqlite3") - mysql_engine = create_engine("psycopg2") + postgres_engine = create_engine("psycopg2") sqlite_store = Store(sqlite_db_pool, sqlite_engine) - mysql_store = Store(mysql_db_pool, mysql_engine) + postgres_store = Store(postgres_db_pool, postgres_engine) - # Step 1. Set up mysql database. + # Step 1. Set up databases. logger.info("Preparing sqlite database...") setup_db(sqlite_config, sqlite_engine) - logger.info("Preparing mysql database...") - setup_db(mysql_config, mysql_engine) + logger.info("Preparing postgres database...") + setup_db(postgress_config, postgres_engine) # Step 2. Get tables. logger.info("Fetching tables...") @@ -319,7 +319,7 @@ def main(sqlite_config, mysql_config): ) try: - yield mysql_store.runInteraction( + yield postgres_store.runInteraction( "create_port_table", create_port_table ) except Exception as e: @@ -328,7 +328,7 @@ def main(sqlite_config, mysql_config): # Process tables. yield defer.gatherResults( [ - handle_table(table, sqlite_store, mysql_store) + handle_table(table, sqlite_store, postgres_store) for table in tables if table not in ["schema_version", "applied_schema_deltas"] and not table.startswith("sqlite_") @@ -336,10 +336,6 @@ def main(sqlite_config, mysql_config): consumeErrors=True, ) - # for table in ["current_state_events"]: # tables: - # if table not in ["schema_version", "applied_schema_deltas"]: - # if not table.startswith("sqlite_"): - # yield handle_table(table, sqlite_store, mysql_store) except: logger.exception("") finally: @@ -350,7 +346,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--sqlite-database") parser.add_argument( - "--mysql-config", type=argparse.FileType('r'), + "--postgres-config", type=argparse.FileType('r'), ) args = parser.parse_args() @@ -366,18 +362,12 @@ if __name__ == "__main__": }, } - mysql_config = yaml.safe_load(args.mysql_config) - # mysql_config["args"].update({ - # "sql_mode": "TRADITIONAL", - # "charset": "utf8mb4", - # "use_unicode": True, - # "collation": "utf8mb4_bin", - # }) + postgres_config = yaml.safe_load(args.postgres_config) reactor.callWhenRunning( main, sqlite_config=sqlite_config, - mysql_config=mysql_config, + postgres_config=postgres_config, ) reactor.run() diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f8a33120b..93500dd79 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -366,14 +366,7 @@ def setup(config_options): } name = db_config.get("name", None) - if name in ["MySQLdb", "mysql.connector"]: - db_config.setdefault("args", {}).update({ - "sql_mode": "TRADITIONAL", - "charset": "utf8mb4", - "use_unicode": True, - "collation": "utf8mb4_bin", - }) - elif name == "psycopg2": + if name == "psycopg2": pass elif name == "sqlite3": db_config.setdefault("args", {}).update({ diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index 548d4e1b4..eb76df7f0 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .maria import MariaEngine from .postgres import PostgresEngine from .sqlite3 import Sqlite3Engine @@ -22,7 +21,6 @@ import importlib SUPPORTED_MODULE = { "sqlite3": Sqlite3Engine, - "mysql.connector": MariaEngine, "psycopg2": PostgresEngine, } diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py deleted file mode 100644 index 90165f684..000000000 --- a/synapse/storage/engines/maria.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from synapse.storage import prepare_database - -import types - - -class MariaEngine(object): - def __init__(self, database_module): - self.module = database_module - - def convert_param_style(self, sql): - return sql.replace("?", "%s") - - def encode_parameter(self, param): - if isinstance(param, types.BufferType): - return bytes(param) - return param - - def on_new_connection(self, db_conn): - pass - - def prepare_database(self, db_conn): - cur = db_conn.cursor() - cur.execute( - "ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_bin" - ) - db_conn.commit() - prepare_database(db_conn, self) - - def is_deadlock(self, error): - if isinstance(error, self.module.DatabaseError): - return error.sqlstate == "40001" and error.errno == 1213 - return False - - def load_unicode(self, v): - return bytes(v).decode("UTF8") diff --git a/synapse/storage/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/delta/16/unique_constraints.sql index f9fbb6b44..3604ea842 100644 --- a/synapse/storage/schema/delta/16/unique_constraints.sql +++ b/synapse/storage/schema/delta/16/unique_constraints.sql @@ -1,5 +1,5 @@ --- We can use SQLite features here, since mysql support was only added in v16 +-- We can use SQLite features here, since other db support was only added in v16 -- DELETE FROM current_state_events WHERE rowid not in ( From bc9e69e16099142acdb06c3ea7c7ef9aec9e4ec9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 16 Apr 2015 11:01:09 +0100 Subject: [PATCH 079/104] Move encoding and decoding of JSON into storage layer --- synapse/push/pusherpool.py | 8 +------- synapse/storage/pusher.py | 26 ++++++-------------------- 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 90babd722..7f3dd50b4 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -19,10 +19,7 @@ from twisted.internet import defer from httppusher import HttpPusher from synapse.push import PusherConfigException -from syutil.jsonutil import encode_canonical_json - import logging -import simplejson as json logger = logging.getLogger(__name__) @@ -52,8 +49,6 @@ class PusherPool: @defer.inlineCallbacks def start(self): pushers = yield self.store.get_all_pushers() - for p in pushers: - p['data'] = json.loads(p['data']) self._start_pushers(pushers) @defer.inlineCallbacks @@ -98,7 +93,7 @@ class PusherPool: pushkey=pushkey, pushkey_ts=self.hs.get_clock().time_msec(), lang=lang, - data=encode_canonical_json(data).decode("UTF-8"), + data=data, ) self._refresh_pusher((app_id, pushkey)) @@ -129,7 +124,6 @@ class PusherPool: p = yield self.store.get_pushers_by_app_id_and_pushkey( app_id_pushkey ) - p['data'] = json.loads(p['data']) self._start_pushers([p]) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 80c3ba154..5c1c3d32e 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -20,7 +20,10 @@ from twisted.internet import defer from synapse.api.errors import StoreError +from syutil.jsonutil import encode_canonical_json + import logging +import simplejson as json logger = logging.getLogger(__name__) @@ -52,7 +55,7 @@ class PusherStore(SQLBaseStore): "device_display_name": r[6], "pushkey": r[7], "pushkey_ts": r[8], - "data": r[9], + "data": json.loads(r[9]), "last_token": r[10], "last_success": r[11], "failing_since": r[12] @@ -84,7 +87,7 @@ class PusherStore(SQLBaseStore): "device_display_name": r[6], "pushkey": r[7], "pushkey_ts": r[8], - "data": r[9], + "data": json.loads(r[9]), "last_token": r[10], "last_success": r[11], "failing_since": r[12] @@ -114,7 +117,7 @@ class PusherStore(SQLBaseStore): device_display_name=device_display_name, ts=pushkey_ts, lang=lang, - data=data + data=encode_canonical_json(data).decode("UTF-8"), ), insertion_values=dict( id=next_id, @@ -165,20 +168,3 @@ class PusherStore(SQLBaseStore): class PushersTable(Table): table_name = "pushers" - fields = [ - "id", - "user_name", - "kind", - "profile_tag", - "app_id", - "app_display_name", - "device_display_name", - "pushkey", - "pushkey_ts", - "data", - "last_token", - "last_success", - "failing_since" - ] - - EntryType = collections.namedtuple("PusherEntry", fields) From b8092fbc82edb3c7d8aa09f0756fa853ad6a6ad8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 16 Apr 2015 11:17:52 +0100 Subject: [PATCH 080/104] Go back to storing JSON in TEXT --- synapse/storage/_base.py | 3 --- synapse/storage/appservice.py | 4 +--- synapse/storage/engines/postgres.py | 3 --- synapse/storage/engines/sqlite3.py | 5 ----- synapse/storage/events.py | 16 ++++++++-------- synapse/storage/profile.py | 5 +---- synapse/storage/registration.py | 5 ----- .../storage/schema/delta/15/appservice_txns.sql | 2 +- synapse/storage/schema/full_schemas/11/im.sql | 8 ++++---- .../full_schemas/16/application_services.sql | 2 +- synapse/storage/schema/full_schemas/16/im.sql | 8 ++++---- 11 files changed, 20 insertions(+), 41 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 5ec1d2613..f5952d1fc 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -818,9 +818,6 @@ class SQLBaseStore(object): internal_metadata, js, redacted, rejected_reason = res - internal_metadata = self.database_engine.load_unicode(internal_metadata) - js = self.database_engine.load_unicode(js) - start_time = update_counter("select_event", start_time) result = self._get_event_from_row_txn( diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 40e05b363..63d1af4e8 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -366,9 +366,7 @@ class ApplicationServiceTransactionStore(SQLBaseStore): new_txn_id = max(highest_txn_id, last_txn_id) + 1 # Insert new txn into txn table - event_ids = buffer( - json.dumps([e.event_id for e in events]).encode("utf8") - ) + event_ids = json.dumps([e.event_id for e in events]) txn.execute( "INSERT INTO application_services_txns(as_id, txn_id, event_ids) " "VALUES(?,?,?)", diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 457c1f70a..6f75245fa 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -39,6 +39,3 @@ class PostgresEngine(object): if isinstance(error, self.module.DatabaseError): return error.pgcode in ["40001", "40P01"] return False - - def load_unicode(self, v): - return bytes(v).decode("UTF8") diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 389df35eb..dd0d8e0e0 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -37,8 +37,3 @@ class Sqlite3Engine(object): def is_deadlock(self, error): return False - - def load_unicode(self, v): - if isinstance(v, types.UnicodeType): - return v - return bytes(v).decode("UTF8") diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 0373d152b..7dbf7a396 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -168,7 +168,7 @@ class EventsStore(SQLBaseStore): metadata_json = encode_canonical_json( event.internal_metadata.get_dict() - ) + ).decode("UTF-8") # If we have already persisted this event, we don't need to do any # more processing. @@ -184,7 +184,7 @@ class EventsStore(SQLBaseStore): ) txn.execute( sql, - (buffer(metadata_json), event.event_id,) + (metadata_json, event.event_id,) ) sql = ( @@ -229,14 +229,14 @@ class EventsStore(SQLBaseStore): values={ "event_id": event.event_id, "room_id": event.room_id, - "internal_metadata": buffer(metadata_json), - "json": buffer(encode_canonical_json(event_dict)), + "internal_metadata": metadata_json, + "json": encode_canonical_json(event_dict).decode("UTF-8"), }, ) - content = buffer(encode_canonical_json( + content = encode_canonical_json( event.content - )) + ).decode("UTF-8") vals = { "topological_ordering": event.depth, @@ -261,9 +261,9 @@ class EventsStore(SQLBaseStore): ] } - vals["unrecognized_keys"] = buffer(encode_canonical_json( + vals["unrecognized_keys"] = encode_canonical_json( unrec - )) + ).decode("UTF-8") sql = ( "INSERT INTO events" diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py index e33963d0b..047698aa1 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py @@ -35,16 +35,13 @@ class ProfileStore(SQLBaseStore): desc="get_profile_displayname", ) - if name: - name = self.database_engine.load_unicode(name) - defer.returnValue(name) def set_profile_displayname(self, user_localpart, new_displayname): return self._simple_update_one( table="profiles", keyvalues={"user_id": user_localpart}, - updatevalues={"displayname": new_displayname.encode("utf8")}, + updatevalues={"displayname": new_displayname}, desc="set_profile_displayname", ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 8a63fe469..2a5c5080e 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -98,11 +98,6 @@ class RegistrationStore(SQLBaseStore): allow_none=True, ) - if user_info: - user_info["password_hash"] = self.database_engine.load_unicode( - user_info["password_hash"] - ) - defer.returnValue(user_info) @cached() diff --git a/synapse/storage/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/delta/15/appservice_txns.sql index ddea8fc69..1c3324f41 100644 --- a/synapse/storage/schema/delta/15/appservice_txns.sql +++ b/synapse/storage/schema/delta/15/appservice_txns.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS application_services_state( CREATE TABLE IF NOT EXISTS application_services_txns( as_id VARCHAR(150) NOT NULL, txn_id INTEGER NOT NULL, - event_ids LONGBLOB NOT NULL, + event_ids TEXT NOT NULL, UNIQUE(as_id, txn_id) ); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index e9e09214d..addbec588 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS events( event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - content bytea NOT NULL, - unrecognized_keys bytea, + content TEXT NOT NULL, + unrecognized_keys TEXT, processed BOOL NOT NULL, outlier BOOL NOT NULL, depth BIGINT DEFAULT 0 NOT NULL, @@ -35,8 +35,8 @@ CREATE INDEX events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - internal_metadata bytea NOT NULL, - json bytea NOT NULL, + internal_metadata TEXT NOT NULL, + json TEXT NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql index f08c5bcf7..5d63d57d5 100644 --- a/synapse/storage/schema/full_schemas/16/application_services.sql +++ b/synapse/storage/schema/full_schemas/16/application_services.sql @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS application_services_state( CREATE TABLE IF NOT EXISTS application_services_txns( as_id VARCHAR(150) NOT NULL, txn_id INTEGER NOT NULL, - event_ids bytea NOT NULL, + event_ids TEXT NOT NULL, UNIQUE(as_id, txn_id) ); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index 17e4c949b..5b4b49448 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS events( event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - content bytea NOT NULL, - unrecognized_keys bytea, + content TEXT NOT NULL, + unrecognized_keys TEXT, processed BOOL NOT NULL, outlier BOOL NOT NULL, depth BIGINT DEFAULT 0 NOT NULL, @@ -39,8 +39,8 @@ CREATE INDEX events_order_room ON events ( CREATE TABLE IF NOT EXISTS event_json( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - internal_metadata bytea NOT NULL, - json bytea NOT NULL, + internal_metadata TEXT NOT NULL, + json TEXT NOT NULL, UNIQUE (event_id) ); From e1c0970c116fe3700fc80401dc50aeb9d52c45a6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 16 Apr 2015 11:18:45 +0100 Subject: [PATCH 081/104] PEP8 --- synapse/handlers/_base.py | 1 - synapse/storage/engines/sqlite3.py | 2 -- synapse/storage/pusher.py | 3 --- 3 files changed, 6 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index e4471dd9b..dffb033fb 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -16,7 +16,6 @@ from twisted.internet import defer from synapse.api.errors import LimitExceededError, SynapseError -from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.api.constants import Membership, EventTypes from synapse.types import UserID diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index dd0d8e0e0..72c11df46 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -15,8 +15,6 @@ from synapse.storage import prepare_database, prepare_sqlite3_database -import types - class Sqlite3Engine(object): def __init__(self, database_module): diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 5c1c3d32e..a44bccdca 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections - from ._base import SQLBaseStore, Table from twisted.internet import defer @@ -167,4 +165,3 @@ class PusherStore(SQLBaseStore): class PushersTable(Table): table_name = "pushers" - From 0ec346d942450ac4a29c74d115428fd856af94b2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 11:10:20 +0100 Subject: [PATCH 082/104] Add unique index to room_aliases, remove duplicates on upgrade. Convert some columns back to TEXT from bytea --- synapse/storage/schema/delta/16/room_alias_index.sql | 3 +++ synapse/storage/schema/delta/16/unique_constraints.sql | 8 ++++++++ synapse/storage/schema/full_schemas/11/transactions.sql | 2 +- synapse/storage/schema/full_schemas/11/users.sql | 2 +- synapse/storage/schema/full_schemas/16/room_aliases.sql | 4 ++++ synapse/storage/schema/full_schemas/16/transactions.sql | 2 +- synapse/storage/schema/full_schemas/16/users.sql | 2 +- 7 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 synapse/storage/schema/delta/16/room_alias_index.sql diff --git a/synapse/storage/schema/delta/16/room_alias_index.sql b/synapse/storage/schema/delta/16/room_alias_index.sql new file mode 100644 index 000000000..f82486132 --- /dev/null +++ b/synapse/storage/schema/delta/16/room_alias_index.sql @@ -0,0 +1,3 @@ + +CREATE INDEX IF NOT EXISTS room_aliases_id ON room_aliases(room_id); +CREATE INDEX IF NOT EXISTS room_alias_servers_alias ON room_alias_servers(room_alias); diff --git a/synapse/storage/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/delta/16/unique_constraints.sql index 3604ea842..fecf11118 100644 --- a/synapse/storage/schema/delta/16/unique_constraints.sql +++ b/synapse/storage/schema/delta/16/unique_constraints.sql @@ -70,3 +70,11 @@ DROP INDEX IF EXISTS presence_list_observers; CREATE UNIQUE INDEX presence_list_observers ON presence_list( user_id, observed_user_id ); + +-- +DELETE FROM room_aliases WHERE rowid not in ( + SELECT MIN(rowid) FROM room_aliases GROUP BY room_alias +); + +DROP INDEX IF EXISTS room_aliases_id; +CREATE INDEX room_aliases_id ON room_aliases(room_id); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index f01d9eba3..524a69692 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, - response_json bytea, + response_json TEXT, ts BIGINT ); diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 2edc13c3b..48a6aecfe 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS user_ips ( access_token VARCHAR(150) NOT NULL, device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, - user_agent VARCHAR(150) NOT NULL, + user_agent TEXT NOT NULL, last_seen BIGINT NOT NULL, UNIQUE (user, access_token, ip, user_agent) ); diff --git a/synapse/storage/schema/full_schemas/16/room_aliases.sql b/synapse/storage/schema/full_schemas/16/room_aliases.sql index 952cae35b..2c0853a2a 100644 --- a/synapse/storage/schema/full_schemas/16/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/16/room_aliases.sql @@ -19,7 +19,11 @@ CREATE TABLE IF NOT EXISTS room_aliases( UNIQUE (room_alias) ); +CREATE INDEX room_aliases_id ON room_aliases(room_id); + CREATE TABLE IF NOT EXISTS room_alias_servers( room_alias VARCHAR(150) NOT NULL, server VARCHAR(150) NOT NULL ); + +CREATE INDEX room_alias_servers_alias ON room_alias_servers(room_alias); diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql index bc6406493..ed431bd3a 100644 --- a/synapse/storage/schema/full_schemas/16/transactions.sql +++ b/synapse/storage/schema/full_schemas/16/transactions.sql @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, - response_json bytea, + response_json TEXT, ts BIGINT ); diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql index 006b249fc..033e3244b 100644 --- a/synapse/storage/schema/full_schemas/16/users.sql +++ b/synapse/storage/schema/full_schemas/16/users.sql @@ -34,7 +34,7 @@ CREATE TABLE IF NOT EXISTS user_ips ( access_token VARCHAR(150) NOT NULL, device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, - user_agent VARCHAR(150) NOT NULL, + user_agent TEXT NOT NULL, last_seen BIGINT NOT NULL ); From 427bcb76085c005069f8163e035ebce395a14b2a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 11:13:05 +0100 Subject: [PATCH 083/104] Fix port script after storage changes. Add very simple (off by default) curses UI to see progress. --- scripts/port_from_sqlite_to_postgres.py | 632 ++++++++++++++++-------- 1 file changed, 435 insertions(+), 197 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 4b3fd9e52..fc1603c1c 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -20,38 +20,17 @@ from synapse.storage._base import LoggingTransaction, SQLBaseStore from synapse.storage.engines import create_engine import argparse -import itertools +import curses import logging -import types +import sys +import time +import traceback import yaml logger = logging.getLogger("port_from_sqlite_to_postgres") -BINARY_COLUMNS = { - "event_content_hashes": ["hash"], - "event_reference_hashes": ["hash"], - "event_signatures": ["signature"], - "event_edge_hashes": ["hash"], - "events": ["content", "unrecognized_keys"], - "event_json": ["internal_metadata", "json"], - "application_services_txns": ["event_ids"], - "received_transactions": ["response_json"], - "sent_transactions": ["response_json"], - "server_tls_certificates": ["tls_certificate"], - "server_signature_keys": ["verify_key"], - "pushers": ["pushkey", "data"], - "user_filters": ["filter_json"], -} - -UNICODE_COLUMNS = { - "events": ["content", "unrecognized_keys"], - "event_json": ["internal_metadata", "json"], - "users": ["password_hash"], -} - - BOOLEAN_COLUMNS = { "events": ["processed", "outlier"], "rooms": ["is_public"], @@ -91,7 +70,15 @@ APPEND_ONLY_TABLES = [ ] +end_error_exec_info = None + + class Store(object): + """This object is used to pull out some of the convenience API from the + Storage layer. + + *All* database interactions should go through this object. + """ def __init__(self, db_pool, engine): self.db_pool = db_pool self.database_engine = engine @@ -130,11 +117,14 @@ class Store(object): continue raise except Exception as e: - logger.debug("[TXN FAIL] {%s}", desc, e) + logger.debug("[TXN FAIL] {%s} %s", desc, e) raise return self.db_pool.runWithConnection(r) + def execute(self, f): + return self.runInteraction(f.__name__, f) + def insert_many_txn(self, txn, table, headers, rows): sql = "INSERT INTO %s (%s) VALUES (%s)" % ( table, @@ -152,205 +142,435 @@ class Store(object): raise +class Progress(object): + """Used to report progress of the port + """ + def __init__(self): + self.tables = {} -def chunks(n): - for i in itertools.count(0, n): - yield range(i, i+n) + self.start_time = int(time.time()) + + def add_table(self, table, cur, size): + self.tables[table] = { + "start": cur, + "num_done": cur, + "total": size, + "perc": int(cur * 100 / size), + } + + def update(self, table, num_done): + data = self.tables[table] + data["num_done"] = num_done + data["perc"] = int(num_done * 100 / data["total"]) + + def done(self): + pass -@defer.inlineCallbacks -def handle_table(table, sqlite_store, postgres_store): - if table in APPEND_ONLY_TABLES: - # It's safe to just carry on inserting. - next_chunk = yield postgres_store._simple_select_one_onecol( - table="port_from_sqlite3", - keyvalues={"table_name": table}, - retcol="rowid", - allow_none=True, - ) +class CursesProgress(Progress): + """Reports progress to a curses window + """ + def __init__(self, stdscr): + self.stdscr = stdscr - if next_chunk is None: - yield postgres_store._simple_insert( - table="port_from_sqlite3", - values={"table_name": table, "rowid": 0} - ) + curses.use_default_colors() + curses.curs_set(0) - next_chunk = 0 - else: - def delete_all(txn): - txn.execute( - "DELETE FROM port_from_sqlite3 WHERE table_name = %s", - (table,) - ) - txn.execute("TRUNCATE %s CASCADE" % (table,)) - postgres_store._simple_insert_txn( - txn, - table="port_from_sqlite3", - values={"table_name": table, "rowid": 0} - ) + curses.init_pair(1, curses.COLOR_RED, -1) + curses.init_pair(2, curses.COLOR_GREEN, -1) - yield postgres_store.runInteraction( - "delete_non_append_only", delete_all - ) + self.last_update = 0 - next_chunk = 0 + self.finished = False - logger.info("next_chunk for %s: %d", table, next_chunk) + super(CursesProgress, self).__init__() - N = 5000 + def update(self, table, num_done): + super(CursesProgress, self).update(table, num_done) - select = "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,) + self.render() - uni_col_names = UNICODE_COLUMNS.get(table, []) - bool_col_names = BOOLEAN_COLUMNS.get(table, []) - bin_col_names = BINARY_COLUMNS.get(table, []) + def render(self, force=False): + now = time.time() - while True: - def r(txn): - txn.execute(select, (next_chunk, N,)) - rows = txn.fetchall() - headers = [column[0] for column in txn.description] - - return headers, rows - - headers, rows = yield sqlite_store.runInteraction("select", r) - - logger.info("Got %d rows for %s", len(rows), table) - - if rows: - uni_cols = [i for i, h in enumerate(headers) if h in uni_col_names] - bool_cols = [i for i, h in enumerate(headers) if h in bool_col_names] - bin_cols = [i for i, h in enumerate(headers) if h in bin_col_names] - next_chunk = rows[-1][0] + 1 - - def conv(j, col): - if j in uni_cols: - col = sqlite_store.database_engine.load_unicode(col) - if j in bool_cols: - return bool(col) - - if j in bin_cols: - if isinstance(col, types.UnicodeType): - col = buffer(col.encode("utf8")) - - return col - - for i, row in enumerate(rows): - rows[i] = tuple( - postgres_store.database_engine.encode_parameter( - conv(j, col) - ) - for j, col in enumerate(row) - if j > 0 - ) - - def ins(txn): - postgres_store.insert_many_txn(txn, table, headers[1:], rows) - - postgres_store._simple_update_one_txn( - txn, - table="port_from_sqlite3", - keyvalues={"table_name": table}, - updatevalues={"rowid": next_chunk}, - ) - - yield postgres_store.runInteraction("insert_many", ins) - else: + if not force and now - self.last_update < 0.2: + # reactor.callLater(1, self.render) return + self.stdscr.clear() -def setup_db(db_config, database_engine): - db_conn = database_engine.module.connect( - **{ - k: v for k, v in db_config.get("args", {}).items() - if not k.startswith("cp_") - } - ) + rows, cols = self.stdscr.getmaxyx() - database_engine.prepare_database(db_conn) + duration = int(now) - int(self.start_time) - db_conn.commit() + minutes, seconds = divmod(duration, 60) + duration_str = '%02dm %02ds' % (minutes, seconds,) - -@defer.inlineCallbacks -def main(sqlite_config, postgress_config): - try: - sqlite_db_pool = adbapi.ConnectionPool( - sqlite_config["name"], - **sqlite_config["args"] - ) - - postgres_db_pool = adbapi.ConnectionPool( - postgress_config["name"], - **postgress_config["args"] - ) - - sqlite_engine = create_engine("sqlite3") - postgres_engine = create_engine("psycopg2") - - sqlite_store = Store(sqlite_db_pool, sqlite_engine) - postgres_store = Store(postgres_db_pool, postgres_engine) - - # Step 1. Set up databases. - logger.info("Preparing sqlite database...") - setup_db(sqlite_config, sqlite_engine) - - logger.info("Preparing postgres database...") - setup_db(postgress_config, postgres_engine) - - # Step 2. Get tables. - logger.info("Fetching tables...") - tables = yield sqlite_store._simple_select_onecol( - table="sqlite_master", - keyvalues={ - "type": "table", - }, - retcol="name", - ) - - logger.info("Found %d tables", len(tables)) - - def create_port_table(txn): - txn.execute( - "CREATE TABLE port_from_sqlite3 (" - " table_name varchar(100) NOT NULL UNIQUE," - " rowid bigint NOT NULL" - ")" + if self.finished: + status = "Time spent: %s (Done!)" % (duration_str,) + else: + min_perc = min( + (v["num_done"] - v["start"]) * 100. / (v["total"] - v["start"]) + if v["total"] - v["start"] else 100 + for v in self.tables.values() + ) + if min_perc > 0: + est_remaining = (int(now) - self.start_time) * 100 / min_perc + est_remaining_str = '%02dm %02ds remaining' % divmod(est_remaining, 60) + else: + est_remaining_str = "Unknown" + status = ( + "Time spent: %s (est. remaining: %s)" + % (duration_str, est_remaining_str,) ) + self.stdscr.addstr( + 0, 0, + status, + curses.A_BOLD, + ) + + max_len = max([len(t) for t in self.tables.keys()]) + + left_margin = 5 + middle_space = 1 + + items = self.tables.items() + items.sort( + key=lambda i: (i[1]["perc"], i[0]), + ) + + for i, (table, data) in enumerate(items): + if i + 2 >= rows: + break + + perc = data["perc"] + + color = curses.color_pair(2) if perc == 100 else curses.color_pair(1) + + self.stdscr.addstr( + i+2, left_margin + max_len - len(table), + table, + curses.A_BOLD | color, + ) + + size = 20 + + progress = "[%s%s]" % ( + "#" * int(perc*size/100), + " " * (size - int(perc*size/100)), + ) + + self.stdscr.addstr( + i+2, left_margin + max_len + middle_space, + "%s %3d%% (%d/%d)" % (progress, perc, data["num_done"], data["total"]), + ) + + if self.finished: + self.stdscr.addstr( + self.rows-1, 0 , + "Press any key to exit...", + ) + + self.stdscr.refresh() + self.last_update = time.time() + + def done(self): + self.finished = True + self.render(True) + self.stdscr.getch() + + def on_prepare_sqlite(self): + self.stdscr.clear() + self.stdscr.addstr( + 0, 0, + "Preparing SQLite database...", + curses.A_BOLD, + ) + self.stdscr.refresh() + + def on_prepare_postgres(self): + self.stdscr.clear() + self.stdscr.addstr( + 0, 0, + "Preparing PostgreSQL database...", + curses.A_BOLD, + ) + self.stdscr.refresh() + + def fetching_tables(self): + self.stdscr.clear() + self.stdscr.addstr( + 0, 0, + "Fetching tables...", + curses.A_BOLD, + ) + self.stdscr.refresh() + + def preparing_tables(self): + self.stdscr.clear() + self.stdscr.addstr( + 0, 0, + "Preparing tables...", + curses.A_BOLD, + ) + self.stdscr.refresh() + + +class TerminalProgress(Progress): + """Just prints progress to the terminal + """ + def update(self, table, num_done): + super(TerminalProgress, self).update(table, num_done) + + data = self.tables[table] + + print "%s: %d%% (%d/%d)" % ( + table, data["perc"], + data["num_done"], data["total"], + ) + + def on_prepare_sqlite(self): + print "Preparing SQLite database..." + + def on_prepare_postgres(self): + print "Preparing PostgreSQL database..." + + def fetching_tables(self): + print "Fetching tables..." + + def preparing_tables(self): + print "Preparing tables..." + + +class Porter(object): + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + @defer.inlineCallbacks + def handle_table(self, table): + if table in APPEND_ONLY_TABLES: + # It's safe to just carry on inserting. + next_chunk = yield self.postgres_store._simple_select_one_onecol( + table="port_from_sqlite3", + keyvalues={"table_name": table}, + retcol="rowid", + allow_none=True, + ) + + if next_chunk is None: + yield self.postgres_store._simple_insert( + table="port_from_sqlite3", + values={"table_name": table, "rowid": 1} + ) + + next_chunk = 1 + else: + def delete_all(txn): + txn.execute( + "DELETE FROM port_from_sqlite3 WHERE table_name = %s", + (table,) + ) + txn.execute("TRUNCATE %s CASCADE" % (table,)) + self.postgres_store._simple_insert_txn( + txn, + table="port_from_sqlite3", + values={"table_name": table, "rowid": 0} + ) + yield self.postgres_store.execute(delete_all) + + next_chunk = 1 + + def get_table_size(txn): + txn.execute("SELECT count(*) FROM %s" % (table,)) + size, = txn.fetchone() + return int(size) + + table_size = yield self.sqlite_store.execute(get_table_size) + postgres_size = yield self.postgres_store.execute(get_table_size) + + if not table_size: + return + + self.progress.add_table(table, postgres_size, table_size) + + select = ( + "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" + % (table,) + ) + + bool_col_names = BOOLEAN_COLUMNS.get(table, []) + + while True: + def r(txn): + txn.execute(select, (next_chunk, self.batch_size,)) + rows = txn.fetchall() + headers = [column[0] for column in txn.description] + + return headers, rows + + headers, rows = yield self.sqlite_store.runInteraction("select", r) + + if rows: + bool_cols = [ + i for i, h in enumerate(headers) if h in bool_col_names + ] + next_chunk = rows[-1][0] + 1 + + def conv(j, col): + if j in bool_cols: + return bool(col) + return col + + for i, row in enumerate(rows): + rows[i] = tuple( + self.postgres_store.database_engine.encode_parameter( + conv(j, col) + ) + for j, col in enumerate(row) + if j > 0 + ) + + def insert(txn): + self.postgres_store.insert_many_txn( + txn, table, headers[1:], rows + ) + + self.postgres_store._simple_update_one_txn( + txn, + table="port_from_sqlite3", + keyvalues={"table_name": table}, + updatevalues={"rowid": next_chunk}, + ) + + yield self.postgres_store.execute(insert) + + postgres_size += len(rows) + + self.progress.update(table, postgres_size) + else: + return + + def setup_db(self, db_config, database_engine): + db_conn = database_engine.module.connect( + **{ + k: v for k, v in db_config.get("args", {}).items() + if not k.startswith("cp_") + } + ) + + database_engine.prepare_database(db_conn) + + db_conn.commit() + + @defer.inlineCallbacks + def run(self): try: - yield postgres_store.runInteraction( - "create_port_table", create_port_table + sqlite_db_pool = adbapi.ConnectionPool( + self.sqlite_config["name"], + **self.sqlite_config["args"] ) - except Exception as e: - logger.info("Failed to create port table: %s", e) - # Process tables. - yield defer.gatherResults( - [ - handle_table(table, sqlite_store, postgres_store) - for table in tables - if table not in ["schema_version", "applied_schema_deltas"] - and not table.startswith("sqlite_") - ], - consumeErrors=True, - ) + postgres_db_pool = adbapi.ConnectionPool( + self.postgres_config["name"], + **self.postgres_config["args"] + ) - except: - logger.exception("") - finally: - reactor.stop() + sqlite_engine = create_engine("sqlite3") + postgres_engine = create_engine("psycopg2") + + self.sqlite_store = Store(sqlite_db_pool, sqlite_engine) + self.postgres_store = Store(postgres_db_pool, postgres_engine) + + # Step 1. Set up databases. + self.progress.on_prepare_sqlite() + self.setup_db(sqlite_config, sqlite_engine) + + self.progress.on_prepare_postgres() + self.setup_db(postgres_config, postgres_engine) + + # Step 2. Get tables. + self.progress.fetching_tables() + sqlite_tables = yield self.sqlite_store._simple_select_onecol( + table="sqlite_master", + keyvalues={ + "type": "table", + }, + retcol="name", + ) + + postgres_tables = yield self.postgres_store._simple_select_onecol( + table="information_schema.tables", + keyvalues={ + "table_schema": "public", + }, + retcol="distinct table_name", + ) + + tables = set(sqlite_tables) & set(postgres_tables) + + self.progress.preparing_tables() + + logger.info("Found %d tables", len(tables)) + + def create_port_table(txn): + txn.execute( + "CREATE TABLE port_from_sqlite3 (" + " table_name varchar(100) NOT NULL UNIQUE," + " rowid bigint NOT NULL" + ")" + ) + + try: + yield self.postgres_store.runInteraction( + "create_port_table", create_port_table + ) + except Exception as e: + logger.info("Failed to create port table: %s", e) + + # Process tables. + yield defer.gatherResults( + [ + self.handle_table(table) + for table in tables + if table not in ["schema_version", "applied_schema_deltas"] + and not table.startswith("sqlite_") + ], + consumeErrors=True, + ) + + self.progress.done() + except: + global end_error_exec_info + end_error_exec_info = sys.exc_info() + logger.exception("") + finally: + reactor.stop() if __name__ == "__main__": parser = argparse.ArgumentParser() + parser.add_argument("-v", action='store_true') + parser.add_argument("--curses", action='store_true') parser.add_argument("--sqlite-database") parser.add_argument( "--postgres-config", type=argparse.FileType('r'), ) + parser.add_argument("--batch-size", type=int, default=1000) + args = parser.parse_args() - logging.basicConfig(level=logging.INFO) + + + logging_config = { + "level": logging.DEBUG if args.v else logging.INFO, + "format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s" + } + + if args.curses: + logging_config["filename"] = "port-synapse.log" + + logging.basicConfig(**logging_config) sqlite_config = { "name": "sqlite3", @@ -364,10 +584,28 @@ if __name__ == "__main__": postgres_config = yaml.safe_load(args.postgres_config) - reactor.callWhenRunning( - main, - sqlite_config=sqlite_config, - postgres_config=postgres_config, - ) + def start(stdscr=None): + if stdscr: + progress = CursesProgress(stdscr) + else: + progress = TerminalProgress() - reactor.run() + porter = Porter( + sqlite_config=sqlite_config, + postgres_config=postgres_config, + progress=progress, + batch_size=args.batch_size, + ) + + reactor.callWhenRunning(porter.run) + + reactor.run() + + if args.curses: + curses.wrapper(start) + else: + start() + + if end_error_exec_info: + exc_type, exc_value, exc_traceback = end_error_exec_info + traceback.print_exception(exc_type, exc_value, exc_traceback) From dfc46c6220b7439262de6fc7034e7a6df99d4e40 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 12:46:29 +0100 Subject: [PATCH 084/104] PEP8 --- scripts/port_from_sqlite_to_postgres.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index fc1603c1c..f4b6ed068 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -268,7 +268,7 @@ class CursesProgress(Progress): if self.finished: self.stdscr.addstr( - self.rows-1, 0 , + self.rows-1, 0, "Press any key to exit...", ) @@ -561,7 +561,6 @@ if __name__ == "__main__": args = parser.parse_args() - logging_config = { "level": logging.DEBUG if args.v else logging.INFO, "format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s" From 5e7a90316d43f0dc42c5b18e505ce9b432e15461 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 16:08:18 +0100 Subject: [PATCH 085/104] Update --database-path metavar to SQLITE_DATABASE_PATH --- synapse/config/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/database.py b/synapse/config/database.py index 8dc9873f8..f3d0898c0 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -37,7 +37,7 @@ class DatabaseConfig(Config): db_group = parser.add_argument_group("database") db_group.add_argument( "-d", "--database-path", default="homeserver.db", - help="The database name." + metavar="SQLITE_DATABASE_PATH", help="The database name." ) db_group.add_argument( "--event-cache-size", default="100K", From e3417bbbe0d40d17eec9f07981359d081c9b1f9d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 16:13:14 +0100 Subject: [PATCH 086/104] Revert needless change to storage.profile --- synapse/storage/profile.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py index 047698aa1..a6e52cb24 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer - from ._base import SQLBaseStore @@ -26,17 +24,14 @@ class ProfileStore(SQLBaseStore): desc="create_profile", ) - @defer.inlineCallbacks def get_profile_displayname(self, user_localpart): - name = yield self._simple_select_one_onecol( + return self._simple_select_one_onecol( table="profiles", keyvalues={"user_id": user_localpart}, retcol="displayname", desc="get_profile_displayname", ) - defer.returnValue(name) - def set_profile_displayname(self, user_localpart, new_displayname): return self._simple_update_one( table="profiles", From 865398b4a953109015a7fa595634070b39e68a6d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 16:14:51 +0100 Subject: [PATCH 087/104] Revert needless change to rest.profile --- synapse/rest/client/v1/profile.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index 7387b4adb..1e77eb49c 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -19,13 +19,9 @@ from twisted.internet import defer from .base import ClientV1RestServlet, client_path_pattern from synapse.types import UserID -import logging import simplejson as json -logger = logging.getLogger(__name__) - - class ProfileDisplaynameRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/profile/(?P[^/]*)/displayname") @@ -51,8 +47,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): defer.returnValue((400, "Unable to parse name")) yield self.handlers.profile_handler.set_displayname( - user, auth_user, new_name - ) + user, auth_user, new_name) defer.returnValue((200, {})) From e4c4664d73abd260264f51a0645281d21afc358e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 12:40:49 +0100 Subject: [PATCH 088/104] Handle the fact that postgres databases can be restarted from under us --- synapse/storage/_base.py | 28 +++++++++++++++++++++++++++- synapse/storage/engines/postgres.py | 3 +++ synapse/storage/engines/sqlite3.py | 3 +++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index f5952d1fc..874d41447 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -300,6 +300,9 @@ class SQLBaseStore(object): def inner_func(conn, *args, **kwargs): with LoggingContext("runInteraction") as context: + if self.database_engine.is_connection_closed(conn): + conn.reconnect() + current_context.copy_to(context) start = time.time() * 1000 txn_id = self._TXN_ID @@ -322,12 +325,35 @@ class SQLBaseStore(object): LoggingTransaction(txn, name, self.database_engine), *args, **kwargs ) + except self.database_engine.module.OperationalError as e: + # This can happen if the database disappears mid + # transaction. + logger.warn( + "[TXN OPERROR] {%s} %s %d/%d", + name, e, i, N + ) + if i < N: + i += 1 + try: + conn.rollback() + except self.database_engine.module.Error as e1: + logger.warn( + "[TXN EROLL] {%s} %s", + name, e1, + ) + continue except self.database_engine.module.DatabaseError as e: if self.database_engine.is_deadlock(e): logger.warn("[TXN DEADLOCK] {%s} %d/%d", name, i, N) if i < N: i += 1 - conn.rollback() + try: + conn.rollback() + except self.database_engine.module.Error as e1: + logger.warn( + "[TXN EROLL] {%s} %s", + name, e1, + ) continue raise except Exception as e: diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 6f75245fa..00dbae7b6 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -39,3 +39,6 @@ class PostgresEngine(object): if isinstance(error, self.module.DatabaseError): return error.pgcode in ["40001", "40P01"] return False + + def is_connection_closed(self, conn): + return bool(conn) diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 72c11df46..39828a597 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -35,3 +35,6 @@ class Sqlite3Engine(object): def is_deadlock(self, error): return False + + def is_connection_closed(self, conn): + return False From 2732be83d9e883184f4a783fb7ba15487f30c20d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 13:22:30 +0100 Subject: [PATCH 089/104] Shuffle operations so that locking upsert happens last in the txn. This ensures the lock is held for the least amount of time possible. --- synapse/storage/_base.py | 3 + synapse/storage/engines/postgres.py | 3 + synapse/storage/engines/sqlite3.py | 3 + synapse/storage/events.py | 88 ++++++++++++++--------------- synapse/storage/transactions.py | 21 ++----- 5 files changed, 59 insertions(+), 59 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 874d41447..6017c2a6e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -466,6 +466,9 @@ class SQLBaseStore(object): ) def _simple_upsert_txn(self, txn, table, keyvalues, values, insertion_values={}): + # We need to lock the table :( + self.database_engine.lock_table(txn, table) + # Try to update sql = "UPDATE %s SET %s WHERE %s" % ( table, diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 00dbae7b6..b8cca9b18 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -42,3 +42,6 @@ class PostgresEngine(object): def is_connection_closed(self, conn): return bool(conn) + + def lock_table(self, txn, table): + txn.execute("LOCK TABLE %s in EXCLUSIVE MODE" % (table,)) diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 39828a597..f62d5d120 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -38,3 +38,6 @@ class Sqlite3Engine(object): def is_connection_closed(self, conn): return False + + def lock_table(self, txn, table): + return diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 7dbf7a396..a3c260ddc 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -283,50 +283,6 @@ class EventsStore(SQLBaseStore): if context.rejected: self._store_rejections_txn(txn, event.event_id, context.rejected) - if event.is_state(): - vals = { - "event_id": event.event_id, - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - } - - # TODO: How does this work with backfilling? - if hasattr(event, "replaces_state"): - vals["prev_state"] = event.replaces_state - - self._simple_insert_txn( - txn, - "state_events", - vals, - ) - - if is_new_state and not context.rejected: - self._simple_upsert_txn( - txn, - "current_state_events", - keyvalues={ - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - }, - values={ - "event_id": event.event_id, - } - ) - - for e_id, h in event.prev_state: - self._simple_insert_txn( - txn, - table="event_edges", - values={ - "event_id": event.event_id, - "prev_event_id": e_id, - "room_id": event.room_id, - "is_state": True, - }, - ) - for hash_alg, hash_base64 in event.hashes.items(): hash_bytes = decode_base64(hash_base64) self._store_event_content_hash_txn( @@ -356,6 +312,50 @@ class EventsStore(SQLBaseStore): txn, event.event_id, ref_alg, ref_hash_bytes ) + if event.is_state(): + vals = { + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + } + + # TODO: How does this work with backfilling? + if hasattr(event, "replaces_state"): + vals["prev_state"] = event.replaces_state + + self._simple_insert_txn( + txn, + "state_events", + vals, + ) + + for e_id, h in event.prev_state: + self._simple_insert_txn( + txn, + table="event_edges", + values={ + "event_id": event.event_id, + "prev_event_id": e_id, + "room_id": event.room_id, + "is_state": True, + }, + ) + + if is_new_state and not context.rejected: + self._simple_upsert_txn( + txn, + "current_state_events", + keyvalues={ + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + }, + values={ + "event_id": event.event_id, + } + ) + def _store_redaction(self, txn, event): # invalidate the cache for the redacted event self._invalidate_get_event_cache(event.redacts) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 7e3add528..89dd7d894 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -76,25 +76,16 @@ class TransactionStore(SQLBaseStore): response_json (str) """ - return self.runInteraction( - "set_received_txn_response", - self._set_received_txn_response, - transaction_id, origin, code, response_dict - ) - - def _set_received_txn_response(self, txn, transaction_id, origin, code, - response_json): - self._simple_upsert_txn( - txn, + return self._simple_insert( table=ReceivedTransactionsTable.table_name, - keyvalues={ + values={ "transaction_id": transaction_id, "origin": origin, - }, - values={ "response_code": code, - "response_json": response_json, - } + "response_json": response_dict, + }, + or_ignore=True, + desc="set_received_txn_response", ) def prep_send_transaction(self, transaction_id, destination, From 56f518d279b642efce92e172b463232937d50b8c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 14:53:35 +0100 Subject: [PATCH 090/104] Add docs on how to use synapse with psycopg2 --- docs/postgres.rst | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 docs/postgres.rst diff --git a/docs/postgres.rst b/docs/postgres.rst new file mode 100644 index 000000000..5bb45d3ed --- /dev/null +++ b/docs/postgres.rst @@ -0,0 +1,34 @@ +Using Postgres +-------------- + +Set up client +============= +We need to have installed the postgres python connector ``psycopg2``. In the +virtual env:: + + sudo apt-get install libpq-dev + pip install psycopg2 + + +Synapse config +============== + +Add the following line to your config file:: + + database_config: + +Where ```` is the file name that points to a yaml file of the +following form:: + + name: psycopg2 + args: + user: + password: + database: + host: + cp_min: 5 + cp_max: 10 + +All key, values in ``args`` are passed to the ``psycopg2.connect(..)`` +function, except keys beginning with ``cp_``, which are consumed by the twisted +adbapi connection pool. From 8558e1ec7338a1f60342024736614dec71d104ce Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 15:19:44 +0100 Subject: [PATCH 091/104] Make get_max_token into inlineCallbacks so that the lock works. --- synapse/storage/util/id_generators.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index e5dec1c94..9d461d5e9 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -105,21 +105,22 @@ class StreamIdGenerator(object): return manager() + @defer.inlineCallbacks def get_max_token(self, store): """Returns the maximum stream id such that all stream ids less than or equal to it have been successfully persisted. """ with self._lock: if self._unfinished_ids: - return self._unfinished_ids[0] - 1 + defer.returnValue(self._unfinished_ids[0] - 1) if not self._current_max: - return store.runInteraction( + yield store.runInteraction( "_compute_current_max", self._compute_current_max, ) - return self._current_max + defer.returnValue(self._current_max) def _compute_current_max(self, txn): txn.execute("SELECT MAX(stream_ordering) FROM events") From 416a3e6c4f40c94cf859a07dbc4341c907aac091 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 15:44:30 +0100 Subject: [PATCH 092/104] Ensure check_same_thread is enabled for sqlite3 --- synapse/app/homeserver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 93500dd79..3709cd7bf 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -372,6 +372,7 @@ def setup(config_options): db_config.setdefault("args", {}).update({ "cp_min": 1, "cp_max": 1, + "check_same_thread": False, }) else: raise RuntimeError("Unsupported database type '%s'" % (name,)) From 1ef66cc3bd541ee1e4a017cfdd008eacaec5bcf8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 15:57:43 +0100 Subject: [PATCH 093/104] Move database configuration into config module --- synapse/app/homeserver.py | 35 ++++------------------------------- synapse/config/database.py | 23 +++++++++++++++++++++-- 2 files changed, 25 insertions(+), 33 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 3709cd7bf..f29f9d702 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -350,42 +350,15 @@ def setup(config_options): tls_context_factory = context_factory.ServerContextFactory(config) - if config.database_config: - with open(config.database_config, 'r') as f: - db_config = yaml.safe_load(f) - else: - db_config = { - "name": "sqlite3", - "args": { - "database": config.database_path, - }, - } - - db_config = { - k: v for k, v in db_config.items() - } - - name = db_config.get("name", None) - if name == "psycopg2": - pass - elif name == "sqlite3": - db_config.setdefault("args", {}).update({ - "cp_min": 1, - "cp_max": 1, - "check_same_thread": False, - }) - else: - raise RuntimeError("Unsupported database type '%s'" % (name,)) - - database_engine = create_engine(name) - db_config["args"]["cp_openfun"] = database_engine.on_new_connection + database_engine = create_engine(config.database_config["name"]) + config.database_config["args"]["cp_openfun"] = database_engine.on_new_connection hs = SynapseHomeServer( config.server_name, domain_with_port=domain_with_port, upload_dir=os.path.abspath("uploads"), db_name=config.database_path, - db_config=db_config, + db_config=config.database_config, tls_context_factory=tls_context_factory, config=config, content_addr=config.content_addr, @@ -404,7 +377,7 @@ def setup(config_options): try: db_conn = database_engine.module.connect( **{ - k: v for k, v in db_config.get("args", {}).items() + k: v for k, v in config.database_config.get("args", {}).items() if not k.startswith("cp_") } ) diff --git a/synapse/config/database.py b/synapse/config/database.py index f3d0898c0..190d119df 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -15,6 +15,7 @@ from ._base import Config import os +import yaml class DatabaseConfig(Config): @@ -27,9 +28,27 @@ class DatabaseConfig(Config): self.event_cache_size = self.parse_size(args.event_cache_size) if args.database_config: - self.database_config = self.abspath(args.database_config) + with open(args.database_config) as f: + self.database_config = yaml.safe_load(f) else: - self.database_config = None + self.database_config = { + "name": "sqlite3", + "args": { + "database": self.database_path, + }, + } + + name = self.database_config.get("name", None) + if name == "psycopg2": + pass + elif name == "sqlite3": + self.database_config.setdefault("args", {}).update({ + "cp_min": 1, + "cp_max": 1, + "check_same_thread": False, + }) + else: + raise RuntimeError("Unsupported database type '%s'" % (name,)) @classmethod def add_arguments(cls, parser): From 42b7139deced5d9614bcb487625f4ce5d99e560b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 15:59:56 +0100 Subject: [PATCH 094/104] Remove unused import --- synapse/app/homeserver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f29f9d702..110566e33 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -56,7 +56,6 @@ import os import re import resource import subprocess -import yaml logger = logging.getLogger("synapse.app.homeserver") From 4e49f52375acb705b32937115210d055b0e1ce38 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 17:36:37 +0100 Subject: [PATCH 095/104] Don't port over all of the sent_transactions table --- scripts/port_from_sqlite_to_postgres.py | 162 ++++++++++++++++++------ 1 file changed, 122 insertions(+), 40 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index f4b6ed068..19e35bf80 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -122,8 +122,8 @@ class Store(object): return self.db_pool.runWithConnection(r) - def execute(self, f): - return self.runInteraction(f.__name__, f) + def execute(self, f, *args, **kwargs): + return self.runInteraction(f.__name__, f, *args, **kwargs) def insert_many_txn(self, txn, table, headers, rows): sql = "INSERT INTO %s (%s) VALUES (%s)" % ( @@ -347,9 +347,118 @@ class Porter(object): def __init__(self, **kwargs): self.__dict__.update(kwargs) + def convert_rows(self, table, headers, rows): + bool_col_names = BOOLEAN_COLUMNS.get(table, []) + + bool_cols = [ + i for i, h in enumerate(headers) if h in bool_col_names + ] + + def conv(j, col): + if j in bool_cols: + return bool(col) + return col + + for i, row in enumerate(rows): + rows[i] = tuple( + self.postgres_store.database_engine.encode_parameter( + conv(j, col) + ) + for j, col in enumerate(row) + if j > 0 + ) + @defer.inlineCallbacks def handle_table(self, table): - if table in APPEND_ONLY_TABLES: + def delete_all(txn): + txn.execute( + "DELETE FROM port_from_sqlite3 WHERE table_name = %s", + (table,) + ) + txn.execute("TRUNCATE %s CASCADE" % (table,)) + + def get_table_size(txn): + txn.execute("SELECT count(*) FROM %s" % (table,)) + size, = txn.fetchone() + return int(size) + + if table == "sent_transactions": + # This is a big table, and we really only need some of the recent + # data + yield self.postgres_store.execute(delete_all) + + # Only save things from the last day + yesterday = 1429114568820 #int(time.time()*1000) - 86400000 + + # And save the max transaction id from each destination + select = ( + "SELECT rowid, * FROM sent_transactions WHERE rowid IN (" + "SELECT max(rowid) FROM sent_transactions" + " GROUP BY destination" + ")" + ) + + def r(txn): + txn.execute(select) + rows = txn.fetchall() + headers = [column[0] for column in txn.description] + + ts_ind = headers.index('ts') + + return headers, [r for r in rows if r[ts_ind] < yesterday] + + headers, rows = yield self.sqlite_store.runInteraction( + "select", r, + ) + + self.convert_rows(table, headers, rows) + + inserted_rows = len(rows) + max_inserted_rowid = max(r[0] for r in rows) + + def insert(txn): + self.postgres_store.insert_many_txn( + txn, table, headers[1:], rows + ) + + yield self.postgres_store.execute(insert) + + def get_start_id(txn): + txn.execute( + "SELECT rowid FROM sent_transactions WHERE ts >= ?" + " ORDER BY rowid ASC LIMIT 1", + (yesterday,) + ) + + rows = txn.fetchall() + if rows: + return rows[0][0] + else: + return 1 + + next_chunk = yield self.sqlite_store.execute(get_start_id) + next_chunk = max(max_inserted_rowid + 1, next_chunk) + + yield self.postgres_store._simple_insert( + table="port_from_sqlite3", + values={"table_name": table, "rowid": next_chunk} + ) + + def get_sent_table_size(txn): + txn.execute( + "SELECT count(*) FROM sent_transactions" + " WHERE ts >= ?", + (yesterday,) + ) + size, = txn.fetchone() + return int(size) + + table_size = yield self.sqlite_store.execute( + get_sent_table_size + ) + + table_size += inserted_rows + elif table in APPEND_ONLY_TABLES: # It's safe to just carry on inserting. next_chunk = yield self.postgres_store._simple_select_one_onecol( table="port_from_sqlite3", @@ -365,28 +474,18 @@ class Porter(object): ) next_chunk = 1 - else: - def delete_all(txn): - txn.execute( - "DELETE FROM port_from_sqlite3 WHERE table_name = %s", - (table,) - ) - txn.execute("TRUNCATE %s CASCADE" % (table,)) - self.postgres_store._simple_insert_txn( - txn, - table="port_from_sqlite3", - values={"table_name": table, "rowid": 0} - ) - yield self.postgres_store.execute(delete_all) + table_size = yield self.sqlite_store.execute(get_table_size) + else: + yield self.postgres_store.execute(delete_all) + self.postgres_store._simple_insert( + table="port_from_sqlite3", + values={"table_name": table, "rowid": 0} + ) + + table_size = yield self.sqlite_store.execute(get_table_size) next_chunk = 1 - def get_table_size(txn): - txn.execute("SELECT count(*) FROM %s" % (table,)) - size, = txn.fetchone() - return int(size) - - table_size = yield self.sqlite_store.execute(get_table_size) postgres_size = yield self.postgres_store.execute(get_table_size) if not table_size: @@ -399,8 +498,6 @@ class Porter(object): % (table,) ) - bool_col_names = BOOLEAN_COLUMNS.get(table, []) - while True: def r(txn): txn.execute(select, (next_chunk, self.batch_size,)) @@ -412,24 +509,9 @@ class Porter(object): headers, rows = yield self.sqlite_store.runInteraction("select", r) if rows: - bool_cols = [ - i for i, h in enumerate(headers) if h in bool_col_names - ] next_chunk = rows[-1][0] + 1 - def conv(j, col): - if j in bool_cols: - return bool(col) - return col - - for i, row in enumerate(rows): - rows[i] = tuple( - self.postgres_store.database_engine.encode_parameter( - conv(j, col) - ) - for j, col in enumerate(row) - if j > 0 - ) + self.convert_rows(table, headers, rows) def insert(txn): self.postgres_store.insert_many_txn( From 40cbd6b6ee809c23750ea818ebd668be78bbf328 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 17:53:15 +0100 Subject: [PATCH 096/104] Shuffle progress stuff --- scripts/port_from_sqlite_to_postgres.py | 375 +++++++++++------------- 1 file changed, 174 insertions(+), 201 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 19e35bf80..845e3fcf3 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -142,207 +142,6 @@ class Store(object): raise -class Progress(object): - """Used to report progress of the port - """ - def __init__(self): - self.tables = {} - - self.start_time = int(time.time()) - - def add_table(self, table, cur, size): - self.tables[table] = { - "start": cur, - "num_done": cur, - "total": size, - "perc": int(cur * 100 / size), - } - - def update(self, table, num_done): - data = self.tables[table] - data["num_done"] = num_done - data["perc"] = int(num_done * 100 / data["total"]) - - def done(self): - pass - - -class CursesProgress(Progress): - """Reports progress to a curses window - """ - def __init__(self, stdscr): - self.stdscr = stdscr - - curses.use_default_colors() - curses.curs_set(0) - - curses.init_pair(1, curses.COLOR_RED, -1) - curses.init_pair(2, curses.COLOR_GREEN, -1) - - self.last_update = 0 - - self.finished = False - - super(CursesProgress, self).__init__() - - def update(self, table, num_done): - super(CursesProgress, self).update(table, num_done) - - self.render() - - def render(self, force=False): - now = time.time() - - if not force and now - self.last_update < 0.2: - # reactor.callLater(1, self.render) - return - - self.stdscr.clear() - - rows, cols = self.stdscr.getmaxyx() - - duration = int(now) - int(self.start_time) - - minutes, seconds = divmod(duration, 60) - duration_str = '%02dm %02ds' % (minutes, seconds,) - - if self.finished: - status = "Time spent: %s (Done!)" % (duration_str,) - else: - min_perc = min( - (v["num_done"] - v["start"]) * 100. / (v["total"] - v["start"]) - if v["total"] - v["start"] else 100 - for v in self.tables.values() - ) - if min_perc > 0: - est_remaining = (int(now) - self.start_time) * 100 / min_perc - est_remaining_str = '%02dm %02ds remaining' % divmod(est_remaining, 60) - else: - est_remaining_str = "Unknown" - status = ( - "Time spent: %s (est. remaining: %s)" - % (duration_str, est_remaining_str,) - ) - - self.stdscr.addstr( - 0, 0, - status, - curses.A_BOLD, - ) - - max_len = max([len(t) for t in self.tables.keys()]) - - left_margin = 5 - middle_space = 1 - - items = self.tables.items() - items.sort( - key=lambda i: (i[1]["perc"], i[0]), - ) - - for i, (table, data) in enumerate(items): - if i + 2 >= rows: - break - - perc = data["perc"] - - color = curses.color_pair(2) if perc == 100 else curses.color_pair(1) - - self.stdscr.addstr( - i+2, left_margin + max_len - len(table), - table, - curses.A_BOLD | color, - ) - - size = 20 - - progress = "[%s%s]" % ( - "#" * int(perc*size/100), - " " * (size - int(perc*size/100)), - ) - - self.stdscr.addstr( - i+2, left_margin + max_len + middle_space, - "%s %3d%% (%d/%d)" % (progress, perc, data["num_done"], data["total"]), - ) - - if self.finished: - self.stdscr.addstr( - self.rows-1, 0, - "Press any key to exit...", - ) - - self.stdscr.refresh() - self.last_update = time.time() - - def done(self): - self.finished = True - self.render(True) - self.stdscr.getch() - - def on_prepare_sqlite(self): - self.stdscr.clear() - self.stdscr.addstr( - 0, 0, - "Preparing SQLite database...", - curses.A_BOLD, - ) - self.stdscr.refresh() - - def on_prepare_postgres(self): - self.stdscr.clear() - self.stdscr.addstr( - 0, 0, - "Preparing PostgreSQL database...", - curses.A_BOLD, - ) - self.stdscr.refresh() - - def fetching_tables(self): - self.stdscr.clear() - self.stdscr.addstr( - 0, 0, - "Fetching tables...", - curses.A_BOLD, - ) - self.stdscr.refresh() - - def preparing_tables(self): - self.stdscr.clear() - self.stdscr.addstr( - 0, 0, - "Preparing tables...", - curses.A_BOLD, - ) - self.stdscr.refresh() - - -class TerminalProgress(Progress): - """Just prints progress to the terminal - """ - def update(self, table, num_done): - super(TerminalProgress, self).update(table, num_done) - - data = self.tables[table] - - print "%s: %d%% (%d/%d)" % ( - table, data["perc"], - data["num_done"], data["total"], - ) - - def on_prepare_sqlite(self): - print "Preparing SQLite database..." - - def on_prepare_postgres(self): - print "Preparing PostgreSQL database..." - - def fetching_tables(self): - print "Fetching tables..." - - def preparing_tables(self): - print "Preparing tables..." - - class Porter(object): def __init__(self, **kwargs): self.__dict__.update(kwargs) @@ -630,6 +429,180 @@ class Porter(object): reactor.stop() +############################################## +###### The following is simply UI stuff ###### +############################################## + + +class Progress(object): + """Used to report progress of the port + """ + def __init__(self): + self.tables = {} + + self.start_time = int(time.time()) + + def add_table(self, table, cur, size): + self.tables[table] = { + "start": cur, + "num_done": cur, + "total": size, + "perc": int(cur * 100 / size), + } + + def update(self, table, num_done): + data = self.tables[table] + data["num_done"] = num_done + data["perc"] = int(num_done * 100 / data["total"]) + + def done(self): + pass + + +class CursesProgress(Progress): + """Reports progress to a curses window + """ + def __init__(self, stdscr): + self.stdscr = stdscr + + curses.use_default_colors() + curses.curs_set(0) + + curses.init_pair(1, curses.COLOR_RED, -1) + curses.init_pair(2, curses.COLOR_GREEN, -1) + + self.last_update = 0 + + self.finished = False + + super(CursesProgress, self).__init__() + + def update(self, table, num_done): + super(CursesProgress, self).update(table, num_done) + + self.render() + + def render(self, force=False): + now = time.time() + + if not force and now - self.last_update < 0.2: + # reactor.callLater(1, self.render) + return + + self.stdscr.clear() + + rows, cols = self.stdscr.getmaxyx() + + duration = int(now) - int(self.start_time) + + minutes, seconds = divmod(duration, 60) + duration_str = '%02dm %02ds' % (minutes, seconds,) + + if self.finished: + status = "Time spent: %s (Done!)" % (duration_str,) + else: + min_perc = min( + (v["num_done"] - v["start"]) * 100. / (v["total"] - v["start"]) + if v["total"] - v["start"] else 100 + for v in self.tables.values() + ) + if min_perc > 0: + est_remaining = (int(now) - self.start_time) * 100 / min_perc + est_remaining_str = '%02dm %02ds remaining' % divmod(est_remaining, 60) + else: + est_remaining_str = "Unknown" + status = ( + "Time spent: %s (est. remaining: %s)" + % (duration_str, est_remaining_str,) + ) + + self.stdscr.addstr( + 0, 0, + status, + curses.A_BOLD, + ) + + max_len = max([len(t) for t in self.tables.keys()]) + + left_margin = 5 + middle_space = 1 + + items = self.tables.items() + items.sort( + key=lambda i: (i[1]["perc"], i[0]), + ) + + for i, (table, data) in enumerate(items): + if i + 2 >= rows: + break + + perc = data["perc"] + + color = curses.color_pair(2) if perc == 100 else curses.color_pair(1) + + self.stdscr.addstr( + i+2, left_margin + max_len - len(table), + table, + curses.A_BOLD | color, + ) + + size = 20 + + progress = "[%s%s]" % ( + "#" * int(perc*size/100), + " " * (size - int(perc*size/100)), + ) + + self.stdscr.addstr( + i+2, left_margin + max_len + middle_space, + "%s %3d%% (%d/%d)" % (progress, perc, data["num_done"], data["total"]), + ) + + if self.finished: + self.stdscr.addstr( + self.rows-1, 0, + "Press any key to exit...", + ) + + self.stdscr.refresh() + self.last_update = time.time() + + def done(self): + self.finished = True + self.render(True) + self.stdscr.getch() + + def set_state(self, state): + self.stdscr.clear() + self.stdscr.addstr( + 0, 0, + state + "...", + curses.A_BOLD, + ) + self.stdscr.refresh() + + +class TerminalProgress(Progress): + """Just prints progress to the terminal + """ + def update(self, table, num_done): + super(TerminalProgress, self).update(table, num_done) + + data = self.tables[table] + + print "%s: %d%% (%d/%d)" % ( + table, data["perc"], + data["num_done"], data["total"], + ) + + def set_state(self, state): + print state + "..." + + +############################################## +############################################## + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("-v", action='store_true') From 5b8b1a43bd392559b1960c4a79f5dcdde0d37f23 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 17:53:40 +0100 Subject: [PATCH 097/104] Split setuping up and processing of tables --- scripts/port_from_sqlite_to_postgres.py | 31 ++++++++++++++++++------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 845e3fcf3..1e7ac072f 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -168,7 +168,7 @@ class Porter(object): ) @defer.inlineCallbacks - def handle_table(self, table): + def setup_table(self, table): def delete_all(txn): txn.execute( "DELETE FROM port_from_sqlite3 WHERE table_name = %s", @@ -287,6 +287,10 @@ class Porter(object): postgres_size = yield self.postgres_store.execute(get_table_size) + defer.returnValue((table, postgres_size, table_size, next_chunk)) + + @defer.inlineCallbacks + def handle_table(self, table, postgres_size, table_size, next_chunk): if not table_size: return @@ -364,14 +368,14 @@ class Porter(object): self.postgres_store = Store(postgres_db_pool, postgres_engine) # Step 1. Set up databases. - self.progress.on_prepare_sqlite() + self.progress.set_state("Preparing SQLite3") self.setup_db(sqlite_config, sqlite_engine) - self.progress.on_prepare_postgres() + self.progress.set_state("Preparing PostgreSQL") self.setup_db(postgres_config, postgres_engine) # Step 2. Get tables. - self.progress.fetching_tables() + self.progress.set_state("Fetching tables") sqlite_tables = yield self.sqlite_store._simple_select_onecol( table="sqlite_master", keyvalues={ @@ -390,7 +394,7 @@ class Porter(object): tables = set(sqlite_tables) & set(postgres_tables) - self.progress.preparing_tables() + self.progress.set_state("Creating tables") logger.info("Found %d tables", len(tables)) @@ -409,10 +413,12 @@ class Porter(object): except Exception as e: logger.info("Failed to create port table: %s", e) - # Process tables. - yield defer.gatherResults( + self.progress.set_state("Preparing tables") + + # Set up tables. + setup_res = yield defer.gatherResults( [ - self.handle_table(table) + self.setup_table(table) for table in tables if table not in ["schema_version", "applied_schema_deltas"] and not table.startswith("sqlite_") @@ -420,6 +426,15 @@ class Porter(object): consumeErrors=True, ) + # Process tables. + yield defer.gatherResults( + [ + self.handle_table(*res) + for res in setup_res + ], + consumeErrors=True, + ) + self.progress.done() except: global end_error_exec_info From 1ccaea5b92e476ebb38a246ea933258a7199bbae Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 10:34:06 +0100 Subject: [PATCH 098/104] Typo in port script --- scripts/port_from_sqlite_to_postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 1e7ac072f..56e149c47 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -575,7 +575,7 @@ class CursesProgress(Progress): if self.finished: self.stdscr.addstr( - self.rows-1, 0, + rows-1, 0, "Press any key to exit...", ) From ce8b0b2868af32fac46a8414393760d326b764b7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 10:45:05 +0100 Subject: [PATCH 099/104] Remove accidentally committed debug hardcode hack --- scripts/port_from_sqlite_to_postgres.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 56e149c47..596be75c4 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -184,10 +184,11 @@ class Porter(object): if table == "sent_transactions": # This is a big table, and we really only need some of the recent # data + yield self.postgres_store.execute(delete_all) # Only save things from the last day - yesterday = 1429114568820 #int(time.time()*1000) - 86400000 + yesterday = int(time.time()*1000) - 86400000 # And save the max transaction id from each destination select = ( From 4a13ae72019655ad0531f93af18382c196fb362d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 11:16:44 +0100 Subject: [PATCH 100/104] Correctly handle total/remaining counts in the presence of sent_transasctions table --- scripts/port_from_sqlite_to_postgres.py | 292 ++++++++++++++---------- 1 file changed, 168 insertions(+), 124 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 596be75c4..1e52d82fe 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -125,6 +125,12 @@ class Store(object): def execute(self, f, *args, **kwargs): return self.runInteraction(f.__name__, f, *args, **kwargs) + def execute_sql(self, sql, *args): + def r(txn): + txn.execute(sql, args) + return txn.fetchall() + return self.runInteraction("execute_sql", r) + def insert_many_txn(self, txn, table, headers, rows): sql = "INSERT INTO %s (%s) VALUES (%s)" % ( table, @@ -146,119 +152,9 @@ class Porter(object): def __init__(self, **kwargs): self.__dict__.update(kwargs) - def convert_rows(self, table, headers, rows): - bool_col_names = BOOLEAN_COLUMNS.get(table, []) - - bool_cols = [ - i for i, h in enumerate(headers) if h in bool_col_names - ] - - def conv(j, col): - if j in bool_cols: - return bool(col) - return col - - for i, row in enumerate(rows): - rows[i] = tuple( - self.postgres_store.database_engine.encode_parameter( - conv(j, col) - ) - for j, col in enumerate(row) - if j > 0 - ) - @defer.inlineCallbacks def setup_table(self, table): - def delete_all(txn): - txn.execute( - "DELETE FROM port_from_sqlite3 WHERE table_name = %s", - (table,) - ) - txn.execute("TRUNCATE %s CASCADE" % (table,)) - - def get_table_size(txn): - txn.execute("SELECT count(*) FROM %s" % (table,)) - size, = txn.fetchone() - return int(size) - - if table == "sent_transactions": - # This is a big table, and we really only need some of the recent - # data - - yield self.postgres_store.execute(delete_all) - - # Only save things from the last day - yesterday = int(time.time()*1000) - 86400000 - - # And save the max transaction id from each destination - select = ( - "SELECT rowid, * FROM sent_transactions WHERE rowid IN (" - "SELECT max(rowid) FROM sent_transactions" - " GROUP BY destination" - ")" - ) - - def r(txn): - txn.execute(select) - rows = txn.fetchall() - headers = [column[0] for column in txn.description] - - ts_ind = headers.index('ts') - - return headers, [r for r in rows if r[ts_ind] < yesterday] - - headers, rows = yield self.sqlite_store.runInteraction( - "select", r, - ) - - self.convert_rows(table, headers, rows) - - inserted_rows = len(rows) - max_inserted_rowid = max(r[0] for r in rows) - - def insert(txn): - self.postgres_store.insert_many_txn( - txn, table, headers[1:], rows - ) - - yield self.postgres_store.execute(insert) - - def get_start_id(txn): - txn.execute( - "SELECT rowid FROM sent_transactions WHERE ts >= ?" - " ORDER BY rowid ASC LIMIT 1", - (yesterday,) - ) - - rows = txn.fetchall() - if rows: - return rows[0][0] - else: - return 1 - - next_chunk = yield self.sqlite_store.execute(get_start_id) - next_chunk = max(max_inserted_rowid + 1, next_chunk) - - yield self.postgres_store._simple_insert( - table="port_from_sqlite3", - values={"table_name": table, "rowid": next_chunk} - ) - - def get_sent_table_size(txn): - txn.execute( - "SELECT count(*) FROM sent_transactions" - " WHERE ts >= ?", - (yesterday,) - ) - size, = txn.fetchone() - return int(size) - - table_size = yield self.sqlite_store.execute( - get_sent_table_size - ) - - table_size += inserted_rows - elif table in APPEND_ONLY_TABLES: + if table in APPEND_ONLY_TABLES: # It's safe to just carry on inserting. next_chunk = yield self.postgres_store._simple_select_one_onecol( table="port_from_sqlite3", @@ -267,28 +163,47 @@ class Porter(object): allow_none=True, ) + total_to_port = None if next_chunk is None: - yield self.postgres_store._simple_insert( - table="port_from_sqlite3", - values={"table_name": table, "rowid": 1} + if table == "sent_transactions": + next_chunk, already_ported, total_to_port = ( + yield self._setup_sent_transactions() + ) + else: + yield self.postgres_store._simple_insert( + table="port_from_sqlite3", + values={"table_name": table, "rowid": 1} + ) + + next_chunk = 1 + already_ported = 0 + + if total_to_port is None: + already_ported, total_to_port = yield self._get_total_count_to_port( + table, next_chunk ) - - next_chunk = 1 - - table_size = yield self.sqlite_store.execute(get_table_size) else: + def delete_all(txn): + txn.execute( + "DELETE FROM port_from_sqlite3 WHERE table_name = %s", + (table,) + ) + txn.execute("TRUNCATE %s CASCADE" % (table,)) + yield self.postgres_store.execute(delete_all) - self.postgres_store._simple_insert( + + yield self.postgres_store._simple_insert( table="port_from_sqlite3", values={"table_name": table, "rowid": 0} ) - table_size = yield self.sqlite_store.execute(get_table_size) next_chunk = 1 - postgres_size = yield self.postgres_store.execute(get_table_size) + already_ported, total_to_port = yield self._get_total_count_to_port( + table, next_chunk + ) - defer.returnValue((table, postgres_size, table_size, next_chunk)) + defer.returnValue((table, already_ported, total_to_port, next_chunk)) @defer.inlineCallbacks def handle_table(self, table, postgres_size, table_size, next_chunk): @@ -315,7 +230,7 @@ class Porter(object): if rows: next_chunk = rows[-1][0] + 1 - self.convert_rows(table, headers, rows) + self._convert_rows(table, headers, rows) def insert(txn): self.postgres_store.insert_many_txn( @@ -414,7 +329,7 @@ class Porter(object): except Exception as e: logger.info("Failed to create port table: %s", e) - self.progress.set_state("Preparing tables") + self.progress.set_state("Setting up") # Set up tables. setup_res = yield defer.gatherResults( @@ -444,6 +359,135 @@ class Porter(object): finally: reactor.stop() + def _convert_rows(self, table, headers, rows): + bool_col_names = BOOLEAN_COLUMNS.get(table, []) + + bool_cols = [ + i for i, h in enumerate(headers) if h in bool_col_names + ] + + def conv(j, col): + if j in bool_cols: + return bool(col) + return col + + for i, row in enumerate(rows): + rows[i] = tuple( + self.postgres_store.database_engine.encode_parameter( + conv(j, col) + ) + for j, col in enumerate(row) + if j > 0 + ) + + @defer.inlineCallbacks + def _setup_sent_transactions(self): + # Only save things from the last day + yesterday = int(time.time()*1000) - 86400000 + + # And save the max transaction id from each destination + select = ( + "SELECT rowid, * FROM sent_transactions WHERE rowid IN (" + "SELECT max(rowid) FROM sent_transactions" + " GROUP BY destination" + ")" + ) + + def r(txn): + txn.execute(select) + rows = txn.fetchall() + headers = [column[0] for column in txn.description] + + ts_ind = headers.index('ts') + + return headers, [r for r in rows if r[ts_ind] < yesterday] + + headers, rows = yield self.sqlite_store.runInteraction( + "select", r, + ) + + self._convert_rows("sent_transactions", headers, rows) + + inserted_rows = len(rows) + max_inserted_rowid = max(r[0] for r in rows) + + def insert(txn): + self.postgres_store.insert_many_txn( + txn, "sent_transactions", headers[1:], rows + ) + + yield self.postgres_store.execute(insert) + + def get_start_id(txn): + txn.execute( + "SELECT rowid FROM sent_transactions WHERE ts >= ?" + " ORDER BY rowid ASC LIMIT 1", + (yesterday,) + ) + + rows = txn.fetchall() + if rows: + return rows[0][0] + else: + return 1 + + next_chunk = yield self.sqlite_store.execute(get_start_id) + next_chunk = max(max_inserted_rowid + 1, next_chunk) + + yield self.postgres_store._simple_insert( + table="port_from_sqlite3", + values={"table_name": "sent_transactions", "rowid": next_chunk} + ) + + def get_sent_table_size(txn): + txn.execute( + "SELECT count(*) FROM sent_transactions" + " WHERE ts >= ?", + (yesterday,) + ) + size, = txn.fetchone() + return int(size) + + remaining_count = yield self.sqlite_store.execute( + get_sent_table_size + ) + + total_count = remaining_count + inserted_rows + + defer.returnValue((next_chunk, remaining_count, total_count)) + + @defer.inlineCallbacks + def _get_remaining_count_to_port(self, table, next_chunk): + rows = yield self.sqlite_store.execute_sql( + "SELECT count(*) FROM %s WHERE rowid >= ?" % (table,), + next_chunk, + ) + + defer.returnValue(rows[0][0]) + + @defer.inlineCallbacks + def _get_already_ported_count(self, table): + rows = yield self.postgres_store.execute_sql( + "SELECT count(*) FROM %s" % (table,), + ) + + defer.returnValue(rows[0][0]) + + @defer.inlineCallbacks + def _get_total_count_to_port(self, table, next_chunk): + remaining, done = yield defer.gatherResults( + [ + self._get_remaining_count_to_port(table, next_chunk), + self._get_already_ported_count(table), + ], + consumeErrors=True, + ) + + remaining = int(remaining) if remaining else 0 + done = int(done) if done else 0 + + defer.returnValue((done, remaining + done)) + ############################################## ###### The following is simply UI stuff ###### From af27b84ff769ceb9fe1aaa10c9435586be4c6867 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 12:40:04 +0100 Subject: [PATCH 101/104] Correctly handle total/remaining counts in the presence of sent_transasctions table --- scripts/port_from_sqlite_to_postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 1e52d82fe..a40a93ca7 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -454,7 +454,7 @@ class Porter(object): total_count = remaining_count + inserted_rows - defer.returnValue((next_chunk, remaining_count, total_count)) + defer.returnValue((next_chunk, inserted_rows, total_count)) @defer.inlineCallbacks def _get_remaining_count_to_port(self, table, next_chunk): From 18f82477011ee9279bde8ad995d654c116d3f652 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 12:41:33 +0100 Subject: [PATCH 102/104] Use TEXT instead of VARCHAR(n), since PostgreSQL treats them the same except for a limit --- synapse/storage/schema/delta/12/v12.sql | 18 +++--- synapse/storage/schema/delta/13/v13.sql | 10 +-- synapse/storage/schema/delta/14/v14.sql | 4 +- .../schema/delta/15/appservice_txns.sql | 4 +- synapse/storage/schema/delta/16/users.sql | 20 +++--- .../schema/full_schemas/11/event_edges.sql | 34 +++++----- .../full_schemas/11/event_signatures.sql | 20 +++--- synapse/storage/schema/full_schemas/11/im.sql | 64 +++++++++---------- .../storage/schema/full_schemas/11/keys.sql | 12 ++-- .../full_schemas/11/media_repository.sql | 34 +++++----- .../schema/full_schemas/11/presence.sql | 12 ++-- .../schema/full_schemas/11/profiles.sql | 6 +- .../schema/full_schemas/11/redactions.sql | 4 +- .../schema/full_schemas/11/room_aliases.sql | 8 +-- .../storage/schema/full_schemas/11/state.sql | 14 ++-- .../schema/full_schemas/11/transactions.sql | 16 ++--- .../storage/schema/full_schemas/11/users.sql | 18 +++--- .../full_schemas/16/application_services.sql | 14 ++-- .../schema/full_schemas/16/event_edges.sql | 34 +++++----- .../full_schemas/16/event_signatures.sql | 20 +++--- synapse/storage/schema/full_schemas/16/im.sql | 64 +++++++++---------- .../storage/schema/full_schemas/16/keys.sql | 12 ++-- .../full_schemas/16/media_repository.sql | 34 +++++----- .../schema/full_schemas/16/presence.sql | 12 ++-- .../schema/full_schemas/16/profiles.sql | 6 +- .../storage/schema/full_schemas/16/push.sql | 22 +++---- .../schema/full_schemas/16/redactions.sql | 4 +- .../schema/full_schemas/16/room_aliases.sql | 8 +-- .../storage/schema/full_schemas/16/state.sql | 14 ++-- .../schema/full_schemas/16/transactions.sql | 16 ++--- .../storage/schema/full_schemas/16/users.sql | 18 +++--- synapse/storage/schema/schema_version.sql | 2 +- 32 files changed, 289 insertions(+), 289 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index a246943f5..878c36260 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -14,16 +14,16 @@ */ CREATE TABLE IF NOT EXISTS rejections( - event_id VARCHAR(150) NOT NULL, - reason VARCHAR(150) NOT NULL, - last_check VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + reason TEXT NOT NULL, + last_check TEXT NOT NULL, UNIQUE (event_id) ); -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name VARCHAR(150) NOT NULL, + user_name TEXT NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, app_id VARCHAR(64) NOT NULL, @@ -41,19 +41,19 @@ CREATE TABLE IF NOT EXISTS pushers ( CREATE TABLE IF NOT EXISTS push_rules ( id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name VARCHAR(150) NOT NULL, - rule_id VARCHAR(150) NOT NULL, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, priority_class TINYINT NOT NULL, priority INTEGER NOT NULL DEFAULT 0, - conditions VARCHAR(150) NOT NULL, - actions VARCHAR(150) NOT NULL, + conditions TEXT NOT NULL, + actions TEXT NOT NULL, UNIQUE(user_name, rule_id) ); CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( - user_id VARCHAR(150), + user_id TEXT, filter_id BIGINT UNSIGNED, filter_json LONGBLOB ); diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index d1da2b48e..326592401 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -15,10 +15,10 @@ CREATE TABLE IF NOT EXISTS application_services( id INTEGER PRIMARY KEY AUTOINCREMENT, - url VARCHAR(150), - token VARCHAR(150), - hs_token VARCHAR(150), - sender VARCHAR(150), + url TEXT, + token TEXT, + hs_token TEXT, + sender TEXT, UNIQUE(token) ); @@ -26,6 +26,6 @@ CREATE TABLE IF NOT EXISTS application_services_regex( id INTEGER PRIMARY KEY AUTOINCREMENT, as_id BIGINT UNSIGNED NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ - regex VARCHAR(150), + regex TEXT, FOREIGN KEY(as_id) REFERENCES application_services(id) ); diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index e5fedc585..1d09ad7a1 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS push_rules_enable ( id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name VARCHAR(150) NOT NULL, - rule_id VARCHAR(150) NOT NULL, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, enabled TINYINT, UNIQUE(user_name, rule_id) ); diff --git a/synapse/storage/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/delta/15/appservice_txns.sql index 1c3324f41..db2e72039 100644 --- a/synapse/storage/schema/delta/15/appservice_txns.sql +++ b/synapse/storage/schema/delta/15/appservice_txns.sql @@ -14,13 +14,13 @@ */ CREATE TABLE IF NOT EXISTS application_services_state( - as_id VARCHAR(150) PRIMARY KEY, + as_id TEXT PRIMARY KEY, state VARCHAR(5), last_txn INTEGER ); CREATE TABLE IF NOT EXISTS application_services_txns( - as_id VARCHAR(150) NOT NULL, + as_id TEXT NOT NULL, txn_id INTEGER NOT NULL, event_ids TEXT NOT NULL, UNIQUE(as_id, txn_id) diff --git a/synapse/storage/schema/delta/16/users.sql b/synapse/storage/schema/delta/16/users.sql index db27bdca0..cd0709250 100644 --- a/synapse/storage/schema/delta/16/users.sql +++ b/synapse/storage/schema/delta/16/users.sql @@ -2,9 +2,9 @@ -- MUST BE DONE BEFORE REMOVING ID COLUMN FROM USERS TABLE BELOW CREATE TABLE IF NOT EXISTS new_access_tokens( id BIGINT UNSIGNED PRIMARY KEY, - user_id VARCHAR(150) NOT NULL, - device_id VARCHAR(150), - token VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, + device_id TEXT, + token TEXT NOT NULL, last_used BIGINT UNSIGNED, UNIQUE(token) ); @@ -20,8 +20,8 @@ ALTER TABLE new_access_tokens RENAME TO access_tokens; -- Remove ID column from `users` table CREATE TABLE IF NOT EXISTS new_users( - name VARCHAR(150), - password_hash VARCHAR(150), + name TEXT, + password_hash TEXT, creation_ts BIGINT UNSIGNED, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) @@ -36,11 +36,11 @@ ALTER TABLE new_users RENAME TO users; -- Remove UNIQUE constraint from `user_ips` table CREATE TABLE IF NOT EXISTS new_user_ips ( - user_id VARCHAR(150) NOT NULL, - access_token VARCHAR(150) NOT NULL, - device_id VARCHAR(150), - ip VARCHAR(150) NOT NULL, - user_agent VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, + access_token TEXT NOT NULL, + device_id TEXT, + ip TEXT NOT NULL, + user_agent TEXT NOT NULL, last_seen BIGINT UNSIGNED NOT NULL ); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 05d0874f0..f7020f779 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS event_forward_extremities( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (event_id, room_id) ); @@ -24,8 +24,8 @@ CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_backward_extremities( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (event_id, room_id) ); @@ -34,9 +34,9 @@ CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_edges( - event_id VARCHAR(150) NOT NULL, - prev_event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + prev_event_id TEXT NOT NULL, + room_id TEXT NOT NULL, is_state BOOL NOT NULL, UNIQUE (event_id, prev_event_id, room_id, is_state) ); @@ -46,7 +46,7 @@ CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id); CREATE TABLE IF NOT EXISTS room_depth( - room_id VARCHAR(150) NOT NULL, + room_id TEXT NOT NULL, min_depth INTEGER NOT NULL, UNIQUE (room_id) ); @@ -55,8 +55,8 @@ CREATE INDEX room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( - event_id VARCHAR(150) NOT NULL, - destination VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + destination TEXT NOT NULL, delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ); @@ -65,10 +65,10 @@ CREATE INDEX event_destinations_id ON event_destinations(event_id); CREATE TABLE IF NOT EXISTS state_forward_extremities( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, UNIQUE (event_id, room_id) ); @@ -79,9 +79,9 @@ CREATE INDEX st_extrem_id ON state_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_auth( - event_id VARCHAR(150) NOT NULL, - auth_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + auth_id TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (event_id, auth_id, room_id) ); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index 429182736..636b2d335 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS event_content_hashes ( - event_id VARCHAR(150), - algorithm VARCHAR(150), + event_id TEXT, + algorithm TEXT, hash bytea, UNIQUE (event_id, algorithm) ); @@ -24,8 +24,8 @@ CREATE INDEX event_content_hashes_id ON event_content_hashes(event_id); CREATE TABLE IF NOT EXISTS event_reference_hashes ( - event_id VARCHAR(150), - algorithm VARCHAR(150), + event_id TEXT, + algorithm TEXT, hash bytea, UNIQUE (event_id, algorithm) ); @@ -34,9 +34,9 @@ CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id); CREATE TABLE IF NOT EXISTS event_signatures ( - event_id VARCHAR(150), - signature_name VARCHAR(150), - key_id VARCHAR(150), + event_id TEXT, + signature_name TEXT, + key_id TEXT, signature bytea, UNIQUE (event_id, signature_name, key_id) ); @@ -45,9 +45,9 @@ CREATE INDEX event_signatures_id ON event_signatures(event_id); CREATE TABLE IF NOT EXISTS event_edge_hashes( - event_id VARCHAR(150), - prev_event_id VARCHAR(150), - algorithm VARCHAR(150), + event_id TEXT, + prev_event_id TEXT, + algorithm TEXT, hash bytea, UNIQUE (event_id, prev_event_id, algorithm) ); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index addbec588..1901654ac 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -16,9 +16,9 @@ CREATE TABLE IF NOT EXISTS events( stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, topological_ordering BIGINT NOT NULL, - event_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + type TEXT NOT NULL, + room_id TEXT NOT NULL, content TEXT NOT NULL, unrecognized_keys TEXT, processed BOOL NOT NULL, @@ -33,8 +33,8 @@ CREATE INDEX events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, internal_metadata TEXT NOT NULL, json TEXT NOT NULL, UNIQUE (event_id) @@ -44,11 +44,11 @@ CREATE INDEX event_json_room_id ON event_json(room_id); CREATE TABLE IF NOT EXISTS state_events( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, - prev_state VARCHAR(150), + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + prev_state TEXT, UNIQUE (event_id) ); @@ -58,10 +58,10 @@ CREATE INDEX state_events_state_key ON state_events (state_key); CREATE TABLE IF NOT EXISTS current_state_events( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, UNIQUE (room_id, type, state_key) ); @@ -71,11 +71,11 @@ CREATE INDEX current_state_events_type ON current_state_events (type); CREATE INDEX current_state_events_state_key ON current_state_events (state_key); CREATE TABLE IF NOT EXISTS room_memberships( - event_id VARCHAR(150) NOT NULL, - user_id VARCHAR(150) NOT NULL, - sender VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - membership VARCHAR(150) NOT NULL + event_id TEXT NOT NULL, + user_id TEXT NOT NULL, + sender TEXT NOT NULL, + room_id TEXT NOT NULL, + membership TEXT NOT NULL ); CREATE INDEX room_memberships_event_id ON room_memberships (event_id); @@ -83,16 +83,16 @@ CREATE INDEX room_memberships_room_id ON room_memberships (room_id); CREATE INDEX room_memberships_user_id ON room_memberships (user_id); CREATE TABLE IF NOT EXISTS feedback( - event_id VARCHAR(150) NOT NULL, - feedback_type VARCHAR(150), - target_event_id VARCHAR(150), - sender VARCHAR(150), - room_id VARCHAR(150) + event_id TEXT NOT NULL, + feedback_type TEXT, + target_event_id TEXT, + sender TEXT, + room_id TEXT ); CREATE TABLE IF NOT EXISTS topics( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, topic TEXT NOT NULL ); @@ -100,8 +100,8 @@ CREATE INDEX topics_event_id ON topics(event_id); CREATE INDEX topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, name TEXT NOT NULL ); @@ -109,14 +109,14 @@ CREATE INDEX room_names_event_id ON room_names(event_id); CREATE INDEX room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( - room_id VARCHAR(150) PRIMARY KEY NOT NULL, + room_id TEXT PRIMARY KEY NOT NULL, is_public BOOL, - creator VARCHAR(150) + creator TEXT ); CREATE TABLE IF NOT EXISTS room_hosts( - room_id VARCHAR(150) NOT NULL, - host VARCHAR(150) NOT NULL, + room_id TEXT NOT NULL, + host TEXT NOT NULL, UNIQUE (room_id, host) ); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index 459b51042..afc142045 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -13,18 +13,18 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS server_tls_certificates( - server_name VARCHAR(150), -- Server name. - fingerprint VARCHAR(150), -- Certificate fingerprint. - from_server VARCHAR(150), -- Which key server the certificate was fetched from. + server_name TEXT, -- Server name. + fingerprint TEXT, -- Certificate fingerprint. + from_server TEXT, -- Which key server the certificate was fetched from. ts_added_ms BIGINT, -- When the certifcate was added. tls_certificate bytea, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); CREATE TABLE IF NOT EXISTS server_signature_keys( - server_name VARCHAR(150), -- Server name. - key_id VARCHAR(150), -- Key version. - from_server VARCHAR(150), -- Which key server the key was fetched form. + server_name TEXT, -- Server name. + key_id TEXT, -- Key version. + from_server TEXT, -- Which key server the key was fetched form. ts_added_ms BIGINT, -- When the key was added. verify_key bytea, -- NACL verification key. UNIQUE (server_name, key_id) diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index 6e0ee0db4..e927e581d 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -14,21 +14,21 @@ */ CREATE TABLE IF NOT EXISTS local_media_repository ( - media_id VARCHAR(150), -- The id used to refer to the media. - media_type VARCHAR(150), -- The MIME-type of the media. + media_id TEXT, -- The id used to refer to the media. + media_type TEXT, -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. created_ts BIGINT, -- When the content was uploaded in ms. - upload_name VARCHAR(150), -- The name the media was uploaded with. - user_id VARCHAR(150), -- The user who uploaded the file. + upload_name TEXT, -- The name the media was uploaded with. + user_id TEXT, -- The user who uploaded the file. UNIQUE (media_id) ); CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( - media_id VARCHAR(150), -- The id used to refer to the media. + media_id TEXT, -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. - thumbnail_method VARCHAR(150), -- The method used to make the thumbnail. + thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_method TEXT, -- The method used to make the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type @@ -39,25 +39,25 @@ CREATE INDEX local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); CREATE TABLE IF NOT EXISTS remote_media_cache ( - media_origin VARCHAR(150), -- The remote HS the media came from. - media_id VARCHAR(150), -- The id used to refer to the media on that server. - media_type VARCHAR(150), -- The MIME-type of the media. + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media on that server. + media_type TEXT, -- The MIME-type of the media. created_ts BIGINT, -- When the content was uploaded in ms. - upload_name VARCHAR(150), -- The name the media was uploaded with. + upload_name TEXT, -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. - filesystem_id VARCHAR(150), -- The name used to store the media on disk. + filesystem_id TEXT, -- The name used to store the media on disk. UNIQUE (media_origin, media_id) ); CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( - media_origin VARCHAR(150), -- The remote HS the media came from. - media_id VARCHAR(150), -- The id used to refer to the media. + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_method VARCHAR(150), -- The method used to make the thumbnail - thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. + thumbnail_method TEXT, -- The method used to make the thumbnail + thumbnail_type TEXT, -- The MIME-type of the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - filesystem_id VARCHAR(150), -- The name used to store the media on disk. + filesystem_id TEXT, -- The name used to store the media on disk. UNIQUE ( media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index fce324b89..d8d82e9fe 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -13,23 +13,23 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS presence( - user_id VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, state VARCHAR(20), - status_msg VARCHAR(150), + status_msg TEXT, mtime BIGINT -- miliseconds since last state change ); -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state CREATE TABLE IF NOT EXISTS presence_allow_inbound( - observed_user_id VARCHAR(150) NOT NULL, - observer_user_id VARCHAR(150) NOT NULL -- a UserID, + observed_user_id TEXT NOT NULL, + observer_user_id TEXT NOT NULL -- a UserID, ); -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? CREATE TABLE IF NOT EXISTS presence_list( - user_id VARCHAR(150) NOT NULL, - observed_user_id VARCHAR(150) NOT NULL, -- a UserID, + user_id TEXT NOT NULL, + observed_user_id TEXT NOT NULL, -- a UserID, accepted BOOLEAN NOT NULL ); diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index ffe75edf9..26e420443 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -13,7 +13,7 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS profiles( - user_id VARCHAR(150) NOT NULL, - displayname VARCHAR(150), - avatar_url VARCHAR(150) + user_id TEXT NOT NULL, + displayname TEXT, + avatar_url TEXT ); diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index 492fd2203..69621955d 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS redactions ( - event_id VARCHAR(150) NOT NULL, - redacts VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + redacts TEXT NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index 622691322..5027b1e3f 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -14,11 +14,11 @@ */ CREATE TABLE IF NOT EXISTS room_aliases( - room_alias VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL + room_alias TEXT NOT NULL, + room_id TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS room_alias_servers( - room_alias VARCHAR(150) NOT NULL, - server VARCHAR(150) NOT NULL + room_alias TEXT NOT NULL, + server TEXT NOT NULL ); diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index 62c20819f..ffd164ab7 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -15,20 +15,20 @@ CREATE TABLE IF NOT EXISTS state_groups( id INTEGER PRIMARY KEY, - room_id VARCHAR(150) NOT NULL, - event_id VARCHAR(150) NOT NULL + room_id TEXT NOT NULL, + event_id TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS state_groups_state( state_group INTEGER NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, - event_id VARCHAR(150) NOT NULL + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + event_id TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS event_to_state_groups( - event_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, state_group INTEGER NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 524a69692..cc5b54f5a 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -14,8 +14,8 @@ */ -- Stores what transaction ids we have received and what our response was CREATE TABLE IF NOT EXISTS received_transactions( - transaction_id VARCHAR(150), - origin VARCHAR(150), + transaction_id TEXT, + origin TEXT, ts BIGINT, response_code INTEGER, response_json bytea, @@ -30,8 +30,8 @@ CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_ref -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( id INTEGER PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering - transaction_id VARCHAR(150), - destination VARCHAR(150), + transaction_id TEXT, + destination TEXT, response_code INTEGER DEFAULT 0, response_json TEXT, ts BIGINT @@ -47,9 +47,9 @@ CREATE INDEX sent_transaction_sent ON sent_transactions(response_code); -- For sent transactions only. CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( transaction_id INTEGER, - destination VARCHAR(150), - pdu_id VARCHAR(150), - pdu_origin VARCHAR(150) + destination TEXT, + pdu_id TEXT, + pdu_origin TEXT ); CREATE INDEX transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination); @@ -57,7 +57,7 @@ CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health CREATE TABLE IF NOT EXISTS destinations( - destination VARCHAR(150) PRIMARY KEY, + destination TEXT PRIMARY KEY, retry_last_ts BIGINT, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 48a6aecfe..eec3da3c3 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS users( id INTEGER PRIMARY KEY AUTOINCREMENT, - name VARCHAR(150), - password_hash VARCHAR(150), + name TEXT, + password_hash TEXT, creation_ts BIGINT, admin SMALLINT DEFAULT 0 NOT NULL, UNIQUE(name) @@ -23,18 +23,18 @@ CREATE TABLE IF NOT EXISTS users( CREATE TABLE IF NOT EXISTS access_tokens( id INTEGER PRIMARY KEY AUTOINCREMENT, - user_id VARCHAR(150) NOT NULL, - device_id VARCHAR(150), - token VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, + device_id TEXT, + token TEXT NOT NULL, last_used BIGINT, UNIQUE(token) ); CREATE TABLE IF NOT EXISTS user_ips ( - user VARCHAR(150) NOT NULL, - access_token VARCHAR(150) NOT NULL, - device_id VARCHAR(150), - ip VARCHAR(150) NOT NULL, + user TEXT NOT NULL, + access_token TEXT NOT NULL, + device_id TEXT, + ip TEXT NOT NULL, user_agent TEXT NOT NULL, last_seen BIGINT NOT NULL, UNIQUE (user, access_token, ip, user_agent) diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql index 5d63d57d5..d382d63fb 100644 --- a/synapse/storage/schema/full_schemas/16/application_services.sql +++ b/synapse/storage/schema/full_schemas/16/application_services.sql @@ -15,10 +15,10 @@ CREATE TABLE IF NOT EXISTS application_services( id BIGINT PRIMARY KEY, - url VARCHAR(150), - token VARCHAR(150), - hs_token VARCHAR(150), - sender VARCHAR(150), + url TEXT, + token TEXT, + hs_token TEXT, + sender TEXT, UNIQUE(token) ); @@ -26,18 +26,18 @@ CREATE TABLE IF NOT EXISTS application_services_regex( id BIGINT PRIMARY KEY, as_id BIGINT NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ - regex VARCHAR(150), + regex TEXT, FOREIGN KEY(as_id) REFERENCES application_services(id) ); CREATE TABLE IF NOT EXISTS application_services_state( - as_id VARCHAR(150) PRIMARY KEY, + as_id TEXT PRIMARY KEY, state VARCHAR(5), last_txn INTEGER ); CREATE TABLE IF NOT EXISTS application_services_txns( - as_id VARCHAR(150) NOT NULL, + as_id TEXT NOT NULL, txn_id INTEGER NOT NULL, event_ids TEXT NOT NULL, UNIQUE(as_id, txn_id) diff --git a/synapse/storage/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/full_schemas/16/event_edges.sql index 05d0874f0..f7020f779 100644 --- a/synapse/storage/schema/full_schemas/16/event_edges.sql +++ b/synapse/storage/schema/full_schemas/16/event_edges.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS event_forward_extremities( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (event_id, room_id) ); @@ -24,8 +24,8 @@ CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_backward_extremities( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (event_id, room_id) ); @@ -34,9 +34,9 @@ CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_edges( - event_id VARCHAR(150) NOT NULL, - prev_event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + prev_event_id TEXT NOT NULL, + room_id TEXT NOT NULL, is_state BOOL NOT NULL, UNIQUE (event_id, prev_event_id, room_id, is_state) ); @@ -46,7 +46,7 @@ CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id); CREATE TABLE IF NOT EXISTS room_depth( - room_id VARCHAR(150) NOT NULL, + room_id TEXT NOT NULL, min_depth INTEGER NOT NULL, UNIQUE (room_id) ); @@ -55,8 +55,8 @@ CREATE INDEX room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( - event_id VARCHAR(150) NOT NULL, - destination VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + destination TEXT NOT NULL, delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ); @@ -65,10 +65,10 @@ CREATE INDEX event_destinations_id ON event_destinations(event_id); CREATE TABLE IF NOT EXISTS state_forward_extremities( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, UNIQUE (event_id, room_id) ); @@ -79,9 +79,9 @@ CREATE INDEX st_extrem_id ON state_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_auth( - event_id VARCHAR(150) NOT NULL, - auth_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + auth_id TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (event_id, auth_id, room_id) ); diff --git a/synapse/storage/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/full_schemas/16/event_signatures.sql index 429182736..636b2d335 100644 --- a/synapse/storage/schema/full_schemas/16/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/16/event_signatures.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS event_content_hashes ( - event_id VARCHAR(150), - algorithm VARCHAR(150), + event_id TEXT, + algorithm TEXT, hash bytea, UNIQUE (event_id, algorithm) ); @@ -24,8 +24,8 @@ CREATE INDEX event_content_hashes_id ON event_content_hashes(event_id); CREATE TABLE IF NOT EXISTS event_reference_hashes ( - event_id VARCHAR(150), - algorithm VARCHAR(150), + event_id TEXT, + algorithm TEXT, hash bytea, UNIQUE (event_id, algorithm) ); @@ -34,9 +34,9 @@ CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id); CREATE TABLE IF NOT EXISTS event_signatures ( - event_id VARCHAR(150), - signature_name VARCHAR(150), - key_id VARCHAR(150), + event_id TEXT, + signature_name TEXT, + key_id TEXT, signature bytea, UNIQUE (event_id, signature_name, key_id) ); @@ -45,9 +45,9 @@ CREATE INDEX event_signatures_id ON event_signatures(event_id); CREATE TABLE IF NOT EXISTS event_edge_hashes( - event_id VARCHAR(150), - prev_event_id VARCHAR(150), - algorithm VARCHAR(150), + event_id TEXT, + prev_event_id TEXT, + algorithm TEXT, hash bytea, UNIQUE (event_id, prev_event_id, algorithm) ); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index 5b4b49448..576653a3c 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -16,9 +16,9 @@ CREATE TABLE IF NOT EXISTS events( stream_ordering INTEGER PRIMARY KEY, topological_ordering BIGINT NOT NULL, - event_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + type TEXT NOT NULL, + room_id TEXT NOT NULL, content TEXT NOT NULL, unrecognized_keys TEXT, processed BOOL NOT NULL, @@ -37,8 +37,8 @@ CREATE INDEX events_order_room ON events ( CREATE TABLE IF NOT EXISTS event_json( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, internal_metadata TEXT NOT NULL, json TEXT NOT NULL, UNIQUE (event_id) @@ -48,11 +48,11 @@ CREATE INDEX event_json_room_id ON event_json(room_id); CREATE TABLE IF NOT EXISTS state_events( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, - prev_state VARCHAR(150), + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + prev_state TEXT, UNIQUE (event_id) ); @@ -62,10 +62,10 @@ CREATE INDEX state_events_state_key ON state_events (state_key); CREATE TABLE IF NOT EXISTS current_state_events( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, UNIQUE (event_id), UNIQUE (room_id, type, state_key) ); @@ -75,11 +75,11 @@ CREATE INDEX current_state_events_type ON current_state_events (type); CREATE INDEX current_state_events_state_key ON current_state_events (state_key); CREATE TABLE IF NOT EXISTS room_memberships( - event_id VARCHAR(150) NOT NULL, - user_id VARCHAR(150) NOT NULL, - sender VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - membership VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + user_id TEXT NOT NULL, + sender TEXT NOT NULL, + room_id TEXT NOT NULL, + membership TEXT NOT NULL, UNIQUE (event_id) ); @@ -87,17 +87,17 @@ CREATE INDEX room_memberships_room_id ON room_memberships (room_id); CREATE INDEX room_memberships_user_id ON room_memberships (user_id); CREATE TABLE IF NOT EXISTS feedback( - event_id VARCHAR(150) NOT NULL, - feedback_type VARCHAR(150), - target_event_id VARCHAR(150), - sender VARCHAR(150), - room_id VARCHAR(150), + event_id TEXT NOT NULL, + feedback_type TEXT, + target_event_id TEXT, + sender TEXT, + room_id TEXT, UNIQUE (event_id) ); CREATE TABLE IF NOT EXISTS topics( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, topic TEXT NOT NULL, UNIQUE (event_id) ); @@ -105,8 +105,8 @@ CREATE TABLE IF NOT EXISTS topics( CREATE INDEX topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, name TEXT NOT NULL, UNIQUE (event_id) ); @@ -114,14 +114,14 @@ CREATE TABLE IF NOT EXISTS room_names( CREATE INDEX room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( - room_id VARCHAR(150) PRIMARY KEY NOT NULL, + room_id TEXT PRIMARY KEY NOT NULL, is_public BOOL, - creator VARCHAR(150) + creator TEXT ); CREATE TABLE IF NOT EXISTS room_hosts( - room_id VARCHAR(150) NOT NULL, - host VARCHAR(150) NOT NULL, + room_id TEXT NOT NULL, + host TEXT NOT NULL, UNIQUE (room_id, host) ); diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql index 459b51042..afc142045 100644 --- a/synapse/storage/schema/full_schemas/16/keys.sql +++ b/synapse/storage/schema/full_schemas/16/keys.sql @@ -13,18 +13,18 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS server_tls_certificates( - server_name VARCHAR(150), -- Server name. - fingerprint VARCHAR(150), -- Certificate fingerprint. - from_server VARCHAR(150), -- Which key server the certificate was fetched from. + server_name TEXT, -- Server name. + fingerprint TEXT, -- Certificate fingerprint. + from_server TEXT, -- Which key server the certificate was fetched from. ts_added_ms BIGINT, -- When the certifcate was added. tls_certificate bytea, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); CREATE TABLE IF NOT EXISTS server_signature_keys( - server_name VARCHAR(150), -- Server name. - key_id VARCHAR(150), -- Key version. - from_server VARCHAR(150), -- Which key server the key was fetched form. + server_name TEXT, -- Server name. + key_id TEXT, -- Key version. + from_server TEXT, -- Which key server the key was fetched form. ts_added_ms BIGINT, -- When the key was added. verify_key bytea, -- NACL verification key. UNIQUE (server_name, key_id) diff --git a/synapse/storage/schema/full_schemas/16/media_repository.sql b/synapse/storage/schema/full_schemas/16/media_repository.sql index 0e819fca3..dacbda40c 100644 --- a/synapse/storage/schema/full_schemas/16/media_repository.sql +++ b/synapse/storage/schema/full_schemas/16/media_repository.sql @@ -14,21 +14,21 @@ */ CREATE TABLE IF NOT EXISTS local_media_repository ( - media_id VARCHAR(150), -- The id used to refer to the media. - media_type VARCHAR(150), -- The MIME-type of the media. + media_id TEXT, -- The id used to refer to the media. + media_type TEXT, -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. created_ts BIGINT, -- When the content was uploaded in ms. - upload_name VARCHAR(150), -- The name the media was uploaded with. - user_id VARCHAR(150), -- The user who uploaded the file. + upload_name TEXT, -- The name the media was uploaded with. + user_id TEXT, -- The user who uploaded the file. UNIQUE (media_id) ); CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( - media_id VARCHAR(150), -- The id used to refer to the media. + media_id TEXT, -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. - thumbnail_method VARCHAR(150), -- The method used to make the thumbnail. + thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_method TEXT, -- The method used to make the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type @@ -39,25 +39,25 @@ CREATE INDEX local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); CREATE TABLE IF NOT EXISTS remote_media_cache ( - media_origin VARCHAR(150), -- The remote HS the media came from. - media_id VARCHAR(150), -- The id used to refer to the media on that server. - media_type VARCHAR(150), -- The MIME-type of the media. + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media on that server. + media_type TEXT, -- The MIME-type of the media. created_ts BIGINT, -- When the content was uploaded in ms. - upload_name VARCHAR(150), -- The name the media was uploaded with. + upload_name TEXT, -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. - filesystem_id VARCHAR(150), -- The name used to store the media on disk. + filesystem_id TEXT, -- The name used to store the media on disk. UNIQUE (media_origin, media_id) ); CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( - media_origin VARCHAR(150), -- The remote HS the media came from. - media_id VARCHAR(150), -- The id used to refer to the media. + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_method VARCHAR(150), -- The method used to make the thumbnail - thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. + thumbnail_method TEXT, -- The method used to make the thumbnail + thumbnail_type TEXT, -- The MIME-type of the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - filesystem_id VARCHAR(150), -- The name used to store the media on disk. + filesystem_id TEXT, -- The name used to store the media on disk. UNIQUE ( media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql index 9c41be296..80088413b 100644 --- a/synapse/storage/schema/full_schemas/16/presence.sql +++ b/synapse/storage/schema/full_schemas/16/presence.sql @@ -13,9 +13,9 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS presence( - user_id VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, state VARCHAR(20), - status_msg VARCHAR(150), + status_msg TEXT, mtime BIGINT, -- miliseconds since last state change UNIQUE (user_id) ); @@ -23,16 +23,16 @@ CREATE TABLE IF NOT EXISTS presence( -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state CREATE TABLE IF NOT EXISTS presence_allow_inbound( - observed_user_id VARCHAR(150) NOT NULL, - observer_user_id VARCHAR(150) NOT NULL, -- a UserID, + observed_user_id TEXT NOT NULL, + observer_user_id TEXT NOT NULL, -- a UserID, UNIQUE (observed_user_id, observer_user_id) ); -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? CREATE TABLE IF NOT EXISTS presence_list( - user_id VARCHAR(150) NOT NULL, - observed_user_id VARCHAR(150) NOT NULL, -- a UserID, + user_id TEXT NOT NULL, + observed_user_id TEXT NOT NULL, -- a UserID, accepted BOOLEAN NOT NULL, UNIQUE (user_id, observed_user_id) ); diff --git a/synapse/storage/schema/full_schemas/16/profiles.sql b/synapse/storage/schema/full_schemas/16/profiles.sql index 21c58a99b..934be8652 100644 --- a/synapse/storage/schema/full_schemas/16/profiles.sql +++ b/synapse/storage/schema/full_schemas/16/profiles.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS profiles( - user_id VARCHAR(150) NOT NULL, - displayname VARCHAR(150), - avatar_url VARCHAR(150), + user_id TEXT NOT NULL, + displayname TEXT, + avatar_url TEXT, UNIQUE(user_id) ); diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql index 5c0c7bc20..db6e05cbd 100644 --- a/synapse/storage/schema/full_schemas/16/push.sql +++ b/synapse/storage/schema/full_schemas/16/push.sql @@ -14,16 +14,16 @@ */ CREATE TABLE IF NOT EXISTS rejections( - event_id VARCHAR(150) NOT NULL, - reason VARCHAR(150) NOT NULL, - last_check VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + reason TEXT NOT NULL, + last_check TEXT NOT NULL, UNIQUE (event_id) ); -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( id BIGINT PRIMARY KEY, - user_name VARCHAR(150) NOT NULL, + user_name TEXT NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, app_id VARCHAR(64) NOT NULL, @@ -41,19 +41,19 @@ CREATE TABLE IF NOT EXISTS pushers ( CREATE TABLE IF NOT EXISTS push_rules ( id BIGINT PRIMARY KEY, - user_name VARCHAR(150) NOT NULL, - rule_id VARCHAR(150) NOT NULL, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, priority_class SMALLINT NOT NULL, priority INTEGER NOT NULL DEFAULT 0, - conditions VARCHAR(150) NOT NULL, - actions VARCHAR(150) NOT NULL, + conditions TEXT NOT NULL, + actions TEXT NOT NULL, UNIQUE(user_name, rule_id) ); CREATE INDEX push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( - user_id VARCHAR(150), + user_id TEXT, filter_id BIGINT, filter_json bytea ); @@ -64,8 +64,8 @@ CREATE INDEX user_filters_by_user_id_filter_id ON user_filters( CREATE TABLE IF NOT EXISTS push_rules_enable ( id BIGINT PRIMARY KEY, - user_name VARCHAR(150) NOT NULL, - rule_id VARCHAR(150) NOT NULL, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, enabled SMALLINT, UNIQUE(user_name, rule_id) ); diff --git a/synapse/storage/schema/full_schemas/16/redactions.sql b/synapse/storage/schema/full_schemas/16/redactions.sql index 492fd2203..69621955d 100644 --- a/synapse/storage/schema/full_schemas/16/redactions.sql +++ b/synapse/storage/schema/full_schemas/16/redactions.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS redactions ( - event_id VARCHAR(150) NOT NULL, - redacts VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + redacts TEXT NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/16/room_aliases.sql b/synapse/storage/schema/full_schemas/16/room_aliases.sql index 2c0853a2a..412bb97fa 100644 --- a/synapse/storage/schema/full_schemas/16/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/16/room_aliases.sql @@ -14,16 +14,16 @@ */ CREATE TABLE IF NOT EXISTS room_aliases( - room_alias VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + room_alias TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (room_alias) ); CREATE INDEX room_aliases_id ON room_aliases(room_id); CREATE TABLE IF NOT EXISTS room_alias_servers( - room_alias VARCHAR(150) NOT NULL, - server VARCHAR(150) NOT NULL + room_alias TEXT NOT NULL, + server TEXT NOT NULL ); CREATE INDEX room_alias_servers_alias ON room_alias_servers(room_alias); diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql index b0cd5ee75..705cac6ce 100644 --- a/synapse/storage/schema/full_schemas/16/state.sql +++ b/synapse/storage/schema/full_schemas/16/state.sql @@ -15,20 +15,20 @@ CREATE TABLE IF NOT EXISTS state_groups( id BIGINT PRIMARY KEY, - room_id VARCHAR(150) NOT NULL, - event_id VARCHAR(150) NOT NULL + room_id TEXT NOT NULL, + event_id TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS state_groups_state( state_group BIGINT NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, - event_id VARCHAR(150) NOT NULL + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + event_id TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS event_to_state_groups( - event_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, state_group BIGINT NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql index ed431bd3a..1ab77cdb6 100644 --- a/synapse/storage/schema/full_schemas/16/transactions.sql +++ b/synapse/storage/schema/full_schemas/16/transactions.sql @@ -14,8 +14,8 @@ */ -- Stores what transaction ids we have received and what our response was CREATE TABLE IF NOT EXISTS received_transactions( - transaction_id VARCHAR(150), - origin VARCHAR(150), + transaction_id TEXT, + origin TEXT, ts BIGINT, response_code INTEGER, response_json bytea, @@ -30,8 +30,8 @@ CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_ref -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering - transaction_id VARCHAR(150), - destination VARCHAR(150), + transaction_id TEXT, + destination TEXT, response_code INTEGER DEFAULT 0, response_json TEXT, ts BIGINT @@ -47,9 +47,9 @@ CREATE INDEX sent_transaction_sent ON sent_transactions(response_code); -- For sent transactions only. CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( transaction_id INTEGER, - destination VARCHAR(150), - pdu_id VARCHAR(150), - pdu_origin VARCHAR(150), + destination TEXT, + pdu_id TEXT, + pdu_origin TEXT, UNIQUE (transaction_id, destination) ); @@ -57,7 +57,7 @@ CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health CREATE TABLE IF NOT EXISTS destinations( - destination VARCHAR(150) PRIMARY KEY, + destination TEXT PRIMARY KEY, retry_last_ts BIGINT, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql index 033e3244b..d2fa3122d 100644 --- a/synapse/storage/schema/full_schemas/16/users.sql +++ b/synapse/storage/schema/full_schemas/16/users.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS users( - name VARCHAR(150), - password_hash VARCHAR(150), + name TEXT, + password_hash TEXT, creation_ts BIGINT, admin SMALLINT DEFAULT 0 NOT NULL, UNIQUE(name) @@ -22,18 +22,18 @@ CREATE TABLE IF NOT EXISTS users( CREATE TABLE IF NOT EXISTS access_tokens( id BIGINT PRIMARY KEY, - user_id VARCHAR(150) NOT NULL, - device_id VARCHAR(150), - token VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, + device_id TEXT, + token TEXT NOT NULL, last_used BIGINT, UNIQUE(token) ); CREATE TABLE IF NOT EXISTS user_ips ( - user_id VARCHAR(150) NOT NULL, - access_token VARCHAR(150) NOT NULL, - device_id VARCHAR(150), - ip VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, + access_token TEXT NOT NULL, + device_id TEXT, + ip TEXT NOT NULL, user_agent TEXT NOT NULL, last_seen BIGINT NOT NULL ); diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql index d9494611e..d682608aa 100644 --- a/synapse/storage/schema/schema_version.sql +++ b/synapse/storage/schema/schema_version.sql @@ -22,6 +22,6 @@ CREATE TABLE IF NOT EXISTS schema_version( CREATE TABLE IF NOT EXISTS applied_schema_deltas( version INTEGER NOT NULL, - file VARCHAR(150) NOT NULL, + file TEXT NOT NULL, UNIQUE(version, file) ); From a1d4813a54ad711d6db7da51f301412e3f139346 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 12:55:29 +0100 Subject: [PATCH 103/104] Quickly fix dodgy est. time remaining --- scripts/port_from_sqlite_to_postgres.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index a40a93ca7..b4ee860a9 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -535,11 +535,20 @@ class CursesProgress(Progress): self.finished = False + self.total_processed = 0 + self.total_remaining = 0 + super(CursesProgress, self).__init__() def update(self, table, num_done): super(CursesProgress, self).update(table, num_done) + self.total_processed = 0 + self.total_remaining = 0 + for table, data in self.tables.items(): + self.total_processed += data["num_done"] - data["start"] + self.total_remaining += data["total"] - data["num_done"] + self.render() def render(self, force=False): @@ -561,13 +570,11 @@ class CursesProgress(Progress): if self.finished: status = "Time spent: %s (Done!)" % (duration_str,) else: - min_perc = min( - (v["num_done"] - v["start"]) * 100. / (v["total"] - v["start"]) - if v["total"] - v["start"] else 100 - for v in self.tables.values() - ) - if min_perc > 0: - est_remaining = (int(now) - self.start_time) * 100 / min_perc + + if self.total_processed > 0: + left = float(self.total_remaining) / self.total_processed + + est_remaining = (int(now) - self.start_time) * left est_remaining_str = '%02dm %02ds remaining' % divmod(est_remaining, 60) else: est_remaining_str = "Unknown" From 073b891ec13e8964759d6f0498cf385fe77729ee Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 13:44:23 +0100 Subject: [PATCH 104/104] Remove unused imports --- synapse/app/homeserver.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 43ce12af7..694a0125a 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -19,8 +19,7 @@ sys.dont_write_bytecode = True from synapse.storage.engines import create_engine from synapse.storage import ( - prepare_database, prepare_sqlite3_database, are_all_users_on_domain, - UpgradeDatabaseException, + are_all_users_on_domain, UpgradeDatabaseException, ) from synapse.server import HomeServer