From 89036579ed475464ccc8619344db513042e1d792 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Mar 2015 11:18:49 +0000 Subject: [PATCH 001/173] Update schema to work with mariadb --- synapse/storage/schema/delta/12/v12.sql | 48 +++++---- synapse/storage/schema/delta/13/v13.sql | 19 ++-- synapse/storage/schema/delta/14/v14.sql | 6 +- .../schema/full_schemas/11/event_edges.sql | 52 +++++----- .../full_schemas/11/event_signatures.sql | 46 ++++----- synapse/storage/schema/full_schemas/11/im.sql | 98 +++++++++---------- .../storage/schema/full_schemas/11/keys.sql | 16 +-- .../full_schemas/11/media_repository.sql | 46 ++++----- .../schema/full_schemas/11/presence.sql | 18 ++-- .../schema/full_schemas/11/profiles.sql | 8 +- .../schema/full_schemas/11/redactions.sql | 6 +- .../schema/full_schemas/11/room_aliases.sql | 12 +-- .../storage/schema/full_schemas/11/state.sql | 29 +++--- .../schema/full_schemas/11/transactions.sql | 32 +++--- .../storage/schema/full_schemas/11/users.sql | 29 +++--- synapse/storage/schema/schema_version.sql | 15 ++- 16 files changed, 224 insertions(+), 256 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index b87ef1fe7..fbe8486b9 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -14,54 +14,50 @@ */ CREATE TABLE IF NOT EXISTS rejections( - event_id TEXT NOT NULL, - reason TEXT NOT NULL, - last_check TEXT NOT NULL, - CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE + event_id VARCHAR(255) NOT NULL, + reason VARCHAR(255) NOT NULL, + last_check VARCHAR(255) NOT NULL, + UNIQUE (event_id) ); -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name TEXT NOT NULL, - profile_tag varchar(32) NOT NULL, - kind varchar(8) NOT NULL, - app_id varchar(64) NOT NULL, - app_display_name varchar(64) NOT NULL, - device_display_name varchar(128) NOT NULL, - pushkey blob NOT NULL, + id INTEGER PRIMARY KEY, + user_name VARCHAR(255) NOT NULL, + profile_tag VARCHAR(32) NOT NULL, + kind VARCHAR(8) NOT NULL, + app_id VARCHAR(64) NOT NULL, + app_display_name VARCHAR(64) NOT NULL, + device_display_name VARCHAR(128) NOT NULL, + pushkey VARBINARY(512) NOT NULL, ts BIGINT NOT NULL, - lang varchar(8), - data blob, + lang VARCHAR(8), + data BLOB, last_token TEXT, last_success BIGINT, failing_since BIGINT, - FOREIGN KEY(user_name) REFERENCES users(name), UNIQUE (app_id, pushkey) ); CREATE TABLE IF NOT EXISTS push_rules ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name TEXT NOT NULL, - rule_id TEXT NOT NULL, + id INTEGER PRIMARY KEY, + user_name VARCHAR(255) NOT NULL, + rule_id VARCHAR(255) NOT NULL, priority_class TINYINT NOT NULL, priority INTEGER NOT NULL DEFAULT 0, - conditions TEXT NOT NULL, - actions TEXT NOT NULL, + conditions VARCHAR(255) NOT NULL, + actions VARCHAR(255) NOT NULL, UNIQUE(user_name, rule_id) ); CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( - user_id TEXT, + user_id VARCHAR(255), filter_id INTEGER, - filter_json TEXT, - FOREIGN KEY(user_id) REFERENCES users(id) + filter_json BLOB ); CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( - user_id, filter_id + user_id, filter_id ); - -PRAGMA user_version = 12; diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index e491ad5ae..68d4ebd89 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -14,21 +14,18 @@ */ CREATE TABLE IF NOT EXISTS application_services( - id INTEGER PRIMARY KEY AUTOINCREMENT, - url TEXT, - token TEXT, - hs_token TEXT, - sender TEXT, - UNIQUE(token) ON CONFLICT ROLLBACK + id INTEGER PRIMARY KEY, + url VARCHAR(255), + token VARCHAR(255), + hs_token VARCHAR(255), + sender VARCHAR(255), + UNIQUE(token) ); CREATE TABLE IF NOT EXISTS application_services_regex( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY, as_id INTEGER NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ - regex TEXT, + regex VARCHAR(255), FOREIGN KEY(as_id) REFERENCES application_services(id) ); - - - diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index 021272644..2b50adbd6 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -1,7 +1,7 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name TEXT NOT NULL, - rule_id TEXT NOT NULL, + id INTEGER PRIMARY KEY, + user_name VARCHAR(255) NOT NULL, + rule_id VARCHAR(255) NOT NULL, enabled TINYINT, UNIQUE(user_name, rule_id) ); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 1e766d6db..823653f2e 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -14,9 +14,9 @@ */ CREATE TABLE IF NOT EXISTS event_forward_extremities( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + UNIQUE (event_id, room_id) ); CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); @@ -24,9 +24,9 @@ CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_backward_extremities( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + UNIQUE (event_id, room_id) ); CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); @@ -34,11 +34,11 @@ CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id CREATE TABLE IF NOT EXISTS event_edges( - event_id TEXT NOT NULL, - prev_event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - is_state INTEGER NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, prev_event_id, room_id, is_state) + event_id VARCHAR(255) NOT NULL, + prev_event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + is_state BOOL NOT NULL, + UNIQUE (event_id, prev_event_id, room_id, is_state) ); CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); @@ -46,30 +46,30 @@ CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); CREATE TABLE IF NOT EXISTS room_depth( - room_id TEXT NOT NULL, + room_id VARCHAR(255) NOT NULL, min_depth INTEGER NOT NULL, - CONSTRAINT uniqueness UNIQUE (room_id) + UNIQUE (room_id) ); CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( - event_id TEXT NOT NULL, - destination TEXT NOT NULL, + event_id VARCHAR(255) NOT NULL, + destination VARCHAR(255) NOT NULL, delivered_ts INTEGER DEFAULT 0, -- or 0 if not delivered - CONSTRAINT uniqueness UNIQUE (event_id, destination) ON CONFLICT REPLACE + UNIQUE (event_id, destination) ); CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); CREATE TABLE IF NOT EXISTS state_forward_extremities( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + type VARCHAR(255) NOT NULL, + state_key VARCHAR(255) NOT NULL, + UNIQUE (event_id, room_id) ); CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( @@ -79,11 +79,11 @@ CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_auth( - event_id TEXT NOT NULL, - auth_id TEXT NOT NULL, - room_id TEXT NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, auth_id, room_id) + event_id VARCHAR(255) NOT NULL, + auth_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + UNIQUE (event_id, auth_id, room_id) ); CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); -CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); \ No newline at end of file +CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index c28c39c48..a7c3acf77 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -14,52 +14,42 @@ */ CREATE TABLE IF NOT EXISTS event_content_hashes ( - event_id TEXT, - algorithm TEXT, + event_id VARCHAR(255), + algorithm VARCHAR(255), hash BLOB, - CONSTRAINT uniqueness UNIQUE (event_id, algorithm) + UNIQUE (event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes( - event_id -); +CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); CREATE TABLE IF NOT EXISTS event_reference_hashes ( - event_id TEXT, - algorithm TEXT, + event_id VARCHAR(255), + algorithm VARCHAR(255), hash BLOB, - CONSTRAINT uniqueness UNIQUE (event_id, algorithm) + UNIQUE (event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes ( - event_id -); +CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); CREATE TABLE IF NOT EXISTS event_signatures ( - event_id TEXT, - signature_name TEXT, - key_id TEXT, + event_id VARCHAR(255), + signature_name VARCHAR(255), + key_id VARCHAR(255), signature BLOB, - CONSTRAINT uniqueness UNIQUE (event_id, signature_name, key_id) + UNIQUE (event_id, signature_name, key_id) ); -CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures ( - event_id -); +CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); CREATE TABLE IF NOT EXISTS event_edge_hashes( - event_id TEXT, - prev_event_id TEXT, - algorithm TEXT, + event_id VARCHAR(255), + prev_event_id VARCHAR(255), + algorithm VARCHAR(255), hash BLOB, - CONSTRAINT uniqueness UNIQUE ( - event_id, prev_event_id, algorithm - ) + UNIQUE (event_id, prev_event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes( - event_id -); +CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index dd00c1cd2..8feacdbe0 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -14,17 +14,17 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, + stream_ordering INTEGER PRIMARY KEY, topological_ordering INTEGER NOT NULL, - event_id TEXT NOT NULL, - type TEXT NOT NULL, - room_id TEXT NOT NULL, - content TEXT NOT NULL, - unrecognized_keys TEXT, + event_id VARCHAR(255) NOT NULL, + type VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + content BLOB NOT NULL, + unrecognized_keys BLOB, processed BOOL NOT NULL, outlier BOOL NOT NULL, depth INTEGER DEFAULT 0 NOT NULL, - CONSTRAINT ev_uniq UNIQUE (event_id) + UNIQUE (event_id) ); CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id); @@ -34,92 +34,92 @@ CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - internal_metadata NOT NULL, + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + internal_metadata BLOB NOT NULL, json BLOB NOT NULL, - CONSTRAINT ev_j_uniq UNIQUE (event_id) + UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS event_json_id ON event_json(event_id); CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); -CREATE TABLE IF NOT EXISTS state_events( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - prev_state TEXT +CREATE TABLE state_events( + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + type VARCHAR(255) NOT NULL, + state_key VARCHAR(255) NOT NULL, + prev_state VARCHAR(255), + UNIQUE (event_id) ); -CREATE UNIQUE INDEX IF NOT EXISTS state_events_event_id ON state_events (event_id); CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); CREATE TABLE IF NOT EXISTS current_state_events( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - CONSTRAINT curr_uniq UNIQUE (room_id, type, state_key) ON CONFLICT REPLACE + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + type VARCHAR(255) NOT NULL, + state_key VARCHAR(255) NOT NULL, + UNIQUE (event_id), + UNIQUE (room_id, type, state_key) ); -CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id); CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); CREATE TABLE IF NOT EXISTS room_memberships( - event_id TEXT NOT NULL, - user_id TEXT NOT NULL, - sender TEXT NOT NULL, - room_id TEXT NOT NULL, - membership TEXT NOT NULL + event_id VARCHAR(255) NOT NULL, + user_id VARCHAR(255) NOT NULL, + sender VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + membership VARCHAR(255) NOT NULL, + UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id); CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); CREATE TABLE IF NOT EXISTS feedback( - event_id TEXT NOT NULL, - feedback_type TEXT, - target_event_id TEXT, - sender TEXT, - room_id TEXT + event_id VARCHAR(255) NOT NULL, + feedback_type VARCHAR(255), + target_event_id VARCHAR(255), + sender VARCHAR(255), + room_id VARCHAR(255), + UNIQUE (event_id) ); CREATE TABLE IF NOT EXISTS topics( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - topic TEXT NOT NULL + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + topic VARCHAR(255) NOT NULL, + UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id); CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - name TEXT NOT NULL + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + name VARCHAR(255) NOT NULL, + UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id); CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( - room_id TEXT PRIMARY KEY NOT NULL, + room_id VARCHAR(255) PRIMARY KEY NOT NULL, is_public INTEGER, - creator TEXT + creator VARCHAR(255) ); CREATE TABLE IF NOT EXISTS room_hosts( - room_id TEXT NOT NULL, - host TEXT NOT NULL, - CONSTRAINT room_hosts_uniq UNIQUE (room_id, host) ON CONFLICT IGNORE + room_id VARCHAR(255) NOT NULL, + host VARCHAR(255) NOT NULL, + UNIQUE (room_id, host) ); CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index a9e0a4fe0..7a5afe094 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -13,19 +13,19 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS server_tls_certificates( - server_name TEXT, -- Server name. - fingerprint TEXT, -- Certificate fingerprint. - from_server TEXT, -- Which key server the certificate was fetched from. + server_name VARCHAR(255), -- Server name. + fingerprint VARCHAR(255), -- Certificate fingerprint. + from_server VARCHAR(255), -- Which key server the certificate was fetched from. ts_added_ms INTEGER, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. - CONSTRAINT uniqueness UNIQUE (server_name, fingerprint) + UNIQUE (server_name, fingerprint) ); CREATE TABLE IF NOT EXISTS server_signature_keys( - server_name TEXT, -- Server name. - key_id TEXT, -- Key version. - from_server TEXT, -- Which key server the key was fetched form. + server_name VARCHAR(255), -- Server name. + key_id VARCHAR(255), -- Key version. + from_server VARCHAR(255), -- Which key server the key was fetched form. ts_added_ms INTEGER, -- When the key was added. verify_key BLOB, -- NACL verification key. - CONSTRAINT uniqueness UNIQUE (server_name, key_id) + UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index afdf48cbf..1e84c1d96 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -14,23 +14,23 @@ */ CREATE TABLE IF NOT EXISTS local_media_repository ( - media_id TEXT, -- The id used to refer to the media. - media_type TEXT, -- The MIME-type of the media. + media_id VARCHAR(255), -- The id used to refer to the media. + media_type VARCHAR(255), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. created_ts INTEGER, -- When the content was uploaded in ms. - upload_name TEXT, -- The name the media was uploaded with. - user_id TEXT, -- The user who uploaded the file. - CONSTRAINT uniqueness UNIQUE (media_id) + upload_name VARCHAR(255), -- The name the media was uploaded with. + user_id VARCHAR(255), -- The user who uploaded the file. + UNIQUE (media_id) ); CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( - media_id TEXT, -- The id used to refer to the media. + media_id VARCHAR(255), -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_type TEXT, -- The MIME-type of the thumbnail. - thumbnail_method TEXT, -- The method used to make the thumbnail. + thumbnail_type VARCHAR(255), -- The MIME-type of the thumbnail. + thumbnail_method VARCHAR(255), -- The method used to make the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - CONSTRAINT uniqueness UNIQUE ( + UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type ) ); @@ -39,29 +39,29 @@ CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); CREATE TABLE IF NOT EXISTS remote_media_cache ( - media_origin TEXT, -- The remote HS the media came from. - media_id TEXT, -- The id used to refer to the media on that server. - media_type TEXT, -- The MIME-type of the media. + media_origin VARCHAR(255), -- The remote HS the media came from. + media_id VARCHAR(255), -- The id used to refer to the media on that server. + media_type VARCHAR(255), -- The MIME-type of the media. created_ts INTEGER, -- When the content was uploaded in ms. - upload_name TEXT, -- The name the media was uploaded with. + upload_name VARCHAR(255), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. - filesystem_id TEXT, -- The name used to store the media on disk. - CONSTRAINT uniqueness UNIQUE (media_origin, media_id) + filesystem_id VARCHAR(255), -- The name used to store the media on disk. + UNIQUE (media_origin, media_id) ); CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( - media_origin TEXT, -- The remote HS the media came from. - media_id TEXT, -- The id used to refer to the media. + media_origin VARCHAR(255), -- The remote HS the media came from. + media_id VARCHAR(255), -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_method TEXT, -- The method used to make the thumbnail - thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_method VARCHAR(255), -- The method used to make the thumbnail + thumbnail_type VARCHAR(255), -- The MIME-type of the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - filesystem_id TEXT, -- The name used to store the media on disk. - CONSTRAINT uniqueness UNIQUE ( + filesystem_id VARCHAR(255), -- The name used to store the media on disk. + UNIQUE ( media_origin, media_id, thumbnail_width, thumbnail_height, - thumbnail_type, thumbnail_type - ) + thumbnail_type + ) ); CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index f9f8db969..519812349 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -13,26 +13,26 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS presence( - user_id INTEGER NOT NULL, + user_id VARCHAR(255) NOT NULL, state INTEGER, - status_msg TEXT, + status_msg VARCHAR(255), mtime INTEGER, -- miliseconds since last state change - FOREIGN KEY(user_id) REFERENCES users(id) + UNIQUE(user_id) ); -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state CREATE TABLE IF NOT EXISTS presence_allow_inbound( - observed_user_id INTEGER NOT NULL, - observer_user_id TEXT, -- a UserID, - FOREIGN KEY(observed_user_id) REFERENCES users(id) + observed_user_id VARCHAR(255) NOT NULL, + observer_user_id VARCHAR(255), -- a UserID, + UNIQUE(observed_user_id) ); -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? CREATE TABLE IF NOT EXISTS presence_list( - user_id INTEGER NOT NULL, - observed_user_id TEXT, -- a UserID, + user_id VARCHAR(255) NOT NULL, + observed_user_id VARCHAR(255), -- a UserID, accepted BOOLEAN, - FOREIGN KEY(user_id) REFERENCES users(id) + UNIQUE(user_id) ); diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index f06a528b4..9b22c9b1b 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS profiles( - user_id INTEGER NOT NULL, - displayname TEXT, - avatar_url TEXT, - FOREIGN KEY(user_id) REFERENCES users(id) + user_id VARCHAR(255) NOT NULL, + displayname VARCHAR(255), + avatar_url VARCHAR(255), + UNIQUE(user_id) ); diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index 5011d95db..774584d7c 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -13,9 +13,9 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS redactions ( - event_id TEXT NOT NULL, - redacts TEXT NOT NULL, - CONSTRAINT ev_uniq UNIQUE (event_id) + event_id VARCHAR(255) NOT NULL, + redacts VARCHAR(255) NOT NULL, + UNIQUE (event_id) ); CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index 0d2df0160..dd51c31c1 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -14,14 +14,12 @@ */ CREATE TABLE IF NOT EXISTS room_aliases( - room_alias TEXT NOT NULL, - room_id TEXT NOT NULL + room_alias VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + UNIQUE (room_alias) ); CREATE TABLE IF NOT EXISTS room_alias_servers( - room_alias TEXT NOT NULL, - server TEXT NOT NULL + room_alias VARCHAR(255) NOT NULL, + server VARCHAR(255) NOT NULL ); - - - diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index 1fe8f1e43..408a2d391 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -15,33 +15,26 @@ CREATE TABLE IF NOT EXISTS state_groups( id INTEGER PRIMARY KEY, - room_id TEXT NOT NULL, - event_id TEXT NOT NULL + room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(255) NOT NULL ); CREATE TABLE IF NOT EXISTS state_groups_state( state_group INTEGER NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - event_id TEXT NOT NULL + room_id VARCHAR(255) NOT NULL, + type VARCHAR(255) NOT NULL, + state_key VARCHAR(255) NOT NULL, + event_id VARCHAR(255) NOT NULL ); CREATE TABLE IF NOT EXISTS event_to_state_groups( - event_id TEXT NOT NULL, + event_id VARCHAR(255) NOT NULL, state_group INTEGER NOT NULL, - CONSTRAINT event_to_state_groups_uniq UNIQUE (event_id) + UNIQUE (event_id) ); CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); -CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state( - state_group -); -CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state( - room_id, type, state_key -); - -CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups( - event_id -); \ No newline at end of file +CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(state_group); +CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(room_id, type, state_key); +CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id); \ No newline at end of file diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 2d30f99b0..0158308a3 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -14,34 +14,31 @@ */ -- Stores what transaction ids we have received and what our response was CREATE TABLE IF NOT EXISTS received_transactions( - transaction_id TEXT, - origin TEXT, + transaction_id VARCHAR(255), + origin VARCHAR(255), ts INTEGER, response_code INTEGER, - response_json TEXT, + response_json BLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx - CONSTRAINT uniquesss UNIQUE (transaction_id, origin) ON CONFLICT REPLACE + UNIQUE (transaction_id, origin) ); -CREATE UNIQUE INDEX IF NOT EXISTS transactions_txid ON received_transactions(transaction_id, origin); CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; -- Stores what transactions we've sent, what their response was (if we got one) and whether we have -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( - id INTEGER PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering - transaction_id TEXT, - destination TEXT, + id INTEGER PRIMARY KEY, -- This is used to apply insertion ordering + transaction_id VARCHAR(255), + destination VARCHAR(255), response_code INTEGER DEFAULT 0, - response_json TEXT, + response_json BLOB, ts INTEGER ); CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); -CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions( - destination -); +CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions(destination); CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); -- So that we can do an efficient look up of all transactions that have yet to be successfully -- sent. @@ -51,18 +48,17 @@ CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_c -- For sent transactions only. CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( transaction_id INTEGER, - destination TEXT, - pdu_id TEXT, - pdu_origin TEXT + destination VARCHAR(255), + pdu_id VARCHAR(255), + pdu_origin VARCHAR(255), + UNIQUE (transaction_id, destination) ); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination); CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_index ON transaction_id_to_pdu(transaction_id, destination); -- To track destination health CREATE TABLE IF NOT EXISTS destinations( - destination TEXT PRIMARY KEY, + destination VARCHAR(255) PRIMARY KEY, retry_last_ts INTEGER, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 08ccfdac0..49a122eed 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -13,33 +13,32 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS users( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT, - password_hash TEXT, + id INTEGER PRIMARY KEY, + name VARCHAR(255), + password_hash VARBINARY(255), creation_ts INTEGER, admin BOOL DEFAULT 0 NOT NULL, - UNIQUE(name) ON CONFLICT ROLLBACK + UNIQUE(name) ); CREATE TABLE IF NOT EXISTS access_tokens( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY, user_id INTEGER NOT NULL, - device_id TEXT, - token TEXT NOT NULL, + device_id VARCHAR(255), + token VARCHAR(255) NOT NULL, last_used INTEGER, FOREIGN KEY(user_id) REFERENCES users(id), - UNIQUE(token) ON CONFLICT ROLLBACK + UNIQUE(token) ); CREATE TABLE IF NOT EXISTS user_ips ( - user TEXT NOT NULL, - access_token TEXT NOT NULL, - device_id TEXT, - ip TEXT NOT NULL, - user_agent TEXT NOT NULL, + user VARCHAR(255) NOT NULL, + access_token VARCHAR(255) NOT NULL, + device_id VARCHAR(255), + ip VARCHAR(255) NOT NULL, + user_agent VARCHAR(255) NOT NULL, last_seen INTEGER NOT NULL, - CONSTRAINT user_ip UNIQUE (user, access_token, ip, user_agent) ON CONFLICT REPLACE + UNIQUE (user, access_token, ip, user_agent) ); CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); - diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql index 0431e2d05..28762861e 100644 --- a/synapse/storage/schema/schema_version.sql +++ b/synapse/storage/schema/schema_version.sql @@ -14,17 +14,16 @@ */ CREATE TABLE IF NOT EXISTS schema_version( - Lock char(1) NOT NULL DEFAULT 'X', -- Makes sure this table only has one row. - version INTEGER NOT NULL, - upgraded BOOL NOT NULL, -- Whether we reached this version from an upgrade or an initial schema. - CONSTRAINT schema_version_lock_x CHECK (Lock='X') - CONSTRAINT schema_version_lock_uniq UNIQUE (Lock) + `Lock` CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row. + `version` INTEGER NOT NULL, + `upgraded` BOOL NOT NULL, -- Whether we reached this version from an upgrade or an initial schema. + CHECK (`Lock`='X') ); CREATE TABLE IF NOT EXISTS applied_schema_deltas( - version INTEGER NOT NULL, - file TEXT NOT NULL, - CONSTRAINT schema_deltas_ver_file UNIQUE (version, file) ON CONFLICT IGNORE + `version` INTEGER NOT NULL, + `file` VARCHAR(255) NOT NULL, + UNIQUE(version, file) ); CREATE INDEX IF NOT EXISTS schema_deltas_ver ON applied_schema_deltas(version); From fae059cc18c0aab25b79d5352aa98ccfda8c637a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Mar 2015 13:42:39 +0000 Subject: [PATCH 002/173] Fix up schemas to work with mariadb --- synapse/storage/schema/delta/12/v12.sql | 16 +++++------ synapse/storage/schema/delta/13/v13.sql | 10 +++---- synapse/storage/schema/delta/14/v14.sql | 4 +-- .../schema/full_schemas/11/event_edges.sql | 18 ++++++------ .../full_schemas/11/event_signatures.sql | 8 +++--- synapse/storage/schema/full_schemas/11/im.sql | 28 +++++++++---------- .../storage/schema/full_schemas/11/keys.sql | 8 +++--- .../full_schemas/11/media_repository.sql | 12 ++++---- .../schema/full_schemas/11/presence.sql | 10 +++---- .../schema/full_schemas/11/profiles.sql | 2 +- .../schema/full_schemas/11/redactions.sql | 2 +- .../schema/full_schemas/11/room_aliases.sql | 4 +-- .../storage/schema/full_schemas/11/state.sql | 12 ++++---- .../schema/full_schemas/11/transactions.sql | 14 +++++----- .../storage/schema/full_schemas/11/users.sql | 18 ++++++------ 15 files changed, 82 insertions(+), 84 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index fbe8486b9..42ea1b2ff 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -18,11 +18,11 @@ CREATE TABLE IF NOT EXISTS rejections( reason VARCHAR(255) NOT NULL, last_check VARCHAR(255) NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( - id INTEGER PRIMARY KEY, + id BIGINT PRIMARY KEY, user_name VARCHAR(255) NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, @@ -37,10 +37,10 @@ CREATE TABLE IF NOT EXISTS pushers ( last_success BIGINT, failing_since BIGINT, UNIQUE (app_id, pushkey) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS push_rules ( - id INTEGER PRIMARY KEY, + id BIGINT PRIMARY KEY, user_name VARCHAR(255) NOT NULL, rule_id VARCHAR(255) NOT NULL, priority_class TINYINT NOT NULL, @@ -48,16 +48,16 @@ CREATE TABLE IF NOT EXISTS push_rules ( conditions VARCHAR(255) NOT NULL, actions VARCHAR(255) NOT NULL, UNIQUE(user_name, rule_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(255), - filter_id INTEGER, + filter_id BIGINT, filter_json BLOB -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( user_id, filter_id -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index 68d4ebd89..f0a5daf44 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -14,18 +14,18 @@ */ CREATE TABLE IF NOT EXISTS application_services( - id INTEGER PRIMARY KEY, + id BIGINT PRIMARY KEY, url VARCHAR(255), token VARCHAR(255), hs_token VARCHAR(255), sender VARCHAR(255), UNIQUE(token) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS application_services_regex( - id INTEGER PRIMARY KEY, - as_id INTEGER NOT NULL, + id BIGINT PRIMARY KEY, + as_id BIGINT NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(255), FOREIGN KEY(as_id) REFERENCES application_services(id) -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index 2b50adbd6..a1260c5c1 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -1,9 +1,9 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( - id INTEGER PRIMARY KEY, + id BIGINT PRIMARY KEY, user_name VARCHAR(255) NOT NULL, rule_id VARCHAR(255) NOT NULL, enabled TINYINT, UNIQUE(user_name, rule_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 823653f2e..910e25dfc 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (event_id, room_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); @@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (event_id, room_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_edges( room_id VARCHAR(255) NOT NULL, is_state BOOL NOT NULL, UNIQUE (event_id, prev_event_id, room_id, is_state) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); @@ -49,7 +49,7 @@ CREATE TABLE IF NOT EXISTS room_depth( room_id VARCHAR(255) NOT NULL, min_depth INTEGER NOT NULL, UNIQUE (room_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); @@ -57,9 +57,9 @@ CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( event_id VARCHAR(255) NOT NULL, destination VARCHAR(255) NOT NULL, - delivered_ts INTEGER DEFAULT 0, -- or 0 if not delivered + delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); @@ -70,11 +70,11 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities( type VARCHAR(255) NOT NULL, state_key VARCHAR(255) NOT NULL, UNIQUE (event_id, room_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( room_id, type, state_key -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); @@ -83,7 +83,7 @@ CREATE TABLE IF NOT EXISTS event_auth( auth_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (event_id, auth_id, room_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index a7c3acf77..334d7c868 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( algorithm VARCHAR(255), hash BLOB, UNIQUE (event_id, algorithm) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); @@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS event_reference_hashes ( algorithm VARCHAR(255), hash BLOB, UNIQUE (event_id, algorithm) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_signatures ( key_id VARCHAR(255), signature BLOB, UNIQUE (event_id, signature_name, key_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); @@ -50,6 +50,6 @@ CREATE TABLE IF NOT EXISTS event_edge_hashes( algorithm VARCHAR(255), hash BLOB, UNIQUE (event_id, prev_event_id, algorithm) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index 8feacdbe0..d58106def 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering INTEGER PRIMARY KEY, - topological_ordering INTEGER NOT NULL, + stream_ordering BIGINT PRIMARY KEY, + topological_ordering BIGINT NOT NULL, event_id VARCHAR(255) NOT NULL, type VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, @@ -23,9 +23,9 @@ CREATE TABLE IF NOT EXISTS events( unrecognized_keys BLOB, processed BOOL NOT NULL, outlier BOOL NOT NULL, - depth INTEGER DEFAULT 0 NOT NULL, + depth BIGINT DEFAULT 0 NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id); CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); @@ -39,19 +39,19 @@ CREATE TABLE IF NOT EXISTS event_json( internal_metadata BLOB NOT NULL, json BLOB NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); -CREATE TABLE state_events( +CREATE TABLE IF NOT EXISTS state_events( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, type VARCHAR(255) NOT NULL, state_key VARCHAR(255) NOT NULL, prev_state VARCHAR(255), UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); @@ -65,7 +65,7 @@ CREATE TABLE IF NOT EXISTS current_state_events( state_key VARCHAR(255) NOT NULL, UNIQUE (event_id), UNIQUE (room_id, type, state_key) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); @@ -78,7 +78,7 @@ CREATE TABLE IF NOT EXISTS room_memberships( room_id VARCHAR(255) NOT NULL, membership VARCHAR(255) NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); @@ -90,14 +90,14 @@ CREATE TABLE IF NOT EXISTS feedback( sender VARCHAR(255), room_id VARCHAR(255), UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS topics( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, topic VARCHAR(255) NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); @@ -112,14 +112,14 @@ CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( room_id VARCHAR(255) PRIMARY KEY NOT NULL, - is_public INTEGER, + is_public BOOL, creator VARCHAR(255) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS room_hosts( room_id VARCHAR(255) NOT NULL, host VARCHAR(255) NOT NULL, UNIQUE (room_id, host) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index 7a5afe094..c0f2ec29b 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -16,16 +16,16 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( server_name VARCHAR(255), -- Server name. fingerprint VARCHAR(255), -- Certificate fingerprint. from_server VARCHAR(255), -- Which key server the certificate was fetched from. - ts_added_ms INTEGER, -- When the certifcate was added. + ts_added_ms BIGINT, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(255), -- Server name. key_id VARCHAR(255), -- Key version. from_server VARCHAR(255), -- Which key server the key was fetched form. - ts_added_ms INTEGER, -- When the key was added. + ts_added_ms BIGINT, -- When the key was added. verify_key BLOB, -- NACL verification key. UNIQUE (server_name, key_id) -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index 1e84c1d96..8bc84dc24 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -17,11 +17,11 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( media_id VARCHAR(255), -- The id used to refer to the media. media_type VARCHAR(255), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. - created_ts INTEGER, -- When the content was uploaded in ms. + created_ts BIGINT, -- When the content was uploaded in ms. upload_name VARCHAR(255), -- The name the media was uploaded with. user_id VARCHAR(255), -- The user who uploaded the file. UNIQUE (media_id) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( media_id VARCHAR(255), -- The id used to refer to the media. @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type ) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); @@ -42,12 +42,12 @@ CREATE TABLE IF NOT EXISTS remote_media_cache ( media_origin VARCHAR(255), -- The remote HS the media came from. media_id VARCHAR(255), -- The id used to refer to the media on that server. media_type VARCHAR(255), -- The MIME-type of the media. - created_ts INTEGER, -- When the content was uploaded in ms. + created_ts BIGINT, -- When the content was uploaded in ms. upload_name VARCHAR(255), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(255), -- The name used to store the media on disk. UNIQUE (media_origin, media_id) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( media_origin VARCHAR(255), -- The remote HS the media came from. @@ -62,7 +62,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type ) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id ON local_media_repository_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index 519812349..803132108 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -14,11 +14,11 @@ */ CREATE TABLE IF NOT EXISTS presence( user_id VARCHAR(255) NOT NULL, - state INTEGER, + state VARCHAR(20), status_msg VARCHAR(255), - mtime INTEGER, -- miliseconds since last state change + mtime BIGINT, -- miliseconds since last state change UNIQUE(user_id) -); +) ENGINE = INNODB; -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS presence_allow_inbound( observed_user_id VARCHAR(255) NOT NULL, observer_user_id VARCHAR(255), -- a UserID, UNIQUE(observed_user_id) -); +) ENGINE = INNODB; -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? @@ -35,4 +35,4 @@ CREATE TABLE IF NOT EXISTS presence_list( observed_user_id VARCHAR(255), -- a UserID, accepted BOOLEAN, UNIQUE(user_id) -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index 9b22c9b1b..32defe2f7 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -17,4 +17,4 @@ CREATE TABLE IF NOT EXISTS profiles( displayname VARCHAR(255), avatar_url VARCHAR(255), UNIQUE(user_id) -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index 774584d7c..ba93e860f 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS redactions ( event_id VARCHAR(255) NOT NULL, redacts VARCHAR(255) NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index dd51c31c1..1e706aac2 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -17,9 +17,9 @@ CREATE TABLE IF NOT EXISTS room_aliases( room_alias VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (room_alias) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS room_alias_servers( room_alias VARCHAR(255) NOT NULL, server VARCHAR(255) NOT NULL -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index 408a2d391..be9dc2920 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -14,24 +14,24 @@ */ CREATE TABLE IF NOT EXISTS state_groups( - id INTEGER PRIMARY KEY, + id VARCHAR(20) PRIMARY KEY, room_id VARCHAR(255) NOT NULL, event_id VARCHAR(255) NOT NULL -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS state_groups_state( - state_group INTEGER NOT NULL, + state_group VARCHAR(20) NOT NULL, room_id VARCHAR(255) NOT NULL, type VARCHAR(255) NOT NULL, state_key VARCHAR(255) NOT NULL, event_id VARCHAR(255) NOT NULL -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS event_to_state_groups( event_id VARCHAR(255) NOT NULL, - state_group INTEGER NOT NULL, + state_group VARCHAR(255) NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 0158308a3..cde6919e4 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -16,12 +16,12 @@ CREATE TABLE IF NOT EXISTS received_transactions( transaction_id VARCHAR(255), origin VARCHAR(255), - ts INTEGER, + ts BIGINT, response_code INTEGER, response_json BLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; @@ -29,13 +29,13 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin -- Stores what transactions we've sent, what their response was (if we got one) and whether we have -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( - id INTEGER PRIMARY KEY, -- This is used to apply insertion ordering + id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering transaction_id VARCHAR(255), destination VARCHAR(255), response_code INTEGER DEFAULT 0, response_json BLOB, ts INTEGER -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions(destination); @@ -52,13 +52,13 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( pdu_id VARCHAR(255), pdu_origin VARCHAR(255), UNIQUE (transaction_id, destination) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(255) PRIMARY KEY, - retry_last_ts INTEGER, + retry_last_ts BIGINT, retry_interval INTEGER -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 49a122eed..55bffb22f 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -13,23 +13,21 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS users( - id INTEGER PRIMARY KEY, name VARCHAR(255), password_hash VARBINARY(255), - creation_ts INTEGER, + creation_ts BIGINT, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS access_tokens( - id INTEGER PRIMARY KEY, - user_id INTEGER NOT NULL, + id INTEGER PRIMARY KEY AUTO_INCREMENT, + user_id VARCHAR(255) NOT NULL, device_id VARCHAR(255), token VARCHAR(255) NOT NULL, - last_used INTEGER, - FOREIGN KEY(user_id) REFERENCES users(id), + last_used BIGINT, UNIQUE(token) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS user_ips ( user VARCHAR(255) NOT NULL, @@ -37,8 +35,8 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(255), ip VARCHAR(255) NOT NULL, user_agent VARCHAR(255) NOT NULL, - last_seen INTEGER NOT NULL, + last_seen BIGINT NOT NULL, UNIQUE (user, access_token, ip, user_agent) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); From 58ed393235381b37182212547af7d65e2f6fa384 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Mar 2015 15:12:05 +0000 Subject: [PATCH 003/173] Remove redundant key --- synapse/storage/schema/full_schemas/11/im.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index d58106def..9849e969b 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -27,7 +27,6 @@ CREATE TABLE IF NOT EXISTS events( UNIQUE (event_id) ) ENGINE = INNODB; -CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id); CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); From d7a0496f3ec534076121632352f44733253e1e16 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Mar 2015 15:59:48 +0000 Subject: [PATCH 004/173] Convert storage layer to be mysql compatible --- synapse/storage/__init__.py | 164 ++++++++++++++++++++-------- synapse/storage/_base.py | 30 ++--- synapse/storage/appservice.py | 4 +- synapse/storage/directory.py | 4 +- synapse/storage/event_federation.py | 25 ++--- synapse/storage/presence.py | 1 - synapse/storage/push_rule.py | 4 +- synapse/storage/registration.py | 14 +-- synapse/storage/room.py | 4 +- synapse/storage/roommember.py | 2 +- synapse/storage/state.py | 10 +- synapse/storage/stream.py | 4 +- synapse/storage/transactions.py | 6 +- 13 files changed, 171 insertions(+), 101 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4b16f445d..30cba4771 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -51,6 +51,8 @@ import logging import os import re +import threading + logger = logging.getLogger(__name__) @@ -89,6 +91,9 @@ class DataStore(RoomMemberStore, RoomStore, self.min_token_deferred = self._get_min_token() self.min_token = None + self._next_stream_id_lock = threading.Lock() + self._next_stream_id = int(hs.get_clock().time_msec()) * 1000 + @defer.inlineCallbacks @log_function def persist_event(self, event, context, backfilled=False, @@ -172,7 +177,6 @@ class DataStore(RoomMemberStore, RoomStore, "type": s.type, "state_key": s.state_key, }, - or_replace=True, ) if event.is_state() and is_new_state: @@ -186,7 +190,6 @@ class DataStore(RoomMemberStore, RoomStore, "type": event.type, "state_key": event.state_key, }, - or_replace=True, ) for prev_state_id, _ in event.prev_state: @@ -285,7 +288,6 @@ class DataStore(RoomMemberStore, RoomStore, "internal_metadata": metadata_json.decode("UTF-8"), "json": encode_canonical_json(event_dict).decode("UTF-8"), }, - or_replace=True, ) content = encode_canonical_json( @@ -303,8 +305,9 @@ class DataStore(RoomMemberStore, RoomStore, "depth": event.depth, } - if stream_ordering is not None: - vals["stream_ordering"] = stream_ordering + if stream_ordering is None: + stream_ordering = self.get_next_stream_id() + unrec = { k: v @@ -322,21 +325,18 @@ class DataStore(RoomMemberStore, RoomStore, unrec ).decode("UTF-8") - try: - self._simple_insert_txn( - txn, - "events", - vals, - or_replace=(not outlier), - or_ignore=bool(outlier), - ) - except: - logger.warn( - "Failed to persist, probably duplicate: %s", - event.event_id, - exc_info=True, - ) - raise _RollbackButIsFineException("_persist_event") + sql = ( + "INSERT INTO events" + " (stream_ordering, topological_ordering, event_id, type," + " room_id, content, processed, outlier, depth)" + " VALUES (%s,?,?,?,?,?,?,?,?)" + ) % (stream_ordering,) + + txn.execute( + sql, + (event.depth, event.event_id, event.type, event.room_id, + content, True, outlier, event.depth) + ) if context.rejected: self._store_rejections_txn(txn, event.event_id, context.rejected) @@ -357,7 +357,6 @@ class DataStore(RoomMemberStore, RoomStore, txn, "state_events", vals, - or_replace=True, ) if is_new_state and not context.rejected: @@ -370,7 +369,6 @@ class DataStore(RoomMemberStore, RoomStore, "type": event.type, "state_key": event.state_key, }, - or_replace=True, ) for e_id, h in event.prev_state: @@ -383,7 +381,6 @@ class DataStore(RoomMemberStore, RoomStore, "room_id": event.room_id, "is_state": 1, }, - or_ignore=True, ) for hash_alg, hash_base64 in event.hashes.items(): @@ -408,7 +405,6 @@ class DataStore(RoomMemberStore, RoomStore, "room_id": event.room_id, "auth_id": auth_id, }, - or_ignore=True, ) (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) @@ -420,8 +416,7 @@ class DataStore(RoomMemberStore, RoomStore, # invalidate the cache for the redacted event self._get_event_cache.pop(event.redacts) txn.execute( - "INSERT OR IGNORE INTO redactions " - "(event_id, redacts) VALUES (?,?)", + "INSERT INTO redactions (event_id, redacts) VALUES (?,?)", (event.event_id, event.redacts) ) @@ -515,7 +510,8 @@ class DataStore(RoomMemberStore, RoomStore, "ip": ip, "user_agent": user_agent, "last_seen": int(self._clock.time_msec()), - } + }, + or_replace=True, ) def get_user_ip_and_agents(self, user): @@ -559,6 +555,12 @@ class DataStore(RoomMemberStore, RoomStore, "have_events", f, ) + def get_next_stream_id(self): + with self._next_stream_id_lock: + i = self._next_stream_id + self._next_stream_id += 1 + return i + def read_schema(path): """ Read the named database schema. @@ -594,7 +596,7 @@ def prepare_database(db_conn): else: _setup_new_database(cur) - cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,)) + # cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,)) cur.close() db_conn.commit() @@ -657,19 +659,17 @@ def _setup_new_database(cur): directory_entries = os.listdir(sql_dir) - sql_script = "BEGIN TRANSACTION;\n" for filename in fnmatch.filter(directory_entries, "*.sql"): sql_loc = os.path.join(sql_dir, filename) logger.debug("Applying schema %s", sql_loc) - sql_script += read_schema(sql_loc) - sql_script += "\n" - sql_script += "COMMIT TRANSACTION;" - cur.executescript(sql_script) + executescript(cur, sql_loc) cur.execute( - "INSERT OR REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)", - (max_current_ver, False) + _convert_param_style( + "REPLACE INTO schema_version (version, upgraded)" + " VALUES (?,?)" + ), + (max_current_ver, False,) ) _upgrade_existing_database( @@ -737,6 +737,8 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, if not upgraded: start_ver += 1 + logger.debug("applied_delta_files: %s", applied_delta_files) + for v in range(start_ver, SCHEMA_VERSION + 1): logger.debug("Upgrading schema to v%d", v) @@ -753,6 +755,7 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, directory_entries.sort() for file_name in directory_entries: relative_path = os.path.join(str(v), file_name) + logger.debug("Found file: %s", relative_path) if relative_path in applied_delta_files: continue @@ -774,9 +777,8 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, module.run_upgrade(cur) elif ext == ".sql": # A plain old .sql file, just read and execute it - delta_schema = read_schema(absolute_path) logger.debug("Applying schema %s", relative_path) - cur.executescript(delta_schema) + executescript(cur, absolute_path) else: # Not a valid delta file. logger.warn( @@ -788,24 +790,85 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, # Mark as done. cur.execute( - "INSERT INTO applied_schema_deltas (version, file)" - " VALUES (?,?)", + _convert_param_style( + "INSERT INTO applied_schema_deltas (version, file)" + " VALUES (?,?)" + ), (v, relative_path) ) cur.execute( - "INSERT OR REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)", + _convert_param_style( + "REPLACE INTO schema_version (version, upgraded)" + " VALUES (?,?)" + ), (v, True) ) +def _convert_param_style(sql): + return sql.replace("?", "%s") + + +def get_statements(f): + statement_buffer = "" + in_comment = False # If we're in a /* ... */ style comment + + for line in f: + line = line.strip() + + if in_comment: + # Check if this line contains an end to the comment + comments = line.split("*/", 1) + if len(comments) == 1: + continue + line = comments[1] + in_comment = False + + # Remove inline block comments + line = re.sub(r"/\*.*\*/", " ", line) + + # Does this line start a comment? + comments = line.split("/*", 1) + if len(comments) > 1: + line = comments[0] + in_comment = True + + # Deal with line comments + line = line.split("--", 1)[0] + line = line.split("//", 1)[0] + + # Find *all* semicolons. We need to treat first and last entry + # specially. + statements = line.split(";") + + # We must prepend statement_buffer to the first statement + first_statement = "%s %s" % ( + statement_buffer.strip(), + statements[0].strip() + ) + statements[0] = first_statement + + # Every entry, except the last, is a full statement + for statement in statements[:-1]: + yield statement.strip() + + # The last entry did *not* end in a semicolon, so we store it for the + # next semicolon we find + statement_buffer = statements[-1].strip() + + +def executescript(txn, schema_path): + with open(schema_path, 'r') as f: + for statement in get_statements(f): + txn.execute(statement) + + def _get_or_create_schema_state(txn): schema_path = os.path.join( dir_path, "schema", "schema_version.sql", ) - create_schema = read_schema(schema_path) - txn.executescript(create_schema) + executescript(txn, schema_path) txn.execute("SELECT version, upgraded FROM schema_version") row = txn.fetchone() @@ -814,10 +877,13 @@ def _get_or_create_schema_state(txn): if current_version: txn.execute( - "SELECT file FROM applied_schema_deltas WHERE version >= ?", + _convert_param_style( + "SELECT file FROM applied_schema_deltas WHERE version >= ?" + ), (current_version,) ) - return current_version, txn.fetchall(), upgraded + applied_deltas = [d for d, in txn.fetchall()] + return current_version, applied_deltas, upgraded return None @@ -849,7 +915,9 @@ def prepare_sqlite3_database(db_conn): if row and row[0]: db_conn.execute( - "INSERT OR REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)", + _convert_param_style( + "REPLACE INTO schema_version (version, upgraded)" + " VALUES (?,?)" + ), (row[0], False) ) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 2979a8352..24ff872da 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -102,6 +102,10 @@ def cached(max_entries=1000): return wrap +def _convert_param_style(sql): + return sql.replace("?", "%s") + + class LoggingTransaction(object): """An object that almost-transparently proxies for the 'txn' object passed to the constructor. Adds logging and metrics to the .execute() @@ -122,6 +126,8 @@ class LoggingTransaction(object): # TODO(paul): Maybe use 'info' and 'debug' for values? sql_logger.debug("[SQL] {%s} %s", self.name, sql) + sql = _convert_param_style(sql) + try: if args and args[0]: values = args[0] @@ -305,11 +311,11 @@ class SQLBaseStore(object): The result of decoder(results) """ def interaction(txn): - cursor = txn.execute(query, args) + txn.execute(query, args) if decoder: - return decoder(cursor) + return decoder(txn) else: - return cursor.fetchall() + return txn.fetchall() return self.runInteraction(desc, interaction) @@ -337,8 +343,7 @@ class SQLBaseStore(object): def _simple_insert_txn(self, txn, table, values, or_replace=False, or_ignore=False): sql = "%s INTO %s (%s) VALUES(%s)" % ( - ("INSERT OR REPLACE" if or_replace else - "INSERT OR IGNORE" if or_ignore else "INSERT"), + ("REPLACE" if or_replace else "INSERT"), table, ", ".join(k for k in values), ", ".join("?" for k in values) @@ -448,8 +453,7 @@ class SQLBaseStore(object): def _simple_select_onecol_txn(self, txn, table, keyvalues, retcol): sql = ( - "SELECT %(retcol)s FROM %(table)s WHERE %(where)s " - "ORDER BY rowid asc" + "SELECT %(retcol)s FROM %(table)s WHERE %(where)s" ) % { "retcol": retcol, "table": table, @@ -505,14 +509,14 @@ class SQLBaseStore(object): retcols : list of strings giving the names of the columns to return """ if keyvalues: - sql = "SELECT %s FROM %s WHERE %s ORDER BY rowid asc" % ( + sql = "SELECT %s FROM %s WHERE %s" % ( ", ".join(retcols), table, " AND ".join("%s = ?" % (k, ) for k in keyvalues) ) txn.execute(sql, keyvalues.values()) else: - sql = "SELECT %s FROM %s ORDER BY rowid asc" % ( + sql = "SELECT %s FROM %s" % ( ", ".join(retcols), table ) @@ -546,7 +550,7 @@ class SQLBaseStore(object): retcols=None, allow_none=False): """ Combined SELECT then UPDATE.""" if retcols: - select_sql = "SELECT %s FROM %s WHERE %s ORDER BY rowid asc" % ( + select_sql = "SELECT %s FROM %s WHERE %s" % ( ", ".join(retcols), table, " AND ".join("%s = ?" % (k) for k in keyvalues) @@ -580,8 +584,8 @@ class SQLBaseStore(object): updatevalues.values() + keyvalues.values() ) - if txn.rowcount == 0: - raise StoreError(404, "No row found") + # if txn.rowcount == 0: + # raise StoreError(404, "No row found") if txn.rowcount > 1: raise StoreError(500, "More than one row matched") @@ -802,7 +806,7 @@ class Table(object): _select_where_clause = "SELECT %s FROM %s WHERE %s" _select_clause = "SELECT %s FROM %s" - _insert_clause = "INSERT OR REPLACE INTO %s (%s) VALUES (%s)" + _insert_clause = "REPLACE INTO %s (%s) VALUES (%s)" @classmethod def select_statement(cls, where_clause=None): diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 850676ce6..375265d66 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -147,11 +147,11 @@ class ApplicationServiceStore(SQLBaseStore): return True def _get_as_id_txn(self, txn, token): - cursor = txn.execute( + txn.execute( "SELECT id FROM application_services WHERE token=?", (token,) ) - res = cursor.fetchone() + res = txn.fetchone() if res: return res[0] diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index 68b7d5969..0c2adffbb 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -111,12 +111,12 @@ class DirectoryStore(SQLBaseStore): ) def _delete_room_alias_txn(self, txn, room_alias): - cursor = txn.execute( + txn.execute( "SELECT room_id FROM room_aliases WHERE room_alias = ?", (room_alias.to_string(),) ) - res = cursor.fetchone() + res = txn.fetchone() if res: room_id = res[0] else: diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 2deda8ac5..5d66b2f24 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -242,7 +242,6 @@ class EventFederationStore(SQLBaseStore): "room_id": room_id, "min_depth": depth, }, - or_replace=True, ) def _handle_prev_events(self, txn, outlier, event_id, prev_events, @@ -262,7 +261,6 @@ class EventFederationStore(SQLBaseStore): "room_id": room_id, "is_state": 0, }, - or_ignore=True, ) # Update the extremities table if this is not an outlier. @@ -281,19 +279,19 @@ class EventFederationStore(SQLBaseStore): # We only insert as a forward extremity the new event if there are # no other events that reference it as a prev event query = ( - "INSERT OR IGNORE INTO %(table)s (event_id, room_id) " - "SELECT ?, ? WHERE NOT EXISTS (" - "SELECT 1 FROM %(event_edges)s WHERE " - "prev_event_id = ? " - ")" - ) % { - "table": "event_forward_extremities", - "event_edges": "event_edges", - } + "SELECT 1 FROM event_edges WHERE prev_event_id = ?" + ) - logger.debug("query: %s", query) + txn.execute(query, (event_id,)) - txn.execute(query, (event_id, room_id, event_id)) + if not txn.fetchone(): + query = ( + "INSERT INTO event_forward_extremities" + " (event_id, room_id)" + " VALUES (?, ?)" + ) + + txn.execute(query, (event_id, room_id)) # Insert all the prev_events as a backwards thing, they'll get # deleted in a second if they're incorrect anyway. @@ -306,7 +304,6 @@ class EventFederationStore(SQLBaseStore): "event_id": e_id, "room_id": room_id, }, - or_ignore=True, ) # Also delete from the backwards extremities table all ones that diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index 1dcd34723..0084d67e5 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -45,7 +45,6 @@ class PresenceStore(SQLBaseStore): updatevalues={"state": new_state["state"], "status_msg": new_state["status_msg"], "mtime": self._clock.time_msec()}, - retcols=["state"], ) def allow_presence_visible(self, observed_localpart, observer_userid): diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index d769db2c7..27a071632 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -153,7 +153,7 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, (user_name, priority_class, new_rule_priority)) # now insert the new rule - sql = "INSERT OR REPLACE INTO "+PushRuleTable.table_name+" (" + sql = "INSERT INTO "+PushRuleTable.table_name+" (" sql += ",".join(new_rule.keys())+") VALUES (" sql += ", ".join(["?" for _ in new_rule.keys()])+")" @@ -182,7 +182,7 @@ class PushRuleStore(SQLBaseStore): new_rule['priority_class'] = priority_class new_rule['priority'] = new_prio - sql = "INSERT OR REPLACE INTO "+PushRuleTable.table_name+" (" + sql = "INSERT INTO "+PushRuleTable.table_name+" (" sql += ",".join(new_rule.keys())+") VALUES (" sql += ", ".join(["?" for _ in new_rule.keys()])+")" diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index adc8fc079..344dd3aaa 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -39,14 +39,10 @@ class RegistrationStore(SQLBaseStore): Raises: StoreError if there was a problem adding this. """ - row = yield self._simple_select_one("users", {"name": user_id}, ["id"]) - if not row: - raise StoreError(400, "Bad user ID supplied.") - row_id = row["id"] yield self._simple_insert( "access_tokens", { - "user_id": row_id, + "user_id": user_id, "token": token } ) @@ -82,7 +78,7 @@ class RegistrationStore(SQLBaseStore): # it's possible for this to get a conflict, but only for a single user # since tokens are namespaced based on their user ID txn.execute("INSERT INTO access_tokens(user_id, token) " + - "VALUES (?,?)", [txn.lastrowid, token]) + "VALUES (?,?)", [user_id, token]) def get_user_by_id(self, user_id): query = ("SELECT users.name, users.password_hash FROM users" @@ -124,12 +120,12 @@ class RegistrationStore(SQLBaseStore): "SELECT users.name, users.admin," " access_tokens.device_id, access_tokens.id as token_id" " FROM users" - " INNER JOIN access_tokens on users.id = access_tokens.user_id" + " INNER JOIN access_tokens on users.name = access_tokens.user_id" " WHERE token = ?" ) - cursor = txn.execute(sql, (token,)) - rows = self.cursor_to_dict(cursor) + txn.execute(sql, (token,)) + rows = self.cursor_to_dict(txn) if rows: return rows[0] diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 549c9af39..3c23f2921 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -114,9 +114,9 @@ class RoomStore(SQLBaseStore): "name": name_subquery, } - c = txn.execute(sql, (is_public,)) + txn.execute(sql, (is_public,)) - return c.fetchall() + return txn.fetchall() rows = yield self.runInteraction( "get_rooms", f diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 65ffb4627..e8ede14cd 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -68,7 +68,7 @@ class RoomMemberStore(SQLBaseStore): # Update room hosts table if event.membership == Membership.JOIN: sql = ( - "INSERT OR IGNORE INTO room_hosts (room_id, host) " + "REPLACE INTO room_hosts (room_id, host) " "VALUES (?, ?)" ) txn.execute(sql, (event.room_id, domain)) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 456e4bd45..888837cd1 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -15,6 +15,8 @@ from ._base import SQLBaseStore +from synapse.util.stringutils import random_string + import logging logger = logging.getLogger(__name__) @@ -89,14 +91,15 @@ class StateStore(SQLBaseStore): state_group = context.state_group if not state_group: + group = _make_group_id(self._clock) state_group = self._simple_insert_txn( txn, table="state_groups", values={ + "id": group, "room_id": event.room_id, "event_id": event.event_id, }, - or_ignore=True, ) for state in state_events.values(): @@ -110,7 +113,6 @@ class StateStore(SQLBaseStore): "state_key": state.state_key, "event_id": state.event_id, }, - or_ignore=True, ) self._simple_insert_txn( @@ -122,3 +124,7 @@ class StateStore(SQLBaseStore): }, or_replace=True, ) + + +def _make_group_id(clock): + return str(int(clock.time_msec())) + random_string(5) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 09bc52221..64adb0c7f 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -110,7 +110,7 @@ class _StreamToken(namedtuple("_StreamToken", "topological stream")): if self.topological is None: return "(%d < %s)" % (self.stream, "stream_ordering") else: - return "(%d < %s OR (%d == %s AND %d < %s))" % ( + return "(%d < %s OR (%d = %s AND %d < %s))" % ( self.topological, "topological_ordering", self.topological, "topological_ordering", self.stream, "stream_ordering", @@ -120,7 +120,7 @@ class _StreamToken(namedtuple("_StreamToken", "topological stream")): if self.topological is None: return "(%d >= %s)" % (self.stream, "stream_ordering") else: - return "(%d > %s OR (%d == %s AND %d >= %s))" % ( + return "(%d > %s OR (%d = %s AND %d >= %s))" % ( self.topological, "topological_ordering", self.topological, "topological_ordering", self.stream, "stream_ordering", diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 0b8a3b7a0..b5ed5453d 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -121,8 +121,8 @@ class TransactionStore(SQLBaseStore): SentTransactions.select_statement("destination = ?"), ) - results = txn.execute(query, (destination,)) - results = SentTransactions.decode_results(results) + txn.execute(query, (destination,)) + results = SentTransactions.decode_results(txn) prev_txns = [r.transaction_id for r in results] @@ -266,7 +266,7 @@ class TransactionStore(SQLBaseStore): retry_last_ts, retry_interval): query = ( - "INSERT OR REPLACE INTO %s " + "REPLACE INTO %s " "(destination, retry_last_ts, retry_interval) " "VALUES (?, ?, ?) " ) % DestinationsTable.table_name From 0d0610870ddd3c47fe397f1d026bf1ca01dbdf74 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 10:55:31 +0000 Subject: [PATCH 005/173] Fix up schemas some more --- synapse/storage/schema/delta/12/v12.sql | 2 +- synapse/storage/schema/full_schemas/11/event_edges.sql | 2 +- synapse/storage/schema/full_schemas/11/transactions.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index 42ea1b2ff..b526109e6 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -60,4 +60,4 @@ CREATE TABLE IF NOT EXISTS user_filters( CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( user_id, filter_id -) ENGINE = INNODB; +); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 910e25dfc..0f53488e9 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -74,7 +74,7 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities( CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( room_id, type, state_key -) ENGINE = INNODB; +); CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index cde6919e4..0570bf95d 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -34,7 +34,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( destination VARCHAR(255), response_code INTEGER DEFAULT 0, response_json BLOB, - ts INTEGER + ts BIGINT ) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); From 455579ca90dd5479dae785b5a1b9bdd201654ea6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 10:55:55 +0000 Subject: [PATCH 006/173] Make database selection configurable --- synapse/app/homeserver.py | 44 ++++++++++++++++++++++++++++---------- synapse/config/database.py | 9 ++++++++ 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 15c454af7..a2fca2e02 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -61,6 +61,7 @@ import resource import subprocess import sqlite3 import syweb +import yaml logger = logging.getLogger(__name__) @@ -108,14 +109,14 @@ class SynapseHomeServer(HomeServer): return None def build_db_pool(self): - return adbapi.ConnectionPool( - "sqlite3", self.get_db_name(), - check_same_thread=False, - cp_min=1, - cp_max=1, - cp_openfun=prepare_database, # Prepare the database for each conn - # so that :memory: sqlite works - ) + name = self.db_config.pop("name", None) + if name == "MySQLdb": + return adbapi.ConnectionPool( + name, + **self.db_config + ) + + raise RuntimeError("Unsupported database type") def create_resource_tree(self, redirect_root_to_web_client): """Create the resource tree for this Home Server. @@ -357,11 +358,29 @@ def setup(config_options): tls_context_factory = context_factory.ServerContextFactory(config) + if config.database_config: + with open(config.database_config, 'r') as f: + db_config = yaml.safe_load(f) + + name = db_config.get("name", None) + if name == "MySQLdb": + db_config.update({ + "sql_mode": "TRADITIONAL", + "charset": "utf8", + "use_unicode": True, + }) + else: + db_config = { + "name": "sqlite3", + "database": config.database_path, + } + hs = SynapseHomeServer( config.server_name, domain_with_port=domain_with_port, upload_dir=os.path.abspath("uploads"), db_name=config.database_path, + db_config=db_config, tls_context_factory=tls_context_factory, config=config, content_addr=config.content_addr, @@ -377,9 +396,12 @@ def setup(config_options): logger.info("Preparing database: %s...", db_name) try: - with sqlite3.connect(db_name) as db_conn: - prepare_sqlite3_database(db_conn) - prepare_database(db_conn) + # with sqlite3.connect(db_name) as db_conn: + # prepare_sqlite3_database(db_conn) + # prepare_database(db_conn) + import MySQLdb + db_conn = MySQLdb.connect(**db_config) + prepare_database(db_conn) except UpgradeDatabaseException: sys.stderr.write( "\nFailed to upgrade database.\n" diff --git a/synapse/config/database.py b/synapse/config/database.py index 87efe5464..8dc9873f8 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -26,6 +26,11 @@ class DatabaseConfig(Config): self.database_path = self.abspath(args.database_path) self.event_cache_size = self.parse_size(args.event_cache_size) + if args.database_config: + self.database_config = self.abspath(args.database_config) + else: + self.database_config = None + @classmethod def add_arguments(cls, parser): super(DatabaseConfig, cls).add_arguments(parser) @@ -38,6 +43,10 @@ class DatabaseConfig(Config): "--event-cache-size", default="100K", help="Number of events to cache in memory." ) + db_group.add_argument( + "--database-config", default=None, + help="Location of the database configuration file." + ) @classmethod def generate_config(cls, args, config_dir_path): From d4f50f3ae5a23886eb271ad8cee92d5fb5aae8b0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 10:57:08 +0000 Subject: [PATCH 007/173] decode_result takes an iterable --- synapse/storage/room.py | 5 ++++- synapse/storage/transactions.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 3c23f2921..b5031f5c7 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -68,7 +68,10 @@ class RoomStore(SQLBaseStore): """ query = RoomsTable.select_statement("room_id=?") return self._execute( - "get_room", RoomsTable.decode_single_result, query, room_id, + "get_room", + lambda txn: RoomsTable.decode_single_result(txn.fetchall()), + query, + room_id, ) @defer.inlineCallbacks diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index b5ed5453d..92eec69ef 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -122,7 +122,7 @@ class TransactionStore(SQLBaseStore): ) txn.execute(query, (destination,)) - results = SentTransactions.decode_results(txn) + results = SentTransactions.decode_results(txn.fetchall()) prev_txns = [r.transaction_id for r in results] From d288d273e15c3686ff73b8cfd8c71707174dc173 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 10:57:44 +0000 Subject: [PATCH 008/173] Generate transaction id in code --- synapse/storage/transactions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 92eec69ef..08e481a36 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -130,7 +130,7 @@ class TransactionStore(SQLBaseStore): query = SentTransactions.insert_statement() txn.execute(query, SentTransactions.EntryType( - None, + self.get_next_stream_id(), transaction_id=transaction_id, destination=destination, ts=origin_server_ts, From cb8162d3d17a97574073d49bd6eef51c93b68157 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 13:52:56 +0000 Subject: [PATCH 009/173] Rearrange storage modules --- synapse/storage/__init__.py | 466 +----------------------------------- synapse/storage/_base.py | 7 + synapse/storage/events.py | 387 ++++++++++++++++++++++++++++++ synapse/storage/feedback.py | 47 ---- synapse/storage/room.py | 37 +++ synapse/storage/state.py | 32 +++ synapse/storage/stream.py | 19 ++ 7 files changed, 486 insertions(+), 509 deletions(-) create mode 100644 synapse/storage/events.py delete mode 100644 synapse/storage/feedback.py diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 30cba4771..8604746fb 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -15,12 +15,9 @@ from twisted.internet import defer -from synapse.util.logutils import log_function -from synapse.api.constants import EventTypes - from .appservice import ApplicationServiceStore from .directory import DirectoryStore -from .feedback import FeedbackStore +from .events import EventsStore from .presence import PresenceStore from .profile import ProfileStore from .registration import RegistrationStore @@ -39,11 +36,6 @@ from .state import StateStore from .signatures import SignatureStore from .filtering import FilteringStore -from syutil.base64util import decode_base64 -from syutil.jsonutil import encode_canonical_json - -from synapse.crypto.event_signing import compute_event_reference_hash - import fnmatch import imp @@ -64,15 +56,8 @@ SCHEMA_VERSION = 14 dir_path = os.path.abspath(os.path.dirname(__file__)) -class _RollbackButIsFineException(Exception): - """ This exception is used to rollback a transaction without implying - something went wrong. - """ - pass - - class DataStore(RoomMemberStore, RoomStore, - RegistrationStore, StreamStore, ProfileStore, FeedbackStore, + RegistrationStore, StreamStore, ProfileStore, PresenceStore, TransactionStore, DirectoryStore, KeyStore, StateStore, SignatureStore, ApplicationServiceStore, @@ -81,7 +66,8 @@ class DataStore(RoomMemberStore, RoomStore, RejectionsStore, FilteringStore, PusherStore, - PushRuleStore + PushRuleStore, + EventsStore, ): def __init__(self, hs): @@ -94,412 +80,6 @@ class DataStore(RoomMemberStore, RoomStore, self._next_stream_id_lock = threading.Lock() self._next_stream_id = int(hs.get_clock().time_msec()) * 1000 - @defer.inlineCallbacks - @log_function - def persist_event(self, event, context, backfilled=False, - is_new_state=True, current_state=None): - stream_ordering = None - if backfilled: - if not self.min_token_deferred.called: - yield self.min_token_deferred - self.min_token -= 1 - stream_ordering = self.min_token - - try: - yield self.runInteraction( - "persist_event", - self._persist_event_txn, - event=event, - context=context, - backfilled=backfilled, - stream_ordering=stream_ordering, - is_new_state=is_new_state, - current_state=current_state, - ) - except _RollbackButIsFineException: - pass - - @defer.inlineCallbacks - def get_event(self, event_id, check_redacted=True, - get_prev_content=False, allow_rejected=False, - allow_none=False): - """Get an event from the database by event_id. - - Args: - event_id (str): The event_id of the event to fetch - check_redacted (bool): If True, check if event has been redacted - and redact it. - get_prev_content (bool): If True and event is a state event, - include the previous states content in the unsigned field. - allow_rejected (bool): If True return rejected events. - allow_none (bool): If True, return None if no event found, if - False throw an exception. - - Returns: - Deferred : A FrozenEvent. - """ - event = yield self.runInteraction( - "get_event", self._get_event_txn, - event_id, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - allow_rejected=allow_rejected, - ) - - if not event and not allow_none: - raise RuntimeError("Could not find event %s" % (event_id,)) - - defer.returnValue(event) - - @log_function - def _persist_event_txn(self, txn, event, context, backfilled, - stream_ordering=None, is_new_state=True, - current_state=None): - - # Remove the any existing cache entries for the event_id - self._get_event_cache.pop(event.event_id) - - # We purposefully do this first since if we include a `current_state` - # key, we *want* to update the `current_state_events` table - if current_state: - txn.execute( - "DELETE FROM current_state_events WHERE room_id = ?", - (event.room_id,) - ) - - for s in current_state: - self._simple_insert_txn( - txn, - "current_state_events", - { - "event_id": s.event_id, - "room_id": s.room_id, - "type": s.type, - "state_key": s.state_key, - }, - ) - - if event.is_state() and is_new_state: - if not backfilled and not context.rejected: - self._simple_insert_txn( - txn, - table="state_forward_extremities", - values={ - "event_id": event.event_id, - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - }, - ) - - for prev_state_id, _ in event.prev_state: - self._simple_delete_txn( - txn, - table="state_forward_extremities", - keyvalues={ - "event_id": prev_state_id, - } - ) - - outlier = event.internal_metadata.is_outlier() - - if not outlier: - self._store_state_groups_txn(txn, event, context) - - self._update_min_depth_for_room_txn( - txn, - event.room_id, - event.depth - ) - - self._handle_prev_events( - txn, - outlier=outlier, - event_id=event.event_id, - prev_events=event.prev_events, - room_id=event.room_id, - ) - - have_persisted = self._simple_select_one_onecol_txn( - txn, - table="event_json", - keyvalues={"event_id": event.event_id}, - retcol="event_id", - allow_none=True, - ) - - metadata_json = encode_canonical_json( - event.internal_metadata.get_dict() - ) - - # If we have already persisted this event, we don't need to do any - # more processing. - # The processing above must be done on every call to persist event, - # since they might not have happened on previous calls. For example, - # if we are persisting an event that we had persisted as an outlier, - # but is no longer one. - if have_persisted: - if not outlier: - sql = ( - "UPDATE event_json SET internal_metadata = ?" - " WHERE event_id = ?" - ) - txn.execute( - sql, - (metadata_json.decode("UTF-8"), event.event_id,) - ) - - sql = ( - "UPDATE events SET outlier = 0" - " WHERE event_id = ?" - ) - txn.execute( - sql, - (event.event_id,) - ) - return - - if event.type == EventTypes.Member: - self._store_room_member_txn(txn, event) - elif event.type == EventTypes.Feedback: - self._store_feedback_txn(txn, event) - elif event.type == EventTypes.Name: - self._store_room_name_txn(txn, event) - elif event.type == EventTypes.Topic: - self._store_room_topic_txn(txn, event) - elif event.type == EventTypes.Redaction: - self._store_redaction(txn, event) - - event_dict = { - k: v - for k, v in event.get_dict().items() - if k not in [ - "redacted", - "redacted_because", - ] - } - - self._simple_insert_txn( - txn, - table="event_json", - values={ - "event_id": event.event_id, - "room_id": event.room_id, - "internal_metadata": metadata_json.decode("UTF-8"), - "json": encode_canonical_json(event_dict).decode("UTF-8"), - }, - ) - - content = encode_canonical_json( - event.content - ).decode("UTF-8") - - vals = { - "topological_ordering": event.depth, - "event_id": event.event_id, - "type": event.type, - "room_id": event.room_id, - "content": content, - "processed": True, - "outlier": outlier, - "depth": event.depth, - } - - if stream_ordering is None: - stream_ordering = self.get_next_stream_id() - - - unrec = { - k: v - for k, v in event.get_dict().items() - if k not in vals.keys() and k not in [ - "redacted", - "redacted_because", - "signatures", - "hashes", - "prev_events", - ] - } - - vals["unrecognized_keys"] = encode_canonical_json( - unrec - ).decode("UTF-8") - - sql = ( - "INSERT INTO events" - " (stream_ordering, topological_ordering, event_id, type," - " room_id, content, processed, outlier, depth)" - " VALUES (%s,?,?,?,?,?,?,?,?)" - ) % (stream_ordering,) - - txn.execute( - sql, - (event.depth, event.event_id, event.type, event.room_id, - content, True, outlier, event.depth) - ) - - if context.rejected: - self._store_rejections_txn(txn, event.event_id, context.rejected) - - if event.is_state(): - vals = { - "event_id": event.event_id, - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - } - - # TODO: How does this work with backfilling? - if hasattr(event, "replaces_state"): - vals["prev_state"] = event.replaces_state - - self._simple_insert_txn( - txn, - "state_events", - vals, - ) - - if is_new_state and not context.rejected: - self._simple_insert_txn( - txn, - "current_state_events", - { - "event_id": event.event_id, - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - }, - ) - - for e_id, h in event.prev_state: - self._simple_insert_txn( - txn, - table="event_edges", - values={ - "event_id": event.event_id, - "prev_event_id": e_id, - "room_id": event.room_id, - "is_state": 1, - }, - ) - - for hash_alg, hash_base64 in event.hashes.items(): - hash_bytes = decode_base64(hash_base64) - self._store_event_content_hash_txn( - txn, event.event_id, hash_alg, hash_bytes, - ) - - for prev_event_id, prev_hashes in event.prev_events: - for alg, hash_base64 in prev_hashes.items(): - hash_bytes = decode_base64(hash_base64) - self._store_prev_event_hash_txn( - txn, event.event_id, prev_event_id, alg, hash_bytes - ) - - for auth_id, _ in event.auth_events: - self._simple_insert_txn( - txn, - table="event_auth", - values={ - "event_id": event.event_id, - "room_id": event.room_id, - "auth_id": auth_id, - }, - ) - - (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) - self._store_event_reference_hash_txn( - txn, event.event_id, ref_alg, ref_hash_bytes - ) - - def _store_redaction(self, txn, event): - # invalidate the cache for the redacted event - self._get_event_cache.pop(event.redacts) - txn.execute( - "INSERT INTO redactions (event_id, redacts) VALUES (?,?)", - (event.event_id, event.redacts) - ) - - @defer.inlineCallbacks - def get_current_state(self, room_id, event_type=None, state_key=""): - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = e.event_id " - "LIMIT 1" - ) - - sql = ( - "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " - "INNER JOIN current_state_events as c ON e.event_id = c.event_id " - "INNER JOIN state_events as s ON e.event_id = s.event_id " - "WHERE c.room_id = ? " - ) % { - "redacted": del_sql, - } - - if event_type and state_key is not None: - sql += " AND s.type = ? AND s.state_key = ? " - args = (room_id, event_type, state_key) - elif event_type: - sql += " AND s.type = ?" - args = (room_id, event_type) - else: - args = (room_id, ) - - results = yield self._execute_and_decode("get_current_state", sql, *args) - - events = yield self._parse_events(results) - defer.returnValue(events) - - @defer.inlineCallbacks - def get_room_name_and_aliases(self, room_id): - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = e.event_id " - "LIMIT 1" - ) - - sql = ( - "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " - "INNER JOIN current_state_events as c ON e.event_id = c.event_id " - "INNER JOIN state_events as s ON e.event_id = s.event_id " - "WHERE c.room_id = ? " - ) % { - "redacted": del_sql, - } - - sql += " AND ((s.type = 'm.room.name' AND s.state_key = '')" - sql += " OR s.type = 'm.room.aliases')" - args = (room_id,) - - results = yield self._execute_and_decode("get_current_state", sql, *args) - - events = yield self._parse_events(results) - - name = None - aliases = [] - - for e in events: - if e.type == 'm.room.name': - if 'name' in e.content: - name = e.content['name'] - elif e.type == 'm.room.aliases': - if 'aliases' in e.content: - aliases.extend(e.content['aliases']) - - defer.returnValue((name, aliases)) - - @defer.inlineCallbacks - def _get_min_token(self): - row = yield self._execute( - "_get_min_token", None, "SELECT MIN(stream_ordering) FROM events" - ) - - self.min_token = row[0][0] if row and row[0] and row[0][0] else -1 - self.min_token = min(self.min_token, -1) - - logger.debug("min_token is: %s", self.min_token) - - defer.returnValue(self.min_token) - def insert_client_ip(self, user, access_token, device_id, ip, user_agent): return self._simple_insert( "user_ips", @@ -523,44 +103,6 @@ class DataStore(RoomMemberStore, RoomStore, ], ) - def have_events(self, event_ids): - """Given a list of event ids, check if we have already processed them. - - Returns: - dict: Has an entry for each event id we already have seen. Maps to - the rejected reason string if we rejected the event, else maps to - None. - """ - if not event_ids: - return defer.succeed({}) - - def f(txn): - sql = ( - "SELECT e.event_id, reason FROM events as e " - "LEFT JOIN rejections as r ON e.event_id = r.event_id " - "WHERE e.event_id = ?" - ) - - res = {} - for event_id in event_ids: - txn.execute(sql, (event_id,)) - row = txn.fetchone() - if row: - _, rejected = row - res[event_id] = rejected - - return res - - return self.runInteraction( - "have_events", f, - ) - - def get_next_stream_id(self): - with self._next_stream_id_lock: - i = self._next_stream_id - self._next_stream_id += 1 - return i - def read_schema(path): """ Read the named database schema. diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 24ff872da..37bb28e6c 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -791,6 +791,13 @@ class SQLBaseStore(object): return result[0] if result else None +class _RollbackButIsFineException(Exception): + """ This exception is used to rollback a transaction without implying + something went wrong. + """ + pass + + class Table(object): """ A base class used to store information about a particular table. """ diff --git a/synapse/storage/events.py b/synapse/storage/events.py new file mode 100644 index 000000000..b222dfb4a --- /dev/null +++ b/synapse/storage/events.py @@ -0,0 +1,387 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from _base import SQLBaseStore, _RollbackButIsFineException + +from twisted.internet import defer + +from synapse.util.logutils import log_function +from synapse.api.constants import EventTypes +from synapse.crypto.event_signing import compute_event_reference_hash + +from syutil.base64util import decode_base64 +from syutil.jsonutil import encode_canonical_json + +import logging + +logger = logging.getLogger(__name__) + + +class EventsStore(SQLBaseStore): + @defer.inlineCallbacks + @log_function + def persist_event(self, event, context, backfilled=False, + is_new_state=True, current_state=None): + stream_ordering = None + if backfilled: + if not self.min_token_deferred.called: + yield self.min_token_deferred + self.min_token -= 1 + stream_ordering = self.min_token + + try: + yield self.runInteraction( + "persist_event", + self._persist_event_txn, + event=event, + context=context, + backfilled=backfilled, + stream_ordering=stream_ordering, + is_new_state=is_new_state, + current_state=current_state, + ) + except _RollbackButIsFineException: + pass + + @defer.inlineCallbacks + def get_event(self, event_id, check_redacted=True, + get_prev_content=False, allow_rejected=False, + allow_none=False): + """Get an event from the database by event_id. + + Args: + event_id (str): The event_id of the event to fetch + check_redacted (bool): If True, check if event has been redacted + and redact it. + get_prev_content (bool): If True and event is a state event, + include the previous states content in the unsigned field. + allow_rejected (bool): If True return rejected events. + allow_none (bool): If True, return None if no event found, if + False throw an exception. + + Returns: + Deferred : A FrozenEvent. + """ + event = yield self.runInteraction( + "get_event", self._get_event_txn, + event_id, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, + ) + + if not event and not allow_none: + raise RuntimeError("Could not find event %s" % (event_id,)) + + defer.returnValue(event) + + @log_function + def _persist_event_txn(self, txn, event, context, backfilled, + stream_ordering=None, is_new_state=True, + current_state=None): + + # Remove the any existing cache entries for the event_id + self._get_event_cache.pop(event.event_id) + + # We purposefully do this first since if we include a `current_state` + # key, we *want* to update the `current_state_events` table + if current_state: + txn.execute( + "DELETE FROM current_state_events WHERE room_id = ?", + (event.room_id,) + ) + + for s in current_state: + self._simple_insert_txn( + txn, + "current_state_events", + { + "event_id": s.event_id, + "room_id": s.room_id, + "type": s.type, + "state_key": s.state_key, + }, + ) + + if event.is_state() and is_new_state: + if not backfilled and not context.rejected: + self._simple_insert_txn( + txn, + table="state_forward_extremities", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + }, + ) + + for prev_state_id, _ in event.prev_state: + self._simple_delete_txn( + txn, + table="state_forward_extremities", + keyvalues={ + "event_id": prev_state_id, + } + ) + + outlier = event.internal_metadata.is_outlier() + + if not outlier: + self._store_state_groups_txn(txn, event, context) + + self._update_min_depth_for_room_txn( + txn, + event.room_id, + event.depth + ) + + self._handle_prev_events( + txn, + outlier=outlier, + event_id=event.event_id, + prev_events=event.prev_events, + room_id=event.room_id, + ) + + have_persisted = self._simple_select_one_onecol_txn( + txn, + table="event_json", + keyvalues={"event_id": event.event_id}, + retcol="event_id", + allow_none=True, + ) + + metadata_json = encode_canonical_json( + event.internal_metadata.get_dict() + ) + + # If we have already persisted this event, we don't need to do any + # more processing. + # The processing above must be done on every call to persist event, + # since they might not have happened on previous calls. For example, + # if we are persisting an event that we had persisted as an outlier, + # but is no longer one. + if have_persisted: + if not outlier: + sql = ( + "UPDATE event_json SET internal_metadata = ?" + " WHERE event_id = ?" + ) + txn.execute( + sql, + (metadata_json.decode("UTF-8"), event.event_id,) + ) + + sql = ( + "UPDATE events SET outlier = 0" + " WHERE event_id = ?" + ) + txn.execute( + sql, + (event.event_id,) + ) + return + + if event.type == EventTypes.Member: + self._store_room_member_txn(txn, event) + elif event.type == EventTypes.Name: + self._store_room_name_txn(txn, event) + elif event.type == EventTypes.Topic: + self._store_room_topic_txn(txn, event) + elif event.type == EventTypes.Redaction: + self._store_redaction(txn, event) + + event_dict = { + k: v + for k, v in event.get_dict().items() + if k not in [ + "redacted", + "redacted_because", + ] + } + + self._simple_insert_txn( + txn, + table="event_json", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "internal_metadata": metadata_json.decode("UTF-8"), + "json": encode_canonical_json(event_dict).decode("UTF-8"), + }, + ) + + content = encode_canonical_json( + event.content + ).decode("UTF-8") + + vals = { + "topological_ordering": event.depth, + "event_id": event.event_id, + "type": event.type, + "room_id": event.room_id, + "content": content, + "processed": True, + "outlier": outlier, + "depth": event.depth, + } + + if stream_ordering is None: + stream_ordering = self.get_next_stream_id() + + + unrec = { + k: v + for k, v in event.get_dict().items() + if k not in vals.keys() and k not in [ + "redacted", + "redacted_because", + "signatures", + "hashes", + "prev_events", + ] + } + + vals["unrecognized_keys"] = encode_canonical_json( + unrec + ).decode("UTF-8") + + sql = ( + "INSERT INTO events" + " (stream_ordering, topological_ordering, event_id, type," + " room_id, content, processed, outlier, depth)" + " VALUES (%s,?,?,?,?,?,?,?,?)" + ) % (stream_ordering,) + + txn.execute( + sql, + (event.depth, event.event_id, event.type, event.room_id, + content, True, outlier, event.depth) + ) + + if context.rejected: + self._store_rejections_txn(txn, event.event_id, context.rejected) + + if event.is_state(): + vals = { + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + } + + # TODO: How does this work with backfilling? + if hasattr(event, "replaces_state"): + vals["prev_state"] = event.replaces_state + + self._simple_insert_txn( + txn, + "state_events", + vals, + ) + + if is_new_state and not context.rejected: + self._simple_insert_txn( + txn, + "current_state_events", + { + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + }, + ) + + for e_id, h in event.prev_state: + self._simple_insert_txn( + txn, + table="event_edges", + values={ + "event_id": event.event_id, + "prev_event_id": e_id, + "room_id": event.room_id, + "is_state": 1, + }, + ) + + for hash_alg, hash_base64 in event.hashes.items(): + hash_bytes = decode_base64(hash_base64) + self._store_event_content_hash_txn( + txn, event.event_id, hash_alg, hash_bytes, + ) + + for prev_event_id, prev_hashes in event.prev_events: + for alg, hash_base64 in prev_hashes.items(): + hash_bytes = decode_base64(hash_base64) + self._store_prev_event_hash_txn( + txn, event.event_id, prev_event_id, alg, hash_bytes + ) + + for auth_id, _ in event.auth_events: + self._simple_insert_txn( + txn, + table="event_auth", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "auth_id": auth_id, + }, + ) + + (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) + self._store_event_reference_hash_txn( + txn, event.event_id, ref_alg, ref_hash_bytes + ) + + def _store_redaction(self, txn, event): + # invalidate the cache for the redacted event + self._get_event_cache.pop(event.redacts) + txn.execute( + "INSERT INTO redactions (event_id, redacts) VALUES (?,?)", + (event.event_id, event.redacts) + ) + + def have_events(self, event_ids): + """Given a list of event ids, check if we have already processed them. + + Returns: + dict: Has an entry for each event id we already have seen. Maps to + the rejected reason string if we rejected the event, else maps to + None. + """ + if not event_ids: + return defer.succeed({}) + + def f(txn): + sql = ( + "SELECT e.event_id, reason FROM events as e " + "LEFT JOIN rejections as r ON e.event_id = r.event_id " + "WHERE e.event_id = ?" + ) + + res = {} + for event_id in event_ids: + txn.execute(sql, (event_id,)) + row = txn.fetchone() + if row: + _, rejected = row + res[event_id] = rejected + + return res + + return self.runInteraction( + "have_events", f, + ) diff --git a/synapse/storage/feedback.py b/synapse/storage/feedback.py deleted file mode 100644 index 8eab769b7..000000000 --- a/synapse/storage/feedback.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from twisted.internet import defer - -from ._base import SQLBaseStore - - -class FeedbackStore(SQLBaseStore): - - def _store_feedback_txn(self, txn, event): - self._simple_insert_txn(txn, "feedback", { - "event_id": event.event_id, - "feedback_type": event.content["type"], - "room_id": event.room_id, - "target_event_id": event.content["target_event_id"], - "sender": event.user_id, - }) - - @defer.inlineCallbacks - def get_feedback_for_event(self, event_id): - sql = ( - "SELECT events.* FROM events INNER JOIN feedback " - "ON events.event_id = feedback.event_id " - "WHERE feedback.target_event_id = ? " - ) - - rows = yield self._execute_and_decode("get_feedback_for_event", sql, event_id) - - defer.returnValue( - [ - (yield self._parse_events(r)) - for r in rows - ] - ) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index b5031f5c7..c64f8f53a 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -161,6 +161,43 @@ class RoomStore(SQLBaseStore): } ) + @defer.inlineCallbacks + def get_room_name_and_aliases(self, room_id): + del_sql = ( + "SELECT event_id FROM redactions WHERE redacts = e.event_id " + "LIMIT 1" + ) + + sql = ( + "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " + "INNER JOIN current_state_events as c ON e.event_id = c.event_id " + "INNER JOIN state_events as s ON e.event_id = s.event_id " + "WHERE c.room_id = ? " + ) % { + "redacted": del_sql, + } + + sql += " AND ((s.type = 'm.room.name' AND s.state_key = '')" + sql += " OR s.type = 'm.room.aliases')" + args = (room_id,) + + results = yield self._execute_and_decode("get_current_state", sql, *args) + + events = yield self._parse_events(results) + + name = None + aliases = [] + + for e in events: + if e.type == 'm.room.name': + if 'name' in e.content: + name = e.content['name'] + elif e.type == 'm.room.aliases': + if 'aliases' in e.content: + aliases.extend(e.content['aliases']) + + defer.returnValue((name, aliases)) + class RoomsTable(Table): table_name = "rooms" diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 888837cd1..012144302 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -15,6 +15,8 @@ from ._base import SQLBaseStore +from twisted.internet import defer + from synapse.util.stringutils import random_string import logging @@ -125,6 +127,36 @@ class StateStore(SQLBaseStore): or_replace=True, ) + @defer.inlineCallbacks + def get_current_state(self, room_id, event_type=None, state_key=""): + del_sql = ( + "SELECT event_id FROM redactions WHERE redacts = e.event_id " + "LIMIT 1" + ) + + sql = ( + "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " + "INNER JOIN current_state_events as c ON e.event_id = c.event_id " + "INNER JOIN state_events as s ON e.event_id = s.event_id " + "WHERE c.room_id = ? " + ) % { + "redacted": del_sql, + } + + if event_type and state_key is not None: + sql += " AND s.type = ? AND s.state_key = ? " + args = (room_id, event_type, state_key) + elif event_type: + sql += " AND s.type = ?" + args = (room_id, event_type) + else: + args = (room_id, ) + + results = yield self._execute_and_decode("get_current_state", sql, *args) + + events = yield self._parse_events(results) + defer.returnValue(events) + def _make_group_id(clock): return str(int(clock.time_msec())) + random_string(5) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 64adb0c7f..9565fc77c 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -419,6 +419,25 @@ class StreamStore(SQLBaseStore): self._get_room_events_max_id_txn ) + @defer.inlineCallbacks + def _get_min_token(self): + row = yield self._execute( + "_get_min_token", None, "SELECT MIN(stream_ordering) FROM events" + ) + + self.min_token = row[0][0] if row and row[0] and row[0][0] else -1 + self.min_token = min(self.min_token, -1) + + logger.debug("min_token is: %s", self.min_token) + + defer.returnValue(self.min_token) + + def get_next_stream_id(self): + with self._next_stream_id_lock: + i = self._next_stream_id + self._next_stream_id += 1 + return i + def _get_room_events_max_id_txn(self, txn): txn.execute( "SELECT MAX(stream_ordering) as m FROM events" From 278149f53392d5e3b08f72106517afe2711c856c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Mar 2015 13:43:21 +0000 Subject: [PATCH 010/173] Sanitize TransactionStore --- synapse/handlers/federation.py | 2 +- synapse/storage/transactions.py | 189 +++++++++++++++++--------------- 2 files changed, 104 insertions(+), 87 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 15ba417e0..9a4773ac0 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -179,7 +179,7 @@ class FederationHandler(BaseHandler): # it's probably a good idea to mark it as not in retry-state # for sending (although this is a bit of a leap) retry_timings = yield self.store.get_destination_retry_timings(origin) - if (retry_timings and retry_timings.retry_last_ts): + if retry_timings and retry_timings["retry_last_ts"]: self.store.set_destination_retry_timings(origin, 0, 0) room = yield self.store.get_room(event.room_id) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 7d2239244..9dec58c21 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._base import SQLBaseStore, Table, cached +from ._base import SQLBaseStore, cached from collections import namedtuple @@ -84,13 +84,18 @@ class TransactionStore(SQLBaseStore): def _set_received_txn_response(self, txn, transaction_id, origin, code, response_json): - query = ( - "UPDATE %s " - "SET response_code = ?, response_json = ? " - "WHERE transaction_id = ? AND origin = ?" - ) % ReceivedTransactionsTable.table_name - - txn.execute(query, (code, response_json, transaction_id, origin)) + self._simple_update_one_txn( + txn, + table=ReceivedTransactionsTable.table_name, + keyvalues={ + "transaction_id": transaction_id, + "origin": origin, + }, + updatevalues={ + "response_code": code, + "response_json": response_json, + } + ) def prep_send_transaction(self, transaction_id, destination, origin_server_ts): @@ -121,38 +126,32 @@ class TransactionStore(SQLBaseStore): # First we find out what the prev_txns should be. # Since we know that we are only sending one transaction at a time, # we can simply take the last one. - query = "%s ORDER BY id DESC LIMIT 1" % ( - SentTransactions.select_statement("destination = ?"), - ) + query = ( + "SELECT * FROM sent_transactions" + " WHERE destination = ?" + " ORDER BY id DESC LIMIT 1" + ) txn.execute(query, (destination,)) - results = SentTransactions.decode_results(txn.fetchall()) + results = self.cursor_to_dict(txn) - prev_txns = [r.transaction_id for r in results] + prev_txns = [r["transaction_id"] for r in results] # Actually add the new transaction to the sent_transactions table. - query = SentTransactions.insert_statement() - txn.execute(query, SentTransactions.EntryType( - self.get_next_stream_id(), - transaction_id=transaction_id, - destination=destination, - ts=origin_server_ts, - response_code=0, - response_json=None - )) + self._simple_insert_txn( + txn, + table=SentTransactions.table_name, + values={ + "transaction_id": self.get_next_stream_id(), + "destination": destination, + "ts": origin_server_ts, + "response_code": 0, + "response_json": None, + } + ) - # Update the tx id -> pdu id mapping - - # values = [ - # (transaction_id, destination, pdu[0], pdu[1]) - # for pdu in pdu_list - # ] - # - # logger.debug("Inserting: %s", repr(values)) - # - # query = TransactionsToPduTable.insert_statement() - # txn.executemany(query, values) + # TODO Update the tx id -> pdu id mapping return prev_txns @@ -171,15 +170,20 @@ class TransactionStore(SQLBaseStore): transaction_id, destination, code, response_dict ) - def _delivered_txn(cls, txn, transaction_id, destination, + def _delivered_txn(self, txn, transaction_id, destination, code, response_json): - query = ( - "UPDATE %s " - "SET response_code = ?, response_json = ? " - "WHERE transaction_id = ? AND destination = ?" - ) % SentTransactions.table_name - - txn.execute(query, (code, response_json, transaction_id, destination)) + self._simple_update_one_txn( + txn, + table=SentTransactions.table_name, + keyvalues={ + "transaction_id": transaction_id, + "destination": destination, + }, + updatevalues={ + "response_code": code, + "response_json": response_json, + } + ) def get_transactions_after(self, transaction_id, destination): """Get all transactions after a given local transaction_id. @@ -189,25 +193,26 @@ class TransactionStore(SQLBaseStore): destination (str) Returns: - list: A list of `ReceivedTransactionsTable.EntryType` + list: A list of dicts """ return self.runInteraction( "get_transactions_after", self._get_transactions_after, transaction_id, destination ) - def _get_transactions_after(cls, txn, transaction_id, destination): - where = ( - "destination = ? AND id > (select id FROM %s WHERE " - "transaction_id = ? AND destination = ?)" - ) % ( - SentTransactions.table_name + def _get_transactions_after(self, txn, transaction_id, destination): + query = ( + "SELECT * FROM sent_transactions" + " WHERE destination = ? AND id >" + " (" + " SELECT id FROM sent_transactions" + " WHERE transaction_id = ? AND destination = ?" + " )" ) - query = SentTransactions.select_statement(where) txn.execute(query, (destination, transaction_id, destination)) - return ReceivedTransactionsTable.decode_results(txn.fetchall()) + return self.cursor_to_dict(txn) @cached() def get_destination_retry_timings(self, destination): @@ -218,22 +223,27 @@ class TransactionStore(SQLBaseStore): Returns: None if not retrying - Otherwise a DestinationsTable.EntryType for the retry scheme + Otherwise a dict for the retry scheme """ return self.runInteraction( "get_destination_retry_timings", self._get_destination_retry_timings, destination) - def _get_destination_retry_timings(cls, txn, destination): - query = DestinationsTable.select_statement("destination = ?") - txn.execute(query, (destination,)) - result = txn.fetchall() - if result: - result = DestinationsTable.decode_single_result(result) - if result.retry_last_ts > 0: - return result - else: - return None + def _get_destination_retry_timings(self, txn, destination): + result = self._simple_select_one_txn( + txn, + table=DestinationsTable.table_name, + keyvalues={ + "destination": destination, + }, + retcols=DestinationsTable.fields, + allow_none=True, + ) + + if result["retry_last_ts"] > 0: + return result + else: + return None def set_destination_retry_timings(self, destination, retry_last_ts, retry_interval): @@ -249,11 +259,11 @@ class TransactionStore(SQLBaseStore): # As this is the new value, we might as well prefill the cache self.get_destination_retry_timings.prefill( destination, - DestinationsTable.EntryType( - destination, - retry_last_ts, - retry_interval - ) + { + "destination": destination, + "retry_last_ts": retry_last_ts, + "retry_interval": retry_interval + }, ) # XXX: we could chose to not bother persisting this if our cache thinks @@ -270,18 +280,27 @@ class TransactionStore(SQLBaseStore): retry_last_ts, retry_interval): query = ( - "REPLACE INTO %s " - "(destination, retry_last_ts, retry_interval) " - "VALUES (?, ?, ?) " - ) % DestinationsTable.table_name + "INSERT INTO destinations" + " (destination, retry_last_ts, retry_interval)" + " VALUES (?, ?, ?)" + " ON DUPLICATE KEY UPDATE" + " retry_last_ts=?, retry_interval=?" + ) - txn.execute(query, (destination, retry_last_ts, retry_interval)) + txn.execute( + query, + ( + destination, + retry_last_ts, retry_interval, + retry_last_ts, retry_interval, + ) + ) def get_destinations_needing_retry(self): """Get all destinations which are due a retry for sending a transaction. Returns: - list: A list of `DestinationsTable.EntryType` + list: A list of dicts """ return self.runInteraction( @@ -289,14 +308,17 @@ class TransactionStore(SQLBaseStore): self._get_destinations_needing_retry ) - def _get_destinations_needing_retry(cls, txn): - where = "retry_last_ts > 0 and retry_next_ts < now()" - query = DestinationsTable.select_statement(where) - txn.execute(query) - return DestinationsTable.decode_results(txn.fetchall()) + def _get_destinations_needing_retry(self, txn): + query = ( + "SELECT * FROM destinations" + " WHERE retry_last_ts > 0 and retry_next_ts < ?" + ) + + txn.execute(query, (self._clock.time_msec(),)) + return self.cursor_to_dict(txn) -class ReceivedTransactionsTable(Table): +class ReceivedTransactionsTable(object): table_name = "received_transactions" fields = [ @@ -308,10 +330,8 @@ class ReceivedTransactionsTable(Table): "has_been_referenced", ] - EntryType = namedtuple("ReceivedTransactionsEntry", fields) - -class SentTransactions(Table): +class SentTransactions(object): table_name = "sent_transactions" fields = [ @@ -326,7 +346,7 @@ class SentTransactions(Table): EntryType = namedtuple("SentTransactionsEntry", fields) -class TransactionsToPduTable(Table): +class TransactionsToPduTable(object): table_name = "transaction_id_to_pdu" fields = [ @@ -336,10 +356,8 @@ class TransactionsToPduTable(Table): "pdu_origin", ] - EntryType = namedtuple("TransactionsToPduEntry", fields) - -class DestinationsTable(Table): +class DestinationsTable(object): table_name = "destinations" fields = [ @@ -348,4 +366,3 @@ class DestinationsTable(Table): "retry_interval", ] - EntryType = namedtuple("DestinationsEntry", fields) From d5272b1d2cf170d2b41d9aa991c3a37e9f7aac34 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Mar 2015 14:02:34 +0000 Subject: [PATCH 011/173] Use 'update or insert' rather than on 'conflict replace' --- synapse/storage/transactions.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 9dec58c21..1a5bb41cb 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -276,26 +276,33 @@ class TransactionStore(SQLBaseStore): retry_interval, ) - def _set_destination_retry_timings(cls, txn, destination, + def _set_destination_retry_timings(self, txn, destination, retry_last_ts, retry_interval): - query = ( - "INSERT INTO destinations" - " (destination, retry_last_ts, retry_interval)" - " VALUES (?, ?, ?)" - " ON DUPLICATE KEY UPDATE" - " retry_last_ts=?, retry_interval=?" + "UPDATE destinations" + " SET retry_last_ts = ?, retry_interval = ?" + " WHERE destinations = ?" ) txn.execute( query, ( - destination, - retry_last_ts, retry_interval, - retry_last_ts, retry_interval, + retry_last_ts, retry_interval, destination, ) ) + if txn.rowcount == 0: + # destination wasn't already in table. Insert it. + self._simple_insert_txn( + txn, + table="destinations", + values={ + "destination": destination, + "retry_last_ts": retry_last_ts, + "retry_interval": retry_interval, + } + ) + def get_destinations_needing_retry(self): """Get all destinations which are due a retry for sending a transaction. From d98660a60daaf1cc8d83cb2d64daa5f20a34139c Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 23 Mar 2015 14:20:28 +0000 Subject: [PATCH 012/173] Implement password changing (finally) along with a start on making client/server auth more general. --- synapse/handlers/__init__.py | 2 + synapse/handlers/auth.py | 109 +++++++++++++++++++++++ synapse/handlers/login.py | 49 ++-------- synapse/rest/client/v2_alpha/__init__.py | 4 +- synapse/rest/client/v2_alpha/_base.py | 12 +++ synapse/rest/client/v2_alpha/password.py | 76 ++++++++++++++++ synapse/storage/registration.py | 33 ++++++- 7 files changed, 236 insertions(+), 49 deletions(-) create mode 100644 synapse/handlers/auth.py create mode 100644 synapse/rest/client/v2_alpha/password.py diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index 8d345bf93..336ce1570 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -29,6 +29,7 @@ from .typing import TypingNotificationHandler from .admin import AdminHandler from .appservice import ApplicationServicesHandler from .sync import SyncHandler +from .auth import AuthHandler class Handlers(object): @@ -58,3 +59,4 @@ class Handlers(object): hs, ApplicationServiceApi(hs) ) self.sync_handler = SyncHandler(hs) + self.auth_handler = AuthHandler(hs) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py new file mode 100644 index 000000000..e4a73da9a --- /dev/null +++ b/synapse/handlers/auth.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from ._base import BaseHandler +from synapse.api.constants import LoginType +from synapse.types import UserID +from synapse.api.errors import LoginError, Codes + +import logging +import bcrypt + + +logger = logging.getLogger(__name__) + + +class AuthHandler(BaseHandler): + + def __init__(self, hs): + super(AuthHandler, self).__init__(hs) + + @defer.inlineCallbacks + def check_auth(self, flows, clientdict): + """ + Takes a dictionary sent by the client in the login / registration + protocol and handles the login flow. + + Args: + flows: list of list of stages + authdict: The dictionary from the client root level, not the + 'auth' key: this method prompts for auth if none is sent. + Returns: + A tuple of authed, dict where authed is true if the client + has successfully completed an auth flow. If it is true, the dict + contains the authenticated credentials of each stage. + If authed is false, the dictionary is the server response to the + login request and should be passed back to the client. + """ + types = { + LoginType.PASSWORD: self.check_password_auth + } + + if 'auth' not in clientdict: + defer.returnValue((False, auth_dict_for_flows(flows))) + + authdict = clientdict['auth'] + + # In future: support sessions & retrieve previously succeeded + # login types + creds = {} + + # check auth type currently being presented + if 'type' not in authdict: + raise LoginError(400, "", Codes.MISSING_PARAM) + if authdict['type'] not in types: + raise LoginError(400, "", Codes.UNRECOGNIZED) + result = yield types[authdict['type']](authdict) + if result: + creds[authdict['type']] = result + + for f in flows: + if len(set(f) - set(creds.keys())) == 0: + logger.info("Auth completed with creds: %r", creds) + defer.returnValue((True, creds)) + + ret = auth_dict_for_flows(flows) + ret['completed'] = creds.keys() + defer.returnValue((False, ret)) + + @defer.inlineCallbacks + def check_password_auth(self, authdict): + if "user" not in authdict or "password" not in authdict: + raise LoginError(400, "", Codes.MISSING_PARAM) + + user = authdict["user"] + password = authdict["password"] + if not user.startswith('@'): + user = UserID.create(user, self.hs.hostname).to_string() + + user_info = yield self.store.get_user_by_id(user_id=user) + if not user_info: + logger.warn("Attempted to login as %s but they do not exist", user) + raise LoginError(403, "", errcode=Codes.FORBIDDEN) + + stored_hash = user_info[0]["password_hash"] + if bcrypt.checkpw(password, stored_hash): + defer.returnValue(user) + else: + logger.warn("Failed password login for user %s", user) + raise LoginError(403, "", errcode=Codes.FORBIDDEN) + + +def auth_dict_for_flows(flows): + return { + "flows": {"stages": f for f in flows} + } diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 744780046..19b560d91 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -69,48 +69,9 @@ class LoginHandler(BaseHandler): raise LoginError(403, "", errcode=Codes.FORBIDDEN) @defer.inlineCallbacks - def reset_password(self, user_id, email): - is_valid = yield self._check_valid_association(user_id, email) - logger.info("reset_password user=%s email=%s valid=%s", user_id, email, - is_valid) - if is_valid: - try: - # send an email out - emailutils.send_email( - smtp_server=self.hs.config.email_smtp_server, - from_addr=self.hs.config.email_from_address, - to_addr=email, - subject="Password Reset", - body="TODO." - ) - except EmailException as e: - logger.exception(e) + def set_password(self, user_id, newpassword, token_id=None): + password_hash = bcrypt.hashpw(newpassword, bcrypt.gensalt()) - @defer.inlineCallbacks - def _check_valid_association(self, user_id, email): - identity = yield self._query_email(email) - if identity and "mxid" in identity: - if identity["mxid"] == user_id: - defer.returnValue(True) - return - defer.returnValue(False) - - @defer.inlineCallbacks - def _query_email(self, email): - http_client = SimpleHttpClient(self.hs) - try: - data = yield http_client.get_json( - # TODO FIXME This should be configurable. - # XXX: ID servers need to use HTTPS - "http://%s%s" % ( - "matrix.org:8090", "/_matrix/identity/api/v1/lookup" - ), - { - 'medium': 'email', - 'address': email - } - ) - defer.returnValue(data) - except CodeMessageException as e: - data = json.loads(e.msg) - defer.returnValue(data) + yield self.store.user_set_password_hash(user_id, password_hash) + yield self.store.user_delete_access_tokens_apart_from(user_id, token_id) + yield self.store.flush_user(user_id) diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py index bca65f2a6..041f538e2 100644 --- a/synapse/rest/client/v2_alpha/__init__.py +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -15,7 +15,8 @@ from . import ( sync, - filter + filter, + password ) from synapse.http.server import JsonResource @@ -32,3 +33,4 @@ class ClientV2AlphaRestResource(JsonResource): def register_servlets(client_resource, hs): sync.register_servlets(hs, client_resource) filter.register_servlets(hs, client_resource) + password.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py index 22dc5cb86..c772cc986 100644 --- a/synapse/rest/client/v2_alpha/_base.py +++ b/synapse/rest/client/v2_alpha/_base.py @@ -17,9 +17,11 @@ """ from synapse.api.urls import CLIENT_V2_ALPHA_PREFIX +from synapse.api.errors import SynapseError import re import logging +import simplejson logger = logging.getLogger(__name__) @@ -36,3 +38,13 @@ def client_v2_pattern(path_regex): SRE_Pattern """ return re.compile("^" + CLIENT_V2_ALPHA_PREFIX + path_regex) + + +def parse_json_dict_from_request(request): + try: + content = simplejson.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.") + return content + except simplejson.JSONDecodeError: + raise SynapseError(400, "Content not JSON.") diff --git a/synapse/rest/client/v2_alpha/password.py b/synapse/rest/client/v2_alpha/password.py new file mode 100644 index 000000000..3663781c9 --- /dev/null +++ b/synapse/rest/client/v2_alpha/password.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.constants import LoginType +from synapse.api.errors import LoginError, SynapseError, Codes +from synapse.http.servlet import RestServlet + +from ._base import client_v2_pattern, parse_json_dict_from_request + +import simplejson as json +import logging + + +logger = logging.getLogger(__name__) + + +class PasswordRestServlet(RestServlet): + PATTERN = client_v2_pattern("/account/password") + + def __init__(self, hs): + super(PasswordRestServlet, self).__init__() + self.hs = hs + self.auth = hs.get_auth() + self.auth_handler = hs.get_handlers().auth_handler + self.login_handler = hs.get_handlers().login_handler + + @defer.inlineCallbacks + def on_POST(self, request): + body = parse_json_dict_from_request(request) + + authed, result = yield self.auth_handler.check_auth([ + [LoginType.PASSWORD] + ], body) + + if not authed: + defer.returnValue((401, result)) + + auth_user = None + + if LoginType.PASSWORD in result: + # if using password, they should also be logged in + auth_user, client = yield self.auth.get_user_by_req(request) + if auth_user.to_string() != result[LoginType.PASSWORD]: + raise LoginError(400, "", Codes.UNKNOWN) + else: + logger.error("Auth succeeded but no known type!", result.keys()) + raise SynapseError(500, "", Codes.UNKNOWN) + + user_id = auth_user.to_string() + + if 'new_password' not in body: + raise SynapseError(400, "", Codes.MISSING_PARAM) + new_password = body['new_password'] + + self.login_handler.set_password( + user_id, new_password, client.token_id + ) + + defer.returnValue((200, {})) + +def register_servlets(hs, http_server): + PasswordRestServlet(hs).register(http_server) \ No newline at end of file diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index f24154f14..7e60dc395 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -95,11 +95,36 @@ class RegistrationStore(SQLBaseStore): "get_user_by_id", self.cursor_to_dict, query, user_id ) + def user_set_password_hash(self, user_id, password_hash): + """ + NB. This does *not* evict any cache because the one use for this + removes most of the entries subsequently anyway so it would be + pointless. Use flush_user separately. + """ + return self._simple_update_one('users', { + 'name': user_id + }, { + 'password_hash': password_hash + }) + + def user_delete_access_tokens_apart_from(self, user_id, token_id): + return self._execute( + "delete_access_tokens_apart_from", None, + "DELETE FROM access_tokens WHERE user_id = ? AND id != ?", + user_id, token_id + ) + + @defer.inlineCallbacks + def flush_user(self, user_id): + rows = yield self._execute( + 'user_delete_access_tokens_apart_from', None, + "SELECT token FROM access_tokens WHERE user_id = ?", + user_id + ) + for r in rows: + self.get_user_by_token.invalidate(r) + @cached() - # TODO(paul): Currently there's no code to invalidate this cache. That - # means if/when we ever add internal ways to invalidate access tokens or - # change whether a user is a server admin, those will need to invoke - # store.get_user_by_token.invalidate(token) def get_user_by_token(self, token): """Get a user from the given access token. From 78adccfaf497dcb75451adfc5d366d5ff26cad52 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 23 Mar 2015 14:23:51 +0000 Subject: [PATCH 013/173] pep8 / pyflakes --- synapse/handlers/login.py | 6 +----- synapse/rest/client/v2_alpha/password.py | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 19b560d91..7aff2e69e 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -16,13 +16,9 @@ from twisted.internet import defer from ._base import BaseHandler -from synapse.api.errors import LoginError, Codes, CodeMessageException -from synapse.http.client import SimpleHttpClient -from synapse.util.emailutils import EmailException -import synapse.util.emailutils as emailutils +from synapse.api.errors import LoginError, Codes import bcrypt -import json import logging logger = logging.getLogger(__name__) diff --git a/synapse/rest/client/v2_alpha/password.py b/synapse/rest/client/v2_alpha/password.py index 3663781c9..127753211 100644 --- a/synapse/rest/client/v2_alpha/password.py +++ b/synapse/rest/client/v2_alpha/password.py @@ -21,7 +21,6 @@ from synapse.http.servlet import RestServlet from ._base import client_v2_pattern, parse_json_dict_from_request -import simplejson as json import logging @@ -72,5 +71,6 @@ class PasswordRestServlet(RestServlet): defer.returnValue((200, {})) + def register_servlets(hs, http_server): - PasswordRestServlet(hs).register(http_server) \ No newline at end of file + PasswordRestServlet(hs).register(http_server) From 9a7f4962982e309877acb777eb358178579cacc9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Mar 2015 15:29:04 +0000 Subject: [PATCH 014/173] Sanitize RoomMemberStore --- synapse/handlers/room.py | 19 ------ synapse/storage/roommember.py | 124 +++++++++++++++++----------------- 2 files changed, 61 insertions(+), 82 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 823affc38..bc7f1c240 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -310,25 +310,6 @@ class RoomMemberHandler(BaseHandler): # paginating defer.returnValue(chunk_data) - @defer.inlineCallbacks - def get_room_member(self, room_id, member_user_id, auth_user_id): - """Retrieve a room member from a room. - - Args: - room_id : The room the member is in. - member_user_id : The member's user ID - auth_user_id : The user ID of the user making this request. - Returns: - The room member, or None if this member does not exist. - Raises: - SynapseError if something goes wrong. - """ - yield self.auth.check_joined_room(room_id, auth_user_id) - - member = yield self.store.get_room_member(user_id=member_user_id, - room_id=room_id) - defer.returnValue(member) - @defer.inlineCallbacks def change_membership(self, event, context, do_auth=True): """ Change the membership status of a user in a room. diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 17ee4bb9e..a22950520 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -74,23 +74,10 @@ class RoomMemberStore(SQLBaseStore): txn.execute(sql, (event.room_id, domain)) elif event.membership != Membership.INVITE: # Check if this was the last person to have left. - member_events = self._get_members_query_txn( - txn, - where_clause=("c.room_id = ? AND m.membership = ?" - " AND m.user_id != ?"), - where_values=(event.room_id, Membership.JOIN, target_user_id,) + joined_domains = self._get_joined_hosts_for_room_txn( + txn, event.room_id ) - joined_domains = set() - for e in member_events: - try: - joined_domains.add( - UserID.from_string(e.state_key).domain - ) - except: - # FIXME: How do we deal with invalid user ids in the db? - logger.exception("Invalid user_id: %s", event.state_key) - if domain not in joined_domains: sql = ( "DELETE FROM room_hosts WHERE room_id = ? AND host = ?" @@ -100,7 +87,6 @@ class RoomMemberStore(SQLBaseStore): self.get_rooms_for_user.invalidate(target_user_id) - @defer.inlineCallbacks def get_room_member(self, user_id, room_id): """Retrieve the current state of a room member. @@ -110,41 +96,27 @@ class RoomMemberStore(SQLBaseStore): Returns: Deferred: Results in a MembershipEvent or None. """ - rows = yield self._get_members_by_dict({ - "e.room_id": room_id, - "m.user_id": user_id, - }) + def f(txn): + events = self._get_members_events_txn( + txn, + room_id, + user_id=user_id, + ) - defer.returnValue(rows[0] if rows else None) + return events[0] if events else None - def _get_room_member(self, txn, user_id, room_id): - sql = ( - "SELECT e.* FROM events as e" - " INNER JOIN room_memberships as m" - " ON e.event_id = m.event_id" - " INNER JOIN current_state_events as c" - " ON m.event_id = c.event_id" - " WHERE m.user_id = ? and e.room_id = ?" - " LIMIT 1" - ) - txn.execute(sql, (user_id, room_id)) - rows = self.cursor_to_dict(txn) - if rows: - return self._parse_events_txn(txn, rows)[0] - else: - return None + return self.runInteraction("get_room_member", f) def get_users_in_room(self, room_id): def f(txn): - sql = ( - "SELECT m.user_id FROM room_memberships as m" - " INNER JOIN current_state_events as c" - " ON m.event_id = c.event_id" - " WHERE m.membership = ? AND m.room_id = ?" + + rows = self._get_members_rows_txn( + txn, + room_id=room_id, + membership=Membership.JOIN, ) - txn.execute(sql, (Membership.JOIN, room_id)) - return [r[0] for r in txn.fetchall()] + return [r["user_id"] for r in rows] return self.runInteraction("get_users_in_room", f) def get_room_members(self, room_id, membership=None): @@ -159,11 +131,14 @@ class RoomMemberStore(SQLBaseStore): list of namedtuples representing the members in this room. """ - where = {"m.room_id": room_id} - if membership: - where["m.membership"] = membership + def f(txn): + return self._get_members_events_txn( + txn, + room_id, + membership=membership, + ) - return self._get_members_by_dict(where) + return self.runInteraction("get_room_members", f) def get_rooms_for_user_where_membership_is(self, user_id, membership_list): """ Get all the rooms for this user where the membership for this user @@ -209,28 +184,52 @@ class RoomMemberStore(SQLBaseStore): ] def get_joined_hosts_for_room(self, room_id): - return self._simple_select_onecol( - "room_hosts", - {"room_id": room_id}, - "host", - desc="get_joined_hosts_for_room", + return self.runInteraction( + "get_joined_hosts_for_room", + self._get_joined_hosts_for_room_txn, + room_id, ) - def _get_members_by_dict(self, where_dict): - clause = " AND ".join("%s = ?" % k for k in where_dict.keys()) - vals = where_dict.values() - return self._get_members_query(clause, vals) + def _get_joined_hosts_for_room_txn(self, txn, room_id): + rows = self._get_members_rows_txn( + txn, + room_id, membership=Membership.JOIN + ) + + joined_domains = set( + UserID.from_string(r["user_id"]).domain + for r in rows + ) + + return joined_domains def _get_members_query(self, where_clause, where_values): return self.runInteraction( - "get_members_query", self._get_members_query_txn, + "get_members_query", self._get_members_events_txn, where_clause, where_values ) - def _get_members_query_txn(self, txn, where_clause, where_values): + def _get_members_events_txn(self, txn, room_id, membership=None, user_id=None): + rows = self._get_members_rows_txn( + txn, + room_id, membership, user_id, + ) + return self._get_events_txn(txn, [r["event_id"] for r in rows]) + + def _get_members_rows_txn(self, txn, room_id, membership=None, user_id=None): + where_clause = "c.room_id = ?" + where_values = [room_id] + + if membership: + where_clause += " AND m.membership = ?" + where_values.append(membership) + + if user_id: + where_clause += " AND m.user_id = ?" + where_values.append(user_id) + sql = ( - "SELECT e.* FROM events as e " - "INNER JOIN room_memberships as m " + "SELECT m.* FROM room_memberships as m " "ON e.event_id = m.event_id " "INNER JOIN current_state_events as c " "ON m.event_id = c.event_id " @@ -242,8 +241,7 @@ class RoomMemberStore(SQLBaseStore): txn.execute(sql, where_values) rows = self.cursor_to_dict(txn) - results = self._parse_events_txn(txn, rows) - return results + return rows @cached() def get_rooms_for_user(self, user_id): From 6e7131f02f33695b6423db9a377f130ffebbdb67 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Mar 2015 15:38:56 +0000 Subject: [PATCH 015/173] Remove uses of REPLACE and ON CONFLICT IGNORE to make the SQL more portable. --- synapse/storage/_base.py | 13 ++++--------- synapse/storage/keys.py | 2 -- synapse/storage/signatures.py | 4 ---- synapse/storage/state.py | 1 - 4 files changed, 4 insertions(+), 16 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index d038c5509..9214e0c49 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -341,26 +341,21 @@ class SQLBaseStore(object): # "Simple" SQL API methods that operate on a single table with no JOINs, # no complex WHERE clauses, just a dict of values for columns. - def _simple_insert(self, table, values, or_replace=False, or_ignore=False, - desc="_simple_insert"): + def _simple_insert(self, table, values, desc="_simple_insert"): """Executes an INSERT query on the named table. Args: table : string giving the table name values : dict of new column names and values for them - or_replace : bool; if True performs an INSERT OR REPLACE """ return self.runInteraction( desc, - self._simple_insert_txn, table, values, or_replace=or_replace, - or_ignore=or_ignore, + self._simple_insert_txn, table, values, ) @log_function - def _simple_insert_txn(self, txn, table, values, or_replace=False, - or_ignore=False): - sql = "%s INTO %s (%s) VALUES(%s)" % ( - ("REPLACE" if or_replace else "INSERT"), + def _simple_insert_txn(self, txn, table, values): + sql = "INSERT INTO %s (%s) VALUES(%s)" % ( table, ", ".join(k for k in values), ", ".join("?" for k in values) diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 09d1e6365..25fef7943 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -66,7 +66,6 @@ class KeyStore(SQLBaseStore): "ts_added_ms": time_now_ms, "tls_certificate": buffer(tls_certificate_bytes), }, - or_ignore=True, ) @defer.inlineCallbacks @@ -116,5 +115,4 @@ class KeyStore(SQLBaseStore): "ts_added_ms": time_now_ms, "verify_key": buffer(verify_key.encode()), }, - or_ignore=True, ) diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index d0d53770f..13ce33510 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -56,7 +56,6 @@ class SignatureStore(SQLBaseStore): "algorithm": algorithm, "hash": buffer(hash_bytes), }, - or_ignore=True, ) def get_event_reference_hashes(self, event_ids): @@ -119,7 +118,6 @@ class SignatureStore(SQLBaseStore): "algorithm": algorithm, "hash": buffer(hash_bytes), }, - or_ignore=True, ) def _get_event_signatures_txn(self, txn, event_id): @@ -164,7 +162,6 @@ class SignatureStore(SQLBaseStore): "key_id": key_id, "signature": buffer(signature_bytes), }, - or_ignore=True, ) def _get_prev_event_hashes_txn(self, txn, event_id): @@ -198,5 +195,4 @@ class SignatureStore(SQLBaseStore): "algorithm": algorithm, "hash": buffer(hash_bytes), }, - or_ignore=True, ) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 721f2862c..65ea9c4d8 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -124,7 +124,6 @@ class StateStore(SQLBaseStore): "state_group": state_group, "event_id": event.event_id, }, - or_replace=True, ) @defer.inlineCallbacks From d19e79ecc956e5ba7ed6b6fd37e80ec6a737b048 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 24 Mar 2015 15:33:48 +0000 Subject: [PATCH 016/173] Make deleting other access tokens when you change your password actually work --- synapse/rest/client/v2_alpha/password.py | 5 ++++- synapse/storage/registration.py | 16 +++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/synapse/rest/client/v2_alpha/password.py b/synapse/rest/client/v2_alpha/password.py index 127753211..85954c71c 100644 --- a/synapse/rest/client/v2_alpha/password.py +++ b/synapse/rest/client/v2_alpha/password.py @@ -65,12 +65,15 @@ class PasswordRestServlet(RestServlet): raise SynapseError(400, "", Codes.MISSING_PARAM) new_password = body['new_password'] - self.login_handler.set_password( + yield self.login_handler.set_password( user_id, new_password, client.token_id ) defer.returnValue((200, {})) + def on_OPTIONS(self, _): + return 200, {} + def register_servlets(hs, http_server): PasswordRestServlet(hs).register(http_server) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 7e60dc395..0364d1085 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -89,35 +89,41 @@ class RegistrationStore(SQLBaseStore): "VALUES (?,?)", [txn.lastrowid, token]) def get_user_by_id(self, user_id): - query = ("SELECT users.name, users.password_hash FROM users" + query = ("SELECT users.id, users.name, users.password_hash FROM users" " WHERE users.name = ?") return self._execute( "get_user_by_id", self.cursor_to_dict, query, user_id ) + @defer.inlineCallbacks def user_set_password_hash(self, user_id, password_hash): """ NB. This does *not* evict any cache because the one use for this removes most of the entries subsequently anyway so it would be pointless. Use flush_user separately. """ - return self._simple_update_one('users', { + yield self._simple_update_one('users', { 'name': user_id }, { 'password_hash': password_hash }) + @defer.inlineCallbacks def user_delete_access_tokens_apart_from(self, user_id, token_id): - return self._execute( + rows = yield self.get_user_by_id(user_id) + if len(rows) == 0: + raise Exception("No such user!") + + yield self._execute( "delete_access_tokens_apart_from", None, "DELETE FROM access_tokens WHERE user_id = ? AND id != ?", - user_id, token_id + rows[0]['id'], token_id ) @defer.inlineCallbacks def flush_user(self, user_id): rows = yield self._execute( - 'user_delete_access_tokens_apart_from', None, + 'flush_user', None, "SELECT token FROM access_tokens WHERE user_id = ?", user_id ) From 9115421ace4cd6cd1080ab99c997994541aea30b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Mar 2015 16:17:39 +0000 Subject: [PATCH 017/173] Use _simple_upsert --- synapse/storage/__init__.py | 8 +++++--- synapse/storage/transactions.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4877f45dc..e03d55b00 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -81,14 +81,16 @@ class DataStore(RoomMemberStore, RoomStore, self._next_stream_id = int(hs.get_clock().time_msec()) * 1000 def insert_client_ip(self, user, access_token, device_id, ip, user_agent): - return self._simple_insert( + return self._simple_upsert( "user_ips", - { + keyvalues={ "user": user.to_string(), "access_token": access_token, - "device_id": device_id, "ip": ip, "user_agent": user_agent, + }, + values={ + "device_id": device_id, "last_seen": int(self._clock.time_msec()), }, desc="insert_client_ip", diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 1a5bb41cb..c6e00ae3f 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -84,14 +84,14 @@ class TransactionStore(SQLBaseStore): def _set_received_txn_response(self, txn, transaction_id, origin, code, response_json): - self._simple_update_one_txn( + self._simple_upsert_txn( txn, table=ReceivedTransactionsTable.table_name, keyvalues={ "transaction_id": transaction_id, "origin": origin, }, - updatevalues={ + values={ "response_code": code, "response_json": response_json, } From 9e98f1022a927adef959e0a76dde2fc6f708917d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Mar 2015 16:19:01 +0000 Subject: [PATCH 018/173] Don't order by rowid --- synapse/storage/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 9214e0c49..1ea39bc0a 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -585,7 +585,7 @@ class SQLBaseStore(object): def _simple_select_one_txn(self, txn, table, keyvalues, retcols, allow_none=False): - select_sql = "SELECT %s FROM %s WHERE %s ORDER BY rowid asc" % ( + select_sql = "SELECT %s FROM %s WHERE %s" % ( ", ".join(retcols), table, " AND ".join("%s = ?" % (k) for k in keyvalues) From 5a447098dd8442c7c69b2db22d42e6ce40c2dc6e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Mar 2015 16:19:24 +0000 Subject: [PATCH 019/173] Don't use room hosts table --- synapse/storage/roommember.py | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index a22950520..df707f812 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -65,26 +65,6 @@ class RoomMemberStore(SQLBaseStore): } ) - # Update room hosts table - if event.membership == Membership.JOIN: - sql = ( - "REPLACE INTO room_hosts (room_id, host) " - "VALUES (?, ?)" - ) - txn.execute(sql, (event.room_id, domain)) - elif event.membership != Membership.INVITE: - # Check if this was the last person to have left. - joined_domains = self._get_joined_hosts_for_room_txn( - txn, event.room_id - ) - - if domain not in joined_domains: - sql = ( - "DELETE FROM room_hosts WHERE room_id = ? AND host = ?" - ) - - txn.execute(sql, (event.room_id, domain)) - self.get_rooms_for_user.invalidate(target_user_id) def get_room_member(self, user_id, room_id): @@ -229,11 +209,10 @@ class RoomMemberStore(SQLBaseStore): where_values.append(user_id) sql = ( - "SELECT m.* FROM room_memberships as m " - "ON e.event_id = m.event_id " - "INNER JOIN current_state_events as c " - "ON m.event_id = c.event_id " - "WHERE %(where)s " + "SELECT m.* FROM room_memberships as m" + " INNER JOIN current_state_events as c" + " ON m.event_id = c.event_id" + " WHERE %(where)s" ) % { "where": where_clause, } From 1aa44939fc94ba1062d53a74dbd9629c32a2ae20 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Mar 2015 16:20:05 +0000 Subject: [PATCH 020/173] Fix bugs in transactions storage --- synapse/storage/transactions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index c6e00ae3f..03e1e3b80 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -143,7 +143,8 @@ class TransactionStore(SQLBaseStore): txn, table=SentTransactions.table_name, values={ - "transaction_id": self.get_next_stream_id(), + "id": self.get_next_stream_id(), + "transaction_id": transaction_id, "destination": destination, "ts": origin_server_ts, "response_code": 0, @@ -240,7 +241,7 @@ class TransactionStore(SQLBaseStore): allow_none=True, ) - if result["retry_last_ts"] > 0: + if result and result["retry_last_ts"] > 0: return result else: return None From f488293d96dc690ce7a4b3163fe8c4e0d3e31899 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Mar 2015 16:20:26 +0000 Subject: [PATCH 021/173] Don't reinsert into event_edges --- synapse/storage/events.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 542559c59..4d636d3f4 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -149,14 +149,6 @@ class EventsStore(SQLBaseStore): event.depth ) - self._handle_prev_events( - txn, - outlier=outlier, - event_id=event.event_id, - prev_events=event.prev_events, - room_id=event.room_id, - ) - have_persisted = self._simple_select_one_onecol_txn( txn, table="event_json", @@ -196,6 +188,14 @@ class EventsStore(SQLBaseStore): ) return + self._handle_prev_events( + txn, + outlier=outlier, + event_id=event.event_id, + prev_events=event.prev_events, + room_id=event.room_id, + ) + if event.type == EventTypes.Member: self._store_room_member_txn(txn, event) elif event.type == EventTypes.Name: From 0ba393924a823698eb5263a29ce6c4b50488ac4b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Mar 2015 16:31:52 +0000 Subject: [PATCH 022/173] Escape non printing ascii character --- synapse/storage/room.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 864103332..501e947ad 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -102,10 +102,10 @@ class RoomStore(SQLBaseStore): "ON c.event_id = room_names.event_id " ) - # We use non printing ascii character US () as a seperator + # We use non printing ascii character US (\x1F) as a separator sql = ( "SELECT r.room_id, n.name, t.topic, " - "group_concat(a.room_alias, '') " + "group_concat(a.room_alias, '\x1F') " "FROM rooms AS r " "LEFT JOIN (%(topic)s) AS t ON t.room_id = r.room_id " "LEFT JOIN (%(name)s) AS n ON n.room_id = r.room_id " @@ -130,7 +130,7 @@ class RoomStore(SQLBaseStore): "room_id": r[0], "name": r[1], "topic": r[2], - "aliases": r[3].split(""), + "aliases": r[3].split("\x1F"), } for r in rows ] From c7023f21555a0adf0d8bb5040c817a8198bbf5a8 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 24 Mar 2015 17:24:15 +0000 Subject: [PATCH 023/173] 1) Pushers are now associated with an access token 2) Change places where we mean unauthenticated to 401, not 403, in C/S v2: hack so it stays as 403 in v1 because web client relies on it. --- synapse/api/auth.py | 43 ++++++++++++++----------- synapse/push/pusherpool.py | 9 +++--- synapse/rest/client/v1/base.py | 2 +- synapse/rest/client/v1/pusher.py | 3 +- synapse/server.py | 10 ++++++ synapse/storage/pusher.py | 3 +- synapse/storage/registration.py | 2 +- synapse/storage/schema/delta/15/v15.sql | 2 ++ 8 files changed, 48 insertions(+), 26 deletions(-) create mode 100644 synapse/storage/schema/delta/15/v15.sql diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 64f605b96..d08faf23f 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -40,6 +40,7 @@ class Auth(object): self.hs = hs self.store = hs.get_datastore() self.state = hs.get_state_handler() + self.TOKEN_NOT_FOUND_HTTP_STATUS = 401 def check(self, event, auth_events): """ Checks if this event is correctly authed. @@ -373,7 +374,9 @@ class Auth(object): defer.returnValue((user, ClientInfo(device_id, token_id))) except KeyError: - raise AuthError(403, "Missing access token.") + raise AuthError( + self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token." + ) @defer.inlineCallbacks def get_user_by_token(self, token): @@ -387,21 +390,20 @@ class Auth(object): Raises: AuthError if no user by that token exists or the token is invalid. """ - try: - ret = yield self.store.get_user_by_token(token) - if not ret: - raise StoreError(400, "Unknown token") - user_info = { - "admin": bool(ret.get("admin", False)), - "device_id": ret.get("device_id"), - "user": UserID.from_string(ret.get("name")), - "token_id": ret.get("token_id", None), - } + ret = yield self.store.get_user_by_token(token) + if not ret: + raise AuthError( + self.TOKEN_NOT_FOUND_HTTP_STATUS, "Unrecognised access token.", + errcode=Codes.UNKNOWN_TOKEN + ) + user_info = { + "admin": bool(ret.get("admin", False)), + "device_id": ret.get("device_id"), + "user": UserID.from_string(ret.get("name")), + "token_id": ret.get("token_id", None), + } - defer.returnValue(user_info) - except StoreError: - raise AuthError(403, "Unrecognised access token.", - errcode=Codes.UNKNOWN_TOKEN) + defer.returnValue(user_info) @defer.inlineCallbacks def get_appservice_by_req(self, request): @@ -409,11 +411,16 @@ class Auth(object): token = request.args["access_token"][0] service = yield self.store.get_app_service_by_token(token) if not service: - raise AuthError(403, "Unrecognised access token.", - errcode=Codes.UNKNOWN_TOKEN) + raise AuthError( + self.TOKEN_NOT_FOUND_HTTP_STATUS, + "Unrecognised access token.", + errcode=Codes.UNKNOWN_TOKEN + ) defer.returnValue(service) except KeyError: - raise AuthError(403, "Missing access token.") + raise AuthError( + self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token." + ) def is_server_admin(self, user): return self.store.is_server_admin(user) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 90babd722..f75eebf8b 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -57,7 +57,7 @@ class PusherPool: self._start_pushers(pushers) @defer.inlineCallbacks - def add_pusher(self, user_name, profile_tag, kind, app_id, + def add_pusher(self, user_name, access_token, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, lang, data): # we try to create the pusher just to validate the config: it # will then get pulled out of the database, @@ -79,17 +79,18 @@ class PusherPool: "failing_since": None }) yield self._add_pusher_to_store( - user_name, profile_tag, kind, app_id, + user_name, access_token, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, lang, data ) @defer.inlineCallbacks - def _add_pusher_to_store(self, user_name, profile_tag, kind, app_id, - app_display_name, device_display_name, + def _add_pusher_to_store(self, user_name, access_token, profile_tag, kind, + app_id, app_display_name, device_display_name, pushkey, lang, data): yield self.store.add_pusher( user_name=user_name, + access_token=access_token, profile_tag=profile_tag, kind=kind, app_id=app_id, diff --git a/synapse/rest/client/v1/base.py b/synapse/rest/client/v1/base.py index 72332bdb1..504a5e432 100644 --- a/synapse/rest/client/v1/base.py +++ b/synapse/rest/client/v1/base.py @@ -48,5 +48,5 @@ class ClientV1RestServlet(RestServlet): self.hs = hs self.handlers = hs.get_handlers() self.builder_factory = hs.get_event_builder_factory() - self.auth = hs.get_auth() + self.auth = hs.get_v1auth() self.txns = HttpTransactionStore() diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 6045e86f3..87e89c930 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -27,7 +27,7 @@ class PusherRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): - user, _ = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -54,6 +54,7 @@ class PusherRestServlet(ClientV1RestServlet): try: yield pusher_pool.add_pusher( user_name=user.to_string(), + access_token=client.token_id, profile_tag=content['profile_tag'], kind=content['kind'], app_id=content['app_id'], diff --git a/synapse/server.py b/synapse/server.py index c7772244b..4c4f6ca23 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -65,6 +65,7 @@ class BaseHomeServer(object): 'replication_layer', 'datastore', 'handlers', + 'v1auth', 'auth', 'rest_servlet_factory', 'state_handler', @@ -182,6 +183,15 @@ class HomeServer(BaseHomeServer): def build_auth(self): return Auth(self) + def build_v1auth(self): + orf = Auth(self) + # Matrix spec makes no reference to what HTTP status code is returned, + # but the V1 API uses 403 where it means 401, and the webclient + # relies on this behaviour, so V1 gets its own copy of the auth + # with backwards compat behaviour. + orf.TOKEN_NOT_FOUND_HTTP_STATUS = 403 + return orf + def build_state_handler(self): return StateHandler(self) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 000502b4f..1ef8e06ac 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -95,7 +95,7 @@ class PusherStore(SQLBaseStore): defer.returnValue(ret) @defer.inlineCallbacks - def add_pusher(self, user_name, profile_tag, kind, app_id, + def add_pusher(self, user_name, access_token, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, lang, data): try: @@ -107,6 +107,7 @@ class PusherStore(SQLBaseStore): ), dict( user_name=user_name, + access_token=access_token, kind=kind, profile_tag=profile_tag, app_display_name=app_display_name, diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 0364d1085..f61d8fdb6 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -174,4 +174,4 @@ class RegistrationStore(SQLBaseStore): if rows: return rows[0] - raise StoreError(404, "Token not found.") + return None diff --git a/synapse/storage/schema/delta/15/v15.sql b/synapse/storage/schema/delta/15/v15.sql new file mode 100644 index 000000000..fc3e43687 --- /dev/null +++ b/synapse/storage/schema/delta/15/v15.sql @@ -0,0 +1,2 @@ +ALTER TABLE pushers ADD COLUMN access_token INTEGER DEFAULT NULL; + From 9aa0224cdf6ae9243903090d0a264e684f557da2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 24 Mar 2015 17:25:59 +0000 Subject: [PATCH 024/173] unused import --- synapse/api/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index d08faf23f..0bf35109c 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership, JoinRules -from synapse.api.errors import AuthError, StoreError, Codes, SynapseError +from synapse.api.errors import AuthError, Codes, SynapseError from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor from synapse.types import UserID, ClientInfo From 438a21c87bec6386be1a9ed92d61b1b467a40bcd Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 24 Mar 2015 18:21:54 +0000 Subject: [PATCH 025/173] Don't test exact equality of the list: as long as it has the fields we expect, that's just fine. I added the user_id (as in database pkey) and it broke: no point testing what that comes out as: it's determined by the db. --- tests/storage/test_registration.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index e0b81f2b5..93be02f8c 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -38,13 +38,12 @@ class RegistrationStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_register(self): yield self.store.register(self.user_id, self.tokens[0], self.pwhash) + u = yield self.store.get_user_by_id(self.user_id)[0] - self.assertEquals( - # TODO(paul): Surely this field should be 'user_id', not 'name' - # Additionally surely it shouldn't come in a 1-element list - [{"name": self.user_id, "password_hash": self.pwhash}], - (yield self.store.get_user_by_id(self.user_id)) - ) + # TODO(paul): Surely this field should be 'user_id', not 'name' + # Additionally surely it shouldn't come in a 1-element list + self.assertEquals(self.user_id, u['name']) + self.assertEquals(self.pwhash, u['password_hash']) self.assertEquals( {"admin": 0, From ce2766d19c985d0100cc143477100c2723be1844 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 24 Mar 2015 18:56:51 +0000 Subject: [PATCH 026/173] Fix tests --- tests/rest/client/v1/test_presence.py | 6 +++--- tests/rest/client/v1/test_profile.py | 2 +- tests/rest/client/v1/test_rooms.py | 16 ++++++++-------- tests/rest/client/v1/test_typing.py | 2 +- tests/storage/test_registration.py | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index b9c03383a..8e0c5fa63 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -75,7 +75,7 @@ class PresenceStateTestCase(unittest.TestCase): "token_id": 1, } - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_token = _get_user_by_token room_member_handler = hs.handlers.room_member_handler = Mock( spec=[ @@ -170,7 +170,7 @@ class PresenceListTestCase(unittest.TestCase): ] ) - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_token = _get_user_by_token presence.register_servlets(hs, self.mock_resource) @@ -277,7 +277,7 @@ class PresenceEventStreamTestCase(unittest.TestCase): def _get_user_by_req(req=None): return (UserID.from_string(myid), "") - hs.get_auth().get_user_by_req = _get_user_by_req + hs.get_v1auth().get_user_by_req = _get_user_by_req presence.register_servlets(hs, self.mock_resource) events.register_servlets(hs, self.mock_resource) diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py index 5cd5767f2..929e5e5dd 100644 --- a/tests/rest/client/v1/test_profile.py +++ b/tests/rest/client/v1/test_profile.py @@ -55,7 +55,7 @@ class ProfileTestCase(unittest.TestCase): def _get_user_by_req(request=None): return (UserID.from_string(myid), "") - hs.get_auth().get_user_by_req = _get_user_by_req + hs.get_v1auth().get_user_by_req = _get_user_by_req hs.get_handlers().profile_handler = self.mock_handler diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 72fb4576b..c83348acf 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -61,7 +61,7 @@ class RoomPermissionsTestCase(RestTestCase): "device_id": None, "token_id": 1, } - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_token = _get_user_by_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -71,7 +71,7 @@ class RoomPermissionsTestCase(RestTestCase): synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) - self.auth = hs.get_auth() + self.auth = hs.get_v1auth() # create some rooms under the name rmcreator_id self.uncreated_rmid = "!aa:test" @@ -448,7 +448,7 @@ class RoomsMemberListTestCase(RestTestCase): "device_id": None, "token_id": 1, } - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_token = _get_user_by_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -528,7 +528,7 @@ class RoomsCreateTestCase(RestTestCase): "device_id": None, "token_id": 1, } - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_token = _get_user_by_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -622,7 +622,7 @@ class RoomTopicTestCase(RestTestCase): "token_id": 1, } - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_token = _get_user_by_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -728,7 +728,7 @@ class RoomMemberStateTestCase(RestTestCase): "device_id": None, "token_id": 1, } - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_token = _get_user_by_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -855,7 +855,7 @@ class RoomMessagesTestCase(RestTestCase): "device_id": None, "token_id": 1, } - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_token = _get_user_by_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -952,7 +952,7 @@ class RoomInitialSyncTestCase(RestTestCase): "device_id": None, "token_id": 1, } - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_token = _get_user_by_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index 80f2ec9dd..3442e6ede 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -67,7 +67,7 @@ class RoomTypingTestCase(RestTestCase): "token_id": 1, } - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_token = _get_user_by_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index 93be02f8c..2f8953f51 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -38,7 +38,7 @@ class RegistrationStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_register(self): yield self.store.register(self.user_id, self.tokens[0], self.pwhash) - u = yield self.store.get_user_by_id(self.user_id)[0] + u = (yield self.store.get_user_by_id(self.user_id))[0] # TODO(paul): Surely this field should be 'user_id', not 'name' # Additionally surely it shouldn't come in a 1-element list From 0e8f5095c7e7075b249ad53a9f60a4d2fdeeaaed Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 25 Mar 2015 17:15:20 +0000 Subject: [PATCH 027/173] Fix unicode database support --- synapse/app/homeserver.py | 47 ++++++++++++------- synapse/handlers/login.py | 2 +- synapse/rest/client/v1/profile.py | 7 ++- synapse/storage/__init__.py | 12 +++-- synapse/storage/_base.py | 4 ++ synapse/storage/events.py | 8 ++-- synapse/storage/keys.py | 4 +- synapse/storage/profile.py | 12 ++++- synapse/storage/registration.py | 18 +++++-- synapse/storage/room.py | 1 + .../full_schemas/11/media_repository.sql | 2 +- .../schema/full_schemas/11/profiles.sql | 2 +- .../schema/full_schemas/11/transactions.sql | 1 - synapse/storage/signatures.py | 10 ++-- synapse/storage/transactions.py | 2 +- 15 files changed, 88 insertions(+), 44 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 394e93e6c..beab6ffc7 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -110,14 +110,12 @@ class SynapseHomeServer(HomeServer): return None def build_db_pool(self): - name = self.db_config.pop("name", None) - if name == "MySQLdb": - return adbapi.ConnectionPool( - name, - **self.db_config - ) + name = self.db_config["name"] - raise RuntimeError("Unsupported database type") + return adbapi.ConnectionPool( + name, + **self.db_config.get("args", {}) + ) def create_resource_tree(self, redirect_root_to_web_client): """Create the resource tree for this Home Server. @@ -323,7 +321,7 @@ def change_resource_limit(soft_file_no): resource.setrlimit(resource.RLIMIT_NOFILE, (soft_file_no, hard)) logger.info("Set file limit to: %d", soft_file_no) - except (ValueError, resource.error) as e: + except ( ValueError, resource.error) as e: logger.warn("Failed to set file limit: %s", e) @@ -363,20 +361,33 @@ def setup(config_options): if config.database_config: with open(config.database_config, 'r') as f: db_config = yaml.safe_load(f) - - name = db_config.get("name", None) - if name == "MySQLdb": - db_config.update({ - "sql_mode": "TRADITIONAL", - "charset": "utf8", - "use_unicode": True, - }) else: db_config = { "name": "sqlite3", "database": config.database_path, } + db_config = { + k: v for k, v in db_config.items() + if not k.startswith("cp_") + } + + name = db_config.get("name", None) + if name in ["MySQLdb", "mysql.connector"]: + db_config.setdefault("args", {}).update({ + "sql_mode": "TRADITIONAL", + "charset": "utf8", + "use_unicode": True, + }) + elif name == "sqlite3": + db_config.setdefault("args", {}).update({ + "cp_min": 1, + "cp_max": 1, + "cp_openfun": prepare_database, + }) + else: + raise RuntimeError("Unsupported database type '%s'" % (name,)) + hs = SynapseHomeServer( config.server_name, domain_with_port=domain_with_port, @@ -401,8 +412,8 @@ def setup(config_options): # with sqlite3.connect(db_name) as db_conn: # prepare_sqlite3_database(db_conn) # prepare_database(db_conn) - import MySQLdb - db_conn = MySQLdb.connect(**db_config) + import mysql.connector + db_conn = mysql.connector.connect(**db_config.get("args", {})) prepare_database(db_conn) except UpgradeDatabaseException: sys.stderr.write( diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 744780046..76647c794 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -57,7 +57,7 @@ class LoginHandler(BaseHandler): logger.warn("Attempted to login as %s but they do not exist", user) raise LoginError(403, "", errcode=Codes.FORBIDDEN) - stored_hash = user_info[0]["password_hash"] + stored_hash = user_info["password_hash"] if bcrypt.checkpw(password, stored_hash): # generate an access token and store it. token = self.reg_handler._generate_token(user) diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index 1e77eb49c..7387b4adb 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -19,9 +19,13 @@ from twisted.internet import defer from .base import ClientV1RestServlet, client_path_pattern from synapse.types import UserID +import logging import simplejson as json +logger = logging.getLogger(__name__) + + class ProfileDisplaynameRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/profile/(?P[^/]*)/displayname") @@ -47,7 +51,8 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): defer.returnValue((400, "Unable to parse name")) yield self.handlers.profile_handler.set_displayname( - user, auth_user, new_name) + user, auth_user, new_name + ) defer.returnValue((200, {})) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index e03d55b00..abde7d0df 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -410,10 +410,14 @@ def executescript(txn, schema_path): def _get_or_create_schema_state(txn): - schema_path = os.path.join( - dir_path, "schema", "schema_version.sql", - ) - executescript(txn, schema_path) + try: + # Bluntly try creating the schema_version tables. + schema_path = os.path.join( + dir_path, "schema", "schema_version.sql", + ) + executescript(txn, schema_path) + except: + pass txn.execute("SELECT version, upgraded FROM schema_version") row = txn.fetchone() diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 1ea39bc0a..76ec3ee93 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -755,6 +755,8 @@ class SQLBaseStore(object): return None internal_metadata, js, redacted, rejected_reason = res + js = js.decode("utf8") + internal_metadata = internal_metadata.decode("utf8") start_time = update_counter("select_event", start_time) @@ -779,9 +781,11 @@ class SQLBaseStore(object): sql_getevents_timer.inc_by(curr_time - last_time, desc) return curr_time + logger.debug("Got js: %r", js) d = json.loads(js) start_time = update_counter("decode_json", start_time) + logger.debug("Got internal_metadata: %r", internal_metadata) internal_metadata = json.loads(internal_metadata) start_time = update_counter("decode_internal", start_time) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 4d636d3f4..69f598967 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -294,15 +294,17 @@ class EventsStore(SQLBaseStore): ) if is_new_state and not context.rejected: - self._simple_insert_txn( + self._simple_upsert_txn( txn, "current_state_events", - { - "event_id": event.event_id, + keyvalues={ "room_id": event.room_id, "type": event.type, "state_key": event.state_key, }, + values={ + "event_id": event.event_id, + } ) for e_id, h in event.prev_state: diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 25fef7943..e6975a945 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -64,7 +64,7 @@ class KeyStore(SQLBaseStore): "fingerprint": fingerprint, "from_server": from_server, "ts_added_ms": time_now_ms, - "tls_certificate": buffer(tls_certificate_bytes), + "tls_certificate": tls_certificate_bytes, }, ) @@ -113,6 +113,6 @@ class KeyStore(SQLBaseStore): "key_id": "%s:%s" % (verify_key.alg, verify_key.version), "from_server": from_server, "ts_added_ms": time_now_ms, - "verify_key": buffer(verify_key.encode()), + "verify_key": verify_key.encode(), }, ) diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py index a6e52cb24..09778045b 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer + from ._base import SQLBaseStore @@ -24,19 +26,25 @@ class ProfileStore(SQLBaseStore): desc="create_profile", ) + @defer.inlineCallbacks def get_profile_displayname(self, user_localpart): - return self._simple_select_one_onecol( + name = yield self._simple_select_one_onecol( table="profiles", keyvalues={"user_id": user_localpart}, retcol="displayname", desc="get_profile_displayname", ) + if name: + name = name.decode("utf8") + + defer.returnValue(name) + def set_profile_displayname(self, user_localpart, new_displayname): return self._simple_update_one( table="profiles", keyvalues={"user_id": user_localpart}, - updatevalues={"displayname": new_displayname}, + updatevalues={"displayname": new_displayname.encode("utf8")}, desc="set_profile_displayname", ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index fe26d6d62..7258f7b2a 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -81,13 +81,23 @@ class RegistrationStore(SQLBaseStore): txn.execute("INSERT INTO access_tokens(user_id, token) " + "VALUES (?,?)", [user_id, token]) + @defer.inlineCallbacks def get_user_by_id(self, user_id): - query = ("SELECT users.name, users.password_hash FROM users" - " WHERE users.name = ?") - return self._execute( - "get_user_by_id", self.cursor_to_dict, query, user_id + user_info = yield self._simple_select_one( + table="users", + keyvalues={ + "name": user_id, + }, + retcols=["name", "password_hash"], + allow_none=True, ) + if user_info: + user_info["password_hash"] = user_info["password_hash"].decode("utf8") + + defer.returnValue(user_info) + + @cached() # TODO(paul): Currently there's no code to invalidate this cache. That # means if/when we ever add internal ways to invalidate access tokens or diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 501e947ad..a1a76280f 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -72,6 +72,7 @@ class RoomStore(SQLBaseStore): keyvalues={"room_id": room_id}, retcols=RoomsTable.fields, desc="get_room", + allow_none=True, ) @defer.inlineCallbacks diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index 8bc84dc24..d9559f590 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -65,4 +65,4 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( ) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id - ON local_media_repository_thumbnails (media_id); + ON remote_media_cache_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index 32defe2f7..552645c56 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS profiles( user_id VARCHAR(255) NOT NULL, - displayname VARCHAR(255), + displayname VARBINARY(255), avatar_url VARCHAR(255), UNIQUE(user_id) ) ENGINE = INNODB; diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 0570bf95d..bd13bba8c 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -38,7 +38,6 @@ CREATE TABLE IF NOT EXISTS sent_transactions( ) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); -CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions(destination); CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); -- So that we can do an efficient look up of all transactions that have yet to be successfully -- sent. diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 13ce33510..35bba854f 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -54,7 +54,7 @@ class SignatureStore(SQLBaseStore): { "event_id": event_id, "algorithm": algorithm, - "hash": buffer(hash_bytes), + "hash": hash_bytes, }, ) @@ -99,7 +99,7 @@ class SignatureStore(SQLBaseStore): " WHERE event_id = ?" ) txn.execute(query, (event_id, )) - return dict(txn.fetchall()) + return {k: v for k, v in txn.fetchall()} def _store_event_reference_hash_txn(self, txn, event_id, algorithm, hash_bytes): @@ -116,7 +116,7 @@ class SignatureStore(SQLBaseStore): { "event_id": event_id, "algorithm": algorithm, - "hash": buffer(hash_bytes), + "hash": hash_bytes, }, ) @@ -160,7 +160,7 @@ class SignatureStore(SQLBaseStore): "event_id": event_id, "signature_name": signature_name, "key_id": key_id, - "signature": buffer(signature_bytes), + "signature": signature_bytes, }, ) @@ -193,6 +193,6 @@ class SignatureStore(SQLBaseStore): "event_id": event_id, "prev_event_id": prev_event_id, "algorithm": algorithm, - "hash": buffer(hash_bytes), + "hash": hash_bytes, }, ) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 03e1e3b80..e3e484fb2 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -282,7 +282,7 @@ class TransactionStore(SQLBaseStore): query = ( "UPDATE destinations" " SET retry_last_ts = ?, retry_interval = ?" - " WHERE destinations = ?" + " WHERE destination = ?" ) txn.execute( From c1a256cc4c82ce746eae8e719b9aa2344fe66177 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 25 Mar 2015 19:06:22 +0000 Subject: [PATCH 028/173] Allow multiple pushers for a single app ID & pushkey, honouring the 'append' flag in the API. --- synapse/push/__init__.py | 10 ++++-- synapse/push/pusherpool.py | 47 +++++++++++++++++++------ synapse/rest/client/v1/pusher.py | 13 ++++++- synapse/storage/pusher.py | 27 +++++++------- synapse/storage/schema/delta/15/v15.sql | 27 ++++++++++++-- 5 files changed, 96 insertions(+), 28 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 0727f772a..5575c847f 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -253,7 +253,8 @@ class Pusher(object): self.user_name, config, timeout=0) self.last_token = chunk['end'] self.store.update_pusher_last_token( - self.app_id, self.pushkey, self.last_token) + self.app_id, self.pushkey, self.user_name, self.last_token + ) logger.info("Pusher %s for user %s starting from token %s", self.pushkey, self.user_name, self.last_token) @@ -314,7 +315,7 @@ class Pusher(object): pk ) yield self.hs.get_pusherpool().remove_pusher( - self.app_id, pk + self.app_id, pk, self.user_name ) if not self.alive: @@ -326,6 +327,7 @@ class Pusher(object): self.store.update_pusher_last_token_and_success( self.app_id, self.pushkey, + self.user_name, self.last_token, self.clock.time_msec() ) @@ -334,6 +336,7 @@ class Pusher(object): self.store.update_pusher_failing_since( self.app_id, self.pushkey, + self.user_name, self.failing_since) else: if not self.failing_since: @@ -341,6 +344,7 @@ class Pusher(object): self.store.update_pusher_failing_since( self.app_id, self.pushkey, + self.user_name, self.failing_since ) @@ -358,6 +362,7 @@ class Pusher(object): self.store.update_pusher_last_token( self.app_id, self.pushkey, + self.user_name, self.last_token ) @@ -365,6 +370,7 @@ class Pusher(object): self.store.update_pusher_failing_since( self.app_id, self.pushkey, + self.user_name, self.failing_since ) else: diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index f75eebf8b..cda072839 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -84,6 +84,21 @@ class PusherPool: pushkey, lang, data ) + @defer.inlineCallbacks + def remove_pushers_by_app_id_and_pushkey_not_user(self, app_id, pushkey, + not_user_id): + to_remove = yield self.store.get_pushers_by_app_id_and_pushkey( + app_id, pushkey + ) + for p in to_remove: + if p['user_name'] != not_user_id: + logger.info( + "Removing pusher for app id %s, pushkey %s, user %s", + app_id, pushkey, p['user_name'] + ) + self.remove_pusher(p['app_id'], p['pushkey'], p['user_name']) + + @defer.inlineCallbacks def _add_pusher_to_store(self, user_name, access_token, profile_tag, kind, app_id, app_display_name, device_display_name, @@ -101,7 +116,7 @@ class PusherPool: lang=lang, data=encode_canonical_json(data).decode("UTF-8"), ) - self._refresh_pusher((app_id, pushkey)) + self._refresh_pusher(app_id, pushkey, user_name) def _create_pusher(self, pusherdict): if pusherdict['kind'] == 'http': @@ -126,30 +141,42 @@ class PusherPool: ) @defer.inlineCallbacks - def _refresh_pusher(self, app_id_pushkey): - p = yield self.store.get_pushers_by_app_id_and_pushkey( - app_id_pushkey + def _refresh_pusher(self, app_id, pushkey, user_name): + resultlist = yield self.store.get_pushers_by_app_id_and_pushkey( + app_id, pushkey ) - p['data'] = json.loads(p['data']) + p = None + for r in resultlist: + if r['user_name'] == user_name: + p = r - self._start_pushers([p]) + if p: + p['data'] = json.loads(p['data']) + + self._start_pushers([p]) def _start_pushers(self, pushers): logger.info("Starting %d pushers", len(pushers)) for pusherdict in pushers: p = self._create_pusher(pusherdict) if p: - fullid = "%s:%s" % (pusherdict['app_id'], pusherdict['pushkey']) + fullid = "%s:%s:%s" % ( + pusherdict['app_id'], + pusherdict['pushkey'], + pusherdict['user_name'] + ) if fullid in self.pushers: self.pushers[fullid].stop() self.pushers[fullid] = p p.start() @defer.inlineCallbacks - def remove_pusher(self, app_id, pushkey): - fullid = "%s:%s" % (app_id, pushkey) + def remove_pusher(self, app_id, pushkey, user_name): + fullid = "%s:%s:%s" % (app_id, pushkey, user_name) if fullid in self.pushers: logger.info("Stopping pusher %s", fullid) self.pushers[fullid].stop() del self.pushers[fullid] - yield self.store.delete_pusher_by_app_id_pushkey(app_id, pushkey) + yield self.store.delete_pusher_by_app_id_pushkey_user_name( + app_id, pushkey, user_name + ) diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 87e89c930..c83287c02 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -37,7 +37,7 @@ class PusherRestServlet(ClientV1RestServlet): and 'kind' in content and content['kind'] is None): yield pusher_pool.remove_pusher( - content['app_id'], content['pushkey'] + content['app_id'], content['pushkey'], user_name=user.to_string() ) defer.returnValue((200, {})) @@ -51,6 +51,17 @@ class PusherRestServlet(ClientV1RestServlet): raise SynapseError(400, "Missing parameters: "+','.join(missing), errcode=Codes.MISSING_PARAM) + append = False + if 'append' in content: + append = content['append'] + + if not append: + yield pusher_pool.remove_pushers_by_app_id_and_pushkey_not_user( + app_id=content['app_id'], + pushkey=content['pushkey'], + not_user_id=user.to_string() + ) + try: yield pusher_pool.add_pusher( user_name=user.to_string(), diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 1ef8e06ac..423878c6a 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) class PusherStore(SQLBaseStore): @defer.inlineCallbacks - def get_pushers_by_app_id_and_pushkey(self, app_id_and_pushkey): + def get_pushers_by_app_id_and_pushkey(self, app_id, pushkey): sql = ( "SELECT id, user_name, kind, profile_tag, app_id," "app_display_name, device_display_name, pushkey, ts, data, " @@ -38,7 +38,7 @@ class PusherStore(SQLBaseStore): rows = yield self._execute( "get_pushers_by_app_id_and_pushkey", None, sql, - app_id_and_pushkey[0], app_id_and_pushkey[1] + app_id, pushkey ) ret = [ @@ -60,7 +60,7 @@ class PusherStore(SQLBaseStore): for r in rows ] - defer.returnValue(ret[0]) + defer.returnValue(ret) @defer.inlineCallbacks def get_all_pushers(self): @@ -104,9 +104,9 @@ class PusherStore(SQLBaseStore): dict( app_id=app_id, pushkey=pushkey, + user_name=user_name, ), dict( - user_name=user_name, access_token=access_token, kind=kind, profile_tag=profile_tag, @@ -123,37 +123,38 @@ class PusherStore(SQLBaseStore): raise StoreError(500, "Problem creating pusher.") @defer.inlineCallbacks - def delete_pusher_by_app_id_pushkey(self, app_id, pushkey): + def delete_pusher_by_app_id_pushkey_user_name(self, app_id, pushkey, user_name): yield self._simple_delete_one( PushersTable.table_name, - {"app_id": app_id, "pushkey": pushkey}, - desc="delete_pusher_by_app_id_pushkey", + {"app_id": app_id, "pushkey": pushkey, 'user_name': user_name}, + desc="delete_pusher_by_app_id_pushkey_user_name", ) @defer.inlineCallbacks - def update_pusher_last_token(self, app_id, pushkey, last_token): + def update_pusher_last_token(self, app_id, pushkey, user_name, last_token): yield self._simple_update_one( PushersTable.table_name, - {'app_id': app_id, 'pushkey': pushkey}, + {'app_id': app_id, 'pushkey': pushkey, 'user_name': user_name}, {'last_token': last_token}, desc="update_pusher_last_token", ) @defer.inlineCallbacks - def update_pusher_last_token_and_success(self, app_id, pushkey, + def update_pusher_last_token_and_success(self, app_id, pushkey, user_name, last_token, last_success): yield self._simple_update_one( PushersTable.table_name, - {'app_id': app_id, 'pushkey': pushkey}, + {'app_id': app_id, 'pushkey': pushkey, 'user_name': user_name}, {'last_token': last_token, 'last_success': last_success}, desc="update_pusher_last_token_and_success", ) @defer.inlineCallbacks - def update_pusher_failing_since(self, app_id, pushkey, failing_since): + def update_pusher_failing_since(self, app_id, pushkey, user_name, + failing_since): yield self._simple_update_one( PushersTable.table_name, - {'app_id': app_id, 'pushkey': pushkey}, + {'app_id': app_id, 'pushkey': pushkey, 'user_name': user_name}, {'failing_since': failing_since}, desc="update_pusher_failing_since", ) diff --git a/synapse/storage/schema/delta/15/v15.sql b/synapse/storage/schema/delta/15/v15.sql index fc3e43687..f5b2a08ca 100644 --- a/synapse/storage/schema/delta/15/v15.sql +++ b/synapse/storage/schema/delta/15/v15.sql @@ -1,2 +1,25 @@ -ALTER TABLE pushers ADD COLUMN access_token INTEGER DEFAULT NULL; - +-- Drop, copy & recreate pushers table to change unique key +-- Also add access_token column at the same time +CREATE TABLE IF NOT EXISTS pushers2 ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + access_token INTEGER DEFAULT NULL, + profile_tag varchar(32) NOT NULL, + kind varchar(8) NOT NULL, + app_id varchar(64) NOT NULL, + app_display_name varchar(64) NOT NULL, + device_display_name varchar(128) NOT NULL, + pushkey blob NOT NULL, + ts BIGINT NOT NULL, + lang varchar(8), + data blob, + last_token TEXT, + last_success BIGINT, + failing_since BIGINT, + FOREIGN KEY(user_name) REFERENCES users(name), + UNIQUE (app_id, pushkey, user_name) +); +INSERT INTO pushers2 (id, user_name, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, ts, lang, data, last_token, last_success, failing_since) + SELECT id, user_name, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, ts, lang, data, last_token, last_success, failing_since FROM pushers; +DROP TABLE pushers; +ALTER TABLE pushers2 RENAME TO pushers; From df4c12c76285fa8d423f1545453356a008690155 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 25 Mar 2015 19:08:17 +0000 Subject: [PATCH 029/173] pep8 blank lines --- synapse/push/pusherpool.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index cda072839..46444157c 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -98,7 +98,6 @@ class PusherPool: ) self.remove_pusher(p['app_id'], p['pushkey'], p['user_name']) - @defer.inlineCallbacks def _add_pusher_to_store(self, user_name, access_token, profile_tag, kind, app_id, app_display_name, device_display_name, From a32e876ef43df22cec37aad748c32c0cda30428a Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 26 Mar 2015 13:40:16 +0000 Subject: [PATCH 030/173] Delete pushers when changing password --- synapse/handlers/login.py | 3 +++ synapse/push/pusherpool.py | 20 +++++++++++++++-- synapse/storage/pusher.py | 45 +++++++++----------------------------- 3 files changed, 31 insertions(+), 37 deletions(-) diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 7aff2e69e..04f6dbb95 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -70,4 +70,7 @@ class LoginHandler(BaseHandler): yield self.store.user_set_password_hash(user_id, password_hash) yield self.store.user_delete_access_tokens_apart_from(user_id, token_id) + yield self.hs.get_pusherpool().remove_pushers_by_user_access_token( + user_id, token_id + ) yield self.store.flush_user(user_id) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 46444157c..0fdd7ea78 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -71,7 +71,7 @@ class PusherPool: "app_display_name": app_display_name, "device_display_name": device_display_name, "pushkey": pushkey, - "pushkey_ts": self.hs.get_clock().time_msec(), + "ts": self.hs.get_clock().time_msec(), "lang": lang, "data": data, "last_token": None, @@ -98,6 +98,22 @@ class PusherPool: ) self.remove_pusher(p['app_id'], p['pushkey'], p['user_name']) + @defer.inlineCallbacks + def remove_pushers_by_user_access_token(self, user_id, not_access_token_id): + all = yield self.store.get_all_pushers() + logger.info( + "Removing all pushers for user %s except access token %s", + user_id, not_access_token_id + ) + for p in all: + if (p['user_name'] == user_id and + p['access_token'] != not_access_token_id): + logger.info( + "Removing pusher for app id %s, pushkey %s, user %s", + p['app_id'], p['pushkey'], p['user_name'] + ) + self.remove_pusher(p['app_id'], p['pushkey'], p['user_name']) + @defer.inlineCallbacks def _add_pusher_to_store(self, user_name, access_token, profile_tag, kind, app_id, app_display_name, device_display_name, @@ -127,7 +143,7 @@ class PusherPool: app_display_name=pusherdict['app_display_name'], device_display_name=pusherdict['device_display_name'], pushkey=pusherdict['pushkey'], - pushkey_ts=pusherdict['pushkey_ts'], + pushkey_ts=pusherdict['ts'], data=pusherdict['data'], last_token=pusherdict['last_token'], last_success=pusherdict['last_success'], diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 423878c6a..1c657bedd 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -28,11 +28,9 @@ logger = logging.getLogger(__name__) class PusherStore(SQLBaseStore): @defer.inlineCallbacks def get_pushers_by_app_id_and_pushkey(self, app_id, pushkey): + cols = ",".join(PushersTable.fields) sql = ( - "SELECT id, user_name, kind, profile_tag, app_id," - "app_display_name, device_display_name, pushkey, ts, data, " - "last_token, last_success, failing_since " - "FROM pushers " + "SELECT "+cols+" FROM pushers " "WHERE app_id = ? AND pushkey = ?" ) @@ -43,51 +41,26 @@ class PusherStore(SQLBaseStore): ret = [ { - "id": r[0], - "user_name": r[1], - "kind": r[2], - "profile_tag": r[3], - "app_id": r[4], - "app_display_name": r[5], - "device_display_name": r[6], - "pushkey": r[7], - "pushkey_ts": r[8], - "data": r[9], - "last_token": r[10], - "last_success": r[11], - "failing_since": r[12] + k: r[i] for i, k in enumerate(PushersTable.fields) } for r in rows ] + print ret defer.returnValue(ret) @defer.inlineCallbacks def get_all_pushers(self): + cols = ",".join(PushersTable.fields) sql = ( - "SELECT id, user_name, kind, profile_tag, app_id," - "app_display_name, device_display_name, pushkey, ts, data, " - "last_token, last_success, failing_since " - "FROM pushers" + "SELECT "+cols+" FROM pushers" ) rows = yield self._execute("get_all_pushers", None, sql) ret = [ { - "id": r[0], - "user_name": r[1], - "kind": r[2], - "profile_tag": r[3], - "app_id": r[4], - "app_display_name": r[5], - "device_display_name": r[6], - "pushkey": r[7], - "pushkey_ts": r[8], - "data": r[9], - "last_token": r[10], - "last_success": r[11], - "failing_since": r[12] + k: r[i] for i, k in enumerate(PushersTable.fields) } for r in rows ] @@ -166,13 +139,15 @@ class PushersTable(Table): fields = [ "id", "user_name", + "access_token", "kind", "profile_tag", "app_id", "app_display_name", "device_display_name", "pushkey", - "pushkey_ts", + "ts", + "lang", "data", "last_token", "last_success", From 6f4f7e4e22522255aca72dc2fe28a636adef481a Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 26 Mar 2015 14:12:06 +0000 Subject: [PATCH 031/173] pep8 --- synapse/push/pusherpool.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 0fdd7ea78..041ce8f22 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -106,8 +106,10 @@ class PusherPool: user_id, not_access_token_id ) for p in all: - if (p['user_name'] == user_id and - p['access_token'] != not_access_token_id): + if ( + p['user_name'] == user_id and + p['access_token'] != not_access_token_id + ): logger.info( "Removing pusher for app id %s, pushkey %s, user %s", p['app_id'], p['pushkey'], p['user_name'] From 59bf16eddcb793705ee6bc243a2158824f7e05c8 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 30 Mar 2015 18:13:10 +0100 Subject: [PATCH 032/173] New registration for C/S API v2. Only ReCAPTCHA working currently. --- synapse/api/constants.py | 2 + synapse/config/captcha.py | 7 +- synapse/handlers/auth.py | 90 ++++++++++++++++++++---- synapse/handlers/register.py | 11 ++- synapse/http/client.py | 2 + synapse/rest/client/v2_alpha/__init__.py | 4 +- synapse/rest/client/v2_alpha/_base.py | 6 ++ synapse/rest/client/v2_alpha/register.py | 86 ++++++++++++++++++++++ 8 files changed, 192 insertions(+), 16 deletions(-) create mode 100644 synapse/rest/client/v2_alpha/register.py diff --git a/synapse/api/constants.py b/synapse/api/constants.py index b16bf4247..3e0ce170a 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -62,6 +62,8 @@ class LoginType(object): APPLICATION_SERVICE = u"m.login.application_service" SHARED_SECRET = u"org.matrix.login.shared_secret" + HIDDEN_TYPES = [APPLICATION_SERVICE, SHARED_SECRET] + class EventTypes(object): Member = "m.room.member" diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index 7e21c7414..07fbfadc0 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -20,6 +20,7 @@ class CaptchaConfig(Config): def __init__(self, args): super(CaptchaConfig, self).__init__(args) self.recaptcha_private_key = args.recaptcha_private_key + self.recaptcha_public_key = args.recaptcha_public_key self.enable_registration_captcha = args.enable_registration_captcha self.captcha_ip_origin_is_x_forwarded = ( args.captcha_ip_origin_is_x_forwarded @@ -30,9 +31,13 @@ class CaptchaConfig(Config): def add_arguments(cls, parser): super(CaptchaConfig, cls).add_arguments(parser) group = parser.add_argument_group("recaptcha") + group.add_argument( + "--recaptcha-public-key", type=str, default="YOUR_PUBLIC_KEY", + help="This Home Server's ReCAPTCHA public key." + ) group.add_argument( "--recaptcha-private-key", type=str, default="YOUR_PRIVATE_KEY", - help="The matching private key for the web client's public key." + help="This Home Server's ReCAPTCHA private key." ) group.add_argument( "--enable-registration-captcha", type=bool, default=False, diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index e4a73da9a..ec625f4ea 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -19,9 +19,12 @@ from ._base import BaseHandler from synapse.api.constants import LoginType from synapse.types import UserID from synapse.api.errors import LoginError, Codes +from synapse.http.client import SimpleHttpClient +from twisted.web.client import PartialDownloadError import logging import bcrypt +import simplejson logger = logging.getLogger(__name__) @@ -33,7 +36,7 @@ class AuthHandler(BaseHandler): super(AuthHandler, self).__init__(hs) @defer.inlineCallbacks - def check_auth(self, flows, clientdict): + def check_auth(self, flows, clientdict, clientip=None): """ Takes a dictionary sent by the client in the login / registration protocol and handles the login flow. @@ -50,11 +53,12 @@ class AuthHandler(BaseHandler): login request and should be passed back to the client. """ types = { - LoginType.PASSWORD: self.check_password_auth + LoginType.PASSWORD: self.check_password_auth, + LoginType.RECAPTCHA: self.check_recaptcha, } - if 'auth' not in clientdict: - defer.returnValue((False, auth_dict_for_flows(flows))) + if not clientdict or 'auth' not in clientdict: + defer.returnValue((False, self.auth_dict_for_flows(flows))) authdict = clientdict['auth'] @@ -67,7 +71,7 @@ class AuthHandler(BaseHandler): raise LoginError(400, "", Codes.MISSING_PARAM) if authdict['type'] not in types: raise LoginError(400, "", Codes.UNRECOGNIZED) - result = yield types[authdict['type']](authdict) + result = yield types[authdict['type']](authdict, clientip) if result: creds[authdict['type']] = result @@ -76,12 +80,12 @@ class AuthHandler(BaseHandler): logger.info("Auth completed with creds: %r", creds) defer.returnValue((True, creds)) - ret = auth_dict_for_flows(flows) + ret = self.auth_dict_for_flows(flows) ret['completed'] = creds.keys() defer.returnValue((False, ret)) @defer.inlineCallbacks - def check_password_auth(self, authdict): + def check_password_auth(self, authdict, _): if "user" not in authdict or "password" not in authdict: raise LoginError(400, "", Codes.MISSING_PARAM) @@ -93,17 +97,77 @@ class AuthHandler(BaseHandler): user_info = yield self.store.get_user_by_id(user_id=user) if not user_info: logger.warn("Attempted to login as %s but they do not exist", user) - raise LoginError(403, "", errcode=Codes.FORBIDDEN) + raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) stored_hash = user_info[0]["password_hash"] if bcrypt.checkpw(password, stored_hash): defer.returnValue(user) else: logger.warn("Failed password login for user %s", user) - raise LoginError(403, "", errcode=Codes.FORBIDDEN) + raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) + @defer.inlineCallbacks + def check_recaptcha(self, authdict, clientip): + try: + user_response = authdict["response"] + except KeyError: + # Client tried to provide captcha but didn't give the parameter: + # bad request. + raise LoginError( + 400, "Captcha response is required", + errcode=Codes.CAPTCHA_NEEDED + ) -def auth_dict_for_flows(flows): - return { - "flows": {"stages": f for f in flows} - } + logger.info( + "Submitting recaptcha response %s with remoteip %s", + user_response, clientip + ) + + # TODO: get this from the homeserver rather than creating a new one for + # each request + try: + client = SimpleHttpClient(self.hs) + data = yield client.post_urlencoded_get_json( + "https://www.google.com/recaptcha/api/siteverify", + args={ + 'secret': self.hs.config.recaptcha_private_key, + 'response': user_response, + 'remoteip': clientip, + } + ) + except PartialDownloadError as pde: + # Twisted is silly + data = pde.response + resp_body = simplejson.loads(data) + if 'success' in resp_body and resp_body['success']: + defer.returnValue(True) + raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) + + def get_params_recaptcha(self): + return {"public_key": self.hs.config.recaptcha_public_key} + + def auth_dict_for_flows(self, flows): + public_flows = [] + for f in flows: + hidden = False + for stagetype in f: + if stagetype in LoginType.HIDDEN_TYPES: + hidden = True + if not hidden: + public_flows.append(f) + + get_params = { + LoginType.RECAPTCHA: self.get_params_recaptcha, + } + + params = {} + + for f in public_flows: + for stage in f: + if stage in get_params and stage not in params: + params[stage] = get_params[stage]() + + return { + "flows": [{"stages": f} for f in public_flows], + "params": params + } \ No newline at end of file diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index c25e32109..542759a82 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -157,7 +157,11 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def check_recaptcha(self, ip, private_key, challenge, response): - """Checks a recaptcha is correct.""" + """ + Checks a recaptcha is correct. + + Used only by c/s api v1 + """ captcha_response = yield self._validate_captcha( ip, @@ -282,6 +286,8 @@ class RegistrationHandler(BaseHandler): def _validate_captcha(self, ip_addr, private_key, challenge, response): """Validates the captcha provided. + Used only by c/s api v1 + Returns: dict: Containing 'valid'(bool) and 'error_url'(str) if invalid. @@ -299,6 +305,9 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def _submit_captcha(self, ip_addr, private_key, challenge, response): + """ + Used only by c/s api v1 + """ # TODO: get this from the homeserver rather than creating a new one for # each request client = CaptchaServerHttpClient(self.hs) diff --git a/synapse/http/client.py b/synapse/http/client.py index 2ae1c4d3a..e8a5dedab 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -200,6 +200,8 @@ class CaptchaServerHttpClient(SimpleHttpClient): """ Separate HTTP client for talking to google's captcha servers Only slightly special because accepts partial download responses + + used only by c/s api v1 """ @defer.inlineCallbacks diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py index 041f538e2..98189ead2 100644 --- a/synapse/rest/client/v2_alpha/__init__.py +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -16,7 +16,8 @@ from . import ( sync, filter, - password + password, + register ) from synapse.http.server import JsonResource @@ -34,3 +35,4 @@ class ClientV2AlphaRestResource(JsonResource): sync.register_servlets(hs, client_resource) filter.register_servlets(hs, client_resource) password.register_servlets(hs, client_resource) + register.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py index c772cc986..db2c9b244 100644 --- a/synapse/rest/client/v2_alpha/_base.py +++ b/synapse/rest/client/v2_alpha/_base.py @@ -40,6 +40,12 @@ def client_v2_pattern(path_regex): return re.compile("^" + CLIENT_V2_ALPHA_PREFIX + path_regex) +def parse_request_allow_empty(request): + content = request.content.read() + if content == None or content == '': + return None + return simplejson.loads(content) + def parse_json_dict_from_request(request): try: content = simplejson.loads(request.content.read()) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py new file mode 100644 index 000000000..84da010c2 --- /dev/null +++ b/synapse/rest/client/v2_alpha/register.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.constants import LoginType +from synapse.api.errors import LoginError, SynapseError, Codes +from synapse.http.servlet import RestServlet + +from ._base import client_v2_pattern, parse_request_allow_empty + +import logging + + +logger = logging.getLogger(__name__) + + +class RegisterRestServlet(RestServlet): + PATTERN = client_v2_pattern("/register") + + def __init__(self, hs): + super(RegisterRestServlet, self).__init__() + self.hs = hs + self.auth = hs.get_auth() + self.auth_handler = hs.get_handlers().auth_handler + self.registration_handler = hs.get_handlers().registration_handler + + @defer.inlineCallbacks + def on_POST(self, request): + body = parse_request_allow_empty(request) + + authed, result = yield self.auth_handler.check_auth([ + [LoginType.RECAPTCHA], + [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA], + [LoginType.APPLICATION_SERVICE] + ], body) + + if not authed: + defer.returnValue((401, result)) + + is_application_server = LoginType.APPLICATION_SERVICE in result + is_using_shared_secret = LoginType.SHARED_SECRET in result + + can_register = ( + not self.hs.config.disable_registration + or is_application_server + or is_using_shared_secret + ) + if not can_register: + raise SynapseError(403, "Registration has been disabled") + + if 'username' not in body or 'password' not in body: + raise SynapseError(400, "", Codes.MISSING_PARAM) + desired_username = body['username'] + new_password = body['password'] + + (user_id, token) = yield self.registration_handler.register( + localpart=desired_username, + password=new_password + ) + result = { + "user_id": user_id, + "access_token": token, + "home_server": self.hs.hostname, + } + + defer.returnValue((200, result)) + + def on_OPTIONS(self, _): + return 200, {} + + +def register_servlets(hs, http_server): + RegisterRestServlet(hs).register(http_server) \ No newline at end of file From 46183cc69ff66af286c29c6f20a28086f93618a5 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 30 Mar 2015 18:18:19 +0100 Subject: [PATCH 033/173] Add original, unmodified CAPTCHA-SETUP from the webclient repo before modifying (captcha setup is now purely on the HS). --- CAPTCHA_SETUP | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 CAPTCHA_SETUP diff --git a/CAPTCHA_SETUP b/CAPTCHA_SETUP new file mode 100644 index 000000000..38089ce09 --- /dev/null +++ b/CAPTCHA_SETUP @@ -0,0 +1,46 @@ +Captcha can be enabled for this web client / home server. This file explains how to do that. +The captcha mechanism used is Google's ReCaptcha. This requires API keys from Google. + +Getting keys +------------ +Requires a public/private key pair from: + +https://developers.google.com/recaptcha/ + + +Setting Private ReCaptcha Key +----------------------------- +The private key is a config option on the home server config. If it is not +visible, you can generate it via --generate-config. Set the following value: + + recaptcha_private_key: YOUR_PRIVATE_KEY + +In addition, you MUST enable captchas via: + + enable_registration_captcha: true + +Setting Public ReCaptcha Key +---------------------------- +The web client will look for the global variable webClientConfig for config +options. You should put your ReCaptcha public key there like so: + +webClientConfig = { + useCaptcha: true, + recaptcha_public_key: "YOUR_PUBLIC_KEY" +}; + +This should be put in webclient/config.js which is already .gitignored, rather +than in the web client source files. You MUST set useCaptcha to true else a +ReCaptcha widget will not be generated. + +Configuring IP used for auth +---------------------------- +The ReCaptcha API requires that the IP address of the user who solved the +captcha is sent. If the client is connecting through a proxy or load balancer, +it may be required to use the X-Forwarded-For (XFF) header instead of the origin +IP address. This can be configured as an option on the home server like so: + + captcha_ip_origin_is_x_forwarded: true + + + From 4891c4ff72af2733627c4bfb29cc0fa5362ab617 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 30 Mar 2015 18:27:42 +0100 Subject: [PATCH 034/173] Update CAPTCHA_SETUP (it continues to ignore fallback, but I guess I should fix it so that doesn't need the key in two different places) --- CAPTCHA_SETUP | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/CAPTCHA_SETUP b/CAPTCHA_SETUP index 38089ce09..75ff80981 100644 --- a/CAPTCHA_SETUP +++ b/CAPTCHA_SETUP @@ -1,4 +1,4 @@ -Captcha can be enabled for this web client / home server. This file explains how to do that. +Captcha can be enabled for this home server. This file explains how to do that. The captcha mechanism used is Google's ReCaptcha. This requires API keys from Google. Getting keys @@ -8,31 +8,18 @@ Requires a public/private key pair from: https://developers.google.com/recaptcha/ -Setting Private ReCaptcha Key ------------------------------ -The private key is a config option on the home server config. If it is not -visible, you can generate it via --generate-config. Set the following value: +Setting ReCaptcha Keys +---------------------- +The keys are a config option on the home server config. If they are not +visible, you can generate them via --generate-config. Set the following value: + recaptcha_public_key: YOUR_PUBLIC_KEY recaptcha_private_key: YOUR_PRIVATE_KEY In addition, you MUST enable captchas via: enable_registration_captcha: true -Setting Public ReCaptcha Key ----------------------------- -The web client will look for the global variable webClientConfig for config -options. You should put your ReCaptcha public key there like so: - -webClientConfig = { - useCaptcha: true, - recaptcha_public_key: "YOUR_PUBLIC_KEY" -}; - -This should be put in webclient/config.js which is already .gitignored, rather -than in the web client source files. You MUST set useCaptcha to true else a -ReCaptcha widget will not be generated. - Configuring IP used for auth ---------------------------- The ReCaptcha API requires that the IP address of the user who solved the @@ -42,5 +29,3 @@ IP address. This can be configured as an option on the home server like so: captcha_ip_origin_is_x_forwarded: true - - From 9f642a93ecab62fb56776ad4b7d7b062b869b66a Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 31 Mar 2015 09:50:44 +0100 Subject: [PATCH 035/173] pep8 --- synapse/handlers/auth.py | 2 +- synapse/rest/client/v2_alpha/_base.py | 3 ++- synapse/rest/client/v2_alpha/register.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index ec625f4ea..26df9fcd8 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -170,4 +170,4 @@ class AuthHandler(BaseHandler): return { "flows": [{"stages": f} for f in public_flows], "params": params - } \ No newline at end of file + } diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py index db2c9b244..8adcc9dd9 100644 --- a/synapse/rest/client/v2_alpha/_base.py +++ b/synapse/rest/client/v2_alpha/_base.py @@ -42,10 +42,11 @@ def client_v2_pattern(path_regex): def parse_request_allow_empty(request): content = request.content.read() - if content == None or content == '': + if content is None or content == '': return None return simplejson.loads(content) + def parse_json_dict_from_request(request): try: content = simplejson.loads(request.content.read()) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 84da010c2..4a53e0374 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -16,7 +16,7 @@ from twisted.internet import defer from synapse.api.constants import LoginType -from synapse.api.errors import LoginError, SynapseError, Codes +from synapse.api.errors import SynapseError, Codes from synapse.http.servlet import RestServlet from ._base import client_v2_pattern, parse_request_allow_empty @@ -83,4 +83,4 @@ class RegisterRestServlet(RestServlet): def register_servlets(hs, http_server): - RegisterRestServlet(hs).register(http_server) \ No newline at end of file + RegisterRestServlet(hs).register(http_server) From d18e7779cae4610d9e1425e4f01681359a20d374 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 31 Mar 2015 14:40:02 +0100 Subject: [PATCH 036/173] Grammar and deduplication --- synapse/http/server.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index dee49b9e1..b5c1a3cee 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -57,10 +57,10 @@ class HttpServer(object): """ def register_path(self, method, path_pattern, callback): - """ Register a callback that get's fired if we receive a http request + """ Register a callback that gets fired if we receive a http request with the given method for a path that matches the given regex. - If the regex contains groups these get's passed to the calback via + If the regex contains groups these gets passed to the calback via an unpacked tuple. Args: @@ -111,9 +111,8 @@ class JsonResource(HttpServer, resource.Resource): interface=self.hs.config.bind_host ) - # Gets called by twisted def render(self, request): - """ This get's called by twisted every time someone sends us a request. + """ This gets called by twisted every time someone sends us a request. """ self._async_render_with_logging_context(request) return server.NOT_DONE_YET @@ -130,7 +129,7 @@ class JsonResource(HttpServer, resource.Resource): @defer.inlineCallbacks def _async_render(self, request): - """ This get's called by twisted every time someone sends us a request. + """ This gets called by twisted every time someone sends us a request. This checks if anyone has registered a callback for that method and path. """ From f129ee1e18e25c3392af2e9633988ae5b87bc6d5 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 31 Mar 2015 18:25:10 +0100 Subject: [PATCH 037/173] Make docs a bit more true --- synapse/http/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index b5c1a3cee..30c3aa5ca 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -129,7 +129,7 @@ class JsonResource(HttpServer, resource.Resource): @defer.inlineCallbacks def _async_render(self, request): - """ This gets called by twisted every time someone sends us a request. + """ This gets called from render() every time someone sends us a request. This checks if anyone has registered a callback for that method and path. """ From 9236136f3a4f0d8119d4a6333f37378f8e259e4a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 1 Apr 2015 14:12:33 +0100 Subject: [PATCH 038/173] Make work in both Maria and SQLite. Fix tests --- synapse/app/homeserver.py | 20 ++++++--- synapse/storage/__init__.py | 43 +++++++++---------- synapse/storage/_base.py | 30 +++++++++---- synapse/storage/engines/__init__.py | 35 +++++++++++++++ synapse/storage/engines/maria.py | 30 +++++++++++++ synapse/storage/engines/sqlite3.py | 25 +++++++++++ synapse/storage/keys.py | 4 +- synapse/storage/registration.py | 8 +++- synapse/storage/schema/delta/12/v12.sql | 8 ++-- synapse/storage/schema/delta/13/v13.sql | 4 +- synapse/storage/schema/delta/14/v14.sql | 2 +- .../schema/full_schemas/11/event_edges.sql | 14 +++--- .../full_schemas/11/event_signatures.sql | 8 ++-- synapse/storage/schema/full_schemas/11/im.sql | 18 ++++---- .../storage/schema/full_schemas/11/keys.sql | 4 +- .../full_schemas/11/media_repository.sql | 8 ++-- .../schema/full_schemas/11/presence.sql | 6 +-- .../schema/full_schemas/11/profiles.sql | 4 +- .../schema/full_schemas/11/redactions.sql | 2 +- .../schema/full_schemas/11/room_aliases.sql | 8 ++-- .../storage/schema/full_schemas/11/state.sql | 6 +-- .../schema/full_schemas/11/transactions.sql | 8 ++-- .../storage/schema/full_schemas/11/users.sql | 10 ++--- synapse/storage/signatures.py | 8 ++-- synapse/storage/stream.py | 6 --- synapse/util/retryutils.py | 2 +- tests/federation/test_federation.py | 10 +++-- tests/handlers/test_federation.py | 9 ++++ tests/handlers/test_presence.py | 7 ++- tests/handlers/test_typing.py | 7 ++- tests/rest/client/v1/test_events.py | 9 ---- tests/storage/test_appservice.py | 12 ++---- tests/storage/test_base.py | 20 +++++---- tests/storage/test_registration.py | 36 ++++++++++------ tests/storage/test_roommember.py | 14 +++--- tests/utils.py | 11 ++++- 36 files changed, 296 insertions(+), 160 deletions(-) create mode 100644 synapse/storage/engines/__init__.py create mode 100644 synapse/storage/engines/maria.py create mode 100644 synapse/storage/engines/sqlite3.py diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index beab6ffc7..b185b2f56 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -20,6 +20,7 @@ sys.dont_write_bytecode = True from synapse.storage import ( prepare_database, prepare_sqlite3_database, UpgradeDatabaseException, ) +from synapse.storage.engines import create_engine from synapse.server import HomeServer @@ -376,7 +377,7 @@ def setup(config_options): if name in ["MySQLdb", "mysql.connector"]: db_config.setdefault("args", {}).update({ "sql_mode": "TRADITIONAL", - "charset": "utf8", + "charset": "utf8mb4", "use_unicode": True, }) elif name == "sqlite3": @@ -388,6 +389,8 @@ def setup(config_options): else: raise RuntimeError("Unsupported database type '%s'" % (name,)) + database_engine = create_engine(name) + hs = SynapseHomeServer( config.server_name, domain_with_port=domain_with_port, @@ -398,6 +401,7 @@ def setup(config_options): config=config, content_addr=config.content_addr, version_string=version_string, + database_engine=database_engine, ) hs.create_resource_tree( @@ -409,12 +413,14 @@ def setup(config_options): logger.info("Preparing database: %s...", db_name) try: - # with sqlite3.connect(db_name) as db_conn: - # prepare_sqlite3_database(db_conn) - # prepare_database(db_conn) - import mysql.connector - db_conn = mysql.connector.connect(**db_config.get("args", {})) - prepare_database(db_conn) + db_conn = database_engine.module.connect(**db_config.get("args", {})) + + if name == "sqlite3": + prepare_sqlite3_database(db_conn) + + prepare_database(db_conn, database_engine) + + db_conn.commit() except UpgradeDatabaseException: sys.stderr.write( "\nFailed to upgrade database.\n" diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index abde7d0df..f8053484c 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -77,9 +77,6 @@ class DataStore(RoomMemberStore, RoomStore, self.min_token_deferred = self._get_min_token() self.min_token = None - self._next_stream_id_lock = threading.Lock() - self._next_stream_id = int(hs.get_clock().time_msec()) * 1000 - def insert_client_ip(self, user, access_token, device_id, ip, user_agent): return self._simple_upsert( "user_ips", @@ -127,19 +124,21 @@ class UpgradeDatabaseException(PrepareDatabaseException): pass -def prepare_database(db_conn): +def prepare_database(db_conn, database_engine): """Prepares a database for usage. Will either create all necessary tables or upgrade from an older schema version. """ try: cur = db_conn.cursor() - version_info = _get_or_create_schema_state(cur) + version_info = _get_or_create_schema_state(cur, database_engine) if version_info: user_version, delta_files, upgraded = version_info - _upgrade_existing_database(cur, user_version, delta_files, upgraded) + _upgrade_existing_database( + cur, user_version, delta_files, upgraded, database_engine + ) else: - _setup_new_database(cur) + _setup_new_database(cur, database_engine) # cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,)) @@ -150,7 +149,7 @@ def prepare_database(db_conn): raise -def _setup_new_database(cur): +def _setup_new_database(cur, database_engine): """Sets up the database by finding a base set of "full schemas" and then applying any necessary deltas. @@ -210,7 +209,7 @@ def _setup_new_database(cur): executescript(cur, sql_loc) cur.execute( - _convert_param_style( + database_engine.convert_param_style( "REPLACE INTO schema_version (version, upgraded)" " VALUES (?,?)" ), @@ -221,12 +220,13 @@ def _setup_new_database(cur): cur, current_version=max_current_ver, applied_delta_files=[], - upgraded=False + upgraded=False, + database_engine=database_engine, ) def _upgrade_existing_database(cur, current_version, applied_delta_files, - upgraded): + upgraded, database_engine): """Upgrades an existing database. Delta files can either be SQL stored in *.sql files, or python modules @@ -335,26 +335,22 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, # Mark as done. cur.execute( - _convert_param_style( + database_engine.convert_param_style( "INSERT INTO applied_schema_deltas (version, file)" - " VALUES (?,?)" + " VALUES (?,?)", ), (v, relative_path) ) cur.execute( - _convert_param_style( + database_engine.convert_param_style( "REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)" + " VALUES (?,?)", ), (v, True) ) -def _convert_param_style(sql): - return sql.replace("?", "%s") - - def get_statements(f): statement_buffer = "" in_comment = False # If we're in a /* ... */ style comment @@ -409,7 +405,7 @@ def executescript(txn, schema_path): txn.execute(statement) -def _get_or_create_schema_state(txn): +def _get_or_create_schema_state(txn, database_engine): try: # Bluntly try creating the schema_version tables. schema_path = os.path.join( @@ -426,7 +422,7 @@ def _get_or_create_schema_state(txn): if current_version: txn.execute( - _convert_param_style( + database_engine.convert_param_style( "SELECT file FROM applied_schema_deltas WHERE version >= ?" ), (current_version,) @@ -446,6 +442,8 @@ def prepare_sqlite3_database(db_conn): new. This only affects sqlite databases since they were the only ones supported at the time. """ + import sqlite3 + with db_conn: schema_path = os.path.join( dir_path, "schema", "schema_version.sql", @@ -466,7 +464,8 @@ def prepare_sqlite3_database(db_conn): db_conn.execute( _convert_param_style( "REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)" + " VALUES (?,?)", + sqlite3 ), (row[0], False) ) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 76ec3ee93..047d100f4 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -29,6 +29,7 @@ import functools import simplejson as json import sys import time +import threading logger = logging.getLogger(__name__) @@ -118,19 +119,16 @@ def cached(max_entries=1000, num_args=1): return wrap -def _convert_param_style(sql): - return sql.replace("?", "%s") - - class LoggingTransaction(object): """An object that almost-transparently proxies for the 'txn' object passed to the constructor. Adds logging and metrics to the .execute() method.""" - __slots__ = ["txn", "name"] + __slots__ = ["txn", "name", "database_engine"] - def __init__(self, txn, name): + def __init__(self, txn, name, database_engine): object.__setattr__(self, "txn", txn) object.__setattr__(self, "name", name) + object.__setattr__(self, "database_engine", database_engine) def __getattr__(self, name): return getattr(self.txn, name) @@ -142,7 +140,7 @@ class LoggingTransaction(object): # TODO(paul): Maybe use 'info' and 'debug' for values? sql_logger.debug("[SQL] {%s} %s", self.name, sql) - sql = _convert_param_style(sql) + sql = self.database_engine.convert_param_style(sql) try: if args and args[0]: @@ -227,9 +225,14 @@ class SQLBaseStore(object): self._get_event_cache = LruCache(hs.config.event_cache_size) + self.database_engine = hs.database_engine + # Pretend the getEventCache is just another named cache caches_by_name["*getEvent*"] = self._get_event_cache + self._next_stream_id_lock = threading.Lock() + self._next_stream_id = int(hs.get_clock().time_msec()) * 1000 + def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -281,7 +284,10 @@ class SQLBaseStore(object): sql_scheduling_timer.inc_by(time.time() * 1000 - start_time) transaction_logger.debug("[TXN START] {%s}", name) try: - return func(LoggingTransaction(txn, name), *args, **kwargs) + return func( + LoggingTransaction(txn, name, self.database_engine), + *args, **kwargs + ) except: logger.exception("[TXN FAIL] {%s}", name) raise @@ -588,7 +594,7 @@ class SQLBaseStore(object): select_sql = "SELECT %s FROM %s WHERE %s" % ( ", ".join(retcols), table, - " AND ".join("%s = ?" % (k) for k in keyvalues) + " AND ".join("%s = ?" % (k,) for k in keyvalues) ) txn.execute(select_sql, keyvalues.values()) @@ -836,6 +842,12 @@ class SQLBaseStore(object): result = txn.fetchone() return result[0] if result else None + def get_next_stream_id(self): + with self._next_stream_id_lock: + i = self._next_stream_id + self._next_stream_id += 1 + return i + class _RollbackButIsFineException(Exception): """ This exception is used to rollback a transaction without implying diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py new file mode 100644 index 000000000..709b6f88a --- /dev/null +++ b/synapse/storage/engines/__init__.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .maria import MariaEngine +from .sqlite3 import Sqlite3Engine + + +SUPPORTED_MODULE = { + "sqlite3": Sqlite3Engine, + "mysql.connector": MariaEngine, +} + + +def create_engine(name): + engine_class = SUPPORTED_MODULE.get(name, None) + + if engine_class: + module = __import__(name) + return engine_class(module) + + raise RuntimeError( + "Unsupported database engine '%s'" % (name,) + ) diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py new file mode 100644 index 000000000..df4776364 --- /dev/null +++ b/synapse/storage/engines/maria.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import types + + +class MariaEngine(object): + def __init__(self, database_module): + self.module = database_module + + def convert_param_style(self, sql): + return sql.replace("?", "%s") + + def encode_parameter(self, param): + if isinstance(param, types.BufferType): + return str(param) + return param diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py new file mode 100644 index 000000000..639cdea41 --- /dev/null +++ b/synapse/storage/engines/sqlite3.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class Sqlite3Engine(object): + def __init__(self, database_module): + self.module = database_module + + def convert_param_style(self, sql): + return sql + + def encode_parameter(self, param): + return param diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index e6975a945..25fef7943 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -64,7 +64,7 @@ class KeyStore(SQLBaseStore): "fingerprint": fingerprint, "from_server": from_server, "ts_added_ms": time_now_ms, - "tls_certificate": tls_certificate_bytes, + "tls_certificate": buffer(tls_certificate_bytes), }, ) @@ -113,6 +113,6 @@ class KeyStore(SQLBaseStore): "key_id": "%s:%s" % (verify_key.alg, verify_key.version), "from_server": from_server, "ts_added_ms": time_now_ms, - "verify_key": verify_key.encode(), + "verify_key": buffer(verify_key.encode()), }, ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 7258f7b2a..0c785ec98 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -42,6 +42,7 @@ class RegistrationStore(SQLBaseStore): yield self._simple_insert( "access_tokens", { + "id": self.get_next_stream_id(), "user_id": user_id, "token": token }, @@ -78,8 +79,11 @@ class RegistrationStore(SQLBaseStore): # it's possible for this to get a conflict, but only for a single user # since tokens are namespaced based on their user ID - txn.execute("INSERT INTO access_tokens(user_id, token) " + - "VALUES (?,?)", [user_id, token]) + txn.execute( + "INSERT INTO access_tokens(id, user_id, token)" + " VALUES (?,?,?)", + (self.get_next_stream_id(), user_id, token,) + ) @defer.inlineCallbacks def get_user_by_id(self, user_id): diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index b526109e6..90ac47485 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS rejections( reason VARCHAR(255) NOT NULL, last_check VARCHAR(255) NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( @@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS pushers ( last_success BIGINT, failing_since BIGINT, UNIQUE (app_id, pushkey) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS push_rules ( id BIGINT PRIMARY KEY, @@ -48,7 +48,7 @@ CREATE TABLE IF NOT EXISTS push_rules ( conditions VARCHAR(255) NOT NULL, actions VARCHAR(255) NOT NULL, UNIQUE(user_name, rule_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); @@ -56,7 +56,7 @@ CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(255), filter_id BIGINT, filter_json BLOB -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( user_id, filter_id diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index f0a5daf44..4953b6323 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS application_services( hs_token VARCHAR(255), sender VARCHAR(255), UNIQUE(token) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS application_services_regex( id BIGINT PRIMARY KEY, @@ -28,4 +28,4 @@ CREATE TABLE IF NOT EXISTS application_services_regex( namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(255), FOREIGN KEY(as_id) REFERENCES application_services(id) -) ENGINE = INNODB; +) ; diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index a1260c5c1..3bda073c9 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -4,6 +4,6 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( rule_id VARCHAR(255) NOT NULL, enabled TINYINT, UNIQUE(user_name, rule_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 0f53488e9..336cd563d 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (event_id, room_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); @@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (event_id, room_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_edges( room_id VARCHAR(255) NOT NULL, is_state BOOL NOT NULL, UNIQUE (event_id, prev_event_id, room_id, is_state) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); @@ -49,7 +49,7 @@ CREATE TABLE IF NOT EXISTS room_depth( room_id VARCHAR(255) NOT NULL, min_depth INTEGER NOT NULL, UNIQUE (room_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); @@ -59,7 +59,7 @@ create TABLE IF NOT EXISTS event_destinations( destination VARCHAR(255) NOT NULL, delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); @@ -70,7 +70,7 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities( type VARCHAR(255) NOT NULL, state_key VARCHAR(255) NOT NULL, UNIQUE (event_id, room_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( room_id, type, state_key @@ -83,7 +83,7 @@ CREATE TABLE IF NOT EXISTS event_auth( auth_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (event_id, auth_id, room_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index 334d7c868..11e611598 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( algorithm VARCHAR(255), hash BLOB, UNIQUE (event_id, algorithm) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); @@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS event_reference_hashes ( algorithm VARCHAR(255), hash BLOB, UNIQUE (event_id, algorithm) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_signatures ( key_id VARCHAR(255), signature BLOB, UNIQUE (event_id, signature_name, key_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); @@ -50,6 +50,6 @@ CREATE TABLE IF NOT EXISTS event_edge_hashes( algorithm VARCHAR(255), hash BLOB, UNIQUE (event_id, prev_event_id, algorithm) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index 9849e969b..a0fb33762 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS events( outlier BOOL NOT NULL, depth BIGINT DEFAULT 0 NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); @@ -38,7 +38,7 @@ CREATE TABLE IF NOT EXISTS event_json( internal_metadata BLOB NOT NULL, json BLOB NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); @@ -50,7 +50,7 @@ CREATE TABLE IF NOT EXISTS state_events( state_key VARCHAR(255) NOT NULL, prev_state VARCHAR(255), UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); @@ -64,7 +64,7 @@ CREATE TABLE IF NOT EXISTS current_state_events( state_key VARCHAR(255) NOT NULL, UNIQUE (event_id), UNIQUE (room_id, type, state_key) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); @@ -77,7 +77,7 @@ CREATE TABLE IF NOT EXISTS room_memberships( room_id VARCHAR(255) NOT NULL, membership VARCHAR(255) NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); @@ -89,14 +89,14 @@ CREATE TABLE IF NOT EXISTS feedback( sender VARCHAR(255), room_id VARCHAR(255), UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS topics( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, topic VARCHAR(255) NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); @@ -113,12 +113,12 @@ CREATE TABLE IF NOT EXISTS rooms( room_id VARCHAR(255) PRIMARY KEY NOT NULL, is_public BOOL, creator VARCHAR(255) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS room_hosts( room_id VARCHAR(255) NOT NULL, host VARCHAR(255) NOT NULL, UNIQUE (room_id, host) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index c0f2ec29b..a785cdb4c 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -19,7 +19,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( ts_added_ms BIGINT, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(255), -- Server name. @@ -28,4 +28,4 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( ts_added_ms BIGINT, -- When the key was added. verify_key BLOB, -- NACL verification key. UNIQUE (server_name, key_id) -) ENGINE = INNODB; +) ; diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index d9559f590..27fe297af 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( upload_name VARCHAR(255), -- The name the media was uploaded with. user_id VARCHAR(255), -- The user who uploaded the file. UNIQUE (media_id) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( media_id VARCHAR(255), -- The id used to refer to the media. @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type ) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache ( media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(255), -- The name used to store the media on disk. UNIQUE (media_origin, media_id) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( media_origin VARCHAR(255), -- The remote HS the media came from. @@ -62,7 +62,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type ) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id ON remote_media_cache_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index 803132108..b48b110ae 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS presence( status_msg VARCHAR(255), mtime BIGINT, -- miliseconds since last state change UNIQUE(user_id) -) ENGINE = INNODB; +) ; -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS presence_allow_inbound( observed_user_id VARCHAR(255) NOT NULL, observer_user_id VARCHAR(255), -- a UserID, UNIQUE(observed_user_id) -) ENGINE = INNODB; +) ; -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? @@ -35,4 +35,4 @@ CREATE TABLE IF NOT EXISTS presence_list( observed_user_id VARCHAR(255), -- a UserID, accepted BOOLEAN, UNIQUE(user_id) -) ENGINE = INNODB; +) ; diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index 552645c56..92da48f97 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS profiles( user_id VARCHAR(255) NOT NULL, - displayname VARBINARY(255), + displayname VARCHAR(255), avatar_url VARCHAR(255), UNIQUE(user_id) -) ENGINE = INNODB; +) ; diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index ba93e860f..9b52a2012 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS redactions ( event_id VARCHAR(255) NOT NULL, redacts VARCHAR(255) NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index 1e706aac2..220df8757 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -14,12 +14,12 @@ */ CREATE TABLE IF NOT EXISTS room_aliases( - room_alias VARCHAR(255) NOT NULL, + room_alias VARBINARY(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (room_alias) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS room_alias_servers( - room_alias VARCHAR(255) NOT NULL, + room_alias VARBINARY(255) NOT NULL, server VARCHAR(255) NOT NULL -) ENGINE = INNODB; +) ; diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index be9dc2920..40584a325 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS state_groups( id VARCHAR(20) PRIMARY KEY, room_id VARCHAR(255) NOT NULL, event_id VARCHAR(255) NOT NULL -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS state_groups_state( state_group VARCHAR(20) NOT NULL, @@ -25,13 +25,13 @@ CREATE TABLE IF NOT EXISTS state_groups_state( type VARCHAR(255) NOT NULL, state_key VARCHAR(255) NOT NULL, event_id VARCHAR(255) NOT NULL -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS event_to_state_groups( event_id VARCHAR(255) NOT NULL, state_group VARCHAR(255) NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index bd13bba8c..d33bdfb30 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( response_json BLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( response_code INTEGER DEFAULT 0, response_json BLOB, ts BIGINT -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); @@ -51,7 +51,7 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( pdu_id VARCHAR(255), pdu_origin VARCHAR(255), UNIQUE (transaction_id, destination) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); @@ -60,4 +60,4 @@ CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(255) PRIMARY KEY, retry_last_ts BIGINT, retry_interval INTEGER -) ENGINE = INNODB; +) ; diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 55bffb22f..28909f580 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -14,20 +14,20 @@ */ CREATE TABLE IF NOT EXISTS users( name VARCHAR(255), - password_hash VARBINARY(255), + password_hash VARCHAR(255), creation_ts BIGINT, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS access_tokens( - id INTEGER PRIMARY KEY AUTO_INCREMENT, + id BIGINT PRIMARY KEY, user_id VARCHAR(255) NOT NULL, device_id VARCHAR(255), token VARCHAR(255) NOT NULL, last_used BIGINT, UNIQUE(token) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS user_ips ( user VARCHAR(255) NOT NULL, @@ -37,6 +37,6 @@ CREATE TABLE IF NOT EXISTS user_ips ( user_agent VARCHAR(255) NOT NULL, last_seen BIGINT NOT NULL, UNIQUE (user, access_token, ip, user_agent) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 35bba854f..f05182863 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -54,7 +54,7 @@ class SignatureStore(SQLBaseStore): { "event_id": event_id, "algorithm": algorithm, - "hash": hash_bytes, + "hash": buffer(hash_bytes), }, ) @@ -116,7 +116,7 @@ class SignatureStore(SQLBaseStore): { "event_id": event_id, "algorithm": algorithm, - "hash": hash_bytes, + "hash": buffer(hash_bytes), }, ) @@ -160,7 +160,7 @@ class SignatureStore(SQLBaseStore): "event_id": event_id, "signature_name": signature_name, "key_id": key_id, - "signature": signature_bytes, + "signature": buffer(signature_bytes), }, ) @@ -193,6 +193,6 @@ class SignatureStore(SQLBaseStore): "event_id": event_id, "prev_event_id": prev_event_id, "algorithm": algorithm, - "hash": hash_bytes, + "hash": buffer(hash_bytes), }, ) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 3a310cd00..e6bb5a807 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -433,12 +433,6 @@ class StreamStore(SQLBaseStore): defer.returnValue(self.min_token) - def get_next_stream_id(self): - with self._next_stream_id_lock: - i = self._next_stream_id - self._next_stream_id += 1 - return i - def _get_room_events_max_id_txn(self, txn): txn.execute( "SELECT MAX(stream_ordering) as m FROM events" diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index 4e8223279..a42138f55 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -60,7 +60,7 @@ def get_retry_limiter(destination, clock, store, **kwargs): if retry_timings: retry_last_ts, retry_interval = ( - retry_timings.retry_last_ts, retry_timings.retry_interval + retry_timings["retry_last_ts"], retry_timings["retry_interval"] ) now = int(clock.time_msec()) diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 2ecd00d2a..a4ef60b91 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -24,8 +24,6 @@ from ..utils import MockHttpResource, MockClock, setup_test_homeserver from synapse.federation import initialize_http_replication from synapse.events import FrozenEvent -from synapse.storage.transactions import DestinationsTable - def make_pdu(prev_pdus=[], **kwargs): """Provide some default fields for making a PduTuple.""" @@ -57,8 +55,14 @@ class FederationTestCase(unittest.TestCase): self.mock_persistence.get_received_txn_response.return_value = ( defer.succeed(None) ) + + retry_timings_res = { + "destination": "", + "retry_last_ts": 0, + "retry_interval": 0, + } self.mock_persistence.get_destination_retry_timings.return_value = ( - defer.succeed(DestinationsTable.EntryType("", 0, 0)) + defer.succeed(retry_timings_res) ) self.mock_persistence.get_auth_chain.return_value = [] self.clock = MockClock() diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index c13ade328..08d2404b6 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -87,6 +87,15 @@ class FederationTestCase(unittest.TestCase): self.datastore.get_room.return_value = defer.succeed(True) self.auth.check_host_in_room.return_value = defer.succeed(True) + retry_timings_res = { + "destination": "", + "retry_last_ts": 0, + "retry_interval": 0, + } + self.datastore.get_destination_retry_timings.return_value = ( + defer.succeed(retry_timings_res) + ) + def have_events(event_ids): return defer.succeed({}) self.datastore.have_events.side_effect = have_events diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 04eba4289..9b0e60691 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -194,8 +194,13 @@ class MockedDatastorePresenceTestCase(PresenceTestCase): return datastore def setUp_datastore_federation_mocks(self, datastore): + retry_timings_res = { + "destination": "", + "retry_last_ts": 0, + "retry_interval": 0, + } datastore.get_destination_retry_timings.return_value = ( - defer.succeed(DestinationsTable.EntryType("", 0, 0)) + defer.succeed(retry_timings_res) ) def get_received_txn_response(*args): diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index bf34b7ccb..2d76b2356 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -96,8 +96,13 @@ class TypingNotificationsTestCase(unittest.TestCase): self.event_source = hs.get_event_sources().sources["typing"] self.datastore = hs.get_datastore() + retry_timings_res = { + "destination": "", + "retry_last_ts": 0, + "retry_interval": 0, + } self.datastore.get_destination_retry_timings.return_value = ( - defer.succeed(DestinationsTable.EntryType("", 0, 0)) + defer.succeed(retry_timings_res) ) def get_received_txn_response(*args): diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py index 36b0f2ff6..445272e32 100644 --- a/tests/rest/client/v1/test_events.py +++ b/tests/rest/client/v1/test_events.py @@ -115,12 +115,6 @@ class EventStreamPermissionsTestCase(RestTestCase): hs = yield setup_test_homeserver( http_client=None, replication_layer=Mock(), - clock=Mock(spec=[ - "call_later", - "cancel_call_later", - "time_msec", - "time" - ]), ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), @@ -132,9 +126,6 @@ class EventStreamPermissionsTestCase(RestTestCase): hs.get_handlers().federation_handler = Mock() - hs.get_clock().time_msec.return_value = 1000000 - hs.get_clock().time.return_value = 1000 - synapse.rest.client.v1.register.register_servlets(hs, self.mock_resource) synapse.rest.client.v1.events.register_servlets(hs, self.mock_resource) synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index ca5b92ec8..2ad55c846 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -16,22 +16,18 @@ from tests import unittest from twisted.internet import defer from synapse.appservice import ApplicationService -from synapse.server import HomeServer from synapse.storage.appservice import ApplicationServiceStore -from mock import Mock -from tests.utils import SQLiteMemoryDbPool, MockClock +from tests.utils import setup_test_homeserver class ApplicationServiceStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def setUp(self): - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - hs = HomeServer( - "test", db_pool=db_pool, clock=MockClock(), config=Mock() - ) + hs = yield setup_test_homeserver() + db_pool = hs.get_db_pool() + self.as_token = "token1" db_pool.runQuery( "INSERT INTO application_services(token) VALUES(?)", diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 7f5845cf0..5c17d3014 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -24,6 +24,7 @@ from collections import OrderedDict from synapse.server import HomeServer from synapse.storage._base import SQLBaseStore +from synapse.storage.engines import create_engine class SQLBaseStoreTestCase(unittest.TestCase): @@ -40,7 +41,12 @@ class SQLBaseStoreTestCase(unittest.TestCase): config = Mock() config.event_cache_size = 1 - hs = HomeServer("test", db_pool=self.db_pool, config=config) + hs = HomeServer( + "test", + db_pool=self.db_pool, + config=config, + database_engine=create_engine("sqlite3"), + ) self.datastore = SQLBaseStore(hs) @@ -86,8 +92,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): self.assertEquals("Value", value) self.mock_txn.execute.assert_called_with( - "SELECT retcol FROM tablename WHERE keycol = ? " - "ORDER BY rowid asc", + "SELECT retcol FROM tablename WHERE keycol = ?", ["TheKey"] ) @@ -104,8 +109,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): self.assertEquals({"colA": 1, "colB": 2, "colC": 3}, ret) self.mock_txn.execute.assert_called_with( - "SELECT colA, colB, colC FROM tablename WHERE keycol = ? " - "ORDER BY rowid asc", + "SELECT colA, colB, colC FROM tablename WHERE keycol = ?", ["TheKey"] ) @@ -139,8 +143,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): self.assertEquals([{"colA": 1}, {"colA": 2}, {"colA": 3}], ret) self.mock_txn.execute.assert_called_with( - "SELECT colA FROM tablename WHERE keycol = ? " - "ORDER BY rowid asc", + "SELECT colA FROM tablename WHERE keycol = ?", ["A set"] ) @@ -189,8 +192,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): self.assertEquals({"columname": "Old Value"}, ret) self.mock_txn.execute.assert_has_calls([ - call('SELECT columname FROM tablename WHERE keycol = ? ' - 'ORDER BY rowid asc', + call('SELECT columname FROM tablename WHERE keycol = ?', ['TheKey']), call("UPDATE tablename SET columname = ? WHERE keycol = ?", ["New Value", "TheKey"]) diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index e0b81f2b5..78f600420 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -42,28 +42,38 @@ class RegistrationStoreTestCase(unittest.TestCase): self.assertEquals( # TODO(paul): Surely this field should be 'user_id', not 'name' # Additionally surely it shouldn't come in a 1-element list - [{"name": self.user_id, "password_hash": self.pwhash}], + {"name": self.user_id, "password_hash": self.pwhash}, (yield self.store.get_user_by_id(self.user_id)) ) - self.assertEquals( - {"admin": 0, - "device_id": None, - "name": self.user_id, - "token_id": 1}, - (yield self.store.get_user_by_token(self.tokens[0])) + result = yield self.store.get_user_by_token(self.tokens[1]) + + self.assertDictContainsSubset( + { + "admin": 0, + "device_id": None, + "name": self.user_id, + }, + result ) + self.assertTrue("token_id" in result) + @defer.inlineCallbacks def test_add_tokens(self): yield self.store.register(self.user_id, self.tokens[0], self.pwhash) yield self.store.add_access_token_to_user(self.user_id, self.tokens[1]) - self.assertEquals( - {"admin": 0, - "device_id": None, - "name": self.user_id, - "token_id": 2}, - (yield self.store.get_user_by_token(self.tokens[1])) + result = yield self.store.get_user_by_token(self.tokens[1]) + + self.assertDictContainsSubset( + { + "admin": 0, + "device_id": None, + "name": self.user_id, + }, + result ) + self.assertTrue("token_id" in result) + diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index 811fea544..785953cc8 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -119,7 +119,7 @@ class RoomMemberStoreTestCase(unittest.TestCase): yield self.inject_room_member(self.room, self.u_alice, Membership.JOIN) self.assertEquals( - ["test"], + {"test"}, (yield self.store.get_joined_hosts_for_room(self.room.to_string())) ) @@ -127,7 +127,7 @@ class RoomMemberStoreTestCase(unittest.TestCase): yield self.inject_room_member(self.room, self.u_bob, Membership.JOIN) self.assertEquals( - ["test"], + {"test"}, (yield self.store.get_joined_hosts_for_room(self.room.to_string())) ) @@ -136,9 +136,9 @@ class RoomMemberStoreTestCase(unittest.TestCase): self.assertEquals( {"test", "elsewhere"}, - set((yield + (yield self.store.get_joined_hosts_for_room(self.room.to_string()) - )) + ) ) # Should still have both hosts @@ -146,15 +146,15 @@ class RoomMemberStoreTestCase(unittest.TestCase): self.assertEquals( {"test", "elsewhere"}, - set((yield + (yield self.store.get_joined_hosts_for_room(self.room.to_string()) - )) + ) ) # Should have only one host after other leaves yield self.inject_room_member(self.room, self.u_charlie, Membership.LEAVE) self.assertEquals( - ["test"], + {"test"}, (yield self.store.get_joined_hosts_for_room(self.room.to_string())) ) diff --git a/tests/utils.py b/tests/utils.py index 81e82a80d..cc038fecf 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -17,6 +17,7 @@ from synapse.http.server import HttpServer from synapse.api.errors import cs_error, CodeMessageException, StoreError from synapse.api.constants import EventTypes from synapse.storage import prepare_database +from synapse.storage.engines import create_engine from synapse.server import HomeServer from synapse.util.logcontext import LoggingContext @@ -44,18 +45,23 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): config.event_cache_size = 1 config.disable_registration = False + if "clock" not in kargs: + kargs["clock"] = MockClock() + if datastore is None: db_pool = SQLiteMemoryDbPool() yield db_pool.prepare() hs = HomeServer( name, db_pool=db_pool, config=config, version_string="Synapse/tests", + database_engine=create_engine("sqlite3"), **kargs ) else: hs = HomeServer( name, db_pool=None, datastore=datastore, config=config, version_string="Synapse/tests", + database_engine=create_engine("sqlite3"), **kargs ) @@ -227,7 +233,10 @@ class SQLiteMemoryDbPool(ConnectionPool, object): ) def prepare(self): - return self.runWithConnection(prepare_database) + engine = create_engine("sqlite3") + return self.runWithConnection( + lambda conn: prepare_database(conn, engine) + ) class MemoryDataStore(object): From e9c908ebc09ccc050bd09692c5413124a8c3c06e Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 1 Apr 2015 15:05:30 +0100 Subject: [PATCH 039/173] Completely replace fallback auth for C/S V2: * Now only the auth part goes to fallback, not the whole operation * Auth fallback is a normal API endpoint, not a static page * Params like the recaptcha pubkey can just live in the config Involves a little engineering on JsonResource so its servlets aren't always forced to return JSON. I should document this more, in fact I'll do that now. --- static/client/register/style.css | 6 +- synapse/handlers/auth.py | 98 +++++++++--- synapse/http/server.py | 7 +- synapse/rest/client/v2_alpha/__init__.py | 4 +- synapse/rest/client/v2_alpha/auth.py | 189 +++++++++++++++++++++++ synapse/rest/client/v2_alpha/register.py | 2 +- 6 files changed, 280 insertions(+), 26 deletions(-) create mode 100644 synapse/rest/client/v2_alpha/auth.py diff --git a/static/client/register/style.css b/static/client/register/style.css index a3398852b..5a7b6eebf 100644 --- a/static/client/register/style.css +++ b/static/client/register/style.css @@ -37,9 +37,13 @@ textarea, input { margin: auto } +.g-recaptcha div { + margin: auto; +} + #registrationForm { text-align: left; - padding: 1em; + padding: 5px; margin-bottom: 40px; display: inline-block; diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 26df9fcd8..3d2461dd7 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -20,12 +20,15 @@ from synapse.api.constants import LoginType from synapse.types import UserID from synapse.api.errors import LoginError, Codes from synapse.http.client import SimpleHttpClient + from twisted.web.client import PartialDownloadError import logging import bcrypt import simplejson +import synapse.util.stringutils as stringutils + logger = logging.getLogger(__name__) @@ -34,6 +37,11 @@ class AuthHandler(BaseHandler): def __init__(self, hs): super(AuthHandler, self).__init__(hs) + self.checkers = { + LoginType.PASSWORD: self._check_password_auth, + LoginType.RECAPTCHA: self._check_recaptcha, + } + self.sessions = {} @defer.inlineCallbacks def check_auth(self, flows, clientdict, clientip=None): @@ -52,40 +60,64 @@ class AuthHandler(BaseHandler): If authed is false, the dictionary is the server response to the login request and should be passed back to the client. """ - types = { - LoginType.PASSWORD: self.check_password_auth, - LoginType.RECAPTCHA: self.check_recaptcha, - } if not clientdict or 'auth' not in clientdict: - defer.returnValue((False, self.auth_dict_for_flows(flows))) + sess = self._get_session_info(None) + defer.returnValue( + (False, self._auth_dict_for_flows(flows, sess)) + ) authdict = clientdict['auth'] - # In future: support sessions & retrieve previously succeeded - # login types - creds = {} + sess = self._get_session_info( + authdict['session'] if 'session' in authdict else None + ) + if 'creds' not in sess: + sess['creds'] = {} + creds = sess['creds'] # check auth type currently being presented - if 'type' not in authdict: - raise LoginError(400, "", Codes.MISSING_PARAM) - if authdict['type'] not in types: - raise LoginError(400, "", Codes.UNRECOGNIZED) - result = yield types[authdict['type']](authdict, clientip) - if result: - creds[authdict['type']] = result + if 'type' in authdict: + if authdict['type'] not in self.checkers: + raise LoginError(400, "", Codes.UNRECOGNIZED) + result = yield self.checkers[authdict['type']](authdict, clientip) + if result: + creds[authdict['type']] = result + self._save_session(sess) for f in flows: if len(set(f) - set(creds.keys())) == 0: logger.info("Auth completed with creds: %r", creds) + self._remove_session(sess) defer.returnValue((True, creds)) - ret = self.auth_dict_for_flows(flows) + ret = self._auth_dict_for_flows(flows, sess) ret['completed'] = creds.keys() defer.returnValue((False, ret)) @defer.inlineCallbacks - def check_password_auth(self, authdict, _): + def add_oob_auth(self, stagetype, authdict, clientip): + if stagetype not in self.checkers: + raise LoginError(400, "", Codes.MISSING_PARAM) + if 'session' not in authdict: + raise LoginError(400, "", Codes.MISSING_PARAM) + + sess = self._get_session_info( + authdict['session'] + ) + if 'creds' not in sess: + sess['creds'] = {} + creds = sess['creds'] + + result = yield self.checkers[stagetype](authdict, clientip) + if result: + creds[stagetype] = result + self._save_session(sess) + defer.returnValue(True) + defer.returnValue(False) + + @defer.inlineCallbacks + def _check_password_auth(self, authdict, _): if "user" not in authdict or "password" not in authdict: raise LoginError(400, "", Codes.MISSING_PARAM) @@ -107,7 +139,7 @@ class AuthHandler(BaseHandler): raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) @defer.inlineCallbacks - def check_recaptcha(self, authdict, clientip): + def _check_recaptcha(self, authdict, clientip): try: user_response = authdict["response"] except KeyError: @@ -143,10 +175,10 @@ class AuthHandler(BaseHandler): defer.returnValue(True) raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) - def get_params_recaptcha(self): + def _get_params_recaptcha(self): return {"public_key": self.hs.config.recaptcha_public_key} - def auth_dict_for_flows(self, flows): + def _auth_dict_for_flows(self, flows, session): public_flows = [] for f in flows: hidden = False @@ -157,7 +189,7 @@ class AuthHandler(BaseHandler): public_flows.append(f) get_params = { - LoginType.RECAPTCHA: self.get_params_recaptcha, + LoginType.RECAPTCHA: self._get_params_recaptcha, } params = {} @@ -168,6 +200,30 @@ class AuthHandler(BaseHandler): params[stage] = get_params[stage]() return { + "session": session['id'], "flows": [{"stages": f} for f in public_flows], "params": params } + + def _get_session_info(self, session_id): + if session_id not in self.sessions: + session_id = None + + if not session_id: + # create a new session + while session_id is None or session_id in self.sessions: + session_id = stringutils.random_string(24) + self.sessions[session_id] = { + "id": session_id, + } + + return self.sessions[session_id] + + def _save_session(self, session): + # TODO: Persistent storage + logger.debug("Saving session %s", session) + self.sessions[session["id"]] = session + + def _remove_session(self, session): + logger.debug("Removing session %s", session) + del self.sessions[session["id"]] diff --git a/synapse/http/server.py b/synapse/http/server.py index 30c3aa5ca..76c561d10 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -170,9 +170,12 @@ class JsonResource(HttpServer, resource.Resource): request.method, request.path ) - code, response = yield callback(request, *args) + callback_return = yield callback(request, *args) + if callback_return is not None: + code, response = callback_return + + self._send_response(request, code, response) - self._send_response(request, code, response) response_timer.inc_by( self.clock.time_msec() - start, request.method, servlet_classname ) diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py index 98189ead2..86e4bc729 100644 --- a/synapse/rest/client/v2_alpha/__init__.py +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -17,7 +17,8 @@ from . import ( sync, filter, password, - register + register, + auth ) from synapse.http.server import JsonResource @@ -36,3 +37,4 @@ class ClientV2AlphaRestResource(JsonResource): filter.register_servlets(hs, client_resource) password.register_servlets(hs, client_resource) register.register_servlets(hs, client_resource) + auth.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py new file mode 100644 index 000000000..7a518e226 --- /dev/null +++ b/synapse/rest/client/v2_alpha/auth.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.constants import LoginType +from synapse.api.errors import SynapseError +from synapse.api.urls import CLIENT_V2_ALPHA_PREFIX +from synapse.http.servlet import RestServlet + +from ._base import client_v2_pattern + +import logging + + +logger = logging.getLogger(__name__) + +RECAPTCHA_TEMPLATE = """ + + +Authentication + + + + + + + +
+
+

+ Hello! We need to prevent computer programs and other automated + things from creating accounts on this server. +

+

+ Please verify that you're not a robot. +

+ +
+
+ +
+ +
+ + +""" + +SUCCESS_TEMPLATE = """ + + +Success! + + + + + +
+

Thank you

+

You may now close this window and return to the application

+
+ + +""" + +class AuthRestServlet(RestServlet): + """ + Handles Client / Server API authentication in any situations where it + cannot be handled in the normal flow (with requests to the same endpoint). + Current use is for web fallback auth. + """ + PATTERN = client_v2_pattern("/auth/(?P[\w\.]*)/fallback/web") + + def __init__(self, hs): + super(AuthRestServlet, self).__init__() + self.hs = hs + self.auth = hs.get_auth() + self.auth_handler = hs.get_handlers().auth_handler + self.registration_handler = hs.get_handlers().registration_handler + + @defer.inlineCallbacks + def on_GET(self, request, stagetype): + yield + if stagetype == LoginType.RECAPTCHA: + if ('session' not in request.args or + len(request.args['session']) == 0): + raise SynapseError(400, "No session supplied") + + session = request.args["session"][0] + + html = RECAPTCHA_TEMPLATE % { + 'session': session, + 'myurl': "%s/auth/%s/fallback/web" % ( + CLIENT_V2_ALPHA_PREFIX, LoginType.RECAPTCHA + ), + 'sitekey': self.hs.config.recaptcha_public_key, + } + html_bytes = html.encode("utf8") + request.setResponseCode(200) + request.setHeader(b"Content-Type", b"text/html; charset=utf-8") + request.setHeader(b"Server", self.hs.version_string) + request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),)) + + request.write(html_bytes) + request.finish() + defer.returnValue(None) + else: + raise SynapseError(404, "Unknown auth stage type") + + @defer.inlineCallbacks + def on_POST(self, request, stagetype): + yield + if stagetype == "m.login.recaptcha": + if ('g-recaptcha-response' not in request.args or + len(request.args['g-recaptcha-response'])) == 0: + raise SynapseError(400, "No captcha response supplied") + if ('session' not in request.args or + len(request.args['session'])) == 0: + raise SynapseError(400, "No session supplied") + + session = request.args['session'][0] + + authdict = { + 'response': request.args['g-recaptcha-response'][0], + 'session': session, + } + + success = yield self.auth_handler.add_oob_auth( + LoginType.RECAPTCHA, + authdict, + self.hs.get_ip_from_request(request) + ) + + if success: + html = SUCCESS_TEMPLATE + else: + html = RECAPTCHA_TEMPLATE % { + 'session': session, + 'myurl': "%s/auth/%s/fallback/web" % ( + CLIENT_V2_ALPHA_PREFIX, LoginType.RECAPTCHA + ), + 'sitekey': self.hs.config.recaptcha_public_key, + } + html_bytes = html.encode("utf8") + request.setResponseCode(200) + request.setHeader(b"Content-Type", b"text/html; charset=utf-8") + request.setHeader(b"Server", self.hs.version_string) + request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),)) + + request.write(html_bytes) + request.finish() + + defer.returnValue(None) + else: + raise SynapseError(404, "Unknown auth stage type") + + def on_OPTIONS(self, _): + return 200, {} + + +def register_servlets(hs, http_server): + AuthRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 4a53e0374..537918ea2 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -45,7 +45,7 @@ class RegisterRestServlet(RestServlet): [LoginType.RECAPTCHA], [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA], [LoginType.APPLICATION_SERVICE] - ], body) + ], body, self.hs.get_ip_from_request(request)) if not authed: defer.returnValue((401, result)) From e24c32e6f3c0d7c75529d05762645fe613085bec Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 1 Apr 2015 15:09:51 +0100 Subject: [PATCH 040/173] Fix SQLite support --- synapse/app/homeserver.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index b185b2f56..1ab6effd5 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -365,7 +365,9 @@ def setup(config_options): else: db_config = { "name": "sqlite3", - "database": config.database_path, + "args": { + "database": config.database_path, + }, } db_config = { @@ -381,10 +383,12 @@ def setup(config_options): "use_unicode": True, }) elif name == "sqlite3": + def open_fun(conn): + prepare_database(conn, database_engine) db_config.setdefault("args", {}).update({ "cp_min": 1, "cp_max": 1, - "cp_openfun": prepare_database, + "cp_openfun": open_fun, }) else: raise RuntimeError("Unsupported database type '%s'" % (name,)) @@ -413,7 +417,12 @@ def setup(config_options): logger.info("Preparing database: %s...", db_name) try: - db_conn = database_engine.module.connect(**db_config.get("args", {})) + db_conn = database_engine.module.connect( + **{ + k: v for k, v in db_config.get("args", {}).items() + if not k.startswith("cp_") + } + ) if name == "sqlite3": prepare_sqlite3_database(db_conn) From c5bf0343e8c9dc5a21a546bfc93f24f9fc4d9737 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 1 Apr 2015 15:13:14 +0100 Subject: [PATCH 041/173] Explain how I justified to myself making JsonResource not always send JSON. --- synapse/http/server.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapse/http/server.py b/synapse/http/server.py index 76c561d10..0dbdce283 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -79,6 +79,13 @@ class JsonResource(HttpServer, resource.Resource): Resources. Register callbacks via register_path() + + Callbacks can return a tuple of status code and a dict in which case the + the dict will automatically be sent to the client as a JSON object. + + The JsonResource is primarily intended for returning JSON, but callbacks + may send something other than JSON, they may do so by using the methods + on the request object and instead returning None. """ isLeaf = True From 779f7b0f443bfab351702161763b62b7bb6a27d8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 2 Apr 2015 10:06:22 +0100 Subject: [PATCH 042/173] Fix unicode support --- synapse/app/homeserver.py | 12 ++-- synapse/storage/_base.py | 22 +++--- synapse/storage/engines/__init__.py | 4 +- synapse/storage/engines/maria.py | 12 ++++ synapse/storage/engines/sqlite3.py | 9 +++ synapse/storage/keys.py | 14 ++-- synapse/storage/schema/delta/12/v12.sql | 18 ++--- synapse/storage/schema/delta/13/v13.sql | 10 +-- synapse/storage/schema/delta/14/v14.sql | 4 +- .../schema/full_schemas/11/event_edges.sql | 34 +++++----- .../full_schemas/11/event_signatures.sql | 20 +++--- synapse/storage/schema/full_schemas/11/im.sql | 68 +++++++++---------- .../storage/schema/full_schemas/11/keys.sql | 12 ++-- .../full_schemas/11/media_repository.sql | 34 +++++----- .../schema/full_schemas/11/presence.sql | 16 ++--- .../schema/full_schemas/11/profiles.sql | 6 +- .../schema/full_schemas/11/redactions.sql | 4 +- .../schema/full_schemas/11/room_aliases.sql | 8 +-- .../storage/schema/full_schemas/11/state.sql | 16 ++--- .../schema/full_schemas/11/transactions.sql | 16 ++--- .../storage/schema/full_schemas/11/users.sql | 20 +++--- synapse/storage/schema/schema_version.sql | 2 +- 22 files changed, 193 insertions(+), 168 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 1ab6effd5..9d7a58080 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -18,7 +18,7 @@ import sys sys.dont_write_bytecode = True from synapse.storage import ( - prepare_database, prepare_sqlite3_database, UpgradeDatabaseException, + prepare_database, UpgradeDatabaseException, ) from synapse.storage.engines import create_engine @@ -381,19 +381,18 @@ def setup(config_options): "sql_mode": "TRADITIONAL", "charset": "utf8mb4", "use_unicode": True, + "collation": "utf8mb4_general_ci", }) elif name == "sqlite3": - def open_fun(conn): - prepare_database(conn, database_engine) db_config.setdefault("args", {}).update({ "cp_min": 1, "cp_max": 1, - "cp_openfun": open_fun, }) else: raise RuntimeError("Unsupported database type '%s'" % (name,)) database_engine = create_engine(name) + db_config["args"]["cp_openfun"] = database_engine.on_new_connection hs = SynapseHomeServer( config.server_name, @@ -424,10 +423,7 @@ def setup(config_options): } ) - if name == "sqlite3": - prepare_sqlite3_database(db_conn) - - prepare_database(db_conn, database_engine) + database_engine.prepare_database(db_conn) db_conn.commit() except UpgradeDatabaseException: diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 047d100f4..de4f66197 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -142,19 +142,23 @@ class LoggingTransaction(object): sql = self.database_engine.convert_param_style(sql) - try: - if args and args[0]: - values = args[0] + if args and args[0]: + args = list(args) + args[0] = [ + self.database_engine.encode_parameter(a) for a in args[0] + ] + try: sql_logger.debug( - "[SQL values] {%s} " + ", ".join(("<%r>",) * len(values)), + "[SQL values] {%s} " + ", ".join(("<%r>",) * len(args[0])), self.name, - *values + *args[0] ) - except: - # Don't let logging failures stop SQL from working - pass + except: + # Don't let logging failures stop SQL from working + pass start = time.time() * 1000 + try: return self.txn.execute( sql, *args, **kwargs @@ -761,8 +765,6 @@ class SQLBaseStore(object): return None internal_metadata, js, redacted, rejected_reason = res - js = js.decode("utf8") - internal_metadata = internal_metadata.decode("utf8") start_time = update_counter("select_event", start_time) diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index 709b6f88a..29702be92 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -16,6 +16,8 @@ from .maria import MariaEngine from .sqlite3 import Sqlite3Engine +import importlib + SUPPORTED_MODULE = { "sqlite3": Sqlite3Engine, @@ -27,7 +29,7 @@ def create_engine(name): engine_class = SUPPORTED_MODULE.get(name, None) if engine_class: - module = __import__(name) + module = importlib.import_module(name) return engine_class(module) raise RuntimeError( diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py index df4776364..7fcb706a6 100644 --- a/synapse/storage/engines/maria.py +++ b/synapse/storage/engines/maria.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from synapse.storage import prepare_database import types @@ -28,3 +29,14 @@ class MariaEngine(object): if isinstance(param, types.BufferType): return str(param) return param + + def on_new_connection(self, db_conn): + pass + + def prepare_database(self, db_conn): + cur = db_conn.cursor() + cur.execute( + "ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci" + ) + db_conn.commit() + prepare_database(db_conn, self) diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 639cdea41..e802b5d5f 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from synapse.storage import prepare_database, prepare_sqlite3_database + class Sqlite3Engine(object): def __init__(self, database_module): @@ -23,3 +25,10 @@ class Sqlite3Engine(object): def encode_parameter(self, param): return param + + def on_new_connection(self, db_conn): + self.prepare_database(db_conn) + + def prepare_database(self, db_conn): + prepare_sqlite3_database(db_conn) + prepare_database(db_conn, self) diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 25fef7943..d3b9b3866 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -57,15 +57,18 @@ class KeyStore(SQLBaseStore): OpenSSL.crypto.FILETYPE_ASN1, tls_certificate ) fingerprint = hashlib.sha256(tls_certificate_bytes).hexdigest() - return self._simple_insert( + return self._simple_upsert( table="server_tls_certificates", - values={ + keyvalues={ "server_name": server_name, "fingerprint": fingerprint, + }, + values={ "from_server": from_server, "ts_added_ms": time_now_ms, "tls_certificate": buffer(tls_certificate_bytes), }, + desc="store_server_certificate", ) @defer.inlineCallbacks @@ -106,13 +109,16 @@ class KeyStore(SQLBaseStore): ts_now_ms (int): The time now in milliseconds verification_key (VerifyKey): The NACL verify key. """ - return self._simple_insert( + return self._simple_upsert( table="server_signature_keys", - values={ + keyvalues={ "server_name": server_name, "key_id": "%s:%s" % (verify_key.alg, verify_key.version), + }, + values={ "from_server": from_server, "ts_added_ms": time_now_ms, "verify_key": buffer(verify_key.encode()), }, + desc="store_server_verify_key", ) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index 90ac47485..717d289f7 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -14,16 +14,16 @@ */ CREATE TABLE IF NOT EXISTS rejections( - event_id VARCHAR(255) NOT NULL, - reason VARCHAR(255) NOT NULL, - last_check VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + reason VARCHAR(150) NOT NULL, + last_check VARCHAR(150) NOT NULL, UNIQUE (event_id) ) ; -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( id BIGINT PRIMARY KEY, - user_name VARCHAR(255) NOT NULL, + user_name VARCHAR(150) NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, app_id VARCHAR(64) NOT NULL, @@ -41,19 +41,19 @@ CREATE TABLE IF NOT EXISTS pushers ( CREATE TABLE IF NOT EXISTS push_rules ( id BIGINT PRIMARY KEY, - user_name VARCHAR(255) NOT NULL, - rule_id VARCHAR(255) NOT NULL, + user_name VARCHAR(150) NOT NULL, + rule_id VARCHAR(150) NOT NULL, priority_class TINYINT NOT NULL, priority INTEGER NOT NULL DEFAULT 0, - conditions VARCHAR(255) NOT NULL, - actions VARCHAR(255) NOT NULL, + conditions VARCHAR(150) NOT NULL, + actions VARCHAR(150) NOT NULL, UNIQUE(user_name, rule_id) ) ; CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( - user_id VARCHAR(255), + user_id VARCHAR(150), filter_id BIGINT, filter_json BLOB ) ; diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index 4953b6323..f5275a59b 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -15,10 +15,10 @@ CREATE TABLE IF NOT EXISTS application_services( id BIGINT PRIMARY KEY, - url VARCHAR(255), - token VARCHAR(255), - hs_token VARCHAR(255), - sender VARCHAR(255), + url VARCHAR(150), + token VARCHAR(150), + hs_token VARCHAR(150), + sender VARCHAR(150), UNIQUE(token) ) ; @@ -26,6 +26,6 @@ CREATE TABLE IF NOT EXISTS application_services_regex( id BIGINT PRIMARY KEY, as_id BIGINT NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ - regex VARCHAR(255), + regex VARCHAR(150), FOREIGN KEY(as_id) REFERENCES application_services(id) ) ; diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index 3bda073c9..1d582cc62 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -1,7 +1,7 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( id BIGINT PRIMARY KEY, - user_name VARCHAR(255) NOT NULL, - rule_id VARCHAR(255) NOT NULL, + user_name VARCHAR(150) NOT NULL, + rule_id VARCHAR(150) NOT NULL, enabled TINYINT, UNIQUE(user_name, rule_id) ) ; diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 336cd563d..124c9a9bd 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS event_forward_extremities( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, UNIQUE (event_id, room_id) ) ; @@ -24,8 +24,8 @@ CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_backward_extremities( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, UNIQUE (event_id, room_id) ) ; @@ -34,9 +34,9 @@ CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id CREATE TABLE IF NOT EXISTS event_edges( - event_id VARCHAR(255) NOT NULL, - prev_event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + prev_event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, is_state BOOL NOT NULL, UNIQUE (event_id, prev_event_id, room_id, is_state) ) ; @@ -46,7 +46,7 @@ CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); CREATE TABLE IF NOT EXISTS room_depth( - room_id VARCHAR(255) NOT NULL, + room_id VARCHAR(150) NOT NULL, min_depth INTEGER NOT NULL, UNIQUE (room_id) ) ; @@ -55,8 +55,8 @@ CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( - event_id VARCHAR(255) NOT NULL, - destination VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + destination VARCHAR(150) NOT NULL, delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ) ; @@ -65,10 +65,10 @@ CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id) CREATE TABLE IF NOT EXISTS state_forward_extremities( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, - type VARCHAR(255) NOT NULL, - state_key VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, UNIQUE (event_id, room_id) ) ; @@ -79,9 +79,9 @@ CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_auth( - event_id VARCHAR(255) NOT NULL, - auth_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + auth_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, UNIQUE (event_id, auth_id, room_id) ) ; diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index 11e611598..30e3f71c5 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS event_content_hashes ( - event_id VARCHAR(255), - algorithm VARCHAR(255), + event_id VARCHAR(150), + algorithm VARCHAR(150), hash BLOB, UNIQUE (event_id, algorithm) ) ; @@ -24,8 +24,8 @@ CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event CREATE TABLE IF NOT EXISTS event_reference_hashes ( - event_id VARCHAR(255), - algorithm VARCHAR(255), + event_id VARCHAR(150), + algorithm VARCHAR(150), hash BLOB, UNIQUE (event_id, algorithm) ) ; @@ -34,9 +34,9 @@ CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(e CREATE TABLE IF NOT EXISTS event_signatures ( - event_id VARCHAR(255), - signature_name VARCHAR(255), - key_id VARCHAR(255), + event_id VARCHAR(150), + signature_name VARCHAR(150), + key_id VARCHAR(150), signature BLOB, UNIQUE (event_id, signature_name, key_id) ) ; @@ -45,9 +45,9 @@ CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); CREATE TABLE IF NOT EXISTS event_edge_hashes( - event_id VARCHAR(255), - prev_event_id VARCHAR(255), - algorithm VARCHAR(255), + event_id VARCHAR(150), + prev_event_id VARCHAR(150), + algorithm VARCHAR(150), hash BLOB, UNIQUE (event_id, prev_event_id, algorithm) ) ; diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index a0fb33762..7cb8f802e 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -16,9 +16,9 @@ CREATE TABLE IF NOT EXISTS events( stream_ordering BIGINT PRIMARY KEY, topological_ordering BIGINT NOT NULL, - event_id VARCHAR(255) NOT NULL, - type VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, content BLOB NOT NULL, unrecognized_keys BLOB, processed BOOL NOT NULL, @@ -33,8 +33,8 @@ CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, internal_metadata BLOB NOT NULL, json BLOB NOT NULL, UNIQUE (event_id) @@ -44,11 +44,11 @@ CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); CREATE TABLE IF NOT EXISTS state_events( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, - type VARCHAR(255) NOT NULL, - state_key VARCHAR(255) NOT NULL, - prev_state VARCHAR(255), + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, + prev_state VARCHAR(150), UNIQUE (event_id) ) ; @@ -58,10 +58,10 @@ CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); CREATE TABLE IF NOT EXISTS current_state_events( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, - type VARCHAR(255) NOT NULL, - state_key VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, UNIQUE (event_id), UNIQUE (room_id, type, state_key) ) ; @@ -71,11 +71,11 @@ CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (ty CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); CREATE TABLE IF NOT EXISTS room_memberships( - event_id VARCHAR(255) NOT NULL, - user_id VARCHAR(255) NOT NULL, - sender VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, - membership VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + user_id VARCHAR(150) NOT NULL, + sender VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + membership VARCHAR(150) NOT NULL, UNIQUE (event_id) ) ; @@ -83,41 +83,41 @@ CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); CREATE TABLE IF NOT EXISTS feedback( - event_id VARCHAR(255) NOT NULL, - feedback_type VARCHAR(255), - target_event_id VARCHAR(255), - sender VARCHAR(255), - room_id VARCHAR(255), + event_id VARCHAR(150) NOT NULL, + feedback_type VARCHAR(150), + target_event_id VARCHAR(150), + sender VARCHAR(150), + room_id VARCHAR(150), UNIQUE (event_id) ) ; CREATE TABLE IF NOT EXISTS topics( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, - topic VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + topic VARCHAR(150) NOT NULL, UNIQUE (event_id) ) ; CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, - name VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + name VARCHAR(150) NOT NULL, UNIQUE (event_id) ); CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( - room_id VARCHAR(255) PRIMARY KEY NOT NULL, + room_id VARCHAR(150) PRIMARY KEY NOT NULL, is_public BOOL, - creator VARCHAR(255) + creator VARCHAR(150) ) ; CREATE TABLE IF NOT EXISTS room_hosts( - room_id VARCHAR(255) NOT NULL, - host VARCHAR(255) NOT NULL, + room_id VARCHAR(150) NOT NULL, + host VARCHAR(150) NOT NULL, UNIQUE (room_id, host) ) ; diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index a785cdb4c..062ca53fe 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -13,18 +13,18 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS server_tls_certificates( - server_name VARCHAR(255), -- Server name. - fingerprint VARCHAR(255), -- Certificate fingerprint. - from_server VARCHAR(255), -- Which key server the certificate was fetched from. + server_name VARCHAR(150), -- Server name. + fingerprint VARCHAR(150), -- Certificate fingerprint. + from_server VARCHAR(150), -- Which key server the certificate was fetched from. ts_added_ms BIGINT, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ) ; CREATE TABLE IF NOT EXISTS server_signature_keys( - server_name VARCHAR(255), -- Server name. - key_id VARCHAR(255), -- Key version. - from_server VARCHAR(255), -- Which key server the key was fetched form. + server_name VARCHAR(150), -- Server name. + key_id VARCHAR(150), -- Key version. + from_server VARCHAR(150), -- Which key server the key was fetched form. ts_added_ms BIGINT, -- When the key was added. verify_key BLOB, -- NACL verification key. UNIQUE (server_name, key_id) diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index 27fe297af..c8c5f1d2f 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -14,21 +14,21 @@ */ CREATE TABLE IF NOT EXISTS local_media_repository ( - media_id VARCHAR(255), -- The id used to refer to the media. - media_type VARCHAR(255), -- The MIME-type of the media. + media_id VARCHAR(150), -- The id used to refer to the media. + media_type VARCHAR(150), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. created_ts BIGINT, -- When the content was uploaded in ms. - upload_name VARCHAR(255), -- The name the media was uploaded with. - user_id VARCHAR(255), -- The user who uploaded the file. + upload_name VARCHAR(150), -- The name the media was uploaded with. + user_id VARCHAR(150), -- The user who uploaded the file. UNIQUE (media_id) ) ; CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( - media_id VARCHAR(255), -- The id used to refer to the media. + media_id VARCHAR(150), -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_type VARCHAR(255), -- The MIME-type of the thumbnail. - thumbnail_method VARCHAR(255), -- The method used to make the thumbnail. + thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. + thumbnail_method VARCHAR(150), -- The method used to make the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type @@ -39,25 +39,25 @@ CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); CREATE TABLE IF NOT EXISTS remote_media_cache ( - media_origin VARCHAR(255), -- The remote HS the media came from. - media_id VARCHAR(255), -- The id used to refer to the media on that server. - media_type VARCHAR(255), -- The MIME-type of the media. + media_origin VARCHAR(150), -- The remote HS the media came from. + media_id VARCHAR(150), -- The id used to refer to the media on that server. + media_type VARCHAR(150), -- The MIME-type of the media. created_ts BIGINT, -- When the content was uploaded in ms. - upload_name VARCHAR(255), -- The name the media was uploaded with. + upload_name VARCHAR(150), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. - filesystem_id VARCHAR(255), -- The name used to store the media on disk. + filesystem_id VARCHAR(150), -- The name used to store the media on disk. UNIQUE (media_origin, media_id) ) ; CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( - media_origin VARCHAR(255), -- The remote HS the media came from. - media_id VARCHAR(255), -- The id used to refer to the media. + media_origin VARCHAR(150), -- The remote HS the media came from. + media_id VARCHAR(150), -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_method VARCHAR(255), -- The method used to make the thumbnail - thumbnail_type VARCHAR(255), -- The MIME-type of the thumbnail. + thumbnail_method VARCHAR(150), -- The method used to make the thumbnail + thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - filesystem_id VARCHAR(255), -- The name used to store the media on disk. + filesystem_id VARCHAR(150), -- The name used to store the media on disk. UNIQUE ( media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index b48b110ae..273e61281 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -13,9 +13,9 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS presence( - user_id VARCHAR(255) NOT NULL, + user_id VARCHAR(150) NOT NULL, state VARCHAR(20), - status_msg VARCHAR(255), + status_msg VARCHAR(150), mtime BIGINT, -- miliseconds since last state change UNIQUE(user_id) ) ; @@ -23,16 +23,14 @@ CREATE TABLE IF NOT EXISTS presence( -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state CREATE TABLE IF NOT EXISTS presence_allow_inbound( - observed_user_id VARCHAR(255) NOT NULL, - observer_user_id VARCHAR(255), -- a UserID, - UNIQUE(observed_user_id) + observed_user_id VARCHAR(150) NOT NULL, + observer_user_id VARCHAR(150) -- a UserID, ) ; -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? CREATE TABLE IF NOT EXISTS presence_list( - user_id VARCHAR(255) NOT NULL, - observed_user_id VARCHAR(255), -- a UserID, - accepted BOOLEAN, - UNIQUE(user_id) + user_id VARCHAR(150) NOT NULL, + observed_user_id VARCHAR(150), -- a UserID, + accepted BOOLEAN ) ; diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index 92da48f97..023060a54 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS profiles( - user_id VARCHAR(255) NOT NULL, - displayname VARCHAR(255), - avatar_url VARCHAR(255), + user_id VARCHAR(150) NOT NULL, + displayname VARCHAR(150), + avatar_url VARCHAR(150), UNIQUE(user_id) ) ; diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index 9b52a2012..5c23188d6 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS redactions ( - event_id VARCHAR(255) NOT NULL, - redacts VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + redacts VARCHAR(150) NOT NULL, UNIQUE (event_id) ) ; diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index 220df8757..63fe0f5c6 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -14,12 +14,12 @@ */ CREATE TABLE IF NOT EXISTS room_aliases( - room_alias VARBINARY(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + room_alias VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, UNIQUE (room_alias) ) ; CREATE TABLE IF NOT EXISTS room_alias_servers( - room_alias VARBINARY(255) NOT NULL, - server VARCHAR(255) NOT NULL + room_alias VARCHAR(150) NOT NULL, + server VARCHAR(150) NOT NULL ) ; diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index 40584a325..acfb76439 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -15,21 +15,21 @@ CREATE TABLE IF NOT EXISTS state_groups( id VARCHAR(20) PRIMARY KEY, - room_id VARCHAR(255) NOT NULL, - event_id VARCHAR(255) NOT NULL + room_id VARCHAR(150) NOT NULL, + event_id VARCHAR(150) NOT NULL ) ; CREATE TABLE IF NOT EXISTS state_groups_state( state_group VARCHAR(20) NOT NULL, - room_id VARCHAR(255) NOT NULL, - type VARCHAR(255) NOT NULL, - state_key VARCHAR(255) NOT NULL, - event_id VARCHAR(255) NOT NULL + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, + event_id VARCHAR(150) NOT NULL ) ; CREATE TABLE IF NOT EXISTS event_to_state_groups( - event_id VARCHAR(255) NOT NULL, - state_group VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + state_group VARCHAR(150) NOT NULL, UNIQUE (event_id) ) ; diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index d33bdfb30..43541661c 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -14,8 +14,8 @@ */ -- Stores what transaction ids we have received and what our response was CREATE TABLE IF NOT EXISTS received_transactions( - transaction_id VARCHAR(255), - origin VARCHAR(255), + transaction_id VARCHAR(150), + origin VARCHAR(150), ts BIGINT, response_code INTEGER, response_json BLOB, @@ -30,8 +30,8 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering - transaction_id VARCHAR(255), - destination VARCHAR(255), + transaction_id VARCHAR(150), + destination VARCHAR(150), response_code INTEGER DEFAULT 0, response_json BLOB, ts BIGINT @@ -47,9 +47,9 @@ CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_c -- For sent transactions only. CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( transaction_id INTEGER, - destination VARCHAR(255), - pdu_id VARCHAR(255), - pdu_origin VARCHAR(255), + destination VARCHAR(150), + pdu_id VARCHAR(150), + pdu_origin VARCHAR(150), UNIQUE (transaction_id, destination) ) ; @@ -57,7 +57,7 @@ CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(d -- To track destination health CREATE TABLE IF NOT EXISTS destinations( - destination VARCHAR(255) PRIMARY KEY, + destination VARCHAR(150) PRIMARY KEY, retry_last_ts BIGINT, retry_interval INTEGER ) ; diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 28909f580..0271de352 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS users( - name VARCHAR(255), - password_hash VARCHAR(255), + name VARCHAR(150), + password_hash VARCHAR(150), creation_ts BIGINT, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) @@ -22,19 +22,19 @@ CREATE TABLE IF NOT EXISTS users( CREATE TABLE IF NOT EXISTS access_tokens( id BIGINT PRIMARY KEY, - user_id VARCHAR(255) NOT NULL, - device_id VARCHAR(255), - token VARCHAR(255) NOT NULL, + user_id VARCHAR(150) NOT NULL, + device_id VARCHAR(150), + token VARCHAR(150) NOT NULL, last_used BIGINT, UNIQUE(token) ) ; CREATE TABLE IF NOT EXISTS user_ips ( - user VARCHAR(255) NOT NULL, - access_token VARCHAR(255) NOT NULL, - device_id VARCHAR(255), - ip VARCHAR(255) NOT NULL, - user_agent VARCHAR(255) NOT NULL, + user VARCHAR(150) NOT NULL, + access_token VARCHAR(150) NOT NULL, + device_id VARCHAR(150), + ip VARCHAR(150) NOT NULL, + user_agent VARCHAR(150) NOT NULL, last_seen BIGINT NOT NULL, UNIQUE (user, access_token, ip, user_agent) ) ; diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql index 28762861e..e7fa6fe56 100644 --- a/synapse/storage/schema/schema_version.sql +++ b/synapse/storage/schema/schema_version.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS schema_version( CREATE TABLE IF NOT EXISTS applied_schema_deltas( `version` INTEGER NOT NULL, - `file` VARCHAR(255) NOT NULL, + `file` VARCHAR(150) NOT NULL, UNIQUE(version, file) ); From 70a84f17f39bbc5c8a68541874ca4767871f2b79 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 2 Apr 2015 17:01:29 +0100 Subject: [PATCH 043/173] Add shared secret auth into register v2 and switch the script over. --- register_new_matrix_user | 5 +- synapse/api/constants.py | 4 +- synapse/rest/client/v2_alpha/register.py | 65 ++++++++++++++++++++---- 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/register_new_matrix_user b/register_new_matrix_user index daddadc30..f833d2a4d 100755 --- a/register_new_matrix_user +++ b/register_new_matrix_user @@ -33,10 +33,9 @@ def request_registration(user, password, server_location, shared_secret): ).hexdigest() data = { - "user": user, + "username": user, "password": password, "mac": mac, - "type": "org.matrix.login.shared_secret", } server_location = server_location.rstrip("/") @@ -44,7 +43,7 @@ def request_registration(user, password, server_location, shared_secret): print "Sending registration request..." req = urllib2.Request( - "%s/_matrix/client/api/v1/register" % (server_location,), + "%s/_matrix/client/v2_alpha/register" % (server_location,), data=json.dumps(data), headers={'Content-Type': 'application/json'} ) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 3e0ce170a..f825c1a58 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -60,9 +60,11 @@ class LoginType(object): EMAIL_IDENTITY = u"m.login.email.identity" RECAPTCHA = u"m.login.recaptcha" APPLICATION_SERVICE = u"m.login.application_service" + + # Only for C/S API v1 SHARED_SECRET = u"org.matrix.login.shared_secret" - HIDDEN_TYPES = [APPLICATION_SERVICE, SHARED_SECRET] + HIDDEN_TYPES = [APPLICATION_SERVICE] class EventTypes(object): diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 537918ea2..a69b45f36 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -22,6 +22,19 @@ from synapse.http.servlet import RestServlet from ._base import client_v2_pattern, parse_request_allow_empty import logging +import hmac +from hashlib import sha1 +from synapse.util.async import run_on_reactor + + +# We ought to be using hmac.compare_digest() but on older pythons it doesn't +# exist. It's a _really minor_ security flaw to use plain string comparison +# because the timing attack is so obscured by all the other code here it's +# unlikely to make much difference +if hasattr(hmac, "compare_digest"): + compare_digest = hmac.compare_digest +else: + compare_digest = lambda a, b: a == b logger = logging.getLogger(__name__) @@ -39,19 +52,30 @@ class RegisterRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request): + yield run_on_reactor() + body = parse_request_allow_empty(request) - authed, result = yield self.auth_handler.check_auth([ - [LoginType.RECAPTCHA], - [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA], - [LoginType.APPLICATION_SERVICE] - ], body, self.hs.get_ip_from_request(request)) + is_using_shared_secret = False + is_application_server = False - if not authed: - defer.returnValue((401, result)) + if 'mac' in body: + # Check registration-specific shared secret auth + if 'username' not in body: + raise SynapseError(400, "", Codes.MISSING_PARAM) + self._check_shared_secret_auth( + body['username'], body['mac'] + ) + is_using_shared_secret = True + else: + authed, result = yield self.auth_handler.check_auth([ + [LoginType.RECAPTCHA], + [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA], + [LoginType.APPLICATION_SERVICE] + ], body, self.hs.get_ip_from_request(request)) - is_application_server = LoginType.APPLICATION_SERVICE in result - is_using_shared_secret = LoginType.SHARED_SECRET in result + if not authed: + defer.returnValue((401, result)) can_register = ( not self.hs.config.disable_registration @@ -81,6 +105,29 @@ class RegisterRestServlet(RestServlet): def on_OPTIONS(self, _): return 200, {} + def _check_shared_secret_auth(self, username, mac): + if not self.hs.config.registration_shared_secret: + raise SynapseError(400, "Shared secret registration is not enabled") + + user = username.encode("utf-8") + + # str() because otherwise hmac complains that 'unicode' does not + # have the buffer interface + got_mac = str(mac) + + want_mac = hmac.new( + key=self.hs.config.registration_shared_secret, + msg=user, + digestmod=sha1, + ).hexdigest() + + if compare_digest(want_mac, got_mac): + return True + else: + raise SynapseError( + 403, "HMAC incorrect", + ) + def register_servlets(hs, http_server): RegisterRestServlet(hs).register(http_server) From 41cd778d6672101c7f18fe9acbca74e5d2dccc04 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 2 Apr 2015 17:06:17 +0100 Subject: [PATCH 044/173] pep8 --- synapse/rest/client/v2_alpha/auth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py index 7a518e226..4c726f05f 100644 --- a/synapse/rest/client/v2_alpha/auth.py +++ b/synapse/rest/client/v2_alpha/auth.py @@ -90,6 +90,7 @@ if (window.onAuthDone != undefined) { """ + class AuthRestServlet(RestServlet): """ Handles Client / Server API authentication in any situations where it From 6b59650753a516404d3bb038f4a55cad5d7ddcd1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 2 Apr 2015 17:45:16 +0100 Subject: [PATCH 045/173] Throw sensible errors on not-json when allowing empty body --- synapse/rest/client/v2_alpha/_base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py index 8adcc9dd9..4540e8dcf 100644 --- a/synapse/rest/client/v2_alpha/_base.py +++ b/synapse/rest/client/v2_alpha/_base.py @@ -44,7 +44,10 @@ def parse_request_allow_empty(request): content = request.content.read() if content is None or content == '': return None - return simplejson.loads(content) + try: + return simplejson.loads(content) + except simplejson.JSONDecodeError: + raise SynapseError(400, "Content not JSON.") def parse_json_dict_from_request(request): From 4eb6d66b45356efcc87089cb52ca6f51c98cd798 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 2 Apr 2015 17:51:19 +0100 Subject: [PATCH 046/173] Add app service auth back in to v2 register --- synapse/api/constants.py | 4 +--- synapse/rest/client/v2_alpha/register.py | 9 +++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index f825c1a58..d29c2dde0 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -59,13 +59,11 @@ class LoginType(object): EMAIL_URL = u"m.login.email.url" EMAIL_IDENTITY = u"m.login.email.identity" RECAPTCHA = u"m.login.recaptcha" - APPLICATION_SERVICE = u"m.login.application_service" # Only for C/S API v1 + APPLICATION_SERVICE = u"m.login.application_service" SHARED_SECRET = u"org.matrix.login.shared_secret" - HIDDEN_TYPES = [APPLICATION_SERVICE] - class EventTypes(object): Member = "m.room.member" diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index a69b45f36..72319a3bb 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -59,7 +59,13 @@ class RegisterRestServlet(RestServlet): is_using_shared_secret = False is_application_server = False - if 'mac' in body: + service = None + if 'access_token' in request.args: + service = yield self.auth.get_appservice_by_req(request) + + if service: + is_application_server = True + elif 'mac' in body: # Check registration-specific shared secret auth if 'username' not in body: raise SynapseError(400, "", Codes.MISSING_PARAM) @@ -71,7 +77,6 @@ class RegisterRestServlet(RestServlet): authed, result = yield self.auth_handler.check_auth([ [LoginType.RECAPTCHA], [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA], - [LoginType.APPLICATION_SERVICE] ], body, self.hs.get_ip_from_request(request)) if not authed: From d0e444a648ec0e4168673665ff37d09a1c2c2292 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:04:02 +0100 Subject: [PATCH 047/173] Explicitly name the __main__ module logger --- synapse/app/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 9d7a58080..95190c1d7 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -64,7 +64,7 @@ import sqlite3 import yaml -logger = logging.getLogger(__name__) +logger = logging.getLogger("synapse.app.homeserver") class SynapseHomeServer(HomeServer): From 304111afd0fdda6a0c58d81238a1bbfa7f318208 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:05:36 +0100 Subject: [PATCH 048/173] Don't use AUTOINCREMENT, use an in memory version --- synapse/storage/_base.py | 54 +++++++++++++++++++++++++++++---- synapse/storage/events.py | 13 ++++---- synapse/storage/registration.py | 10 ++++-- synapse/storage/state.py | 6 ++-- synapse/storage/transactions.py | 4 ++- 5 files changed, 68 insertions(+), 19 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index de4f66197..9f63f0708 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -163,8 +163,8 @@ class LoggingTransaction(object): return self.txn.execute( sql, *args, **kwargs ) - except: - logger.exception("[SQL FAIL] {%s}", self.name) + except Exception as e: + logger.debug("[SQL FAIL] {%s} %s", self.name, e) raise finally: msecs = (time.time() * 1000) - start @@ -209,6 +209,46 @@ class PerformanceCounters(object): return top_n_counters +class IdGenerator(object): + def __init__(self, table, column, store): + self.table = table + self.column = column + self.store = store + self._lock = threading.Lock() + self._next_id = None + + @defer.inlineCallbacks + def get_next(self): + with self._lock: + if not self._next_id: + res = yield self.store._execute_and_decode( + "IdGenerator_%s" % (self.table,), + "SELECT MAX(%s) as mx FROM %s" % (self.column, self.table,) + ) + + self._next_id = (res and res[0] and res[0]["mx"]) or 1 + + i = self._next_id + self._next_id += 1 + defer.returnValue(i) + + def get_next_txn(self, txn): + with self._lock: + if self._next_id: + i = self._next_id + self._next_id += 1 + return i + else: + txn.execute( + "SELECT MAX(%s) FROM %s" % (self.column, self.table,) + ) + + val, = txn.fetchone() + self._next_id = val or 2 + + return 1 + + class SQLBaseStore(object): _TXN_ID = 0 @@ -234,8 +274,10 @@ class SQLBaseStore(object): # Pretend the getEventCache is just another named cache caches_by_name["*getEvent*"] = self._get_event_cache - self._next_stream_id_lock = threading.Lock() - self._next_stream_id = int(hs.get_clock().time_msec()) * 1000 + self._stream_id_gen = IdGenerator("events", "stream_ordering", self) + self._transaction_id_gen = IdGenerator("sent_transactions", "id", self) + self._state_groups_id_gen = IdGenerator("state_groups", "id", self) + self._access_tokens_id_gen = IdGenerator("access_tokens", "id", self) def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -292,8 +334,8 @@ class SQLBaseStore(object): LoggingTransaction(txn, name, self.database_engine), *args, **kwargs ) - except: - logger.exception("[TXN FAIL] {%s}", name) + except Exception as e: + logger.debug("[TXN FAIL] {%s}", name, e) raise finally: end = time.time() * 1000 diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 69f598967..514feebcb 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -96,12 +96,16 @@ class EventsStore(SQLBaseStore): # Remove the any existing cache entries for the event_id self._get_event_cache.pop(event.event_id) + if stream_ordering is None: + stream_ordering = self._stream_id_gen.get_next_txn(txn) + # We purposefully do this first since if we include a `current_state` # key, we *want* to update the `current_state_events` table if current_state: - txn.execute( - "DELETE FROM current_state_events WHERE room_id = ?", - (event.room_id,) + self._simple_delete_txn( + txn, + table="current_state_events", + keyvalues={"room_id": event.room_id}, ) for s in current_state: @@ -240,9 +244,6 @@ class EventsStore(SQLBaseStore): "depth": event.depth, } - if stream_ordering is None: - stream_ordering = self.get_next_stream_id() - unrec = { k: v for k, v in event.get_dict().items() diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 0c785ec98..b62b4a341 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -39,10 +39,12 @@ class RegistrationStore(SQLBaseStore): Raises: StoreError if there was a problem adding this. """ - yield self._simple_insert( + next_id = yield self._access_tokens_id_gen.get_next() + + self._simple_insert( "access_tokens", { - "id": self.get_next_stream_id(), + "id": next_id, "user_id": user_id, "token": token }, @@ -68,6 +70,8 @@ class RegistrationStore(SQLBaseStore): def _register(self, txn, user_id, token, password_hash): now = int(self.clock.time()) + next_id = self._access_tokens_id_gen.get_next_txn(txn) + try: txn.execute("INSERT INTO users(name, password_hash, creation_ts) " "VALUES (?,?,?)", @@ -82,7 +86,7 @@ class RegistrationStore(SQLBaseStore): txn.execute( "INSERT INTO access_tokens(id, user_id, token)" " VALUES (?,?,?)", - (self.get_next_stream_id(), user_id, token,) + (next_id, user_id, token,) ) @defer.inlineCallbacks diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 65ea9c4d8..3e55cb81b 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -93,12 +93,12 @@ class StateStore(SQLBaseStore): state_group = context.state_group if not state_group: - group = _make_group_id(self._clock) - state_group = self._simple_insert_txn( + state_group = _make_group_id(self._clock) + self._simple_insert_txn( txn, table="state_groups", values={ - "id": group, + "id": state_group, "room_id": event.room_id, "event_id": event.event_id, }, diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index e3e484fb2..9594fe1f2 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -123,6 +123,8 @@ class TransactionStore(SQLBaseStore): def _prep_send_transaction(self, txn, transaction_id, destination, origin_server_ts): + next_id = self._transaction_id_gen.get_next_txn(txn) + # First we find out what the prev_txns should be. # Since we know that we are only sending one transaction at a time, # we can simply take the last one. @@ -143,7 +145,7 @@ class TransactionStore(SQLBaseStore): txn, table=SentTransactions.table_name, values={ - "id": self.get_next_stream_id(), + "id": next_id, "transaction_id": transaction_id, "destination": destination, "ts": origin_server_ts, From c8d3f6486da29b1e0e36b447948b8971bdf05fd3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:06:01 +0100 Subject: [PATCH 049/173] Implement or_ignore flag on inserts --- synapse/storage/_base.py | 13 +++++++++---- synapse/storage/presence.py | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 9f63f0708..a0c1718c2 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -393,7 +393,8 @@ class SQLBaseStore(object): # "Simple" SQL API methods that operate on a single table with no JOINs, # no complex WHERE clauses, just a dict of values for columns. - def _simple_insert(self, table, values, desc="_simple_insert"): + def _simple_insert(self, table, values, or_ignore=False, + desc="_simple_insert"): """Executes an INSERT query on the named table. Args: @@ -403,10 +404,11 @@ class SQLBaseStore(object): return self.runInteraction( desc, self._simple_insert_txn, table, values, + or_ignore=or_ignore ) @log_function - def _simple_insert_txn(self, txn, table, values): + def _simple_insert_txn(self, txn, table, values, or_ignore=False): sql = "INSERT INTO %s (%s) VALUES(%s)" % ( table, ", ".join(k for k in values), @@ -418,8 +420,11 @@ class SQLBaseStore(object): sql, values.values(), ) - txn.execute(sql, values.values()) - return txn.lastrowid + try: + txn.execute(sql, values.values()) + except self.database_engine.module.IntegrityError: + if not or_ignore: + raise def _simple_upsert(self, table, keyvalues, values, desc="_simple_upsert"): """ diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index 87fba5543..22ec94bc1 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -57,6 +57,7 @@ class PresenceStore(SQLBaseStore): values={"observed_user_id": observed_localpart, "observer_user_id": observer_userid}, desc="allow_presence_visible", + or_ignore=True, ) def disallow_presence_visible(self, observed_localpart, observer_userid): From 0af5f5efaf0b24187514cf78d7982ef9b85a208c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:08:35 +0100 Subject: [PATCH 050/173] Don't use multiple UNIQUE constraints; it will cause deadlocks --- synapse/storage/_base.py | 4 ++-- synapse/storage/events.py | 14 +++++++------- .../storage/schema/full_schemas/11/presence.sql | 10 ++++++---- synapse/storage/schema/full_schemas/11/users.sql | 4 ++-- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index a0c1718c2..4ac61be89 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -837,11 +837,11 @@ class SQLBaseStore(object): return curr_time logger.debug("Got js: %r", js) - d = json.loads(js) + d = json.loads(str(js).decode("utf8")) start_time = update_counter("decode_json", start_time) logger.debug("Got internal_metadata: %r", internal_metadata) - internal_metadata = json.loads(internal_metadata) + internal_metadata = json.loads(str(internal_metadata).decode("utf8")) start_time = update_counter("decode_internal", start_time) ev = FrozenEvent(d, internal_metadata_dict=internal_metadata) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 514feebcb..3b3416716 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -179,7 +179,7 @@ class EventsStore(SQLBaseStore): ) txn.execute( sql, - (metadata_json.decode("UTF-8"), event.event_id,) + (buffer(metadata_json), event.event_id,) ) sql = ( @@ -224,14 +224,14 @@ class EventsStore(SQLBaseStore): values={ "event_id": event.event_id, "room_id": event.room_id, - "internal_metadata": metadata_json.decode("UTF-8"), - "json": encode_canonical_json(event_dict).decode("UTF-8"), + "internal_metadata": buffer(metadata_json), + "json": buffer(encode_canonical_json(event_dict)), }, ) - content = encode_canonical_json( + content = buffer(encode_canonical_json( event.content - ).decode("UTF-8") + )) vals = { "topological_ordering": event.depth, @@ -256,9 +256,9 @@ class EventsStore(SQLBaseStore): ] } - vals["unrecognized_keys"] = encode_canonical_json( + vals["unrecognized_keys"] = buffer(encode_canonical_json( unrec - ).decode("UTF-8") + )) sql = ( "INSERT INTO events" diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index 273e61281..00d803a5c 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -17,20 +17,22 @@ CREATE TABLE IF NOT EXISTS presence( state VARCHAR(20), status_msg VARCHAR(150), mtime BIGINT, -- miliseconds since last state change - UNIQUE(user_id) + UNIQUE (user_id) ) ; -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state CREATE TABLE IF NOT EXISTS presence_allow_inbound( observed_user_id VARCHAR(150) NOT NULL, - observer_user_id VARCHAR(150) -- a UserID, + observer_user_id VARCHAR(150) NOT NULL, -- a UserID, + UNIQUE (observed_user_id, observer_user_id) ) ; -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? CREATE TABLE IF NOT EXISTS presence_list( user_id VARCHAR(150) NOT NULL, - observed_user_id VARCHAR(150), -- a UserID, - accepted BOOLEAN + observed_user_id VARCHAR(150) NOT NULL, -- a UserID, + accepted BOOLEAN NOT NULL, + UNIQUE (user_id, observed_user_id) ) ; diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 0271de352..ba0f42d45 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -35,8 +35,8 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT NOT NULL, - UNIQUE (user, access_token, ip, user_agent) + last_seen BIGINT NOT NULL ) ; CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); +CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); From 8574bf62dc27ec3c2059c57a32c955e85d7e51df Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:09:36 +0100 Subject: [PATCH 051/173] Add index to presence table --- synapse/storage/schema/delta/15/presence_indices.sql | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 synapse/storage/schema/delta/15/presence_indices.sql diff --git a/synapse/storage/schema/delta/15/presence_indices.sql b/synapse/storage/schema/delta/15/presence_indices.sql new file mode 100644 index 000000000..6b8d0f1ca --- /dev/null +++ b/synapse/storage/schema/delta/15/presence_indices.sql @@ -0,0 +1,2 @@ + +CREATE INDEX IF NOT EXISTS presence_list_user_id ON presence_list (user_id); From 05a35d62b6c57d144cb0ca6b3c1071380d69b40c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:10:15 +0100 Subject: [PATCH 052/173] Bump database version --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index f8053484c..9b30cd6ad 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -51,7 +51,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 14 +SCHEMA_VERSION = 15 dir_path = os.path.abspath(os.path.dirname(__file__)) From 6bc9edd8b22309796470950a855fc43b88171bea Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:13:58 +0100 Subject: [PATCH 053/173] Fix prepare_sqlite3_database's convert_param_style --- synapse/storage/__init__.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 9b30cd6ad..e16414d73 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -442,8 +442,6 @@ def prepare_sqlite3_database(db_conn): new. This only affects sqlite databases since they were the only ones supported at the time. """ - import sqlite3 - with db_conn: schema_path = os.path.join( dir_path, "schema", "schema_version.sql", @@ -462,10 +460,7 @@ def prepare_sqlite3_database(db_conn): if row and row[0]: db_conn.execute( - _convert_param_style( - "REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)", - sqlite3 - ), + "REPLACE INTO schema_version (version, upgraded)" + " VALUES (?,?)", (row[0], False) ) From 0bfa78b39bf95ee24e78166c9545f59b34fd1d81 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:16:05 +0100 Subject: [PATCH 054/173] PEP8 --- synapse/app/homeserver.py | 7 ++----- synapse/storage/__init__.py | 4 ---- synapse/storage/registration.py | 1 - synapse/storage/roommember.py | 1 - synapse/storage/state.py | 1 - synapse/storage/transactions.py | 1 - 6 files changed, 2 insertions(+), 13 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 95190c1d7..f7c724c4b 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -17,9 +17,7 @@ import sys sys.dont_write_bytecode = True -from synapse.storage import ( - prepare_database, UpgradeDatabaseException, -) +from synapse.storage import UpgradeDatabaseException from synapse.storage.engines import create_engine from synapse.server import HomeServer @@ -60,7 +58,6 @@ import os import re import resource import subprocess -import sqlite3 import yaml @@ -322,7 +319,7 @@ def change_resource_limit(soft_file_no): resource.setrlimit(resource.RLIMIT_NOFILE, (soft_file_no, hard)) logger.info("Set file limit to: %d", soft_file_no) - except ( ValueError, resource.error) as e: + except (ValueError, resource.error) as e: logger.warn("Failed to set file limit: %s", e) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index e16414d73..231ec8169 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer - from .appservice import ApplicationServiceStore from .directory import DirectoryStore from .events import EventsStore @@ -43,8 +41,6 @@ import logging import os import re -import threading - logger = logging.getLogger(__name__) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index b62b4a341..0f9d898e5 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -105,7 +105,6 @@ class RegistrationStore(SQLBaseStore): defer.returnValue(user_info) - @cached() # TODO(paul): Currently there's no code to invalidate this cache. That # means if/when we ever add internal ways to invalidate access tokens or diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index df707f812..8ea5756d6 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -40,7 +40,6 @@ class RoomMemberStore(SQLBaseStore): """ try: target_user_id = event.state_key - domain = UserID.from_string(target_user_id).domain except: logger.exception( "Failed to parse target_user_id=%s", target_user_id diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 3e55cb81b..4994bacd6 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -159,4 +159,3 @@ class StateStore(SQLBaseStore): def _make_group_id(clock): return str(int(clock.time_msec())) + random_string(5) - diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 9594fe1f2..4c3dc5866 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -375,4 +375,3 @@ class DestinationsTable(object): "retry_last_ts", "retry_interval", ] - From 49d6aa1394bba42d52cae4013f7103d682756b28 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 15:28:37 +0100 Subject: [PATCH 055/173] Retry on deadlock --- synapse/storage/_base.py | 16 +++++++++++++--- synapse/storage/engines/maria.py | 5 +++++ synapse/storage/engines/sqlite3.py | 3 +++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 4ac61be89..c15cec0c7 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -160,9 +160,19 @@ class LoggingTransaction(object): start = time.time() * 1000 try: - return self.txn.execute( - sql, *args, **kwargs - ) + i = 0 + N = 5 + while True: + try: + return self.txn.execute( + sql, *args, **kwargs + ) + except self.database_engine.module.DatabaseError as e: + if self.database_engine.is_deadlock(e) and i < N: + i += 1 + logger.warn("[SQL DEADLOCK] {%s}", self.name) + continue + raise except Exception as e: logger.debug("[SQL FAIL] {%s} %s", self.name, e) raise diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py index 7fcb706a6..a279bfaf4 100644 --- a/synapse/storage/engines/maria.py +++ b/synapse/storage/engines/maria.py @@ -40,3 +40,8 @@ class MariaEngine(object): ) db_conn.commit() prepare_database(db_conn, self) + + def is_deadlock(self, error): + if isinstance(error, self.module.InternalError): + return error.sqlstate == 40001 and error.errno == 1213 + return False diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index e802b5d5f..72c11df46 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -32,3 +32,6 @@ class Sqlite3Engine(object): def prepare_database(self, db_conn): prepare_sqlite3_database(db_conn) prepare_database(db_conn, self) + + def is_deadlock(self, error): + return False From 24d8134ac135cbbd1f49b8cefacad431ed8f9d71 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Apr 2015 13:10:54 +0100 Subject: [PATCH 056/173] Fix maria engine to correctly recognize deadlocks --- synapse/storage/engines/maria.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py index a279bfaf4..230b32858 100644 --- a/synapse/storage/engines/maria.py +++ b/synapse/storage/engines/maria.py @@ -42,6 +42,6 @@ class MariaEngine(object): prepare_database(db_conn, self) def is_deadlock(self, error): - if isinstance(error, self.module.InternalError): - return error.sqlstate == 40001 and error.errno == 1213 + if isinstance(error, self.module.DatabaseError): + return error.sqlstate == "40001" and error.errno == 1213 return False From 9a05795619a4472460a791617efaee4ce1af110e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Apr 2015 13:11:28 +0100 Subject: [PATCH 057/173] Retry transaction, not SQL query --- synapse/storage/_base.py | 46 ++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 20fc1d0bb..fee713eb2 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -182,22 +182,12 @@ class LoggingTransaction(object): start = time.time() * 1000 try: - i = 0 - N = 5 - while True: - try: - return self.txn.execute( - sql, *args, **kwargs - ) - except self.database_engine.module.DatabaseError as e: - if self.database_engine.is_deadlock(e) and i < N: - i += 1 - logger.warn("[SQL DEADLOCK] {%s}", self.name) - continue - raise + return self.txn.execute( + sql, *args, **kwargs + ) except Exception as e: - logger.debug("[SQL FAIL] {%s} %s", self.name, e) - raise + logger.debug("[SQL FAIL] {%s} %s", self.name, e) + raise finally: msecs = (time.time() * 1000) - start sql_logger.debug("[SQL time] {%s} %f", self.name, msecs) @@ -347,7 +337,7 @@ class SQLBaseStore(object): start_time = time.time() * 1000 - def inner_func(txn, *args, **kwargs): + def inner_func(conn, *args, **kwargs): with LoggingContext("runInteraction") as context: current_context.copy_to(context) start = time.time() * 1000 @@ -362,10 +352,24 @@ class SQLBaseStore(object): sql_scheduling_timer.inc_by(time.time() * 1000 - start_time) transaction_logger.debug("[TXN START] {%s}", name) try: - return func( - LoggingTransaction(txn, name, self.database_engine), - *args, **kwargs - ) + i = 0 + N = 5 + while True: + try: + txn = conn.cursor() + return func( + LoggingTransaction(txn, name, self.database_engine), + *args, **kwargs + ) + except self.database_engine.module.DatabaseError as e: + logger.warn("[TXN DEADLOCK] {%s} %r, %r", name, e.errno, e.sqlstate) + if self.database_engine.is_deadlock(e): + logger.warn("[TXN DEADLOCK] {%s} %d/%d", name, i, N) + if i < N: + i += 1 + conn.rollback() + continue + raise except Exception as e: logger.debug("[TXN FAIL] {%s}", name, e) raise @@ -380,7 +384,7 @@ class SQLBaseStore(object): sql_txn_timer.inc_by(duration, desc) with PreserveLoggingContext(): - result = yield self._db_pool.runInteraction( + result = yield self._db_pool.runWithConnection( inner_func, *args, **kwargs ) defer.returnValue(result) From 279a547a8b7f7e9b9e8f4ed1ae25686824198107 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Apr 2015 16:53:48 +0100 Subject: [PATCH 058/173] Use generic db exceptions rather than sqlite3 specific ones --- synapse/storage/directory.py | 4 +--- synapse/storage/registration.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index cfb200570..2b2bdf861 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -21,8 +21,6 @@ from twisted.internet import defer from collections import namedtuple -import sqlite3 - RoomAliasMapping = namedtuple( "RoomAliasMapping", @@ -91,7 +89,7 @@ class DirectoryStore(SQLBaseStore): }, desc="create_room_alias_association", ) - except sqlite3.IntegrityError: + except self.database_engine.module.IntegrityError: raise SynapseError( 409, "Room alias %s already exists" % room_alias.to_string() ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 0f9d898e5..f7d829128 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -15,8 +15,6 @@ from twisted.internet import defer -from sqlite3 import IntegrityError - from synapse.api.errors import StoreError, Codes from ._base import SQLBaseStore, cached @@ -76,7 +74,7 @@ class RegistrationStore(SQLBaseStore): txn.execute("INSERT INTO users(name, password_hash, creation_ts) " "VALUES (?,?,?)", [user_id, password_hash, now]) - except IntegrityError: + except self.database_engine.module.IntegrityError: raise StoreError( 400, "User ID already taken.", errcode=Codes.USER_IN_USE ) From 22d7a593062eb06081d0cd1dc71e4fc79fc1cb85 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Apr 2015 16:57:14 +0100 Subject: [PATCH 059/173] Fix tests after commit 9a0579 --- tests/storage/test_base.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 5c17d3014..a64d2b821 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -33,12 +33,18 @@ class SQLBaseStoreTestCase(unittest.TestCase): def setUp(self): self.db_pool = Mock(spec=["runInteraction"]) self.mock_txn = Mock() + self.mock_conn = Mock(spec_set=["cursor"]) + self.mock_conn.cursor.return_value = self.mock_txn # Our fake runInteraction just runs synchronously inline def runInteraction(func, *args, **kwargs): return defer.succeed(func(self.mock_txn, *args, **kwargs)) self.db_pool.runInteraction = runInteraction + def runWithConnection(func, *args, **kwargs): + return defer.succeed(func(self.mock_conn, *args, **kwargs)) + self.db_pool.runWithConnection = runWithConnection + config = Mock() config.event_cache_size = 1 hs = HomeServer( From 8ad0f4912ed72daced74ae4d1c939ebdbc517476 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Apr 2015 11:41:36 +0100 Subject: [PATCH 060/173] Stream ordering and out of order insertions. Handle the fact that events can be persisted out of order, and so to get the "current max" stream token becomes non trivial - as we need to make sure that *all* stream tokens less than the current max have also successfully been persisted. --- synapse/storage/_base.py | 46 +--------- synapse/storage/events.py | 9 +- synapse/storage/stream.py | 23 +---- synapse/storage/util/__init__.py | 14 +++ synapse/storage/util/id_generators.py | 126 ++++++++++++++++++++++++++ 5 files changed, 153 insertions(+), 65 deletions(-) create mode 100644 synapse/storage/util/__init__.py create mode 100644 synapse/storage/util/id_generators.py diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 23289bbdd..badf9a5f4 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -22,6 +22,8 @@ from synapse.util.logcontext import PreserveLoggingContext, LoggingContext from synapse.util.lrucache import LruCache import synapse.metrics +from util.id_generators import IdGenerator, StreamIdGenerator + from twisted.internet import defer from collections import namedtuple, OrderedDict @@ -29,7 +31,6 @@ import functools import simplejson as json import sys import time -import threading logger = logging.getLogger(__name__) @@ -232,46 +233,6 @@ class PerformanceCounters(object): return top_n_counters -class IdGenerator(object): - def __init__(self, table, column, store): - self.table = table - self.column = column - self.store = store - self._lock = threading.Lock() - self._next_id = None - - @defer.inlineCallbacks - def get_next(self): - with self._lock: - if not self._next_id: - res = yield self.store._execute_and_decode( - "IdGenerator_%s" % (self.table,), - "SELECT MAX(%s) as mx FROM %s" % (self.column, self.table,) - ) - - self._next_id = (res and res[0] and res[0]["mx"]) or 1 - - i = self._next_id - self._next_id += 1 - defer.returnValue(i) - - def get_next_txn(self, txn): - with self._lock: - if self._next_id: - i = self._next_id - self._next_id += 1 - return i - else: - txn.execute( - "SELECT MAX(%s) FROM %s" % (self.column, self.table,) - ) - - val, = txn.fetchone() - self._next_id = val or 2 - - return 1 - - class SQLBaseStore(object): _TXN_ID = 0 @@ -297,7 +258,7 @@ class SQLBaseStore(object): # Pretend the getEventCache is just another named cache caches_by_name["*getEvent*"] = self._get_event_cache - self._stream_id_gen = IdGenerator("events", "stream_ordering", self) + self._stream_id_gen = StreamIdGenerator() self._transaction_id_gen = IdGenerator("sent_transactions", "id", self) self._state_groups_id_gen = IdGenerator("state_groups", "id", self) self._access_tokens_id_gen = IdGenerator("access_tokens", "id", self) @@ -363,7 +324,6 @@ class SQLBaseStore(object): *args, **kwargs ) except self.database_engine.module.DatabaseError as e: - logger.warn("[TXN DEADLOCK] {%s} %r, %r", name, e.errno, e.sqlstate) if self.database_engine.is_deadlock(e): logger.warn("[TXN DEADLOCK] {%s} %d/%d", name, i, N) if i < N: diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 3b3416716..f066484c7 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -52,7 +52,6 @@ class EventsStore(SQLBaseStore): is_new_state=is_new_state, current_state=current_state, ) - self.get_room_events_max_id.invalidate() except _RollbackButIsFineException: pass @@ -97,7 +96,13 @@ class EventsStore(SQLBaseStore): self._get_event_cache.pop(event.event_id) if stream_ordering is None: - stream_ordering = self._stream_id_gen.get_next_txn(txn) + with self._stream_id_gen.get_next_txn(txn) as stream_ordering: + return self._persist_event_txn( + txn, event, context, backfilled, + stream_ordering=stream_ordering, + is_new_state=is_new_state, + current_state=current_state, + ) # We purposefully do this first since if we include a `current_state` # key, we *want* to update the `current_state_events` table diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index e6bb5a807..9925f04bf 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -413,12 +413,10 @@ class StreamStore(SQLBaseStore): "get_recent_events_for_room", get_recent_events_for_room_txn ) - @cached(num_args=0) + @defer.inlineCallbacks def get_room_events_max_id(self): - return self.runInteraction( - "get_room_events_max_id", - self._get_room_events_max_id_txn - ) + token = yield self._stream_id_gen.get_max_token(self) + defer.returnValue("s%d" % (token,)) @defer.inlineCallbacks def _get_min_token(self): @@ -433,21 +431,6 @@ class StreamStore(SQLBaseStore): defer.returnValue(self.min_token) - def _get_room_events_max_id_txn(self, txn): - txn.execute( - "SELECT MAX(stream_ordering) as m FROM events" - ) - - res = self.cursor_to_dict(txn) - - logger.debug("get_room_events_max_id: %s", res) - - if not res or not res[0] or not res[0]["m"]: - return "s0" - - key = res[0]["m"] - return "s%d" % (key,) - @staticmethod def _set_before_and_after(events, rows): for event, row in zip(events, rows): diff --git a/synapse/storage/util/__init__.py b/synapse/storage/util/__init__.py new file mode 100644 index 000000000..c488b10d3 --- /dev/null +++ b/synapse/storage/util/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py new file mode 100644 index 000000000..8f419323a --- /dev/null +++ b/synapse/storage/util/id_generators.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from collections import deque +import contextlib +import threading + + +class IdGenerator(object): + def __init__(self, table, column, store): + self.table = table + self.column = column + self.store = store + self._lock = threading.Lock() + self._next_id = None + + @defer.inlineCallbacks + def get_next(self): + with self._lock: + if not self._next_id: + res = yield self.store._execute_and_decode( + "IdGenerator_%s" % (self.table,), + "SELECT MAX(%s) as mx FROM %s" % (self.column, self.table,) + ) + + self._next_id = (res and res[0] and res[0]["mx"]) or 1 + + i = self._next_id + self._next_id += 1 + defer.returnValue(i) + + def get_next_txn(self, txn): + with self._lock: + if self._next_id: + i = self._next_id + self._next_id += 1 + return i + else: + txn.execute( + "SELECT MAX(%s) FROM %s" % (self.column, self.table,) + ) + + val, = txn.fetchone() + self._next_id = val or 2 + + return 1 + + +class StreamIdGenerator(object): + """Used to generate new stream ids when persisting events while keeping + track of which transactions have been completed. + + This allows us to get the "current" stream id, i.e. the stream id such that + all ids less than or equal to it have completed. This handles the fact that + persistence of events can complete out of order. + + Usage: + with stream_id_gen.get_next_txn(txn) as stream_id: + # ... persist event ... + """ + def __init__(self): + self._lock = threading.Lock() + + self._current_max = None + self._unfinished_ids = deque() + + def get_next_txn(self, txn): + """ + Usage: + with stream_id_gen.get_next_txn(txn) as stream_id: + # ... persist event ... + """ + with self._lock: + if not self._current_max: + self._compute_current_max(txn) + + self._current_max += 1 + next_id = self._current_max + + self._unfinished_ids.append(next_id) + + @contextlib.contextmanager + def manager(): + yield next_id + with self._lock: + self._unfinished_ids.remove(next_id) + + return manager() + + def get_max_token(self, store): + """Returns the maximum stream id such that all stream ids less than or + equal to it have been successfully persisted. + """ + with self._lock: + if self._unfinished_ids: + return self._unfinished_ids[0] - 1 + + if not self._current_max: + return store.runInteraction( + "_compute_current_max", + self._compute_current_max, + ) + + return self._current_max + + def _compute_current_max(self, txn): + txn.execute("SELECT MAX(stream_ordering) FROM events") + val, = txn.fetchone() + + self._current_max = int(val) if val else 1 + + return self._current_max From 9707acfc40547277144438fd8f077bd3bcbf7001 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Apr 2015 13:45:20 +0100 Subject: [PATCH 061/173] Remove spurious spaces --- synapse/storage/schema/delta/12/v12.sql | 8 ++++---- synapse/storage/schema/delta/13/v13.sql | 4 ++-- synapse/storage/schema/delta/14/v14.sql | 2 +- .../schema/full_schemas/11/event_edges.sql | 14 +++++++------- .../full_schemas/11/event_signatures.sql | 8 ++++---- synapse/storage/schema/full_schemas/11/im.sql | 18 +++++++++--------- .../storage/schema/full_schemas/11/keys.sql | 4 ++-- .../full_schemas/11/media_repository.sql | 8 ++++---- .../schema/full_schemas/11/presence.sql | 6 +++--- .../schema/full_schemas/11/profiles.sql | 2 +- .../schema/full_schemas/11/redactions.sql | 2 +- .../schema/full_schemas/11/room_aliases.sql | 4 ++-- .../storage/schema/full_schemas/11/state.sql | 6 +++--- .../schema/full_schemas/11/transactions.sql | 8 ++++---- .../storage/schema/full_schemas/11/users.sql | 6 +++--- 15 files changed, 50 insertions(+), 50 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index 717d289f7..ed8a2f3f1 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS rejections( reason VARCHAR(150) NOT NULL, last_check VARCHAR(150) NOT NULL, UNIQUE (event_id) -) ; +); -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( @@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS pushers ( last_success BIGINT, failing_since BIGINT, UNIQUE (app_id, pushkey) -) ; +); CREATE TABLE IF NOT EXISTS push_rules ( id BIGINT PRIMARY KEY, @@ -48,7 +48,7 @@ CREATE TABLE IF NOT EXISTS push_rules ( conditions VARCHAR(150) NOT NULL, actions VARCHAR(150) NOT NULL, UNIQUE(user_name, rule_id) -) ; +); CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); @@ -56,7 +56,7 @@ CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(150), filter_id BIGINT, filter_json BLOB -) ; +); CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( user_id, filter_id diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index f5275a59b..ef6551f3a 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS application_services( hs_token VARCHAR(150), sender VARCHAR(150), UNIQUE(token) -) ; +); CREATE TABLE IF NOT EXISTS application_services_regex( id BIGINT PRIMARY KEY, @@ -28,4 +28,4 @@ CREATE TABLE IF NOT EXISTS application_services_regex( namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(150), FOREIGN KEY(as_id) REFERENCES application_services(id) -) ; +); diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index 1d582cc62..f1208c731 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -4,6 +4,6 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( rule_id VARCHAR(150) NOT NULL, enabled TINYINT, UNIQUE(user_name, rule_id) -) ; +); CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 124c9a9bd..602c11411 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, UNIQUE (event_id, room_id) -) ; +); CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); @@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, UNIQUE (event_id, room_id) -) ; +); CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_edges( room_id VARCHAR(150) NOT NULL, is_state BOOL NOT NULL, UNIQUE (event_id, prev_event_id, room_id, is_state) -) ; +); CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); @@ -49,7 +49,7 @@ CREATE TABLE IF NOT EXISTS room_depth( room_id VARCHAR(150) NOT NULL, min_depth INTEGER NOT NULL, UNIQUE (room_id) -) ; +); CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); @@ -59,7 +59,7 @@ create TABLE IF NOT EXISTS event_destinations( destination VARCHAR(150) NOT NULL, delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) -) ; +); CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); @@ -70,7 +70,7 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities( type VARCHAR(150) NOT NULL, state_key VARCHAR(150) NOT NULL, UNIQUE (event_id, room_id) -) ; +); CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( room_id, type, state_key @@ -83,7 +83,7 @@ CREATE TABLE IF NOT EXISTS event_auth( auth_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, UNIQUE (event_id, auth_id, room_id) -) ; +); CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index 30e3f71c5..0c27bc1f3 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( algorithm VARCHAR(150), hash BLOB, UNIQUE (event_id, algorithm) -) ; +); CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); @@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS event_reference_hashes ( algorithm VARCHAR(150), hash BLOB, UNIQUE (event_id, algorithm) -) ; +); CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_signatures ( key_id VARCHAR(150), signature BLOB, UNIQUE (event_id, signature_name, key_id) -) ; +); CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); @@ -50,6 +50,6 @@ CREATE TABLE IF NOT EXISTS event_edge_hashes( algorithm VARCHAR(150), hash BLOB, UNIQUE (event_id, prev_event_id, algorithm) -) ; +); CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index 7cb8f802e..363e2c842 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS events( outlier BOOL NOT NULL, depth BIGINT DEFAULT 0 NOT NULL, UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); @@ -38,7 +38,7 @@ CREATE TABLE IF NOT EXISTS event_json( internal_metadata BLOB NOT NULL, json BLOB NOT NULL, UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); @@ -50,7 +50,7 @@ CREATE TABLE IF NOT EXISTS state_events( state_key VARCHAR(150) NOT NULL, prev_state VARCHAR(150), UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); @@ -64,7 +64,7 @@ CREATE TABLE IF NOT EXISTS current_state_events( state_key VARCHAR(150) NOT NULL, UNIQUE (event_id), UNIQUE (room_id, type, state_key) -) ; +); CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); @@ -77,7 +77,7 @@ CREATE TABLE IF NOT EXISTS room_memberships( room_id VARCHAR(150) NOT NULL, membership VARCHAR(150) NOT NULL, UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); @@ -89,14 +89,14 @@ CREATE TABLE IF NOT EXISTS feedback( sender VARCHAR(150), room_id VARCHAR(150), UNIQUE (event_id) -) ; +); CREATE TABLE IF NOT EXISTS topics( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, topic VARCHAR(150) NOT NULL, UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); @@ -113,12 +113,12 @@ CREATE TABLE IF NOT EXISTS rooms( room_id VARCHAR(150) PRIMARY KEY NOT NULL, is_public BOOL, creator VARCHAR(150) -) ; +); CREATE TABLE IF NOT EXISTS room_hosts( room_id VARCHAR(150) NOT NULL, host VARCHAR(150) NOT NULL, UNIQUE (room_id, host) -) ; +); CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index 062ca53fe..0e80bf761 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -19,7 +19,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( ts_added_ms BIGINT, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) -) ; +); CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(150), -- Server name. @@ -28,4 +28,4 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( ts_added_ms BIGINT, -- When the key was added. verify_key BLOB, -- NACL verification key. UNIQUE (server_name, key_id) -) ; +); diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index c8c5f1d2f..08c42722d 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( upload_name VARCHAR(150), -- The name the media was uploaded with. user_id VARCHAR(150), -- The user who uploaded the file. UNIQUE (media_id) -) ; +); CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( media_id VARCHAR(150), -- The id used to refer to the media. @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type ) -) ; +); CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache ( media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(150), -- The name used to store the media on disk. UNIQUE (media_origin, media_id) -) ; +); CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( media_origin VARCHAR(150), -- The remote HS the media came from. @@ -62,7 +62,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type ) -) ; +); CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id ON remote_media_cache_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index 00d803a5c..f3f86e563 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS presence( status_msg VARCHAR(150), mtime BIGINT, -- miliseconds since last state change UNIQUE (user_id) -) ; +); -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS presence_allow_inbound( observed_user_id VARCHAR(150) NOT NULL, observer_user_id VARCHAR(150) NOT NULL, -- a UserID, UNIQUE (observed_user_id, observer_user_id) -) ; +); -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? @@ -35,4 +35,4 @@ CREATE TABLE IF NOT EXISTS presence_list( observed_user_id VARCHAR(150) NOT NULL, -- a UserID, accepted BOOLEAN NOT NULL, UNIQUE (user_id, observed_user_id) -) ; +); diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index 023060a54..21c58a99b 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -17,4 +17,4 @@ CREATE TABLE IF NOT EXISTS profiles( displayname VARCHAR(150), avatar_url VARCHAR(150), UNIQUE(user_id) -) ; +); diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index 5c23188d6..b81451eab 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS redactions ( event_id VARCHAR(150) NOT NULL, redacts VARCHAR(150) NOT NULL, UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index 63fe0f5c6..952cae35b 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -17,9 +17,9 @@ CREATE TABLE IF NOT EXISTS room_aliases( room_alias VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, UNIQUE (room_alias) -) ; +); CREATE TABLE IF NOT EXISTS room_alias_servers( room_alias VARCHAR(150) NOT NULL, server VARCHAR(150) NOT NULL -) ; +); diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index acfb76439..737c3e35c 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS state_groups( id VARCHAR(20) PRIMARY KEY, room_id VARCHAR(150) NOT NULL, event_id VARCHAR(150) NOT NULL -) ; +); CREATE TABLE IF NOT EXISTS state_groups_state( state_group VARCHAR(20) NOT NULL, @@ -25,13 +25,13 @@ CREATE TABLE IF NOT EXISTS state_groups_state( type VARCHAR(150) NOT NULL, state_key VARCHAR(150) NOT NULL, event_id VARCHAR(150) NOT NULL -) ; +); CREATE TABLE IF NOT EXISTS event_to_state_groups( event_id VARCHAR(150) NOT NULL, state_group VARCHAR(150) NOT NULL, UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 43541661c..c908109b6 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( response_json BLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) -) ; +); CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( response_code INTEGER DEFAULT 0, response_json BLOB, ts BIGINT -) ; +); CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); @@ -51,7 +51,7 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( pdu_id VARCHAR(150), pdu_origin VARCHAR(150), UNIQUE (transaction_id, destination) -) ; +); CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); @@ -60,4 +60,4 @@ CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(150) PRIMARY KEY, retry_last_ts BIGINT, retry_interval INTEGER -) ; +); diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index ba0f42d45..32dfc5b18 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS users( creation_ts BIGINT, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) -) ; +); CREATE TABLE IF NOT EXISTS access_tokens( id BIGINT PRIMARY KEY, @@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS access_tokens( token VARCHAR(150) NOT NULL, last_used BIGINT, UNIQUE(token) -) ; +); CREATE TABLE IF NOT EXISTS user_ips ( user VARCHAR(150) NOT NULL, @@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS user_ips ( ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, last_seen BIGINT NOT NULL -) ; +); CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); From 2ded3446201f833a69cdb7cf269c65e5f9de1f27 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Apr 2015 13:46:06 +0100 Subject: [PATCH 062/173] Remove unused import --- synapse/storage/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 9925f04bf..57c2e4dfe 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -35,7 +35,7 @@ what sort order was used: from twisted.internet import defer -from ._base import SQLBaseStore, cached +from ._base import SQLBaseStore from synapse.api.constants import EventTypes from synapse.api.errors import SynapseError from synapse.util.logutils import log_function From a1665c50949aaa257f3ee57fa02d1934b101642e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 10:05:44 +0100 Subject: [PATCH 063/173] Revert non-trivial schema changes and move them to a new schema version. --- synapse/storage/schema/full_schemas/11/im.sql | 19 ++- .../full_schemas/11/media_repository.sql | 3 - .../schema/full_schemas/11/presence.sql | 9 +- .../schema/full_schemas/11/profiles.sql | 3 +- .../schema/full_schemas/11/room_aliases.sql | 3 +- .../schema/full_schemas/11/transactions.sql | 4 +- .../storage/schema/full_schemas/11/users.sql | 5 +- .../schema/full_schemas/16/event_edges.sql | 89 +++++++++++++ .../full_schemas/16/event_signatures.sql | 55 ++++++++ synapse/storage/schema/full_schemas/16/im.sql | 124 ++++++++++++++++++ .../storage/schema/full_schemas/16/keys.sql | 31 +++++ .../full_schemas/16/media_repository.sql | 68 ++++++++++ .../schema/full_schemas/16/presence.sql | 38 ++++++ .../schema/full_schemas/16/profiles.sql | 20 +++ .../schema/full_schemas/16/redactions.sql | 22 ++++ .../schema/full_schemas/16/room_aliases.sql | 25 ++++ .../storage/schema/full_schemas/16/state.sql | 40 ++++++ .../schema/full_schemas/16/transactions.sql | 63 +++++++++ .../storage/schema/full_schemas/16/users.sql | 42 ++++++ 19 files changed, 636 insertions(+), 27 deletions(-) create mode 100644 synapse/storage/schema/full_schemas/16/event_edges.sql create mode 100644 synapse/storage/schema/full_schemas/16/event_signatures.sql create mode 100644 synapse/storage/schema/full_schemas/16/im.sql create mode 100644 synapse/storage/schema/full_schemas/16/keys.sql create mode 100644 synapse/storage/schema/full_schemas/16/media_repository.sql create mode 100644 synapse/storage/schema/full_schemas/16/presence.sql create mode 100644 synapse/storage/schema/full_schemas/16/profiles.sql create mode 100644 synapse/storage/schema/full_schemas/16/redactions.sql create mode 100644 synapse/storage/schema/full_schemas/16/room_aliases.sql create mode 100644 synapse/storage/schema/full_schemas/16/state.sql create mode 100644 synapse/storage/schema/full_schemas/16/transactions.sql create mode 100644 synapse/storage/schema/full_schemas/16/users.sql diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index 363e2c842..eac9c6af4 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering BIGINT PRIMARY KEY, + stream_ordering BIGINT PRIMARY KEY AUTOINCREMENT, topological_ordering BIGINT NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, @@ -62,10 +62,10 @@ CREATE TABLE IF NOT EXISTS current_state_events( room_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, state_key VARCHAR(150) NOT NULL, - UNIQUE (event_id), UNIQUE (room_id, type, state_key) ); +CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id); CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); @@ -75,10 +75,10 @@ CREATE TABLE IF NOT EXISTS room_memberships( user_id VARCHAR(150) NOT NULL, sender VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - membership VARCHAR(150) NOT NULL, - UNIQUE (event_id) + membership VARCHAR(150) NOT NULL ); +CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id); CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); @@ -87,26 +87,25 @@ CREATE TABLE IF NOT EXISTS feedback( feedback_type VARCHAR(150), target_event_id VARCHAR(150), sender VARCHAR(150), - room_id VARCHAR(150), - UNIQUE (event_id) + room_id VARCHAR(150) ); CREATE TABLE IF NOT EXISTS topics( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - topic VARCHAR(150) NOT NULL, - UNIQUE (event_id) + topic VARCHAR(150) NOT NULL ); +CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id); CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - name VARCHAR(150) NOT NULL, - UNIQUE (event_id) + name VARCHAR(150) NOT NULL ); +CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id); CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index 08c42722d..c6ce6cf69 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -63,6 +63,3 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( thumbnail_type ) ); - -CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id - ON remote_media_cache_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index f3f86e563..fce324b89 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -16,16 +16,14 @@ CREATE TABLE IF NOT EXISTS presence( user_id VARCHAR(150) NOT NULL, state VARCHAR(20), status_msg VARCHAR(150), - mtime BIGINT, -- miliseconds since last state change - UNIQUE (user_id) + mtime BIGINT -- miliseconds since last state change ); -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state CREATE TABLE IF NOT EXISTS presence_allow_inbound( observed_user_id VARCHAR(150) NOT NULL, - observer_user_id VARCHAR(150) NOT NULL, -- a UserID, - UNIQUE (observed_user_id, observer_user_id) + observer_user_id VARCHAR(150) NOT NULL -- a UserID, ); -- For each of /my/ users (watcher), which possibly-remote users are they @@ -33,6 +31,5 @@ CREATE TABLE IF NOT EXISTS presence_allow_inbound( CREATE TABLE IF NOT EXISTS presence_list( user_id VARCHAR(150) NOT NULL, observed_user_id VARCHAR(150) NOT NULL, -- a UserID, - accepted BOOLEAN NOT NULL, - UNIQUE (user_id, observed_user_id) + accepted BOOLEAN NOT NULL ); diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index 21c58a99b..ffe75edf9 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -15,6 +15,5 @@ CREATE TABLE IF NOT EXISTS profiles( user_id VARCHAR(150) NOT NULL, displayname VARCHAR(150), - avatar_url VARCHAR(150), - UNIQUE(user_id) + avatar_url VARCHAR(150) ); diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index 952cae35b..622691322 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -15,8 +15,7 @@ CREATE TABLE IF NOT EXISTS room_aliases( room_alias VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - UNIQUE (room_alias) + room_id VARCHAR(150) NOT NULL ); CREATE TABLE IF NOT EXISTS room_alias_servers( diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index c908109b6..c3d46763a 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -49,10 +49,10 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( transaction_id INTEGER, destination VARCHAR(150), pdu_id VARCHAR(150), - pdu_origin VARCHAR(150), - UNIQUE (transaction_id, destination) + pdu_origin VARCHAR(150) ); +CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination) CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 32dfc5b18..94c91e529 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -13,6 +13,7 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS users( + id BIGINT PRIMARY KEY AUTOINCREMENT, name VARCHAR(150), password_hash VARCHAR(150), creation_ts BIGINT, @@ -35,8 +36,8 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT NOT NULL + last_seen BIGINT NOT NULL, + UNIQUE (user, access_token, ip, user_agent) ); CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); -CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); diff --git a/synapse/storage/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/full_schemas/16/event_edges.sql new file mode 100644 index 000000000..602c11411 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/event_edges.sql @@ -0,0 +1,89 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS event_forward_extremities( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + UNIQUE (event_id, room_id) +); + +CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); +CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); + + +CREATE TABLE IF NOT EXISTS event_backward_extremities( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + UNIQUE (event_id, room_id) +); + +CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); +CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); + + +CREATE TABLE IF NOT EXISTS event_edges( + event_id VARCHAR(150) NOT NULL, + prev_event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + is_state BOOL NOT NULL, + UNIQUE (event_id, prev_event_id, room_id, is_state) +); + +CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); +CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); + + +CREATE TABLE IF NOT EXISTS room_depth( + room_id VARCHAR(150) NOT NULL, + min_depth INTEGER NOT NULL, + UNIQUE (room_id) +); + +CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); + + +create TABLE IF NOT EXISTS event_destinations( + event_id VARCHAR(150) NOT NULL, + destination VARCHAR(150) NOT NULL, + delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered + UNIQUE (event_id, destination) +); + +CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); + + +CREATE TABLE IF NOT EXISTS state_forward_extremities( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, + UNIQUE (event_id, room_id) +); + +CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( + room_id, type, state_key +); +CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); + + +CREATE TABLE IF NOT EXISTS event_auth( + event_id VARCHAR(150) NOT NULL, + auth_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + UNIQUE (event_id, auth_id, room_id) +); + +CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); +CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/full_schemas/16/event_signatures.sql new file mode 100644 index 000000000..0c27bc1f3 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/event_signatures.sql @@ -0,0 +1,55 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS event_content_hashes ( + event_id VARCHAR(150), + algorithm VARCHAR(150), + hash BLOB, + UNIQUE (event_id, algorithm) +); + +CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); + + +CREATE TABLE IF NOT EXISTS event_reference_hashes ( + event_id VARCHAR(150), + algorithm VARCHAR(150), + hash BLOB, + UNIQUE (event_id, algorithm) +); + +CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); + + +CREATE TABLE IF NOT EXISTS event_signatures ( + event_id VARCHAR(150), + signature_name VARCHAR(150), + key_id VARCHAR(150), + signature BLOB, + UNIQUE (event_id, signature_name, key_id) +); + +CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); + + +CREATE TABLE IF NOT EXISTS event_edge_hashes( + event_id VARCHAR(150), + prev_event_id VARCHAR(150), + algorithm VARCHAR(150), + hash BLOB, + UNIQUE (event_id, prev_event_id, algorithm) +); + +CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql new file mode 100644 index 000000000..363e2c842 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -0,0 +1,124 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS events( + stream_ordering BIGINT PRIMARY KEY, + topological_ordering BIGINT NOT NULL, + event_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + content BLOB NOT NULL, + unrecognized_keys BLOB, + processed BOOL NOT NULL, + outlier BOOL NOT NULL, + depth BIGINT DEFAULT 0 NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); +CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); +CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); + + +CREATE TABLE IF NOT EXISTS event_json( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + internal_metadata BLOB NOT NULL, + json BLOB NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); + + +CREATE TABLE IF NOT EXISTS state_events( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, + prev_state VARCHAR(150), + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); +CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); +CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); + + +CREATE TABLE IF NOT EXISTS current_state_events( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, + UNIQUE (event_id), + UNIQUE (room_id, type, state_key) +); + +CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); +CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); +CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); + +CREATE TABLE IF NOT EXISTS room_memberships( + event_id VARCHAR(150) NOT NULL, + user_id VARCHAR(150) NOT NULL, + sender VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + membership VARCHAR(150) NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); +CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); + +CREATE TABLE IF NOT EXISTS feedback( + event_id VARCHAR(150) NOT NULL, + feedback_type VARCHAR(150), + target_event_id VARCHAR(150), + sender VARCHAR(150), + room_id VARCHAR(150), + UNIQUE (event_id) +); + +CREATE TABLE IF NOT EXISTS topics( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + topic VARCHAR(150) NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); + +CREATE TABLE IF NOT EXISTS room_names( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + name VARCHAR(150) NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); + +CREATE TABLE IF NOT EXISTS rooms( + room_id VARCHAR(150) PRIMARY KEY NOT NULL, + is_public BOOL, + creator VARCHAR(150) +); + +CREATE TABLE IF NOT EXISTS room_hosts( + room_id VARCHAR(150) NOT NULL, + host VARCHAR(150) NOT NULL, + UNIQUE (room_id, host) +); + +CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql new file mode 100644 index 000000000..0e80bf761 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/keys.sql @@ -0,0 +1,31 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS server_tls_certificates( + server_name VARCHAR(150), -- Server name. + fingerprint VARCHAR(150), -- Certificate fingerprint. + from_server VARCHAR(150), -- Which key server the certificate was fetched from. + ts_added_ms BIGINT, -- When the certifcate was added. + tls_certificate BLOB, -- DER encoded x509 certificate. + UNIQUE (server_name, fingerprint) +); + +CREATE TABLE IF NOT EXISTS server_signature_keys( + server_name VARCHAR(150), -- Server name. + key_id VARCHAR(150), -- Key version. + from_server VARCHAR(150), -- Which key server the key was fetched form. + ts_added_ms BIGINT, -- When the key was added. + verify_key BLOB, -- NACL verification key. + UNIQUE (server_name, key_id) +); diff --git a/synapse/storage/schema/full_schemas/16/media_repository.sql b/synapse/storage/schema/full_schemas/16/media_repository.sql new file mode 100644 index 000000000..08c42722d --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/media_repository.sql @@ -0,0 +1,68 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS local_media_repository ( + media_id VARCHAR(150), -- The id used to refer to the media. + media_type VARCHAR(150), -- The MIME-type of the media. + media_length INTEGER, -- Length of the media in bytes. + created_ts BIGINT, -- When the content was uploaded in ms. + upload_name VARCHAR(150), -- The name the media was uploaded with. + user_id VARCHAR(150), -- The user who uploaded the file. + UNIQUE (media_id) +); + +CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( + media_id VARCHAR(150), -- The id used to refer to the media. + thumbnail_width INTEGER, -- The width of the thumbnail in pixels. + thumbnail_height INTEGER, -- The height of the thumbnail in pixels. + thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. + thumbnail_method VARCHAR(150), -- The method used to make the thumbnail. + thumbnail_length INTEGER, -- The length of the thumbnail in bytes. + UNIQUE ( + media_id, thumbnail_width, thumbnail_height, thumbnail_type + ) +); + +CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id + ON local_media_repository_thumbnails (media_id); + +CREATE TABLE IF NOT EXISTS remote_media_cache ( + media_origin VARCHAR(150), -- The remote HS the media came from. + media_id VARCHAR(150), -- The id used to refer to the media on that server. + media_type VARCHAR(150), -- The MIME-type of the media. + created_ts BIGINT, -- When the content was uploaded in ms. + upload_name VARCHAR(150), -- The name the media was uploaded with. + media_length INTEGER, -- Length of the media in bytes. + filesystem_id VARCHAR(150), -- The name used to store the media on disk. + UNIQUE (media_origin, media_id) +); + +CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( + media_origin VARCHAR(150), -- The remote HS the media came from. + media_id VARCHAR(150), -- The id used to refer to the media. + thumbnail_width INTEGER, -- The width of the thumbnail in pixels. + thumbnail_height INTEGER, -- The height of the thumbnail in pixels. + thumbnail_method VARCHAR(150), -- The method used to make the thumbnail + thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. + thumbnail_length INTEGER, -- The length of the thumbnail in bytes. + filesystem_id VARCHAR(150), -- The name used to store the media on disk. + UNIQUE ( + media_origin, media_id, thumbnail_width, thumbnail_height, + thumbnail_type + ) +); + +CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id + ON remote_media_cache_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql new file mode 100644 index 000000000..f3f86e563 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/presence.sql @@ -0,0 +1,38 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS presence( + user_id VARCHAR(150) NOT NULL, + state VARCHAR(20), + status_msg VARCHAR(150), + mtime BIGINT, -- miliseconds since last state change + UNIQUE (user_id) +); + +-- For each of /my/ users which possibly-remote users are allowed to see their +-- presence state +CREATE TABLE IF NOT EXISTS presence_allow_inbound( + observed_user_id VARCHAR(150) NOT NULL, + observer_user_id VARCHAR(150) NOT NULL, -- a UserID, + UNIQUE (observed_user_id, observer_user_id) +); + +-- For each of /my/ users (watcher), which possibly-remote users are they +-- watching? +CREATE TABLE IF NOT EXISTS presence_list( + user_id VARCHAR(150) NOT NULL, + observed_user_id VARCHAR(150) NOT NULL, -- a UserID, + accepted BOOLEAN NOT NULL, + UNIQUE (user_id, observed_user_id) +); diff --git a/synapse/storage/schema/full_schemas/16/profiles.sql b/synapse/storage/schema/full_schemas/16/profiles.sql new file mode 100644 index 000000000..21c58a99b --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/profiles.sql @@ -0,0 +1,20 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS profiles( + user_id VARCHAR(150) NOT NULL, + displayname VARCHAR(150), + avatar_url VARCHAR(150), + UNIQUE(user_id) +); diff --git a/synapse/storage/schema/full_schemas/16/redactions.sql b/synapse/storage/schema/full_schemas/16/redactions.sql new file mode 100644 index 000000000..b81451eab --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/redactions.sql @@ -0,0 +1,22 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS redactions ( + event_id VARCHAR(150) NOT NULL, + redacts VARCHAR(150) NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); +CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/full_schemas/16/room_aliases.sql b/synapse/storage/schema/full_schemas/16/room_aliases.sql new file mode 100644 index 000000000..952cae35b --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/room_aliases.sql @@ -0,0 +1,25 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS room_aliases( + room_alias VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + UNIQUE (room_alias) +); + +CREATE TABLE IF NOT EXISTS room_alias_servers( + room_alias VARCHAR(150) NOT NULL, + server VARCHAR(150) NOT NULL +); diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql new file mode 100644 index 000000000..737c3e35c --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/state.sql @@ -0,0 +1,40 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS state_groups( + id VARCHAR(20) PRIMARY KEY, + room_id VARCHAR(150) NOT NULL, + event_id VARCHAR(150) NOT NULL +); + +CREATE TABLE IF NOT EXISTS state_groups_state( + state_group VARCHAR(20) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, + event_id VARCHAR(150) NOT NULL +); + +CREATE TABLE IF NOT EXISTS event_to_state_groups( + event_id VARCHAR(150) NOT NULL, + state_group VARCHAR(150) NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); + +CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(state_group); +CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(room_id, type, state_key); +CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id); \ No newline at end of file diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql new file mode 100644 index 000000000..c908109b6 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/transactions.sql @@ -0,0 +1,63 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +-- Stores what transaction ids we have received and what our response was +CREATE TABLE IF NOT EXISTS received_transactions( + transaction_id VARCHAR(150), + origin VARCHAR(150), + ts BIGINT, + response_code INTEGER, + response_json BLOB, + has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx + UNIQUE (transaction_id, origin) +); + +CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; + + +-- Stores what transactions we've sent, what their response was (if we got one) and whether we have +-- since referenced the transaction in another outgoing transaction +CREATE TABLE IF NOT EXISTS sent_transactions( + id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering + transaction_id VARCHAR(150), + destination VARCHAR(150), + response_code INTEGER DEFAULT 0, + response_json BLOB, + ts BIGINT +); + +CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); +CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); +-- So that we can do an efficient look up of all transactions that have yet to be successfully +-- sent. +CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code); + + +-- For sent transactions only. +CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( + transaction_id INTEGER, + destination VARCHAR(150), + pdu_id VARCHAR(150), + pdu_origin VARCHAR(150), + UNIQUE (transaction_id, destination) +); + +CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); + +-- To track destination health +CREATE TABLE IF NOT EXISTS destinations( + destination VARCHAR(150) PRIMARY KEY, + retry_last_ts BIGINT, + retry_interval INTEGER +); diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql new file mode 100644 index 000000000..32dfc5b18 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/users.sql @@ -0,0 +1,42 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS users( + name VARCHAR(150), + password_hash VARCHAR(150), + creation_ts BIGINT, + admin BOOL DEFAULT 0 NOT NULL, + UNIQUE(name) +); + +CREATE TABLE IF NOT EXISTS access_tokens( + id BIGINT PRIMARY KEY, + user_id VARCHAR(150) NOT NULL, + device_id VARCHAR(150), + token VARCHAR(150) NOT NULL, + last_used BIGINT, + UNIQUE(token) +); + +CREATE TABLE IF NOT EXISTS user_ips ( + user VARCHAR(150) NOT NULL, + access_token VARCHAR(150) NOT NULL, + device_id VARCHAR(150), + ip VARCHAR(150) NOT NULL, + user_agent VARCHAR(150) NOT NULL, + last_seen BIGINT NOT NULL +); + +CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); +CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); From e2722f58ee98e4af7b54dfa230f63520b1ba8558 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 10:16:29 +0100 Subject: [PATCH 064/173] Fix schema again --- synapse/storage/schema/full_schemas/11/transactions.sql | 4 ++-- synapse/storage/schema/full_schemas/11/users.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index c3d46763a..093bb275b 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -29,7 +29,7 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin -- Stores what transactions we've sent, what their response was (if we got one) and whether we have -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( - id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering + id BIGINT PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, @@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( pdu_origin VARCHAR(150) ); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination) +CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination); CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 94c91e529..e9a9fc94f 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS users( ); CREATE TABLE IF NOT EXISTS access_tokens( - id BIGINT PRIMARY KEY, + id BIGINT PRIMARY KEY AUTOINCREMENT, user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, From cda4a6f93f695cf35eea334ba17e8694aeeab58c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 10:19:50 +0100 Subject: [PATCH 065/173] Revert non-trivial changes to upgrade scripts --- synapse/storage/schema/delta/12/v12.sql | 4 ++-- synapse/storage/schema/delta/13/v13.sql | 4 ++-- synapse/storage/schema/delta/14/v14.sql | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index ed8a2f3f1..f0d7e74b8 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS rejections( -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( - id BIGINT PRIMARY KEY, + id BIGINT PRIMARY KEY AUTOINCREMENT, user_name VARCHAR(150) NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, @@ -40,7 +40,7 @@ CREATE TABLE IF NOT EXISTS pushers ( ); CREATE TABLE IF NOT EXISTS push_rules ( - id BIGINT PRIMARY KEY, + id BIGINT PRIMARY KEY AUTOINCREMENT, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, priority_class TINYINT NOT NULL, diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index ef6551f3a..202adc720 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS application_services( - id BIGINT PRIMARY KEY, + id BIGINT PRIMARY KEY AUTOINCREMENT, url VARCHAR(150), token VARCHAR(150), hs_token VARCHAR(150), @@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS application_services( ); CREATE TABLE IF NOT EXISTS application_services_regex( - id BIGINT PRIMARY KEY, + id BIGINT PRIMARY KEY AUTOINCREMENT, as_id BIGINT NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(150), diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index f1208c731..db091fcd5 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -1,5 +1,5 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( - id BIGINT PRIMARY KEY, + id BIGINT PRIMARY KEY AUTOINCREMENT, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, enabled TINYINT, From d5d4281647246f14630bebba1820057db7d81c87 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 10:59:46 +0100 Subject: [PATCH 066/173] Update full_schemas/16 to match delta files. Add delta/16 scripts --- .../delta/16/remote_media_cache_index.sql | 2 + .../schema/delta/16/unique_constraints.sql | 64 ++++++++++++++++ synapse/storage/schema/delta/16/users.sql | 56 ++++++++++++++ .../full_schemas/16/application_services.sql | 48 ++++++++++++ .../schema/full_schemas/16/presence.sql | 2 + .../storage/schema/full_schemas/16/push.sql | 73 +++++++++++++++++++ 6 files changed, 245 insertions(+) create mode 100644 synapse/storage/schema/delta/16/remote_media_cache_index.sql create mode 100644 synapse/storage/schema/delta/16/unique_constraints.sql create mode 100644 synapse/storage/schema/delta/16/users.sql create mode 100644 synapse/storage/schema/full_schemas/16/application_services.sql create mode 100644 synapse/storage/schema/full_schemas/16/push.sql diff --git a/synapse/storage/schema/delta/16/remote_media_cache_index.sql b/synapse/storage/schema/delta/16/remote_media_cache_index.sql new file mode 100644 index 000000000..7a15265cb --- /dev/null +++ b/synapse/storage/schema/delta/16/remote_media_cache_index.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id + ON remote_media_cache_thumbnails (media_id); \ No newline at end of file diff --git a/synapse/storage/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/delta/16/unique_constraints.sql new file mode 100644 index 000000000..b2b677e51 --- /dev/null +++ b/synapse/storage/schema/delta/16/unique_constraints.sql @@ -0,0 +1,64 @@ + +-- We can use SQLite features here, since mysql support was only added in v16 + +-- +DELETE FROM current_state_events WHERE rowid not in ( + SELECT MIN(rowid) FROM current_state_events GROUP BY event_id +); + +CREATE UNIQUE INDEX current_state_events_event_id ON current_state_events(event_id); + +-- +DELETE FROM room_memberships WHERE rowid not in ( + SELECT MIN(rowid) FROM room_memberships GROUP BY event_id +); + +CREATE UNIQUE INDEX room_memberships_event_id ON room_memberships(event_id); + +-- +DELETE FROM feedback WHERE rowid not in ( + SELECT MIN(rowid) FROM feedback GROUP BY event_id +); + +CREATE UNIQUE INDEX feedback_event_id ON feedback(event_id); + +-- +DELETE FROM topics WHERE rowid not in ( + SELECT MIN(rowid) FROM topics GROUP BY event_id +); + +CREATE UNIQUE INDEX topics_event_id ON topics(event_id); + +-- +DELETE FROM room_names WHERE rowid not in ( + SELECT MIN(rowid) FROM room_names GROUP BY event_id +); + +CREATE UNIQUE INDEX room_names_id ON room_names(event_id); + +-- +DELETE FROM presence WHERE rowid not in ( + SELECT MIN(rowid) FROM presence GROUP BY user_id +); + +CREATE UNIQUE INDEX presence_id ON presence(user_id); + +-- +DELETE FROM presence_allow_inbound WHERE rowid not in ( + SELECT MIN(rowid) FROM presence_allow_inbound + GROUP BY observed_user_id, observer_user_id +); + +CREATE UNIQUE INDEX presence_allow_inbound_observers ON presence_allow_inbound( + observed_user_id, observer_user_id +); + +-- +DELETE FROM presence_list WHERE rowid not in ( + SELECT MIN(rowid) FROM presence_list + GROUP BY user_id, observed_user_id +); + +CREATE UNIQUE INDEX presence_list_observers ON presence_list( + user_id, observed_user_id +); diff --git a/synapse/storage/schema/delta/16/users.sql b/synapse/storage/schema/delta/16/users.sql new file mode 100644 index 000000000..1bbd1c0a1 --- /dev/null +++ b/synapse/storage/schema/delta/16/users.sql @@ -0,0 +1,56 @@ +-- Convert `access_tokens`.user from rowids to user strings. +-- MUST BE DONE BEFORE REMOVING ID COLUMN FROM USERS TABLE BELOW +CREATE TABLE IF NOT EXISTS new_access_tokens( + id BIGINT PRIMARY KEY, + user_id VARCHAR(150) NOT NULL, + device_id VARCHAR(150), + token VARCHAR(150) NOT NULL, + last_used BIGINT, + UNIQUE(token) +); + +INSERT INTO new_access_tokens + SELECT a.id, u.name, a.device_id, a.token, a.last_used + FROM access_tokens as a + INNER JOIN users as u ON u.id = a.user_id; + +DROP TABLE access_tokens; + +ALTER TABLE new_access_tokens RENAME TO access_tokens; + +-- Remove ID column from `users` table +CREATE TABLE IF NOT EXISTS new_users( + name VARCHAR(150), + password_hash VARCHAR(150), + creation_ts BIGINT, + admin BOOL DEFAULT 0 NOT NULL, + UNIQUE(name) +); + +INSERT INTO new_users SELECT name, password_hash, creation_ts, admin FROM users; + +DROP TABLE users; + +ALTER TABLE new_users RENAME TO users; + + +-- Remove UNIQUE constraint from `user_ips` table +CREATE TABLE IF NOT EXISTS new_user_ips ( + user VARCHAR(150) NOT NULL, + access_token VARCHAR(150) NOT NULL, + device_id VARCHAR(150), + ip VARCHAR(150) NOT NULL, + user_agent VARCHAR(150) NOT NULL, + last_seen BIGINT NOT NULL +); + +INSERT INTO new_user_ips + SELECT user, access_token, device_id, ip, user_agent, last_seen FROM user_ips; + +DROP TABLE user_ips; + +ALTER TABLE new_user_ips RENAME TO user_ips; + +CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); +CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); + diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql new file mode 100644 index 000000000..7a5c2510d --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/application_services.sql @@ -0,0 +1,48 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS application_services( + id BIGINT PRIMARY KEY, + url VARCHAR(150), + token VARCHAR(150), + hs_token VARCHAR(150), + sender VARCHAR(150), + UNIQUE(token) +); + +CREATE TABLE IF NOT EXISTS application_services_regex( + id BIGINT PRIMARY KEY, + as_id BIGINT NOT NULL, + namespace INTEGER, /* enum[room_id|room_alias|user_id] */ + regex VARCHAR(150), + FOREIGN KEY(as_id) REFERENCES application_services(id) +); + +CREATE TABLE IF NOT EXISTS application_services_state( + as_id VARCHAR(150) PRIMARY KEY, + state VARCHAR(5), + last_txn INTEGER +); + +CREATE TABLE IF NOT EXISTS application_services_txns( + as_id VARCHAR(150) NOT NULL, + txn_id INTEGER NOT NULL, + event_ids BLOB NOT NULL, + UNIQUE(as_id, txn_id) +); + +CREATE INDEX IF NOT EXISTS application_services_txns_id ON application_services_txns ( + as_id +); diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql index f3f86e563..15a6ad3c7 100644 --- a/synapse/storage/schema/full_schemas/16/presence.sql +++ b/synapse/storage/schema/full_schemas/16/presence.sql @@ -36,3 +36,5 @@ CREATE TABLE IF NOT EXISTS presence_list( accepted BOOLEAN NOT NULL, UNIQUE (user_id, observed_user_id) ); + +CREATE INDEX IF NOT EXISTS presence_list_user_id ON presence_list (user_id); diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql new file mode 100644 index 000000000..4c8faa921 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/push.sql @@ -0,0 +1,73 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS rejections( + event_id VARCHAR(150) NOT NULL, + reason VARCHAR(150) NOT NULL, + last_check VARCHAR(150) NOT NULL, + UNIQUE (event_id) +); + +-- Push notification endpoints that users have configured +CREATE TABLE IF NOT EXISTS pushers ( + id BIGINT PRIMARY KEY, + user_name VARCHAR(150) NOT NULL, + profile_tag VARCHAR(32) NOT NULL, + kind VARCHAR(8) NOT NULL, + app_id VARCHAR(64) NOT NULL, + app_display_name VARCHAR(64) NOT NULL, + device_display_name VARCHAR(128) NOT NULL, + pushkey VARBINARY(512) NOT NULL, + ts BIGINT NOT NULL, + lang VARCHAR(8), + data BLOB, + last_token TEXT, + last_success BIGINT, + failing_since BIGINT, + UNIQUE (app_id, pushkey) +); + +CREATE TABLE IF NOT EXISTS push_rules ( + id BIGINT PRIMARY KEY, + user_name VARCHAR(150) NOT NULL, + rule_id VARCHAR(150) NOT NULL, + priority_class TINYINT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + conditions VARCHAR(150) NOT NULL, + actions VARCHAR(150) NOT NULL, + UNIQUE(user_name, rule_id) +); + +CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); + +CREATE TABLE IF NOT EXISTS user_filters( + user_id VARCHAR(150), + filter_id BIGINT, + filter_json BLOB +); + +CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( + user_id, filter_id +); + +CREATE TABLE IF NOT EXISTS push_rules_enable ( + id BIGINT PRIMARY KEY, + user_name VARCHAR(150) NOT NULL, + rule_id VARCHAR(150) NOT NULL, + enabled TINYINT, + UNIQUE(user_name, rule_id) +); + +CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name); From 0f12772e320291408c8c7b475a391556db4c6d2b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 11:16:09 +0100 Subject: [PATCH 067/173] SQLite wants INTEGER and not BIGINT for primary keys --- synapse/storage/schema/delta/12/v12.sql | 4 ++-- synapse/storage/schema/delta/13/v13.sql | 4 ++-- synapse/storage/schema/delta/14/v14.sql | 2 +- synapse/storage/schema/full_schemas/11/im.sql | 2 +- synapse/storage/schema/full_schemas/11/transactions.sql | 2 +- synapse/storage/schema/full_schemas/11/users.sql | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index f0d7e74b8..4107cba71 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS rejections( -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, user_name VARCHAR(150) NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, @@ -40,7 +40,7 @@ CREATE TABLE IF NOT EXISTS pushers ( ); CREATE TABLE IF NOT EXISTS push_rules ( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, priority_class TINYINT NOT NULL, diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index 202adc720..17a139b86 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS application_services( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, url VARCHAR(150), token VARCHAR(150), hs_token VARCHAR(150), @@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS application_services( ); CREATE TABLE IF NOT EXISTS application_services_regex( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, as_id BIGINT NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(150), diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index db091fcd5..8c47d4b0f 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -1,5 +1,5 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, enabled TINYINT, diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index eac9c6af4..91ae9b9ee 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering BIGINT PRIMARY KEY AUTOINCREMENT, + stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, topological_ordering BIGINT NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 093bb275b..a9bf301ef 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -29,7 +29,7 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin -- Stores what transactions we've sent, what their response was (if we got one) and whether we have -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( - id BIGINT PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering + id INTEGER PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index e9a9fc94f..ad5e403f9 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -13,7 +13,7 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS users( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(150), password_hash VARCHAR(150), creation_ts BIGINT, @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS users( ); CREATE TABLE IF NOT EXISTS access_tokens( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, From 7e863c51e6a6a5b6f757cb279f5d7308db1432d4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 11:22:04 +0100 Subject: [PATCH 068/173] Use unsigned bigint --- synapse/storage/schema/delta/12/v12.sql | 8 ++++---- synapse/storage/schema/delta/13/v13.sql | 2 +- synapse/storage/schema/delta/16/users.sql | 8 ++++---- .../storage/schema/full_schemas/11/event_edges.sql | 2 +- synapse/storage/schema/full_schemas/11/im.sql | 4 ++-- synapse/storage/schema/full_schemas/11/keys.sql | 4 ++-- .../schema/full_schemas/11/media_repository.sql | 4 ++-- .../storage/schema/full_schemas/11/presence.sql | 2 +- .../schema/full_schemas/11/transactions.sql | 6 +++--- synapse/storage/schema/full_schemas/11/users.sql | 6 +++--- .../full_schemas/16/application_services.sql | 6 +++--- .../storage/schema/full_schemas/16/event_edges.sql | 2 +- synapse/storage/schema/full_schemas/16/im.sql | 6 +++--- synapse/storage/schema/full_schemas/16/keys.sql | 4 ++-- .../schema/full_schemas/16/media_repository.sql | 4 ++-- .../storage/schema/full_schemas/16/presence.sql | 2 +- synapse/storage/schema/full_schemas/16/push.sql | 14 +++++++------- .../schema/full_schemas/16/transactions.sql | 8 ++++---- synapse/storage/schema/full_schemas/16/users.sql | 8 ++++---- 19 files changed, 50 insertions(+), 50 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index 4107cba71..1247fd175 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -30,12 +30,12 @@ CREATE TABLE IF NOT EXISTS pushers ( app_display_name VARCHAR(64) NOT NULL, device_display_name VARCHAR(128) NOT NULL, pushkey VARBINARY(512) NOT NULL, - ts BIGINT NOT NULL, + ts BIGINT UNSIGNED NOT NULL, lang VARCHAR(8), data BLOB, last_token TEXT, - last_success BIGINT, - failing_since BIGINT, + last_success BIGINT UNSIGNED, + failing_since BIGINT UNSIGNED, UNIQUE (app_id, pushkey) ); @@ -54,7 +54,7 @@ CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(150), - filter_id BIGINT, + filter_id BIGINT UNSIGNED, filter_json BLOB ); diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index 17a139b86..d1da2b48e 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS application_services( CREATE TABLE IF NOT EXISTS application_services_regex( id INTEGER PRIMARY KEY AUTOINCREMENT, - as_id BIGINT NOT NULL, + as_id BIGINT UNSIGNED NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(150), FOREIGN KEY(as_id) REFERENCES application_services(id) diff --git a/synapse/storage/schema/delta/16/users.sql b/synapse/storage/schema/delta/16/users.sql index 1bbd1c0a1..f04705cd7 100644 --- a/synapse/storage/schema/delta/16/users.sql +++ b/synapse/storage/schema/delta/16/users.sql @@ -1,11 +1,11 @@ -- Convert `access_tokens`.user from rowids to user strings. -- MUST BE DONE BEFORE REMOVING ID COLUMN FROM USERS TABLE BELOW CREATE TABLE IF NOT EXISTS new_access_tokens( - id BIGINT PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, - last_used BIGINT, + last_used BIGINT UNSIGNED, UNIQUE(token) ); @@ -22,7 +22,7 @@ ALTER TABLE new_access_tokens RENAME TO access_tokens; CREATE TABLE IF NOT EXISTS new_users( name VARCHAR(150), password_hash VARCHAR(150), - creation_ts BIGINT, + creation_ts BIGINT UNSIGNED, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) ); @@ -41,7 +41,7 @@ CREATE TABLE IF NOT EXISTS new_user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT NOT NULL + last_seen BIGINT UNSIGNED NOT NULL ); INSERT INTO new_user_ips diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 602c11411..bdb110909 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -57,7 +57,7 @@ CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( event_id VARCHAR(150) NOT NULL, destination VARCHAR(150) NOT NULL, - delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered + delivered_ts BIGINT UNSIGNED DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index 91ae9b9ee..e75093a44 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS events( stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, - topological_ordering BIGINT NOT NULL, + topological_ordering BIGINT UNSIGNED NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, @@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS events( unrecognized_keys BLOB, processed BOOL NOT NULL, outlier BOOL NOT NULL, - depth BIGINT DEFAULT 0 NOT NULL, + depth BIGINT UNSIGNED DEFAULT 0 NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index 0e80bf761..551650cf1 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( server_name VARCHAR(150), -- Server name. fingerprint VARCHAR(150), -- Certificate fingerprint. from_server VARCHAR(150), -- Which key server the certificate was fetched from. - ts_added_ms BIGINT, -- When the certifcate was added. + ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(150), -- Server name. key_id VARCHAR(150), -- Key version. from_server VARCHAR(150), -- Which key server the key was fetched form. - ts_added_ms BIGINT, -- When the key was added. + ts_added_ms BIGINT UNSIGNED, -- When the key was added. verify_key BLOB, -- NACL verification key. UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index c6ce6cf69..134e7fbce 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( media_id VARCHAR(150), -- The id used to refer to the media. media_type VARCHAR(150), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. - created_ts BIGINT, -- When the content was uploaded in ms. + created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. user_id VARCHAR(150), -- The user who uploaded the file. UNIQUE (media_id) @@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache ( media_origin VARCHAR(150), -- The remote HS the media came from. media_id VARCHAR(150), -- The id used to refer to the media on that server. media_type VARCHAR(150), -- The MIME-type of the media. - created_ts BIGINT, -- When the content was uploaded in ms. + created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(150), -- The name used to store the media on disk. diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index fce324b89..c617ebea7 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS presence( user_id VARCHAR(150) NOT NULL, state VARCHAR(20), status_msg VARCHAR(150), - mtime BIGINT -- miliseconds since last state change + mtime BIGINT UNSIGNED -- miliseconds since last state change ); -- For each of /my/ users which possibly-remote users are allowed to see their diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index a9bf301ef..67c79db3e 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( transaction_id VARCHAR(150), origin VARCHAR(150), - ts BIGINT, + ts BIGINT UNSIGNED, response_code INTEGER, response_json BLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx @@ -34,7 +34,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( destination VARCHAR(150), response_code INTEGER DEFAULT 0, response_json BLOB, - ts BIGINT + ts BIGINT UNSIGNED ); CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); @@ -58,6 +58,6 @@ CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(d -- To track destination health CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(150) PRIMARY KEY, - retry_last_ts BIGINT, + retry_last_ts BIGINT UNSIGNED, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index ad5e403f9..0ddfccd41 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS users( id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(150), password_hash VARCHAR(150), - creation_ts BIGINT, + creation_ts BIGINT UNSIGNED, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) ); @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS access_tokens( user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, - last_used BIGINT, + last_used BIGINT UNSIGNED, UNIQUE(token) ); @@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT NOT NULL, + last_seen BIGINT UNSIGNED NOT NULL, UNIQUE (user, access_token, ip, user_agent) ); diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql index 7a5c2510d..fbc3e08a1 100644 --- a/synapse/storage/schema/full_schemas/16/application_services.sql +++ b/synapse/storage/schema/full_schemas/16/application_services.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS application_services( - id BIGINT PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, url VARCHAR(150), token VARCHAR(150), hs_token VARCHAR(150), @@ -23,8 +23,8 @@ CREATE TABLE IF NOT EXISTS application_services( ); CREATE TABLE IF NOT EXISTS application_services_regex( - id BIGINT PRIMARY KEY, - as_id BIGINT NOT NULL, + id BIGINT UNSIGNED PRIMARY KEY, + as_id BIGINT UNSIGNED NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(150), FOREIGN KEY(as_id) REFERENCES application_services(id) diff --git a/synapse/storage/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/full_schemas/16/event_edges.sql index 602c11411..bdb110909 100644 --- a/synapse/storage/schema/full_schemas/16/event_edges.sql +++ b/synapse/storage/schema/full_schemas/16/event_edges.sql @@ -57,7 +57,7 @@ CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( event_id VARCHAR(150) NOT NULL, destination VARCHAR(150) NOT NULL, - delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered + delivered_ts BIGINT UNSIGNED DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index 363e2c842..81dae2a91 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering BIGINT PRIMARY KEY, - topological_ordering BIGINT NOT NULL, + stream_ordering BIGINT UNSIGNED PRIMARY KEY, + topological_ordering BIGINT UNSIGNED NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, @@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS events( unrecognized_keys BLOB, processed BOOL NOT NULL, outlier BOOL NOT NULL, - depth BIGINT DEFAULT 0 NOT NULL, + depth BIGINT UNSIGNED DEFAULT 0 NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql index 0e80bf761..551650cf1 100644 --- a/synapse/storage/schema/full_schemas/16/keys.sql +++ b/synapse/storage/schema/full_schemas/16/keys.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( server_name VARCHAR(150), -- Server name. fingerprint VARCHAR(150), -- Certificate fingerprint. from_server VARCHAR(150), -- Which key server the certificate was fetched from. - ts_added_ms BIGINT, -- When the certifcate was added. + ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(150), -- Server name. key_id VARCHAR(150), -- Key version. from_server VARCHAR(150), -- Which key server the key was fetched form. - ts_added_ms BIGINT, -- When the key was added. + ts_added_ms BIGINT UNSIGNED, -- When the key was added. verify_key BLOB, -- NACL verification key. UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/16/media_repository.sql b/synapse/storage/schema/full_schemas/16/media_repository.sql index 08c42722d..014bce4ae 100644 --- a/synapse/storage/schema/full_schemas/16/media_repository.sql +++ b/synapse/storage/schema/full_schemas/16/media_repository.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( media_id VARCHAR(150), -- The id used to refer to the media. media_type VARCHAR(150), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. - created_ts BIGINT, -- When the content was uploaded in ms. + created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. user_id VARCHAR(150), -- The user who uploaded the file. UNIQUE (media_id) @@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache ( media_origin VARCHAR(150), -- The remote HS the media came from. media_id VARCHAR(150), -- The id used to refer to the media on that server. media_type VARCHAR(150), -- The MIME-type of the media. - created_ts BIGINT, -- When the content was uploaded in ms. + created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(150), -- The name used to store the media on disk. diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql index 15a6ad3c7..fbe5b0af6 100644 --- a/synapse/storage/schema/full_schemas/16/presence.sql +++ b/synapse/storage/schema/full_schemas/16/presence.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS presence( user_id VARCHAR(150) NOT NULL, state VARCHAR(20), status_msg VARCHAR(150), - mtime BIGINT, -- miliseconds since last state change + mtime BIGINT UNSIGNED, -- miliseconds since last state change UNIQUE (user_id) ); diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql index 4c8faa921..99175b963 100644 --- a/synapse/storage/schema/full_schemas/16/push.sql +++ b/synapse/storage/schema/full_schemas/16/push.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS rejections( -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( - id BIGINT PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, user_name VARCHAR(150) NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, @@ -30,17 +30,17 @@ CREATE TABLE IF NOT EXISTS pushers ( app_display_name VARCHAR(64) NOT NULL, device_display_name VARCHAR(128) NOT NULL, pushkey VARBINARY(512) NOT NULL, - ts BIGINT NOT NULL, + ts BIGINT UNSIGNED NOT NULL, lang VARCHAR(8), data BLOB, last_token TEXT, - last_success BIGINT, - failing_since BIGINT, + last_success BIGINT UNSIGNED, + failing_since BIGINT UNSIGNED, UNIQUE (app_id, pushkey) ); CREATE TABLE IF NOT EXISTS push_rules ( - id BIGINT PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, priority_class TINYINT NOT NULL, @@ -54,7 +54,7 @@ CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(150), - filter_id BIGINT, + filter_id BIGINT UNSIGNED, filter_json BLOB ); @@ -63,7 +63,7 @@ CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( ); CREATE TABLE IF NOT EXISTS push_rules_enable ( - id BIGINT PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, enabled TINYINT, diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql index c908109b6..e48eb4414 100644 --- a/synapse/storage/schema/full_schemas/16/transactions.sql +++ b/synapse/storage/schema/full_schemas/16/transactions.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( transaction_id VARCHAR(150), origin VARCHAR(150), - ts BIGINT, + ts BIGINT UNSIGNED, response_code INTEGER, response_json BLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx @@ -29,12 +29,12 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin -- Stores what transactions we've sent, what their response was (if we got one) and whether we have -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( - id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering + id BIGINT UNSIGNED PRIMARY KEY, -- This is used to apply insertion ordering transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, response_json BLOB, - ts BIGINT + ts BIGINT UNSIGNED ); CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); @@ -58,6 +58,6 @@ CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(d -- To track destination health CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(150) PRIMARY KEY, - retry_last_ts BIGINT, + retry_last_ts BIGINT UNSIGNED, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql index 32dfc5b18..d0011c04b 100644 --- a/synapse/storage/schema/full_schemas/16/users.sql +++ b/synapse/storage/schema/full_schemas/16/users.sql @@ -15,17 +15,17 @@ CREATE TABLE IF NOT EXISTS users( name VARCHAR(150), password_hash VARCHAR(150), - creation_ts BIGINT, + creation_ts BIGINT UNSIGNED, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) ); CREATE TABLE IF NOT EXISTS access_tokens( - id BIGINT PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, - last_used BIGINT, + last_used BIGINT UNSIGNED, UNIQUE(token) ); @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT NOT NULL + last_seen BIGINT UNSIGNED NOT NULL ); CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); From ce797ad3738afdef681bdca48e9cabe0baa7517b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 11:22:30 +0100 Subject: [PATCH 069/173] Bump schema version --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 87db382fb..b46cafd25 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -51,7 +51,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 15 +SCHEMA_VERSION = 16 dir_path = os.path.abspath(os.path.dirname(__file__)) From 7ed2ec3061b12ad79f3bd07da8758c5e41af8e5e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 13:41:54 +0100 Subject: [PATCH 070/173] Handle the fact that in sqlite binary data might be stored as unicode or bytes --- synapse/storage/_base.py | 7 +++++-- synapse/storage/engines/maria.py | 5 ++++- synapse/storage/engines/sqlite3.py | 7 +++++++ synapse/storage/profile.py | 2 +- synapse/storage/registration.py | 4 +++- 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index c1bf98cdc..e30514cd5 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -812,6 +812,9 @@ class SQLBaseStore(object): internal_metadata, js, redacted, rejected_reason = res + internal_metadata = self.database_engine.load_unicode(internal_metadata) + js = self.database_engine.load_unicode(js) + start_time = update_counter("select_event", start_time) result = self._get_event_from_row_txn( @@ -839,11 +842,11 @@ class SQLBaseStore(object): return curr_time logger.debug("Got js: %r", js) - d = json.loads(str(js).decode("utf8")) + d = json.loads(js) start_time = update_counter("decode_json", start_time) logger.debug("Got internal_metadata: %r", internal_metadata) - internal_metadata = json.loads(str(internal_metadata).decode("utf8")) + internal_metadata = json.loads(internal_metadata) start_time = update_counter("decode_internal", start_time) ev = FrozenEvent( diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py index 230b32858..c93437693 100644 --- a/synapse/storage/engines/maria.py +++ b/synapse/storage/engines/maria.py @@ -27,7 +27,7 @@ class MariaEngine(object): def encode_parameter(self, param): if isinstance(param, types.BufferType): - return str(param) + return bytes(param) return param def on_new_connection(self, db_conn): @@ -45,3 +45,6 @@ class MariaEngine(object): if isinstance(error, self.module.DatabaseError): return error.sqlstate == "40001" and error.errno == 1213 return False + + def load_unicode(self, v): + return bytes(v).decode("UTF8") diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 72c11df46..389df35eb 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -15,6 +15,8 @@ from synapse.storage import prepare_database, prepare_sqlite3_database +import types + class Sqlite3Engine(object): def __init__(self, database_module): @@ -35,3 +37,8 @@ class Sqlite3Engine(object): def is_deadlock(self, error): return False + + def load_unicode(self, v): + if isinstance(v, types.UnicodeType): + return v + return bytes(v).decode("UTF8") diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py index 09778045b..e33963d0b 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py @@ -36,7 +36,7 @@ class ProfileStore(SQLBaseStore): ) if name: - name = name.decode("utf8") + name = self.database_engine.load_unicode(name) defer.returnValue(name) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index f7d829128..86fd2600f 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -99,7 +99,9 @@ class RegistrationStore(SQLBaseStore): ) if user_info: - user_info["password_hash"] = user_info["password_hash"].decode("utf8") + user_info["password_hash"] = self.database_engine.load_unicode( + user_info["password_hash"] + ) defer.returnValue(user_info) From 4d5b0986261db7e9a4389c6fe2eb4808e5946e91 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 18:45:09 +0100 Subject: [PATCH 071/173] Use LONGBLOB and TEXT for arbitary length rows --- synapse/storage/schema/delta/12/v12.sql | 4 ++-- synapse/storage/schema/delta/15/appservice_txns.sql | 2 +- .../schema/full_schemas/11/event_signatures.sql | 8 ++++---- synapse/storage/schema/full_schemas/11/im.sql | 12 ++++++------ synapse/storage/schema/full_schemas/11/keys.sql | 4 ++-- .../storage/schema/full_schemas/11/transactions.sql | 4 ++-- .../schema/full_schemas/16/application_services.sql | 2 +- .../schema/full_schemas/16/event_signatures.sql | 8 ++++---- synapse/storage/schema/full_schemas/16/im.sql | 12 ++++++------ synapse/storage/schema/full_schemas/16/keys.sql | 4 ++-- synapse/storage/schema/full_schemas/16/push.sql | 4 ++-- .../storage/schema/full_schemas/16/transactions.sql | 4 ++-- 12 files changed, 34 insertions(+), 34 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index 1247fd175..a246943f5 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -32,7 +32,7 @@ CREATE TABLE IF NOT EXISTS pushers ( pushkey VARBINARY(512) NOT NULL, ts BIGINT UNSIGNED NOT NULL, lang VARCHAR(8), - data BLOB, + data LONGBLOB, last_token TEXT, last_success BIGINT UNSIGNED, failing_since BIGINT UNSIGNED, @@ -55,7 +55,7 @@ CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(150), filter_id BIGINT UNSIGNED, - filter_json BLOB + filter_json LONGBLOB ); CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( diff --git a/synapse/storage/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/delta/15/appservice_txns.sql index 2f4c3eae5..ddea8fc69 100644 --- a/synapse/storage/schema/delta/15/appservice_txns.sql +++ b/synapse/storage/schema/delta/15/appservice_txns.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS application_services_state( CREATE TABLE IF NOT EXISTS application_services_txns( as_id VARCHAR(150) NOT NULL, txn_id INTEGER NOT NULL, - event_ids BLOB NOT NULL, + event_ids LONGBLOB NOT NULL, UNIQUE(as_id, txn_id) ); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index 0c27bc1f3..09886f607 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash BLOB, + hash LONGBLOB, UNIQUE (event_id, algorithm) ); @@ -26,7 +26,7 @@ CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event CREATE TABLE IF NOT EXISTS event_reference_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash BLOB, + hash LONGBLOB, UNIQUE (event_id, algorithm) ); @@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS event_signatures ( event_id VARCHAR(150), signature_name VARCHAR(150), key_id VARCHAR(150), - signature BLOB, + signature LONGBLOB, UNIQUE (event_id, signature_name, key_id) ); @@ -48,7 +48,7 @@ CREATE TABLE IF NOT EXISTS event_edge_hashes( event_id VARCHAR(150), prev_event_id VARCHAR(150), algorithm VARCHAR(150), - hash BLOB, + hash LONGBLOB, UNIQUE (event_id, prev_event_id, algorithm) ); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index e75093a44..9c47f5174 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS events( event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - content BLOB NOT NULL, - unrecognized_keys BLOB, + content LONGBLOB NOT NULL, + unrecognized_keys LONGBLOB, processed BOOL NOT NULL, outlier BOOL NOT NULL, depth BIGINT UNSIGNED DEFAULT 0 NOT NULL, @@ -35,8 +35,8 @@ CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - internal_metadata BLOB NOT NULL, - json BLOB NOT NULL, + internal_metadata LONGBLOB NOT NULL, + json LONGBLOB NOT NULL, UNIQUE (event_id) ); @@ -93,7 +93,7 @@ CREATE TABLE IF NOT EXISTS feedback( CREATE TABLE IF NOT EXISTS topics( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - topic VARCHAR(150) NOT NULL + topic TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id); @@ -102,7 +102,7 @@ CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - name VARCHAR(150) NOT NULL + name TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index 551650cf1..35f141c28 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( fingerprint VARCHAR(150), -- Certificate fingerprint. from_server VARCHAR(150), -- Which key server the certificate was fetched from. ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added. - tls_certificate BLOB, -- DER encoded x509 certificate. + tls_certificate LONGBLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); @@ -26,6 +26,6 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( key_id VARCHAR(150), -- Key version. from_server VARCHAR(150), -- Which key server the key was fetched form. ts_added_ms BIGINT UNSIGNED, -- When the key was added. - verify_key BLOB, -- NACL verification key. + verify_key LONGBLOB, -- NACL verification key. UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 67c79db3e..c2fab10aa 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( origin VARCHAR(150), ts BIGINT UNSIGNED, response_code INTEGER, - response_json BLOB, + response_json LONGBLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) ); @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, - response_json BLOB, + response_json LONGBLOB, ts BIGINT UNSIGNED ); diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql index fbc3e08a1..bc709df92 100644 --- a/synapse/storage/schema/full_schemas/16/application_services.sql +++ b/synapse/storage/schema/full_schemas/16/application_services.sql @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS application_services_state( CREATE TABLE IF NOT EXISTS application_services_txns( as_id VARCHAR(150) NOT NULL, txn_id INTEGER NOT NULL, - event_ids BLOB NOT NULL, + event_ids LONGBLOB NOT NULL, UNIQUE(as_id, txn_id) ); diff --git a/synapse/storage/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/full_schemas/16/event_signatures.sql index 0c27bc1f3..09886f607 100644 --- a/synapse/storage/schema/full_schemas/16/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/16/event_signatures.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash BLOB, + hash LONGBLOB, UNIQUE (event_id, algorithm) ); @@ -26,7 +26,7 @@ CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event CREATE TABLE IF NOT EXISTS event_reference_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash BLOB, + hash LONGBLOB, UNIQUE (event_id, algorithm) ); @@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS event_signatures ( event_id VARCHAR(150), signature_name VARCHAR(150), key_id VARCHAR(150), - signature BLOB, + signature LONGBLOB, UNIQUE (event_id, signature_name, key_id) ); @@ -48,7 +48,7 @@ CREATE TABLE IF NOT EXISTS event_edge_hashes( event_id VARCHAR(150), prev_event_id VARCHAR(150), algorithm VARCHAR(150), - hash BLOB, + hash LONGBLOB, UNIQUE (event_id, prev_event_id, algorithm) ); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index 81dae2a91..19f0f3414 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS events( event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - content BLOB NOT NULL, - unrecognized_keys BLOB, + content LONGBLOB NOT NULL, + unrecognized_keys LONGBLOB, processed BOOL NOT NULL, outlier BOOL NOT NULL, depth BIGINT UNSIGNED DEFAULT 0 NOT NULL, @@ -35,8 +35,8 @@ CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - internal_metadata BLOB NOT NULL, - json BLOB NOT NULL, + internal_metadata LONGBLOB NOT NULL, + json LONGBLOB NOT NULL, UNIQUE (event_id) ); @@ -94,7 +94,7 @@ CREATE TABLE IF NOT EXISTS feedback( CREATE TABLE IF NOT EXISTS topics( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - topic VARCHAR(150) NOT NULL, + topic TEXT NOT NULL, UNIQUE (event_id) ); @@ -103,7 +103,7 @@ CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - name VARCHAR(150) NOT NULL, + name TEXT NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql index 551650cf1..35f141c28 100644 --- a/synapse/storage/schema/full_schemas/16/keys.sql +++ b/synapse/storage/schema/full_schemas/16/keys.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( fingerprint VARCHAR(150), -- Certificate fingerprint. from_server VARCHAR(150), -- Which key server the certificate was fetched from. ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added. - tls_certificate BLOB, -- DER encoded x509 certificate. + tls_certificate LONGBLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); @@ -26,6 +26,6 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( key_id VARCHAR(150), -- Key version. from_server VARCHAR(150), -- Which key server the key was fetched form. ts_added_ms BIGINT UNSIGNED, -- When the key was added. - verify_key BLOB, -- NACL verification key. + verify_key LONGBLOB, -- NACL verification key. UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql index 99175b963..33300736f 100644 --- a/synapse/storage/schema/full_schemas/16/push.sql +++ b/synapse/storage/schema/full_schemas/16/push.sql @@ -32,7 +32,7 @@ CREATE TABLE IF NOT EXISTS pushers ( pushkey VARBINARY(512) NOT NULL, ts BIGINT UNSIGNED NOT NULL, lang VARCHAR(8), - data BLOB, + data LONGBLOB, last_token TEXT, last_success BIGINT UNSIGNED, failing_since BIGINT UNSIGNED, @@ -55,7 +55,7 @@ CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(150), filter_id BIGINT UNSIGNED, - filter_json BLOB + filter_json LONGBLOB ); CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql index e48eb4414..f381e6760 100644 --- a/synapse/storage/schema/full_schemas/16/transactions.sql +++ b/synapse/storage/schema/full_schemas/16/transactions.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( origin VARCHAR(150), ts BIGINT UNSIGNED, response_code INTEGER, - response_json BLOB, + response_json LONGBLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) ); @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, - response_json BLOB, + response_json LONGBLOB, ts BIGINT UNSIGNED ); From 4103b1c4709b374785ffb7eca75c8f338218bc17 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 18:46:09 +0100 Subject: [PATCH 072/173] DROP indexes before recreating them --- synapse/storage/schema/delta/16/unique_constraints.sql | 8 ++++++++ synapse/storage/schema/full_schemas/16/state.sql | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/synapse/storage/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/delta/16/unique_constraints.sql index b2b677e51..f9fbb6b44 100644 --- a/synapse/storage/schema/delta/16/unique_constraints.sql +++ b/synapse/storage/schema/delta/16/unique_constraints.sql @@ -6,6 +6,7 @@ DELETE FROM current_state_events WHERE rowid not in ( SELECT MIN(rowid) FROM current_state_events GROUP BY event_id ); +DROP INDEX IF EXISTS current_state_events_event_id; CREATE UNIQUE INDEX current_state_events_event_id ON current_state_events(event_id); -- @@ -13,6 +14,7 @@ DELETE FROM room_memberships WHERE rowid not in ( SELECT MIN(rowid) FROM room_memberships GROUP BY event_id ); +DROP INDEX IF EXISTS room_memberships_event_id; CREATE UNIQUE INDEX room_memberships_event_id ON room_memberships(event_id); -- @@ -20,6 +22,7 @@ DELETE FROM feedback WHERE rowid not in ( SELECT MIN(rowid) FROM feedback GROUP BY event_id ); +DROP INDEX IF EXISTS feedback_event_id; CREATE UNIQUE INDEX feedback_event_id ON feedback(event_id); -- @@ -27,6 +30,7 @@ DELETE FROM topics WHERE rowid not in ( SELECT MIN(rowid) FROM topics GROUP BY event_id ); +DROP INDEX IF EXISTS topics_event_id; CREATE UNIQUE INDEX topics_event_id ON topics(event_id); -- @@ -34,6 +38,7 @@ DELETE FROM room_names WHERE rowid not in ( SELECT MIN(rowid) FROM room_names GROUP BY event_id ); +DROP INDEX IF EXISTS room_names_id; CREATE UNIQUE INDEX room_names_id ON room_names(event_id); -- @@ -41,6 +46,7 @@ DELETE FROM presence WHERE rowid not in ( SELECT MIN(rowid) FROM presence GROUP BY user_id ); +DROP INDEX IF EXISTS presence_id; CREATE UNIQUE INDEX presence_id ON presence(user_id); -- @@ -49,6 +55,7 @@ DELETE FROM presence_allow_inbound WHERE rowid not in ( GROUP BY observed_user_id, observer_user_id ); +DROP INDEX IF EXISTS presence_allow_inbound_observers; CREATE UNIQUE INDEX presence_allow_inbound_observers ON presence_allow_inbound( observed_user_id, observer_user_id ); @@ -59,6 +66,7 @@ DELETE FROM presence_list WHERE rowid not in ( GROUP BY user_id, observed_user_id ); +DROP INDEX IF EXISTS presence_list_observers; CREATE UNIQUE INDEX presence_list_observers ON presence_list( user_id, observed_user_id ); diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql index 737c3e35c..8d6b94a61 100644 --- a/synapse/storage/schema/full_schemas/16/state.sql +++ b/synapse/storage/schema/full_schemas/16/state.sql @@ -37,4 +37,4 @@ CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(state_group); CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(room_id, type, state_key); -CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id); \ No newline at end of file +CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id); From c5365dee564ffcf46300d1f979212e0ebaf98692 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 18:46:33 +0100 Subject: [PATCH 073/173] Use case sensitive collations --- synapse/app/homeserver.py | 3 +-- synapse/storage/engines/maria.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index fbc9a43d6..a47e548d6 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -363,7 +363,6 @@ def setup(config_options): db_config = { k: v for k, v in db_config.items() - if not k.startswith("cp_") } name = db_config.get("name", None) @@ -372,7 +371,7 @@ def setup(config_options): "sql_mode": "TRADITIONAL", "charset": "utf8mb4", "use_unicode": True, - "collation": "utf8mb4_general_ci", + "collation": "utf8mb4_bin", }) elif name == "sqlite3": db_config.setdefault("args", {}).update({ diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py index c93437693..90165f684 100644 --- a/synapse/storage/engines/maria.py +++ b/synapse/storage/engines/maria.py @@ -36,7 +36,7 @@ class MariaEngine(object): def prepare_database(self, db_conn): cur = db_conn.cursor() cur.execute( - "ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci" + "ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_bin" ) db_conn.commit() prepare_database(db_conn, self) From 93937b2b313e5b1020d21f3f2cad404ecff659c2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 18:47:03 +0100 Subject: [PATCH 074/173] Remove duplicate rows --- synapse/storage/schema/delta/16/remove_duplicates.sql | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 synapse/storage/schema/delta/16/remove_duplicates.sql diff --git a/synapse/storage/schema/delta/16/remove_duplicates.sql b/synapse/storage/schema/delta/16/remove_duplicates.sql new file mode 100644 index 000000000..65c97b5e2 --- /dev/null +++ b/synapse/storage/schema/delta/16/remove_duplicates.sql @@ -0,0 +1,9 @@ + + +DELETE FROM event_to_state_groups WHERE state_group not in ( + SELECT MAX(state_group) FROM event_to_state_groups GROUP BY event_id +); + +DELETE FROM event_to_state_groups WHERE rowid not in ( + SELECT MIN(rowid) FROM event_to_state_groups GROUP BY event_id +); From 1bede478430c7a008ddaae195a81eca4ea155630 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 18:47:20 +0100 Subject: [PATCH 075/173] Add beginnings of migration sqlite->maria db script --- scripts/port_to_maria.py | 271 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 271 insertions(+) create mode 100644 scripts/port_to_maria.py diff --git a/scripts/port_to_maria.py b/scripts/port_to_maria.py new file mode 100644 index 000000000..6e0adf703 --- /dev/null +++ b/scripts/port_to_maria.py @@ -0,0 +1,271 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer, reactor +from twisted.enterprise import adbapi + +from synapse.storage._base import LoggingTransaction, SQLBaseStore +from synapse.storage.engines import create_engine + +import argparse +import itertools +import logging +import yaml + + +logger = logging.getLogger("port_to_maria") + + +BINARY_COLUMNS = { + "event_content_hashes": ["hash"], + "event_reference_hashes": ["hash"], + "event_signatures": ["signature"], + "event_edge_hashes": ["hash"], + "events": ["content", "unrecognized_keys"], + "event_json": ["internal_metadata", "json"], + "application_services_txns": ["event_ids"], + "received_transactions": ["response_json"], + "sent_transactions": ["response_json"], + "server_tls_certificates": ["tls_certificate"], + "server_signature_keys": ["verify_key"], + "pushers": ["pushkey", "data"], + "user_filters": ["filter_json"], +} + +UNICODE_COLUMNS = { + "events": ["content", "unrecognized_keys"], + "event_json": ["internal_metadata", "json"], + "users": ["password_hash"], +} + + +class Store(object): + def __init__(self, db_pool, engine): + self.db_pool = db_pool + self.engine = engine + + _simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"] + _simple_insert = SQLBaseStore.__dict__["_simple_insert"] + + _simple_select_onecol_txn = SQLBaseStore.__dict__["_simple_select_onecol_txn"] + _simple_select_onecol = SQLBaseStore.__dict__["_simple_select_onecol"] + + _execute_and_decode = SQLBaseStore.__dict__["_execute_and_decode"] + + def runInteraction(self, desc, func, *args, **kwargs): + def r(conn): + try: + i = 0 + N = 5 + while True: + try: + txn = conn.cursor() + return func( + LoggingTransaction(txn, desc, self.engine), + *args, **kwargs + ) + except self.engine.module.DatabaseError as e: + if self.engine.is_deadlock(e): + logger.warn("[TXN DEADLOCK] {%s} %d/%d", desc, i, N) + if i < N: + i += 1 + conn.rollback() + continue + raise + except Exception as e: + logger.debug("[TXN FAIL] {%s}", desc, e) + raise + + return self.db_pool.runWithConnection(r) + + def insert_many(self, table, headers, rows): + sql = "INSERT INTO %s (%s) VALUES (%s)" % ( + table, + ", ".join(k for k in headers), + ", ".join("%s" for _ in headers) + ) + + def t(txn): + try: + txn.executemany(sql, rows) + except: + logger.exception( + "Failed to insert: %s", + table, + ) + raise + + return self.runInteraction("insert_many", t) + + +def chunks(n): + for i in itertools.count(0, n): + yield range(i, i+n) + + +@defer.inlineCallbacks +def handle_table(table, sqlite_store, mysql_store): + N = 1000 + + select = "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,) + + uni_col_names = UNICODE_COLUMNS.get(table, []) + + def conv_uni(c): + return sqlite_store.engine.load_unicode(c) + + next_chunk = 0 + while True: + def r(txn): + txn.execute(select, (next_chunk, N,)) + rows = txn.fetchall() + headers = [column[0] for column in txn.description] + + return headers, rows + + headers, rows = yield sqlite_store.runInteraction("select", r) + + logger.info("Got %d rows for %s", len(rows), table) + + if rows: + uni_cols = [i for i, h in enumerate(headers) if h in uni_col_names] + next_chunk = rows[-1][0] + 1 + + for i, row in enumerate(rows): + rows[i] = tuple( + mysql_store.engine.encode_parameter( + conv_uni(col) if j in uni_cols else col + ) + for j, col in enumerate(row) + if j > 0 + ) + + yield mysql_store.insert_many(table, headers[1:], rows) + else: + return + + +def setup_db(db_config, database_engine): + db_conn = database_engine.module.connect( + **{ + k: v for k, v in db_config.get("args", {}).items() + if not k.startswith("cp_") + } + ) + + database_engine.prepare_database(db_conn) + + db_conn.commit() + + +@defer.inlineCallbacks +def main(sqlite_config, mysql_config): + try: + sqlite_db_pool = adbapi.ConnectionPool( + sqlite_config["name"], + **sqlite_config["args"] + ) + + mysql_db_pool = adbapi.ConnectionPool( + mysql_config["name"], + **mysql_config["args"] + ) + + sqlite_engine = create_engine("sqlite3") + mysql_engine = create_engine("mysql.connector") + + sqlite_store = Store(sqlite_db_pool, sqlite_engine) + mysql_store = Store(mysql_db_pool, mysql_engine) + + # Step 1. Set up mysql database. + logger.info("Preparing sqlite database...") + setup_db(sqlite_config, sqlite_engine) + + logger.info("Preparing mysql database...") + setup_db(mysql_config, mysql_engine) + + # Step 2. Get tables. + logger.info("Fetching tables...") + tables = yield sqlite_store._simple_select_onecol( + table="sqlite_master", + keyvalues={ + "type": "table", + }, + retcol="name", + ) + + logger.info("Found %d tables", len(tables)) + + # Process tables. + yield defer.gatherResults( + [ + handle_table(table, sqlite_store, mysql_store) + for table in tables + if table not in ["schema_version", "applied_schema_deltas"] + and not table.startswith("sqlite_") + ], + consumeErrors=True, + ) + + # for table in ["current_state_events"]: # tables: + # if table not in ["schema_version", "applied_schema_deltas"]: + # if not table.startswith("sqlite_"): + # yield handle_table(table, sqlite_store, mysql_store) + except: + logger.exception("") + finally: + reactor.stop() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--sqlite-database") + parser.add_argument( + "--mysql-config", type=argparse.FileType('r'), + ) + + args = parser.parse_args() + logging.basicConfig(level=logging.INFO) + + sqlite_config = { + "name": "sqlite3", + "args": { + "database": args.sqlite_database, + "cp_min": 1, + "cp_max": 1, + "check_same_thread": False, + }, + } + + mysql_config = yaml.safe_load(args.mysql_config) + mysql_config["args"].update({ + "sql_mode": "TRADITIONAL", + "charset": "utf8mb4", + "use_unicode": True, + "collation": "utf8mb4_bin", + }) + + import codecs + codecs.register( + lambda name: codecs.lookup('utf8') if name == "utf8mb4" else None + ) + + reactor.callWhenRunning( + main, + sqlite_config=sqlite_config, + mysql_config=mysql_config, + ) + + reactor.run() From 90bcb869579fe9c33c223851fece8bd3eeabf942 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 13 Apr 2015 13:45:04 +0100 Subject: [PATCH 076/173] Support running porting script multiple times --- scripts/port_to_maria.py | 108 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 100 insertions(+), 8 deletions(-) diff --git a/scripts/port_to_maria.py b/scripts/port_to_maria.py index 6e0adf703..b14cca891 100644 --- a/scripts/port_to_maria.py +++ b/scripts/port_to_maria.py @@ -51,16 +51,52 @@ UNICODE_COLUMNS = { } +APPEND_ONLY_TABLES = [ + "event_content_hashes", + "event_reference_hashes", + "event_signatures", + "event_edge_hashes", + "events", + "event_json", + "state_events", + "room_memberships", + "feedback", + "topics", + "room_names", + "rooms", + "local_media_repository", + "local_media_repository_thumbnails", + "remote_media_cache", + "remote_media_cache_thumbnails", + "redactions", + "event_edges", + "event_auth", + "received_transactions", + "sent_transactions", + "transaction_id_to_pdu", + "users", + "state_groups", + "state_groups_state", + "event_to_state_groups", + "rejections", +] + + class Store(object): def __init__(self, db_pool, engine): self.db_pool = db_pool - self.engine = engine + self.database_engine = engine _simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"] _simple_insert = SQLBaseStore.__dict__["_simple_insert"] _simple_select_onecol_txn = SQLBaseStore.__dict__["_simple_select_onecol_txn"] _simple_select_onecol = SQLBaseStore.__dict__["_simple_select_onecol"] + _simple_select_one_onecol = SQLBaseStore.__dict__["_simple_select_one_onecol"] + _simple_select_one_onecol_txn = SQLBaseStore.__dict__["_simple_select_one_onecol_txn"] + + _simple_update_one = SQLBaseStore.__dict__["_simple_update_one"] + _simple_update_one_txn = SQLBaseStore.__dict__["_simple_update_one_txn"] _execute_and_decode = SQLBaseStore.__dict__["_execute_and_decode"] @@ -73,11 +109,11 @@ class Store(object): try: txn = conn.cursor() return func( - LoggingTransaction(txn, desc, self.engine), + LoggingTransaction(txn, desc, self.database_engine), *args, **kwargs ) - except self.engine.module.DatabaseError as e: - if self.engine.is_deadlock(e): + except self.database_engine.module.DatabaseError as e: + if self.database_engine.is_deadlock(e): logger.warn("[TXN DEADLOCK] {%s} %d/%d", desc, i, N) if i < N: i += 1 @@ -117,16 +153,50 @@ def chunks(n): @defer.inlineCallbacks def handle_table(table, sqlite_store, mysql_store): - N = 1000 + if table in APPEND_ONLY_TABLES: + # It's safe to just carry on inserting. + next_chunk = yield mysql_store._simple_select_one_onecol( + table="port_from_sqlite3", + keyvalues={"table_name": table}, + retcol="rowid", + allow_none=True, + ) + + if next_chunk is None: + yield mysql_store._simple_insert( + table="port_from_sqlite3", + values={"table_name": table, "rowid": 0} + ) + + next_chunk = 0 + else: + def delete_all(txn): + txn.execute( + "DELETE FROM port_from_sqlite3 WHERE table_name = %s", + (table,) + ) + txn.execute("DELETE FROM %s" % (table,)) + mysql_store._simple_insert_txn( + txn, + table="port_from_sqlite3", + values={"table_name": table, "rowid": 0} + ) + + yield mysql_store.runInteraction( + "delete_non_append_only", delete_all + ) + + next_chunk = 0 + + N = 5000 select = "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,) uni_col_names = UNICODE_COLUMNS.get(table, []) def conv_uni(c): - return sqlite_store.engine.load_unicode(c) + return sqlite_store.database_engine.load_unicode(c) - next_chunk = 0 while True: def r(txn): txn.execute(select, (next_chunk, N,)) @@ -145,7 +215,7 @@ def handle_table(table, sqlite_store, mysql_store): for i, row in enumerate(rows): rows[i] = tuple( - mysql_store.engine.encode_parameter( + mysql_store.database_engine.encode_parameter( conv_uni(col) if j in uni_cols else col ) for j, col in enumerate(row) @@ -153,6 +223,12 @@ def handle_table(table, sqlite_store, mysql_store): ) yield mysql_store.insert_many(table, headers[1:], rows) + + yield mysql_store._simple_update_one( + table="port_from_sqlite3", + keyvalues={"table_name": table}, + updatevalues={"rowid": next_chunk}, + ) else: return @@ -208,6 +284,22 @@ def main(sqlite_config, mysql_config): logger.info("Found %d tables", len(tables)) + def create_port_table(txn): + try: + txn.execute( + "CREATE TABLE port_from_sqlite3 (" + " `table_name` varchar(100) NOT NULL UNIQUE," + " `rowid` bigint unsigned NOT NULL" + ")" + ) + except mysql_engine.module.DatabaseError as e: + if e.errno != mysql_engine.module.errorcode.ER_TABLE_EXISTS_ERROR: + raise + + yield mysql_store.runInteraction( + "create_port_table", create_port_table + ) + # Process tables. yield defer.gatherResults( [ From 86fc9b617c5eb91186a8466370ddd97f3acd34a2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 13 Apr 2015 17:03:49 +0100 Subject: [PATCH 077/173] For backwards compat, make state_groups.id have a type of int, not varchar --- synapse/storage/schema/full_schemas/16/state.sql | 2 +- synapse/storage/state.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql index 8d6b94a61..8c5161039 100644 --- a/synapse/storage/schema/full_schemas/16/state.sql +++ b/synapse/storage/schema/full_schemas/16/state.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS state_groups( - id VARCHAR(20) PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, room_id VARCHAR(150) NOT NULL, event_id VARCHAR(150) NOT NULL ); diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 4994bacd6..553ba9dd1 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -93,7 +93,7 @@ class StateStore(SQLBaseStore): state_group = context.state_group if not state_group: - state_group = _make_group_id(self._clock) + state_group = self._state_groups_id_gen.get_next_txn(txn) self._simple_insert_txn( txn, table="state_groups", From 3c741682e59a41fcc45a5b9a370c7f268be7729e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Apr 2015 09:54:44 +0100 Subject: [PATCH 078/173] Correctly increment the _next_id initially --- synapse/storage/util/id_generators.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index 8f419323a..2e2a40898 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -55,9 +55,11 @@ class IdGenerator(object): ) val, = txn.fetchone() - self._next_id = val or 2 + cur = val or 0 + cur += 1 + self._next_id = cur + 1 - return 1 + return cur class StreamIdGenerator(object): From 58d83399663a080c123d2f112b4f4d84accbc638 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Apr 2015 13:53:20 +0100 Subject: [PATCH 079/173] Add support for postgres instead of mysql. Change sql accourdingly. blob + varbinary -> bytea. No support for UNSIGNED or CREATE INDEX IF NOT EXISTS. --- synapse/app/homeserver.py | 2 + synapse/storage/__init__.py | 15 +++---- synapse/storage/_base.py | 2 +- synapse/storage/engines/__init__.py | 2 + synapse/storage/event_federation.py | 10 ++--- synapse/storage/events.py | 4 +- synapse/storage/room.py | 34 +++++++++----- .../full_schemas/16/application_services.sql | 10 ++--- .../schema/full_schemas/16/event_edges.sql | 26 +++++------ .../full_schemas/16/event_signatures.sql | 16 +++---- synapse/storage/schema/full_schemas/16/im.sql | 44 +++++++++---------- .../storage/schema/full_schemas/16/keys.sql | 8 ++-- .../full_schemas/16/media_repository.sql | 8 ++-- .../schema/full_schemas/16/presence.sql | 4 +- .../storage/schema/full_schemas/16/push.sql | 30 ++++++------- .../schema/full_schemas/16/redactions.sql | 4 +- .../storage/schema/full_schemas/16/state.sql | 10 ++--- .../schema/full_schemas/16/transactions.sql | 24 +++++----- .../storage/schema/full_schemas/16/users.sql | 10 ++--- synapse/storage/schema/schema_version.sql | 14 +++--- synapse/storage/stream.py | 16 +++---- 21 files changed, 153 insertions(+), 140 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index a47e548d6..033011e1d 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -373,6 +373,8 @@ def setup(config_options): "use_unicode": True, "collation": "utf8mb4_bin", }) + elif name == "psycopg2": + pass elif name == "sqlite3": db_config.setdefault("args", {}).update({ "cp_min": 1, diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index b46cafd25..272420194 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -236,7 +236,7 @@ def _setup_new_database(cur, database_engine): cur.execute( database_engine.convert_param_style( - "REPLACE INTO schema_version (version, upgraded)" + "INSERT INTO schema_version (version, upgraded)" " VALUES (?,?)" ), (max_current_ver, False,) @@ -432,14 +432,11 @@ def executescript(txn, schema_path): def _get_or_create_schema_state(txn, database_engine): - try: - # Bluntly try creating the schema_version tables. - schema_path = os.path.join( - dir_path, "schema", "schema_version.sql", - ) - executescript(txn, schema_path) - except: - pass + # Bluntly try creating the schema_version tables. + schema_path = os.path.join( + dir_path, "schema", "schema_version.sql", + ) + executescript(txn, schema_path) txn.execute("SELECT version, upgraded FROM schema_version") row = txn.fetchone() diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index e30514cd5..fa5199104 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -330,7 +330,7 @@ class SQLBaseStore(object): continue raise except Exception as e: - logger.debug("[TXN FAIL] {%s}", name, e) + logger.debug("[TXN FAIL] {%s} %s", name, e) raise finally: end = time.time() * 1000 diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index 29702be92..548d4e1b4 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -14,6 +14,7 @@ # limitations under the License. from .maria import MariaEngine +from .postgres import PostgresEngine from .sqlite3 import Sqlite3Engine import importlib @@ -22,6 +23,7 @@ import importlib SUPPORTED_MODULE = { "sqlite3": Sqlite3Engine, "mysql.connector": MariaEngine, + "psycopg2": PostgresEngine, } diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 79ad5ddc9..54a3c9d80 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -153,7 +153,7 @@ class EventFederationStore(SQLBaseStore): results = self._get_prev_events_and_state( txn, event_id, - is_state=1, + is_state=True, ) return [(e_id, h, ) for e_id, h, _ in results] @@ -164,7 +164,7 @@ class EventFederationStore(SQLBaseStore): } if is_state is not None: - keyvalues["is_state"] = is_state + keyvalues["is_state"] = bool(is_state) res = self._simple_select_list_txn( txn, @@ -259,7 +259,7 @@ class EventFederationStore(SQLBaseStore): "event_id": event_id, "prev_event_id": e_id, "room_id": room_id, - "is_state": 0, + "is_state": False, }, ) @@ -397,7 +397,7 @@ class EventFederationStore(SQLBaseStore): query = ( "SELECT prev_event_id FROM event_edges " - "WHERE room_id = ? AND event_id = ? AND is_state = 0 " + "WHERE room_id = ? AND event_id = ? AND is_state = ? " "LIMIT ?" ) @@ -406,7 +406,7 @@ class EventFederationStore(SQLBaseStore): for event_id in front: txn.execute( query, - (room_id, event_id, limit - len(event_results)) + (room_id, event_id, False, limit - len(event_results)) ) for e_id, in txn.fetchall(): diff --git a/synapse/storage/events.py b/synapse/storage/events.py index a2e87c27c..9fe2effb4 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -188,12 +188,12 @@ class EventsStore(SQLBaseStore): ) sql = ( - "UPDATE events SET outlier = 0" + "UPDATE events SET outlier = ?" " WHERE event_id = ?" ) txn.execute( sql, - (event.event_id,) + (False, event.event_id,) ) return diff --git a/synapse/storage/room.py b/synapse/storage/room.py index a1a76280f..48ebb3305 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -105,14 +105,12 @@ class RoomStore(SQLBaseStore): # We use non printing ascii character US (\x1F) as a separator sql = ( - "SELECT r.room_id, n.name, t.topic, " - "group_concat(a.room_alias, '\x1F') " - "FROM rooms AS r " - "LEFT JOIN (%(topic)s) AS t ON t.room_id = r.room_id " - "LEFT JOIN (%(name)s) AS n ON n.room_id = r.room_id " - "INNER JOIN room_aliases AS a ON a.room_id = r.room_id " - "WHERE r.is_public = ? " - "GROUP BY r.room_id " + "SELECT r.room_id, max(n.name), max(t.topic)" + " FROM rooms AS r" + " LEFT JOIN (%(topic)s) AS t ON t.room_id = r.room_id" + " LEFT JOIN (%(name)s) AS n ON n.room_id = r.room_id" + " WHERE r.is_public = ?" + " GROUP BY r.room_id" ) % { "topic": topic_subquery, "name": name_subquery, @@ -120,7 +118,22 @@ class RoomStore(SQLBaseStore): txn.execute(sql, (is_public,)) - return txn.fetchall() + rows = txn.fetchall() + + for i, row in enumerate(rows): + room_id = row[0] + aliases = self._simple_select_onecol_txn( + txn, + table="room_aliases", + keyvalues={ + "room_id": room_id + }, + retcol="room_alias", + ) + + rows[i] = list(row) + [aliases] + + return rows rows = yield self.runInteraction( "get_rooms", f @@ -131,9 +144,10 @@ class RoomStore(SQLBaseStore): "room_id": r[0], "name": r[1], "topic": r[2], - "aliases": r[3].split("\x1F"), + "aliases": r[3], } for r in rows + if r[3] # We only return rooms that have at least one alias. ] defer.returnValue(ret) diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql index bc709df92..f08c5bcf7 100644 --- a/synapse/storage/schema/full_schemas/16/application_services.sql +++ b/synapse/storage/schema/full_schemas/16/application_services.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS application_services( - id BIGINT UNSIGNED PRIMARY KEY, + id BIGINT PRIMARY KEY, url VARCHAR(150), token VARCHAR(150), hs_token VARCHAR(150), @@ -23,8 +23,8 @@ CREATE TABLE IF NOT EXISTS application_services( ); CREATE TABLE IF NOT EXISTS application_services_regex( - id BIGINT UNSIGNED PRIMARY KEY, - as_id BIGINT UNSIGNED NOT NULL, + id BIGINT PRIMARY KEY, + as_id BIGINT NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(150), FOREIGN KEY(as_id) REFERENCES application_services(id) @@ -39,10 +39,10 @@ CREATE TABLE IF NOT EXISTS application_services_state( CREATE TABLE IF NOT EXISTS application_services_txns( as_id VARCHAR(150) NOT NULL, txn_id INTEGER NOT NULL, - event_ids LONGBLOB NOT NULL, + event_ids bytea NOT NULL, UNIQUE(as_id, txn_id) ); -CREATE INDEX IF NOT EXISTS application_services_txns_id ON application_services_txns ( +CREATE INDEX application_services_txns_id ON application_services_txns ( as_id ); diff --git a/synapse/storage/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/full_schemas/16/event_edges.sql index bdb110909..05d0874f0 100644 --- a/synapse/storage/schema/full_schemas/16/event_edges.sql +++ b/synapse/storage/schema/full_schemas/16/event_edges.sql @@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities( UNIQUE (event_id, room_id) ); -CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); -CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); +CREATE INDEX ev_extrem_room ON event_forward_extremities(room_id); +CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_backward_extremities( @@ -29,8 +29,8 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities( UNIQUE (event_id, room_id) ); -CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); -CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); +CREATE INDEX ev_b_extrem_room ON event_backward_extremities(room_id); +CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_edges( @@ -41,8 +41,8 @@ CREATE TABLE IF NOT EXISTS event_edges( UNIQUE (event_id, prev_event_id, room_id, is_state) ); -CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); -CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); +CREATE INDEX ev_edges_id ON event_edges(event_id); +CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id); CREATE TABLE IF NOT EXISTS room_depth( @@ -51,17 +51,17 @@ CREATE TABLE IF NOT EXISTS room_depth( UNIQUE (room_id) ); -CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); +CREATE INDEX room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( event_id VARCHAR(150) NOT NULL, destination VARCHAR(150) NOT NULL, - delivered_ts BIGINT UNSIGNED DEFAULT 0, -- or 0 if not delivered + delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ); -CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); +CREATE INDEX event_destinations_id ON event_destinations(event_id); CREATE TABLE IF NOT EXISTS state_forward_extremities( @@ -72,10 +72,10 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities( UNIQUE (event_id, room_id) ); -CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( +CREATE INDEX st_extrem_keys ON state_forward_extremities( room_id, type, state_key ); -CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); +CREATE INDEX st_extrem_id ON state_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_auth( @@ -85,5 +85,5 @@ CREATE TABLE IF NOT EXISTS event_auth( UNIQUE (event_id, auth_id, room_id) ); -CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); -CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); +CREATE INDEX evauth_edges_id ON event_auth(event_id); +CREATE INDEX evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/full_schemas/16/event_signatures.sql index 09886f607..429182736 100644 --- a/synapse/storage/schema/full_schemas/16/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/16/event_signatures.sql @@ -16,40 +16,40 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash LONGBLOB, + hash bytea, UNIQUE (event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); +CREATE INDEX event_content_hashes_id ON event_content_hashes(event_id); CREATE TABLE IF NOT EXISTS event_reference_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash LONGBLOB, + hash bytea, UNIQUE (event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); +CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id); CREATE TABLE IF NOT EXISTS event_signatures ( event_id VARCHAR(150), signature_name VARCHAR(150), key_id VARCHAR(150), - signature LONGBLOB, + signature bytea, UNIQUE (event_id, signature_name, key_id) ); -CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); +CREATE INDEX event_signatures_id ON event_signatures(event_id); CREATE TABLE IF NOT EXISTS event_edge_hashes( event_id VARCHAR(150), prev_event_id VARCHAR(150), algorithm VARCHAR(150), - hash LONGBLOB, + hash bytea, UNIQUE (event_id, prev_event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); +CREATE INDEX event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index 19f0f3414..a661fc160 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -14,33 +14,33 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering BIGINT UNSIGNED PRIMARY KEY, - topological_ordering BIGINT UNSIGNED NOT NULL, + stream_ordering BIGINT PRIMARY KEY, + topological_ordering BIGINT NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - content LONGBLOB NOT NULL, - unrecognized_keys LONGBLOB, + content bytea NOT NULL, + unrecognized_keys bytea, processed BOOL NOT NULL, outlier BOOL NOT NULL, - depth BIGINT UNSIGNED DEFAULT 0 NOT NULL, + depth BIGINT DEFAULT 0 NOT NULL, UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); -CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); -CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); +CREATE INDEX events_stream_ordering ON events (stream_ordering); +CREATE INDEX events_topological_ordering ON events (topological_ordering); +CREATE INDEX events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - internal_metadata LONGBLOB NOT NULL, - json LONGBLOB NOT NULL, + internal_metadata bytea NOT NULL, + json bytea NOT NULL, UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); +CREATE INDEX event_json_room_id ON event_json(room_id); CREATE TABLE IF NOT EXISTS state_events( @@ -52,9 +52,9 @@ CREATE TABLE IF NOT EXISTS state_events( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); -CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); -CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); +CREATE INDEX state_events_room_id ON state_events (room_id); +CREATE INDEX state_events_type ON state_events (type); +CREATE INDEX state_events_state_key ON state_events (state_key); CREATE TABLE IF NOT EXISTS current_state_events( @@ -66,9 +66,9 @@ CREATE TABLE IF NOT EXISTS current_state_events( UNIQUE (room_id, type, state_key) ); -CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); -CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); -CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); +CREATE INDEX current_state_events_room_id ON current_state_events (room_id); +CREATE INDEX current_state_events_type ON current_state_events (type); +CREATE INDEX current_state_events_state_key ON current_state_events (state_key); CREATE TABLE IF NOT EXISTS room_memberships( event_id VARCHAR(150) NOT NULL, @@ -79,8 +79,8 @@ CREATE TABLE IF NOT EXISTS room_memberships( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); -CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); +CREATE INDEX room_memberships_room_id ON room_memberships (room_id); +CREATE INDEX room_memberships_user_id ON room_memberships (user_id); CREATE TABLE IF NOT EXISTS feedback( event_id VARCHAR(150) NOT NULL, @@ -98,7 +98,7 @@ CREATE TABLE IF NOT EXISTS topics( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); +CREATE INDEX topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( event_id VARCHAR(150) NOT NULL, @@ -107,7 +107,7 @@ CREATE TABLE IF NOT EXISTS room_names( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); +CREATE INDEX room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( room_id VARCHAR(150) PRIMARY KEY NOT NULL, @@ -121,4 +121,4 @@ CREATE TABLE IF NOT EXISTS room_hosts( UNIQUE (room_id, host) ); -CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); +CREATE INDEX room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql index 35f141c28..459b51042 100644 --- a/synapse/storage/schema/full_schemas/16/keys.sql +++ b/synapse/storage/schema/full_schemas/16/keys.sql @@ -16,8 +16,8 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( server_name VARCHAR(150), -- Server name. fingerprint VARCHAR(150), -- Certificate fingerprint. from_server VARCHAR(150), -- Which key server the certificate was fetched from. - ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added. - tls_certificate LONGBLOB, -- DER encoded x509 certificate. + ts_added_ms BIGINT, -- When the certifcate was added. + tls_certificate bytea, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(150), -- Server name. key_id VARCHAR(150), -- Key version. from_server VARCHAR(150), -- Which key server the key was fetched form. - ts_added_ms BIGINT UNSIGNED, -- When the key was added. - verify_key LONGBLOB, -- NACL verification key. + ts_added_ms BIGINT, -- When the key was added. + verify_key bytea, -- NACL verification key. UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/16/media_repository.sql b/synapse/storage/schema/full_schemas/16/media_repository.sql index 014bce4ae..0e819fca3 100644 --- a/synapse/storage/schema/full_schemas/16/media_repository.sql +++ b/synapse/storage/schema/full_schemas/16/media_repository.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( media_id VARCHAR(150), -- The id used to refer to the media. media_type VARCHAR(150), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. - created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. + created_ts BIGINT, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. user_id VARCHAR(150), -- The user who uploaded the file. UNIQUE (media_id) @@ -35,14 +35,14 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( ) ); -CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id +CREATE INDEX local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); CREATE TABLE IF NOT EXISTS remote_media_cache ( media_origin VARCHAR(150), -- The remote HS the media came from. media_id VARCHAR(150), -- The id used to refer to the media on that server. media_type VARCHAR(150), -- The MIME-type of the media. - created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. + created_ts BIGINT, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(150), -- The name used to store the media on disk. @@ -64,5 +64,5 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( ) ); -CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id +CREATE INDEX remote_media_cache_thumbnails_media_id ON remote_media_cache_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql index fbe5b0af6..9c41be296 100644 --- a/synapse/storage/schema/full_schemas/16/presence.sql +++ b/synapse/storage/schema/full_schemas/16/presence.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS presence( user_id VARCHAR(150) NOT NULL, state VARCHAR(20), status_msg VARCHAR(150), - mtime BIGINT UNSIGNED, -- miliseconds since last state change + mtime BIGINT, -- miliseconds since last state change UNIQUE (user_id) ); @@ -37,4 +37,4 @@ CREATE TABLE IF NOT EXISTS presence_list( UNIQUE (user_id, observed_user_id) ); -CREATE INDEX IF NOT EXISTS presence_list_user_id ON presence_list (user_id); +CREATE INDEX presence_list_user_id ON presence_list (user_id); diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql index 33300736f..5c0c7bc20 100644 --- a/synapse/storage/schema/full_schemas/16/push.sql +++ b/synapse/storage/schema/full_schemas/16/push.sql @@ -22,52 +22,52 @@ CREATE TABLE IF NOT EXISTS rejections( -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( - id BIGINT UNSIGNED PRIMARY KEY, + id BIGINT PRIMARY KEY, user_name VARCHAR(150) NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, app_id VARCHAR(64) NOT NULL, app_display_name VARCHAR(64) NOT NULL, device_display_name VARCHAR(128) NOT NULL, - pushkey VARBINARY(512) NOT NULL, - ts BIGINT UNSIGNED NOT NULL, + pushkey bytea NOT NULL, + ts BIGINT NOT NULL, lang VARCHAR(8), - data LONGBLOB, + data bytea, last_token TEXT, - last_success BIGINT UNSIGNED, - failing_since BIGINT UNSIGNED, + last_success BIGINT, + failing_since BIGINT, UNIQUE (app_id, pushkey) ); CREATE TABLE IF NOT EXISTS push_rules ( - id BIGINT UNSIGNED PRIMARY KEY, + id BIGINT PRIMARY KEY, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, - priority_class TINYINT NOT NULL, + priority_class SMALLINT NOT NULL, priority INTEGER NOT NULL DEFAULT 0, conditions VARCHAR(150) NOT NULL, actions VARCHAR(150) NOT NULL, UNIQUE(user_name, rule_id) ); -CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); +CREATE INDEX push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(150), - filter_id BIGINT UNSIGNED, - filter_json LONGBLOB + filter_id BIGINT, + filter_json bytea ); -CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( +CREATE INDEX user_filters_by_user_id_filter_id ON user_filters( user_id, filter_id ); CREATE TABLE IF NOT EXISTS push_rules_enable ( - id BIGINT UNSIGNED PRIMARY KEY, + id BIGINT PRIMARY KEY, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, - enabled TINYINT, + enabled SMALLINT, UNIQUE(user_name, rule_id) ); -CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name); +CREATE INDEX push_rules_enable_user_name on push_rules_enable (user_name); diff --git a/synapse/storage/schema/full_schemas/16/redactions.sql b/synapse/storage/schema/full_schemas/16/redactions.sql index b81451eab..492fd2203 100644 --- a/synapse/storage/schema/full_schemas/16/redactions.sql +++ b/synapse/storage/schema/full_schemas/16/redactions.sql @@ -18,5 +18,5 @@ CREATE TABLE IF NOT EXISTS redactions ( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); -CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); +CREATE INDEX redactions_event_id ON redactions (event_id); +CREATE INDEX redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql index 8c5161039..3c54595e6 100644 --- a/synapse/storage/schema/full_schemas/16/state.sql +++ b/synapse/storage/schema/full_schemas/16/state.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS state_groups( - id BIGINT UNSIGNED PRIMARY KEY, + id BIGINT PRIMARY KEY, room_id VARCHAR(150) NOT NULL, event_id VARCHAR(150) NOT NULL ); @@ -33,8 +33,8 @@ CREATE TABLE IF NOT EXISTS event_to_state_groups( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); +CREATE INDEX state_groups_id ON state_groups(id); -CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(state_group); -CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(room_id, type, state_key); -CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id); +CREATE INDEX state_groups_state_id ON state_groups_state(state_group); +CREATE INDEX state_groups_state_tuple ON state_groups_state(room_id, type, state_key); +CREATE INDEX event_to_state_groups_id ON event_to_state_groups(event_id); diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql index f381e6760..bc6406493 100644 --- a/synapse/storage/schema/full_schemas/16/transactions.sql +++ b/synapse/storage/schema/full_schemas/16/transactions.sql @@ -16,32 +16,32 @@ CREATE TABLE IF NOT EXISTS received_transactions( transaction_id VARCHAR(150), origin VARCHAR(150), - ts BIGINT UNSIGNED, + ts BIGINT, response_code INTEGER, - response_json LONGBLOB, - has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx + response_json bytea, + has_been_referenced smallint default 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) ); -CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; +CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; -- Stores what transactions we've sent, what their response was (if we got one) and whether we have -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( - id BIGINT UNSIGNED PRIMARY KEY, -- This is used to apply insertion ordering + id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, - response_json LONGBLOB, - ts BIGINT UNSIGNED + response_json bytea, + ts BIGINT ); -CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); -CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); +CREATE INDEX sent_transaction_dest ON sent_transactions(destination); +CREATE INDEX sent_transaction_txn_id ON sent_transactions(transaction_id); -- So that we can do an efficient look up of all transactions that have yet to be successfully -- sent. -CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code); +CREATE INDEX sent_transaction_sent ON sent_transactions(response_code); -- For sent transactions only. @@ -53,11 +53,11 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( UNIQUE (transaction_id, destination) ); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); +CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(150) PRIMARY KEY, - retry_last_ts BIGINT UNSIGNED, + retry_last_ts BIGINT, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql index d0011c04b..267284d07 100644 --- a/synapse/storage/schema/full_schemas/16/users.sql +++ b/synapse/storage/schema/full_schemas/16/users.sql @@ -15,17 +15,17 @@ CREATE TABLE IF NOT EXISTS users( name VARCHAR(150), password_hash VARCHAR(150), - creation_ts BIGINT UNSIGNED, - admin BOOL DEFAULT 0 NOT NULL, + creation_ts BIGINT, + admin SMALLINT DEFAULT 0 NOT NULL, UNIQUE(name) ); CREATE TABLE IF NOT EXISTS access_tokens( - id BIGINT UNSIGNED PRIMARY KEY, + id BIGINT PRIMARY KEY, user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, - last_used BIGINT UNSIGNED, + last_used BIGINT, UNIQUE(token) ); @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT UNSIGNED NOT NULL + last_seen BIGINT NOT NULL ); CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql index e7fa6fe56..d9494611e 100644 --- a/synapse/storage/schema/schema_version.sql +++ b/synapse/storage/schema/schema_version.sql @@ -14,16 +14,14 @@ */ CREATE TABLE IF NOT EXISTS schema_version( - `Lock` CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row. - `version` INTEGER NOT NULL, - `upgraded` BOOL NOT NULL, -- Whether we reached this version from an upgrade or an initial schema. - CHECK (`Lock`='X') + Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row. + version INTEGER NOT NULL, + upgraded BOOL NOT NULL, -- Whether we reached this version from an upgrade or an initial schema. + CHECK (Lock='X') ); CREATE TABLE IF NOT EXISTS applied_schema_deltas( - `version` INTEGER NOT NULL, - `file` VARCHAR(150) NOT NULL, + version INTEGER NOT NULL, + file VARCHAR(150) NOT NULL, UNIQUE(version, file) ); - -CREATE INDEX IF NOT EXISTS schema_deltas_ver ON applied_schema_deltas(version); diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 57c2e4dfe..df6de7cbc 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -240,7 +240,7 @@ class StreamStore(SQLBaseStore): sql = ( "SELECT e.event_id, e.stream_ordering FROM events AS e WHERE " - "(e.outlier = 0 AND (room_id IN (%(current)s)) OR " + "(e.outlier = ? AND (room_id IN (%(current)s)) OR " "(event_id IN (%(invites)s))) " "AND e.stream_ordering > ? AND e.stream_ordering <= ? " "ORDER BY stream_ordering ASC LIMIT %(limit)d " @@ -251,7 +251,7 @@ class StreamStore(SQLBaseStore): } def f(txn): - txn.execute(sql, (user_id, user_id, from_id.stream, to_id.stream,)) + txn.execute(sql, (False, user_id, user_id, from_id.stream, to_id.stream,)) rows = self.cursor_to_dict(txn) @@ -283,7 +283,7 @@ class StreamStore(SQLBaseStore): # Tokens really represent positions between elements, but we use # the convention of pointing to the event before the gap. Hence # we have a bit of asymmetry when it comes to equalities. - args = [room_id] + args = [False, room_id] if direction == 'b': order = "DESC" bounds = _StreamToken.parse(from_key).upper_bound() @@ -307,7 +307,7 @@ class StreamStore(SQLBaseStore): sql = ( "SELECT * FROM events" - " WHERE outlier = 0 AND room_id = ? AND %(bounds)s" + " WHERE outlier = ? AND room_id = ? AND %(bounds)s" " ORDER BY topological_ordering %(order)s," " stream_ordering %(order)s %(limit)s" ) % { @@ -358,7 +358,7 @@ class StreamStore(SQLBaseStore): sql = ( "SELECT stream_ordering, topological_ordering, event_id" " FROM events" - " WHERE room_id = ? AND stream_ordering <= ? AND outlier = 0" + " WHERE room_id = ? AND stream_ordering <= ? AND outlier = ?" " ORDER BY topological_ordering DESC, stream_ordering DESC" " LIMIT ?" ) @@ -368,17 +368,17 @@ class StreamStore(SQLBaseStore): "SELECT stream_ordering, topological_ordering, event_id" " FROM events" " WHERE room_id = ? AND stream_ordering > ?" - " AND stream_ordering <= ? AND outlier = 0" + " AND stream_ordering <= ? AND outlier = ?" " ORDER BY topological_ordering DESC, stream_ordering DESC" " LIMIT ?" ) def get_recent_events_for_room_txn(txn): if from_token is None: - txn.execute(sql, (room_id, end_token.stream, limit,)) + txn.execute(sql, (room_id, end_token.stream, False, limit,)) else: txn.execute(sql, ( - room_id, from_token.stream, end_token.stream, limit + room_id, from_token.stream, end_token.stream, False, limit )) rows = self.cursor_to_dict(txn) From 5a95cd444297dfc2aab98259d6fc6a07bed837b9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Apr 2015 13:54:09 +0100 Subject: [PATCH 080/173] Rename user_ips.user -> user_id --- synapse/storage/__init__.py | 4 ++-- synapse/storage/schema/delta/16/users.sql | 6 +++--- synapse/storage/schema/full_schemas/16/users.sql | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 272420194..995114e40 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -107,7 +107,7 @@ class DataStore(RoomMemberStore, RoomStore, yield self._simple_upsert( "user_ips", keyvalues={ - "user": user.to_string(), + "user_id": user.to_string(), "access_token": access_token, "ip": ip, "user_agent": user_agent, @@ -122,7 +122,7 @@ class DataStore(RoomMemberStore, RoomStore, def get_user_ip_and_agents(self, user): return self._simple_select_list( table="user_ips", - keyvalues={"user": user.to_string()}, + keyvalues={"user_id": user.to_string()}, retcols=[ "device_id", "access_token", "ip", "user_agent", "last_seen" ], diff --git a/synapse/storage/schema/delta/16/users.sql b/synapse/storage/schema/delta/16/users.sql index f04705cd7..db27bdca0 100644 --- a/synapse/storage/schema/delta/16/users.sql +++ b/synapse/storage/schema/delta/16/users.sql @@ -36,7 +36,7 @@ ALTER TABLE new_users RENAME TO users; -- Remove UNIQUE constraint from `user_ips` table CREATE TABLE IF NOT EXISTS new_user_ips ( - user VARCHAR(150) NOT NULL, + user_id VARCHAR(150) NOT NULL, access_token VARCHAR(150) NOT NULL, device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, @@ -51,6 +51,6 @@ DROP TABLE user_ips; ALTER TABLE new_user_ips RENAME TO user_ips; -CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); -CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); +CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user_id); +CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user_id, access_token, ip); diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql index 267284d07..006b249fc 100644 --- a/synapse/storage/schema/full_schemas/16/users.sql +++ b/synapse/storage/schema/full_schemas/16/users.sql @@ -30,7 +30,7 @@ CREATE TABLE IF NOT EXISTS access_tokens( ); CREATE TABLE IF NOT EXISTS user_ips ( - user VARCHAR(150) NOT NULL, + user_id VARCHAR(150) NOT NULL, access_token VARCHAR(150) NOT NULL, device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, @@ -38,5 +38,5 @@ CREATE TABLE IF NOT EXISTS user_ips ( last_seen BIGINT NOT NULL ); -CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); -CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); +CREATE INDEX user_ips_user ON user_ips(user_id); +CREATE INDEX user_ips_user_ip ON user_ips(user_id, access_token, ip); From 127fad17dda0ba967d7e334e972cbf95bb689bf7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Apr 2015 14:50:29 +0100 Subject: [PATCH 081/173] Add postgres database engine --- synapse/storage/engines/postgres.py | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 synapse/storage/engines/postgres.py diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py new file mode 100644 index 000000000..9910de5dc --- /dev/null +++ b/synapse/storage/engines/postgres.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.storage import prepare_database + + +class PostgresEngine(object): + def __init__(self, database_module): + self.module = database_module + self.module.extensions.register_type(self.module.extensions.UNICODE) + + def convert_param_style(self, sql): + return sql.replace("?", "%s") + + def encode_parameter(self, param): + return param + + def on_new_connection(self, db_conn): + db_conn.set_isolation_level( + self.module.extensions.ISOLATION_LEVEL_REPEATABLE_READ + ) + + def prepare_database(self, db_conn): + prepare_database(db_conn, self) + + def is_deadlock(self, error): + if isinstance(error, self.module.DatabaseError): + return error.pgcode == "40P01" + return False + + def load_unicode(self, v): + return bytes(v).decode("UTF8") From 63677d1f4752fd928fb2b6f1e97c819896bc5323 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 10:23:24 +0100 Subject: [PATCH 082/173] Change port script to work with postgres --- scripts/port_to_maria.py | 114 +++++++++++++++++++++++---------------- 1 file changed, 67 insertions(+), 47 deletions(-) diff --git a/scripts/port_to_maria.py b/scripts/port_to_maria.py index b14cca891..0d7ba9235 100644 --- a/scripts/port_to_maria.py +++ b/scripts/port_to_maria.py @@ -22,6 +22,7 @@ from synapse.storage.engines import create_engine import argparse import itertools import logging +import types import yaml @@ -51,6 +52,14 @@ UNICODE_COLUMNS = { } +BOOLEAN_COLUMNS = { + "events": ["processed", "outlier"], + "rooms": ["is_public"], + "event_edges": ["is_state"], + "presence_list": ["accepted"], +} + + APPEND_ONLY_TABLES = [ "event_content_hashes", "event_reference_hashes", @@ -126,24 +135,22 @@ class Store(object): return self.db_pool.runWithConnection(r) - def insert_many(self, table, headers, rows): + def insert_many_txn(self, txn, table, headers, rows): sql = "INSERT INTO %s (%s) VALUES (%s)" % ( table, ", ".join(k for k in headers), ", ".join("%s" for _ in headers) ) - def t(txn): - try: - txn.executemany(sql, rows) - except: - logger.exception( - "Failed to insert: %s", - table, - ) - raise + try: + txn.executemany(sql, rows) + except: + logger.exception( + "Failed to insert: %s", + table, + ) + raise - return self.runInteraction("insert_many", t) def chunks(n): @@ -175,7 +182,7 @@ def handle_table(table, sqlite_store, mysql_store): "DELETE FROM port_from_sqlite3 WHERE table_name = %s", (table,) ) - txn.execute("DELETE FROM %s" % (table,)) + txn.execute("TRUNCATE %s CASCADE" % (table,)) mysql_store._simple_insert_txn( txn, table="port_from_sqlite3", @@ -188,14 +195,15 @@ def handle_table(table, sqlite_store, mysql_store): next_chunk = 0 + logger.info("next_chunk for %s: %d", table, next_chunk) + N = 5000 select = "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,) uni_col_names = UNICODE_COLUMNS.get(table, []) - - def conv_uni(c): - return sqlite_store.database_engine.load_unicode(c) + bool_col_names = BOOLEAN_COLUMNS.get(table, []) + bin_col_names = BINARY_COLUMNS.get(table, []) while True: def r(txn): @@ -211,24 +219,42 @@ def handle_table(table, sqlite_store, mysql_store): if rows: uni_cols = [i for i, h in enumerate(headers) if h in uni_col_names] + bool_cols = [i for i, h in enumerate(headers) if h in bool_col_names] + bin_cols = [i for i, h in enumerate(headers) if h in bin_col_names] next_chunk = rows[-1][0] + 1 + def conv(j, col): + if j in uni_cols: + col = sqlite_store.database_engine.load_unicode(col) + if j in bool_cols: + return bool(col) + + if j in bin_cols: + if isinstance(col, types.UnicodeType): + col = buffer(col.encode("utf8")) + + return col + for i, row in enumerate(rows): rows[i] = tuple( mysql_store.database_engine.encode_parameter( - conv_uni(col) if j in uni_cols else col + conv(j, col) ) for j, col in enumerate(row) if j > 0 ) - yield mysql_store.insert_many(table, headers[1:], rows) + def ins(txn): + mysql_store.insert_many_txn(txn, table, headers[1:], rows) - yield mysql_store._simple_update_one( - table="port_from_sqlite3", - keyvalues={"table_name": table}, - updatevalues={"rowid": next_chunk}, - ) + mysql_store._simple_update_one_txn( + txn, + table="port_from_sqlite3", + keyvalues={"table_name": table}, + updatevalues={"rowid": next_chunk}, + ) + + yield mysql_store.runInteraction("insert_many", ins) else: return @@ -260,7 +286,7 @@ def main(sqlite_config, mysql_config): ) sqlite_engine = create_engine("sqlite3") - mysql_engine = create_engine("mysql.connector") + mysql_engine = create_engine("psycopg2") sqlite_store = Store(sqlite_db_pool, sqlite_engine) mysql_store = Store(mysql_db_pool, mysql_engine) @@ -285,20 +311,19 @@ def main(sqlite_config, mysql_config): logger.info("Found %d tables", len(tables)) def create_port_table(txn): - try: - txn.execute( - "CREATE TABLE port_from_sqlite3 (" - " `table_name` varchar(100) NOT NULL UNIQUE," - " `rowid` bigint unsigned NOT NULL" - ")" - ) - except mysql_engine.module.DatabaseError as e: - if e.errno != mysql_engine.module.errorcode.ER_TABLE_EXISTS_ERROR: - raise + txn.execute( + "CREATE TABLE port_from_sqlite3 (" + " table_name varchar(100) NOT NULL UNIQUE," + " rowid bigint NOT NULL" + ")" + ) - yield mysql_store.runInteraction( - "create_port_table", create_port_table - ) + try: + yield mysql_store.runInteraction( + "create_port_table", create_port_table + ) + except Exception as e: + logger.info("Failed to create port table: %s", e) # Process tables. yield defer.gatherResults( @@ -342,17 +367,12 @@ if __name__ == "__main__": } mysql_config = yaml.safe_load(args.mysql_config) - mysql_config["args"].update({ - "sql_mode": "TRADITIONAL", - "charset": "utf8mb4", - "use_unicode": True, - "collation": "utf8mb4_bin", - }) - - import codecs - codecs.register( - lambda name: codecs.lookup('utf8') if name == "utf8mb4" else None - ) + # mysql_config["args"].update({ + # "sql_mode": "TRADITIONAL", + # "charset": "utf8mb4", + # "use_unicode": True, + # "collation": "utf8mb4_bin", + # }) reactor.callWhenRunning( main, From c756dfeb143d61410303dbd51e17303d0b08e7d4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 10:23:42 +0100 Subject: [PATCH 083/173] Correctly identify deadlocks --- synapse/storage/engines/postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 9910de5dc..457c1f70a 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -37,7 +37,7 @@ class PostgresEngine(object): def is_deadlock(self, error): if isinstance(error, self.module.DatabaseError): - return error.pgcode == "40P01" + return error.pgcode in ["40001", "40P01"] return False def load_unicode(self, v): From f9e12f79ca48bc87b395537438eaa74fa6d96745 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 10:24:07 +0100 Subject: [PATCH 084/173] Add missing yield in storage func --- synapse/storage/registration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 86fd2600f..8a63fe469 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -39,7 +39,7 @@ class RegistrationStore(SQLBaseStore): """ next_id = yield self._access_tokens_id_gen.get_next() - self._simple_insert( + yield self._simple_insert( "access_tokens", { "id": next_id, From ded41289655d083194061678c24d3e5551e64a7d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 10:24:24 +0100 Subject: [PATCH 085/173] Use True for True rather than 1 --- synapse/storage/events.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 9fe2effb4..0373d152b 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -269,13 +269,15 @@ class EventsStore(SQLBaseStore): "INSERT INTO events" " (stream_ordering, topological_ordering, event_id, type," " room_id, content, processed, outlier, depth)" - " VALUES (%s,?,?,?,?,?,?,?,?)" - ) % (stream_ordering,) + " VALUES (?,?,?,?,?,?,?,?,?)" + ) txn.execute( sql, - (event.depth, event.event_id, event.type, event.room_id, - content, True, outlier, event.depth) + ( + stream_ordering, event.depth, event.event_id, event.type, + event.room_id, content, True, outlier, event.depth + ) ) if context.rejected: @@ -321,7 +323,7 @@ class EventsStore(SQLBaseStore): "event_id": event.event_id, "prev_event_id": e_id, "room_id": event.room_id, - "is_state": 1, + "is_state": True, }, ) From a971fa9d584b35c35b675b65ed86faed77b46cf7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 10:25:43 +0100 Subject: [PATCH 086/173] Use try..finally in contextlib.contextmanager --- synapse/storage/util/id_generators.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index 2e2a40898..e5dec1c94 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -97,9 +97,11 @@ class StreamIdGenerator(object): @contextlib.contextmanager def manager(): - yield next_id - with self._lock: - self._unfinished_ids.remove(next_id) + try: + yield next_id + finally: + with self._lock: + self._unfinished_ids.remove(next_id) return manager() From cce5d057d34ac6fc953d1987bbce3bce2ce42bd8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 11:43:25 +0100 Subject: [PATCH 087/173] Add index on events (topological_ordering, stream_ordering) to help with ORDER BY clauses --- synapse/storage/schema/delta/16/events_order_index.sql | 1 + synapse/storage/schema/full_schemas/16/im.sql | 1 + 2 files changed, 2 insertions(+) create mode 100644 synapse/storage/schema/delta/16/events_order_index.sql diff --git a/synapse/storage/schema/delta/16/events_order_index.sql b/synapse/storage/schema/delta/16/events_order_index.sql new file mode 100644 index 000000000..c01541466 --- /dev/null +++ b/synapse/storage/schema/delta/16/events_order_index.sql @@ -0,0 +1 @@ +CREATE INDEX events_order ON events (topological_ordering, stream_ordering); \ No newline at end of file diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index a661fc160..b8faed506 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -29,6 +29,7 @@ CREATE TABLE IF NOT EXISTS events( CREATE INDEX events_stream_ordering ON events (stream_ordering); CREATE INDEX events_topological_ordering ON events (topological_ordering); +CREATE INDEX events_order ON events (topological_ordering, stream_ordering); CREATE INDEX events_room_id ON events (room_id); From 25d2914fbaa5c3670464af5f9a2430a277fdda1d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 14:09:35 +0100 Subject: [PATCH 088/173] Turn off persistance of sent_transactions.response_json --- synapse/storage/transactions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 4c3dc5866..7e3add528 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -184,7 +184,7 @@ class TransactionStore(SQLBaseStore): }, updatevalues={ "response_code": code, - "response_json": response_json, + "response_json": None, # For now, don't persist response_json } ) From 1ec6fa98c93114f1799a26100a970273e97576bb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 14:17:16 +0100 Subject: [PATCH 089/173] Parellelize initial sync --- synapse/handlers/message.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 7b9685be7..9c2ef9429 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -274,7 +274,8 @@ class MessageHandler(BaseHandler): if limit is None: limit = 10 - for event in room_list: + @defer.inlineCallbacks + def handle_room(event): d = { "room_id": event.room_id, "membership": event.membership, @@ -290,7 +291,7 @@ class MessageHandler(BaseHandler): rooms_ret.append(d) if event.membership != Membership.JOIN: - continue + return try: messages, token = yield self.store.get_recent_events_for_room( event.room_id, @@ -321,6 +322,11 @@ class MessageHandler(BaseHandler): except: logger.exception("Failed to get snapshot") + yield defer.gatherResults( + [handle_room(e) for e in room_list], + consumeErrors=True + ) + ret = { "rooms": rooms_ret, "presence": presence, From bc6cef823f20f3d14c25ee5409fdea44e518087e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 14:21:59 +0100 Subject: [PATCH 090/173] Do more parellelization for initialSync --- synapse/handlers/message.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 9c2ef9429..9667bb867 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -293,10 +293,17 @@ class MessageHandler(BaseHandler): if event.membership != Membership.JOIN: return try: - messages, token = yield self.store.get_recent_events_for_room( - event.room_id, - limit=limit, - end_token=now_token.room_key, + (messages, token), current_state = yield defer.gatherResults( + [ + self.store.get_recent_events_for_room( + event.room_id, + limit=limit, + end_token=now_token.room_key, + ), + self.state_handler.get_current_state( + event.room_id + ), + ] ) start_token = now_token.copy_and_replace("room_key", token[0]) @@ -312,9 +319,6 @@ class MessageHandler(BaseHandler): "end": end_token.to_string(), } - current_state = yield self.state_handler.get_current_state( - event.room_id - ) d["state"] = [ serialize_event(c, time_now, as_client_event) for c in current_state.values() From 4af32a2817f4a71ee960dc4000299de80bb1efb9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 14:51:21 +0100 Subject: [PATCH 091/173] Postgres does not allow you to continue using a cursor after a DB exception has been raised, so move _simple_insert or_ignore flag out of transaction --- synapse/storage/_base.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index fa5199104..c9677710b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -388,6 +388,7 @@ class SQLBaseStore(object): # "Simple" SQL API methods that operate on a single table with no JOINs, # no complex WHERE clauses, just a dict of values for columns. + @defer.inlineCallbacks def _simple_insert(self, table, values, or_ignore=False, desc="_simple_insert"): """Executes an INSERT query on the named table. @@ -396,14 +397,20 @@ class SQLBaseStore(object): table : string giving the table name values : dict of new column names and values for them """ - return self.runInteraction( - desc, - self._simple_insert_txn, table, values, - or_ignore=or_ignore - ) + try: + yield self.runInteraction( + desc, + self._simple_insert_txn, table, values, + or_ignore=or_ignore + ) + except self.database_engine.module.IntegrityError: + # We have to do or_ignore flag at this layer, since we can't reuse + # a cursor after we receive an error from the db. + if not or_ignore: + raise @log_function - def _simple_insert_txn(self, txn, table, values, or_ignore=False): + def _simple_insert_txn(self, txn, table, values): sql = "INSERT INTO %s (%s) VALUES(%s)" % ( table, ", ".join(k for k in values), @@ -415,11 +422,7 @@ class SQLBaseStore(object): sql, values.values(), ) - try: - txn.execute(sql, values.values()) - except self.database_engine.module.IntegrityError: - if not or_ignore: - raise + txn.execute(sql, values.values()) def _simple_upsert(self, table, keyvalues, values, desc="_simple_upsert"): """ From e845434028ff52f25c5b967c2d1dffcb716ed31e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 15:05:45 +0100 Subject: [PATCH 092/173] Remove run_on_reactor()s --- synapse/handlers/_base.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 48816a242..d0c027de3 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -58,8 +58,6 @@ class BaseHandler(object): @defer.inlineCallbacks def _create_new_client_event(self, builder): - yield run_on_reactor() - latest_ret = yield self.store.get_latest_events_in_room( builder.room_id, ) @@ -101,8 +99,6 @@ class BaseHandler(object): @defer.inlineCallbacks def handle_new_client_event(self, event, context, extra_destinations=[], extra_users=[], suppress_auth=False): - yield run_on_reactor() - # We now need to go and hit out to wherever we need to hit out to. if not suppress_auth: From d5ff9effcf59b55009eb8226d5f1bbac5a95bdff Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 15:05:57 +0100 Subject: [PATCH 093/173] Don't wait on federation_handler.handle_new_event --- synapse/handlers/_base.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index d0c027de3..e4471dd9b 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -139,7 +139,9 @@ class BaseHandler(object): ) # Don't block waiting on waking up all the listeners. - d = self.notifier.on_new_room_event(event, extra_users=extra_users) + notify_d = self.notifier.on_new_room_event( + event, extra_users=extra_users + ) def log_failure(f): logger.warn( @@ -147,8 +149,10 @@ class BaseHandler(object): event.event_id, f.value ) - d.addErrback(log_failure) + notify_d.addErrback(log_failure) - yield federation_handler.handle_new_event( + fed_d = federation_handler.handle_new_event( event, destinations=destinations, ) + + fed_d.addErrback(log_failure) From e19f794fee8fe32215a3dcd49ac37ed7b17a4608 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 15:12:57 +0100 Subject: [PATCH 094/173] Change from exception to warn --- synapse/notifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index d750a6fcf..ea854482b 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -98,7 +98,7 @@ class _NotificationListener(object): try: notifier.clock.cancel_call_later(self.timer) except: - logger.exception("Failed to cancel notifier timer") + logger.warn("Failed to cancel notifier timer") class Notifier(object): From a5c72780e6fdf12c003d059f685632541932e3ed Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 15:13:22 +0100 Subject: [PATCH 095/173] Don't pass in removed flag --- synapse/storage/_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index c9677710b..ede25c43b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -401,7 +401,6 @@ class SQLBaseStore(object): yield self.runInteraction( desc, self._simple_insert_txn, table, values, - or_ignore=or_ignore ) except self.database_engine.module.IntegrityError: # We have to do or_ignore flag at this layer, since we can't reuse From a19b73990962ff3bfe8b2cae59446bbe7f93ec5c Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 15 Apr 2015 15:50:38 +0100 Subject: [PATCH 096/173] Regstration with email in v2 --- synapse/handlers/__init__.py | 2 + synapse/handlers/auth.py | 64 +++++++++++++++-------- synapse/handlers/identity.py | 66 ++++++++++++++++++++++++ synapse/handlers/register.py | 6 ++- synapse/rest/client/v2_alpha/password.py | 6 +-- synapse/rest/client/v2_alpha/register.py | 8 +-- 6 files changed, 123 insertions(+), 29 deletions(-) create mode 100644 synapse/handlers/identity.py diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index 336ce1570..d1b0e032a 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -30,6 +30,7 @@ from .admin import AdminHandler from .appservice import ApplicationServicesHandler from .sync import SyncHandler from .auth import AuthHandler +from .identity import IdentityHandler class Handlers(object): @@ -60,3 +61,4 @@ class Handlers(object): ) self.sync_handler = SyncHandler(hs) self.auth_handler = AuthHandler(hs) + self.identity_handler = IdentityHandler(hs) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 3d2461dd7..2cc54707a 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -20,6 +20,7 @@ from synapse.api.constants import LoginType from synapse.types import UserID from synapse.api.errors import LoginError, Codes from synapse.http.client import SimpleHttpClient +from synapse.util.async import run_on_reactor from twisted.web.client import PartialDownloadError @@ -40,6 +41,7 @@ class AuthHandler(BaseHandler): self.checkers = { LoginType.PASSWORD: self._check_password_auth, LoginType.RECAPTCHA: self._check_recaptcha, + LoginType.EMAIL_IDENTITY: self._check_email_identity, } self.sessions = {} @@ -54,24 +56,37 @@ class AuthHandler(BaseHandler): authdict: The dictionary from the client root level, not the 'auth' key: this method prompts for auth if none is sent. Returns: - A tuple of authed, dict where authed is true if the client - has successfully completed an auth flow. If it is true, the dict - contains the authenticated credentials of each stage. - If authed is false, the dictionary is the server response to the - login request and should be passed back to the client. + A tuple of authed, dict, dict where authed is true if the client + has successfully completed an auth flow. If it is true, the first + dict contains the authenticated credentials of each stage. + + If authed is false, the first dictionary is the server response to + the login request and should be passed back to the client. + + In either case, the second dict contains the parameters for this + request (which may have been given only in a previous call). """ - if not clientdict or 'auth' not in clientdict: - sess = self._get_session_info(None) + authdict = None + sid = None + if clientdict and 'auth' in clientdict: + authdict = clientdict['auth'] + del clientdict['auth'] + if 'session' in authdict: + sid = authdict['session'] + sess = self._get_session_info(sid) + + if len(clientdict) > 0: + sess['clientdict'] = clientdict + self._save_session(sess) + elif 'clientdict' in sess: + clientdict = sess['clientdict'] + + if not authdict: defer.returnValue( - (False, self._auth_dict_for_flows(flows, sess)) + (False, self._auth_dict_for_flows(flows, sess), clientdict) ) - authdict = clientdict['auth'] - - sess = self._get_session_info( - authdict['session'] if 'session' in authdict else None - ) if 'creds' not in sess: sess['creds'] = {} creds = sess['creds'] @@ -89,11 +104,11 @@ class AuthHandler(BaseHandler): if len(set(f) - set(creds.keys())) == 0: logger.info("Auth completed with creds: %r", creds) self._remove_session(sess) - defer.returnValue((True, creds)) + defer.returnValue((True, creds, clientdict)) ret = self._auth_dict_for_flows(flows, sess) ret['completed'] = creds.keys() - defer.returnValue((False, ret)) + defer.returnValue((False, ret, clientdict)) @defer.inlineCallbacks def add_oob_auth(self, stagetype, authdict, clientip): @@ -175,18 +190,25 @@ class AuthHandler(BaseHandler): defer.returnValue(True) raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) + @defer.inlineCallbacks + def _check_email_identity(self, authdict, _): + yield run_on_reactor() + + threepidCreds = authdict['threepidCreds'] + identity_handler = self.hs.get_handlers().identity_handler + + logger.debug("Getting validated threepid. threepidcreds: %r" % (threepidCreds,)) + threepid = yield identity_handler.threepid_from_creds(threepidCreds) + + defer.returnValue(threepid) + def _get_params_recaptcha(self): return {"public_key": self.hs.config.recaptcha_public_key} def _auth_dict_for_flows(self, flows, session): public_flows = [] for f in flows: - hidden = False - for stagetype in f: - if stagetype in LoginType.HIDDEN_TYPES: - hidden = True - if not hidden: - public_flows.append(f) + public_flows.append(f) get_params = { LoginType.RECAPTCHA: self._get_params_recaptcha, diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py new file mode 100644 index 000000000..671d366e4 --- /dev/null +++ b/synapse/handlers/identity.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utilities for interacting with Identity Servers""" +from twisted.internet import defer + +from synapse.api.errors import ( + CodeMessageException +) +from ._base import BaseHandler +from synapse.http.client import SimpleHttpClient +from synapse.util.async import run_on_reactor + +import json +import logging + +logger = logging.getLogger(__name__) + + +class IdentityHandler(BaseHandler): + + def __init__(self, hs): + super(IdentityHandler, self).__init__(hs) + + @defer.inlineCallbacks + def threepid_from_creds(self, creds): + yield run_on_reactor() + + # TODO: get this from the homeserver rather than creating a new one for + # each request + http_client = SimpleHttpClient(self.hs) + # XXX: make this configurable! + #trustedIdServers = ['matrix.org', 'localhost:8090'] + trustedIdServers = ['matrix.org'] + if not creds['idServer'] in trustedIdServers: + logger.warn('%s is not a trusted ID server: rejecting 3pid ' + + 'credentials', creds['idServer']) + defer.returnValue(None) + + data = {} + try: + data = yield http_client.get_json( + "https://%s%s" % ( + creds['idServer'], + "/_matrix/identity/api/v1/3pid/getValidated3pid" + ), + {'sid': creds['sid'], 'clientSecret': creds['clientSecret']} + ) + except CodeMessageException as e: + data = json.loads(e.msg) + + if 'medium' in data: + defer.returnValue(data) + defer.returnValue(None) \ No newline at end of file diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 542759a82..6759a8c58 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -180,7 +180,11 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def register_email(self, threepidCreds): - """Registers emails with an identity server.""" + """ + Registers emails with an identity server. + + Used only by c/s api v1 + """ for c in threepidCreds: logger.info("validating theeepidcred sid %s on id server %s", diff --git a/synapse/rest/client/v2_alpha/password.py b/synapse/rest/client/v2_alpha/password.py index 85954c71c..cb0c8cfb5 100644 --- a/synapse/rest/client/v2_alpha/password.py +++ b/synapse/rest/client/v2_alpha/password.py @@ -41,7 +41,7 @@ class PasswordRestServlet(RestServlet): def on_POST(self, request): body = parse_json_dict_from_request(request) - authed, result = yield self.auth_handler.check_auth([ + authed, result, params = yield self.auth_handler.check_auth([ [LoginType.PASSWORD] ], body) @@ -61,9 +61,9 @@ class PasswordRestServlet(RestServlet): user_id = auth_user.to_string() - if 'new_password' not in body: + if 'new_password' not in params: raise SynapseError(400, "", Codes.MISSING_PARAM) - new_password = body['new_password'] + new_password = params['new_password'] yield self.login_handler.set_password( user_id, new_password, client.token_id diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 72319a3bb..d7a20fc96 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -74,7 +74,7 @@ class RegisterRestServlet(RestServlet): ) is_using_shared_secret = True else: - authed, result = yield self.auth_handler.check_auth([ + authed, result, params = yield self.auth_handler.check_auth([ [LoginType.RECAPTCHA], [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA], ], body, self.hs.get_ip_from_request(request)) @@ -90,10 +90,10 @@ class RegisterRestServlet(RestServlet): if not can_register: raise SynapseError(403, "Registration has been disabled") - if 'username' not in body or 'password' not in body: + if 'username' not in params or 'password' not in params: raise SynapseError(400, "", Codes.MISSING_PARAM) - desired_username = body['username'] - new_password = body['password'] + desired_username = params['username'] + new_password = params['password'] (user_id, token) = yield self.registration_handler.register( localpart=desired_username, From 806f380a8b448ce03671f1ce07f7dfcbdc5261e1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 16:07:59 +0100 Subject: [PATCH 097/173] Make LruCache thread safe, as its used for event cache --- synapse/util/lrucache.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/synapse/util/lrucache.py b/synapse/util/lrucache.py index 2f7b615f7..96163c90f 100644 --- a/synapse/util/lrucache.py +++ b/synapse/util/lrucache.py @@ -14,6 +14,10 @@ # limitations under the License. +from functools import wraps +import threading + + class LruCache(object): """Least-recently-used cache.""" # TODO(mjark) Add mutex for linked list for thread safety. @@ -24,6 +28,16 @@ class LruCache(object): PREV, NEXT, KEY, VALUE = 0, 1, 2, 3 + lock = threading.Lock() + + def synchronized(f): + @wraps(f) + def inner(*args, **kwargs): + with lock: + return f(*args, **kwargs) + + return inner + def add_node(key, value): prev_node = list_root next_node = prev_node[NEXT] @@ -51,6 +65,7 @@ class LruCache(object): next_node[PREV] = prev_node cache.pop(node[KEY], None) + @synchronized def cache_get(key, default=None): node = cache.get(key, None) if node is not None: @@ -59,6 +74,7 @@ class LruCache(object): else: return default + @synchronized def cache_set(key, value): node = cache.get(key, None) if node is not None: @@ -69,6 +85,7 @@ class LruCache(object): if len(cache) > max_size: delete_node(list_root[PREV]) + @synchronized def cache_set_default(key, value): node = cache.get(key, None) if node is not None: @@ -79,6 +96,7 @@ class LruCache(object): delete_node(list_root[PREV]) return value + @synchronized def cache_pop(key, default=None): node = cache.get(key, None) if node: @@ -87,9 +105,11 @@ class LruCache(object): else: return default + @synchronized def cache_len(): return len(cache) + @synchronized def cache_contains(key): return key in cache From ed26e4012bb68cbde33f020b2756efe2bc87455a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 16:24:14 +0100 Subject: [PATCH 098/173] pushers table requires a unique id. --- synapse/storage/_base.py | 10 +++++++--- synapse/storage/pusher.py | 4 ++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index ede25c43b..43a923d91 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -260,6 +260,7 @@ class SQLBaseStore(object): self._transaction_id_gen = IdGenerator("sent_transactions", "id", self) self._state_groups_id_gen = IdGenerator("state_groups", "id", self) self._access_tokens_id_gen = IdGenerator("access_tokens", "id", self) + self._pushers_id_gen = IdGenerator("pushers", "id", self) def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -423,20 +424,22 @@ class SQLBaseStore(object): txn.execute(sql, values.values()) - def _simple_upsert(self, table, keyvalues, values, desc="_simple_upsert"): + def _simple_upsert(self, table, keyvalues, values, + insertion_values={}, desc="_simple_upsert"): """ Args: table (str): The table to upsert into keyvalues (dict): The unique key tables and their new values values (dict): The nonunique columns and their new values + insertion_values (dict): key/values to use when inserting Returns: A deferred """ return self.runInteraction( desc, - self._simple_upsert_txn, table, keyvalues, values + self._simple_upsert_txn, table, keyvalues, values, insertion_values, ) - def _simple_upsert_txn(self, txn, table, keyvalues, values): + def _simple_upsert_txn(self, txn, table, keyvalues, values, insertion_values={}): # Try to update sql = "UPDATE %s SET %s WHERE %s" % ( table, @@ -455,6 +458,7 @@ class SQLBaseStore(object): allvalues = {} allvalues.update(keyvalues) allvalues.update(values) + allvalues.update(insertion_values) sql = "INSERT INTO %s (%s) VALUES (%s)" % ( table, diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 000502b4f..e4135ba92 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -99,6 +99,7 @@ class PusherStore(SQLBaseStore): app_display_name, device_display_name, pushkey, pushkey_ts, lang, data): try: + next_id = self._pushers_id_gen.get_next() yield self._simple_upsert( PushersTable.table_name, dict( @@ -115,6 +116,9 @@ class PusherStore(SQLBaseStore): lang=lang, data=data ), + insertion_values=dict( + id=next_id, + ), desc="add_pusher", ) except Exception as e: From 6e91f14d09cf64100dc8fecc83c333f59fe8e77a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 16:25:07 +0100 Subject: [PATCH 099/173] Add missing yield --- synapse/storage/pusher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index e4135ba92..80c3ba154 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -99,7 +99,7 @@ class PusherStore(SQLBaseStore): app_display_name, device_display_name, pushkey, pushkey_ts, lang, data): try: - next_id = self._pushers_id_gen.get_next() + next_id = yield self._pushers_id_gen.get_next() yield self._simple_upsert( PushersTable.table_name, dict( From 5b31afcbd1c2016d03a29169e3fdc22a1adf7f52 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 16:27:04 +0100 Subject: [PATCH 100/173] Remove debug logging --- synapse/storage/_base.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 43a923d91..5ec1d2613 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -847,11 +847,9 @@ class SQLBaseStore(object): sql_getevents_timer.inc_by(curr_time - last_time, desc) return curr_time - logger.debug("Got js: %r", js) d = json.loads(js) start_time = update_counter("decode_json", start_time) - logger.debug("Got internal_metadata: %r", internal_metadata) internal_metadata = json.loads(internal_metadata) start_time = update_counter("decode_internal", start_time) From cf04cedf216a21f36a476703470b7ffe1a727967 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 16:53:47 +0100 Subject: [PATCH 101/173] Change full_schemas/11 to work with postgres --- .../schema/delta/16/events_order_index.sql | 5 +- .../schema/full_schemas/11/event_edges.sql | 26 +++++----- .../full_schemas/11/event_signatures.sql | 16 +++--- synapse/storage/schema/full_schemas/11/im.sql | 50 +++++++++---------- .../storage/schema/full_schemas/11/keys.sql | 8 +-- .../full_schemas/11/media_repository.sql | 6 +-- .../schema/full_schemas/11/presence.sql | 2 +- .../schema/full_schemas/11/redactions.sql | 4 +- .../storage/schema/full_schemas/11/state.sql | 14 +++--- .../schema/full_schemas/11/transactions.sql | 24 ++++----- .../storage/schema/full_schemas/11/users.sql | 10 ++-- synapse/storage/schema/full_schemas/16/im.sql | 5 +- .../storage/schema/full_schemas/16/state.sql | 4 +- 13 files changed, 90 insertions(+), 84 deletions(-) diff --git a/synapse/storage/schema/delta/16/events_order_index.sql b/synapse/storage/schema/delta/16/events_order_index.sql index c01541466..a48f21517 100644 --- a/synapse/storage/schema/delta/16/events_order_index.sql +++ b/synapse/storage/schema/delta/16/events_order_index.sql @@ -1 +1,4 @@ -CREATE INDEX events_order ON events (topological_ordering, stream_ordering); \ No newline at end of file +CREATE INDEX events_order ON events (topological_ordering, stream_ordering); +CREATE INDEX events_order_room ON events ( + room_id, topological_ordering, stream_ordering +); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index bdb110909..05d0874f0 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities( UNIQUE (event_id, room_id) ); -CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); -CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); +CREATE INDEX ev_extrem_room ON event_forward_extremities(room_id); +CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_backward_extremities( @@ -29,8 +29,8 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities( UNIQUE (event_id, room_id) ); -CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); -CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); +CREATE INDEX ev_b_extrem_room ON event_backward_extremities(room_id); +CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_edges( @@ -41,8 +41,8 @@ CREATE TABLE IF NOT EXISTS event_edges( UNIQUE (event_id, prev_event_id, room_id, is_state) ); -CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); -CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); +CREATE INDEX ev_edges_id ON event_edges(event_id); +CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id); CREATE TABLE IF NOT EXISTS room_depth( @@ -51,17 +51,17 @@ CREATE TABLE IF NOT EXISTS room_depth( UNIQUE (room_id) ); -CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); +CREATE INDEX room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( event_id VARCHAR(150) NOT NULL, destination VARCHAR(150) NOT NULL, - delivered_ts BIGINT UNSIGNED DEFAULT 0, -- or 0 if not delivered + delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ); -CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); +CREATE INDEX event_destinations_id ON event_destinations(event_id); CREATE TABLE IF NOT EXISTS state_forward_extremities( @@ -72,10 +72,10 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities( UNIQUE (event_id, room_id) ); -CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( +CREATE INDEX st_extrem_keys ON state_forward_extremities( room_id, type, state_key ); -CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); +CREATE INDEX st_extrem_id ON state_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_auth( @@ -85,5 +85,5 @@ CREATE TABLE IF NOT EXISTS event_auth( UNIQUE (event_id, auth_id, room_id) ); -CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); -CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); +CREATE INDEX evauth_edges_id ON event_auth(event_id); +CREATE INDEX evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index 09886f607..429182736 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -16,40 +16,40 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash LONGBLOB, + hash bytea, UNIQUE (event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); +CREATE INDEX event_content_hashes_id ON event_content_hashes(event_id); CREATE TABLE IF NOT EXISTS event_reference_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash LONGBLOB, + hash bytea, UNIQUE (event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); +CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id); CREATE TABLE IF NOT EXISTS event_signatures ( event_id VARCHAR(150), signature_name VARCHAR(150), key_id VARCHAR(150), - signature LONGBLOB, + signature bytea, UNIQUE (event_id, signature_name, key_id) ); -CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); +CREATE INDEX event_signatures_id ON event_signatures(event_id); CREATE TABLE IF NOT EXISTS event_edge_hashes( event_id VARCHAR(150), prev_event_id VARCHAR(150), algorithm VARCHAR(150), - hash LONGBLOB, + hash bytea, UNIQUE (event_id, prev_event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); +CREATE INDEX event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index 9c47f5174..e9e09214d 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -15,32 +15,32 @@ CREATE TABLE IF NOT EXISTS events( stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, - topological_ordering BIGINT UNSIGNED NOT NULL, + topological_ordering BIGINT NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - content LONGBLOB NOT NULL, - unrecognized_keys LONGBLOB, + content bytea NOT NULL, + unrecognized_keys bytea, processed BOOL NOT NULL, outlier BOOL NOT NULL, - depth BIGINT UNSIGNED DEFAULT 0 NOT NULL, + depth BIGINT DEFAULT 0 NOT NULL, UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); -CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); -CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); +CREATE INDEX events_stream_ordering ON events (stream_ordering); +CREATE INDEX events_topological_ordering ON events (topological_ordering); +CREATE INDEX events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - internal_metadata LONGBLOB NOT NULL, - json LONGBLOB NOT NULL, + internal_metadata bytea NOT NULL, + json bytea NOT NULL, UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); +CREATE INDEX event_json_room_id ON event_json(room_id); CREATE TABLE IF NOT EXISTS state_events( @@ -52,9 +52,9 @@ CREATE TABLE IF NOT EXISTS state_events( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); -CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); -CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); +CREATE INDEX state_events_room_id ON state_events (room_id); +CREATE INDEX state_events_type ON state_events (type); +CREATE INDEX state_events_state_key ON state_events (state_key); CREATE TABLE IF NOT EXISTS current_state_events( @@ -65,10 +65,10 @@ CREATE TABLE IF NOT EXISTS current_state_events( UNIQUE (room_id, type, state_key) ); -CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id); -CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); -CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); -CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); +CREATE INDEX curr_events_event_id ON current_state_events (event_id); +CREATE INDEX current_state_events_room_id ON current_state_events (room_id); +CREATE INDEX current_state_events_type ON current_state_events (type); +CREATE INDEX current_state_events_state_key ON current_state_events (state_key); CREATE TABLE IF NOT EXISTS room_memberships( event_id VARCHAR(150) NOT NULL, @@ -78,9 +78,9 @@ CREATE TABLE IF NOT EXISTS room_memberships( membership VARCHAR(150) NOT NULL ); -CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id); -CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); -CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); +CREATE INDEX room_memberships_event_id ON room_memberships (event_id); +CREATE INDEX room_memberships_room_id ON room_memberships (room_id); +CREATE INDEX room_memberships_user_id ON room_memberships (user_id); CREATE TABLE IF NOT EXISTS feedback( event_id VARCHAR(150) NOT NULL, @@ -96,8 +96,8 @@ CREATE TABLE IF NOT EXISTS topics( topic TEXT NOT NULL ); -CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id); -CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); +CREATE INDEX topics_event_id ON topics(event_id); +CREATE INDEX topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( event_id VARCHAR(150) NOT NULL, @@ -105,8 +105,8 @@ CREATE TABLE IF NOT EXISTS room_names( name TEXT NOT NULL ); -CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id); -CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); +CREATE INDEX room_names_event_id ON room_names(event_id); +CREATE INDEX room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( room_id VARCHAR(150) PRIMARY KEY NOT NULL, @@ -120,4 +120,4 @@ CREATE TABLE IF NOT EXISTS room_hosts( UNIQUE (room_id, host) ); -CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); +CREATE INDEX room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index 35f141c28..459b51042 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -16,8 +16,8 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( server_name VARCHAR(150), -- Server name. fingerprint VARCHAR(150), -- Certificate fingerprint. from_server VARCHAR(150), -- Which key server the certificate was fetched from. - ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added. - tls_certificate LONGBLOB, -- DER encoded x509 certificate. + ts_added_ms BIGINT, -- When the certifcate was added. + tls_certificate bytea, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(150), -- Server name. key_id VARCHAR(150), -- Key version. from_server VARCHAR(150), -- Which key server the key was fetched form. - ts_added_ms BIGINT UNSIGNED, -- When the key was added. - verify_key LONGBLOB, -- NACL verification key. + ts_added_ms BIGINT, -- When the key was added. + verify_key bytea, -- NACL verification key. UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index 134e7fbce..6e0ee0db4 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( media_id VARCHAR(150), -- The id used to refer to the media. media_type VARCHAR(150), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. - created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. + created_ts BIGINT, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. user_id VARCHAR(150), -- The user who uploaded the file. UNIQUE (media_id) @@ -35,14 +35,14 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( ) ); -CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id +CREATE INDEX local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); CREATE TABLE IF NOT EXISTS remote_media_cache ( media_origin VARCHAR(150), -- The remote HS the media came from. media_id VARCHAR(150), -- The id used to refer to the media on that server. media_type VARCHAR(150), -- The MIME-type of the media. - created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. + created_ts BIGINT, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(150), -- The name used to store the media on disk. diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index c617ebea7..fce324b89 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS presence( user_id VARCHAR(150) NOT NULL, state VARCHAR(20), status_msg VARCHAR(150), - mtime BIGINT UNSIGNED -- miliseconds since last state change + mtime BIGINT -- miliseconds since last state change ); -- For each of /my/ users which possibly-remote users are allowed to see their diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index b81451eab..492fd2203 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -18,5 +18,5 @@ CREATE TABLE IF NOT EXISTS redactions ( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); -CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); +CREATE INDEX redactions_event_id ON redactions (event_id); +CREATE INDEX redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index 737c3e35c..62c20819f 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -14,13 +14,13 @@ */ CREATE TABLE IF NOT EXISTS state_groups( - id VARCHAR(20) PRIMARY KEY, + id INTEGER PRIMARY KEY, room_id VARCHAR(150) NOT NULL, event_id VARCHAR(150) NOT NULL ); CREATE TABLE IF NOT EXISTS state_groups_state( - state_group VARCHAR(20) NOT NULL, + state_group INTEGER NOT NULL, room_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, state_key VARCHAR(150) NOT NULL, @@ -29,12 +29,12 @@ CREATE TABLE IF NOT EXISTS state_groups_state( CREATE TABLE IF NOT EXISTS event_to_state_groups( event_id VARCHAR(150) NOT NULL, - state_group VARCHAR(150) NOT NULL, + state_group INTEGER NOT NULL, UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); +CREATE INDEX state_groups_id ON state_groups(id); -CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(state_group); -CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(room_id, type, state_key); -CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id); \ No newline at end of file +CREATE INDEX state_groups_state_id ON state_groups_state(state_group); +CREATE INDEX state_groups_state_tuple ON state_groups_state(room_id, type, state_key); +CREATE INDEX event_to_state_groups_id ON event_to_state_groups(event_id); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index c2fab10aa..f01d9eba3 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -16,14 +16,14 @@ CREATE TABLE IF NOT EXISTS received_transactions( transaction_id VARCHAR(150), origin VARCHAR(150), - ts BIGINT UNSIGNED, + ts BIGINT, response_code INTEGER, - response_json LONGBLOB, - has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx + response_json bytea, + has_been_referenced SMALLINT DEFAULT 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) ); -CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; +CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; -- Stores what transactions we've sent, what their response was (if we got one) and whether we have @@ -33,15 +33,15 @@ CREATE TABLE IF NOT EXISTS sent_transactions( transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, - response_json LONGBLOB, - ts BIGINT UNSIGNED + response_json bytea, + ts BIGINT ); -CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); -CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); +CREATE INDEX sent_transaction_dest ON sent_transactions(destination); +CREATE INDEX sent_transaction_txn_id ON sent_transactions(transaction_id); -- So that we can do an efficient look up of all transactions that have yet to be successfully -- sent. -CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code); +CREATE INDEX sent_transaction_sent ON sent_transactions(response_code); -- For sent transactions only. @@ -52,12 +52,12 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( pdu_origin VARCHAR(150) ); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); +CREATE INDEX transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination); +CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(150) PRIMARY KEY, - retry_last_ts BIGINT UNSIGNED, + retry_last_ts BIGINT, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 0ddfccd41..2edc13c3b 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -16,8 +16,8 @@ CREATE TABLE IF NOT EXISTS users( id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(150), password_hash VARCHAR(150), - creation_ts BIGINT UNSIGNED, - admin BOOL DEFAULT 0 NOT NULL, + creation_ts BIGINT, + admin SMALLINT DEFAULT 0 NOT NULL, UNIQUE(name) ); @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS access_tokens( user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, - last_used BIGINT UNSIGNED, + last_used BIGINT, UNIQUE(token) ); @@ -36,8 +36,8 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT UNSIGNED NOT NULL, + last_seen BIGINT NOT NULL, UNIQUE (user, access_token, ip, user_agent) ); -CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); +CREATE INDEX user_ips_user ON user_ips(user); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index b8faed506..17e4c949b 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering BIGINT PRIMARY KEY, + stream_ordering INTEGER PRIMARY KEY, topological_ordering BIGINT NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, @@ -31,6 +31,9 @@ CREATE INDEX events_stream_ordering ON events (stream_ordering); CREATE INDEX events_topological_ordering ON events (topological_ordering); CREATE INDEX events_order ON events (topological_ordering, stream_ordering); CREATE INDEX events_room_id ON events (room_id); +CREATE INDEX events_order_room ON events ( + room_id, topological_ordering, stream_ordering +); CREATE TABLE IF NOT EXISTS event_json( diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql index 3c54595e6..b0cd5ee75 100644 --- a/synapse/storage/schema/full_schemas/16/state.sql +++ b/synapse/storage/schema/full_schemas/16/state.sql @@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS state_groups( ); CREATE TABLE IF NOT EXISTS state_groups_state( - state_group VARCHAR(20) NOT NULL, + state_group BIGINT NOT NULL, room_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, state_key VARCHAR(150) NOT NULL, @@ -29,7 +29,7 @@ CREATE TABLE IF NOT EXISTS state_groups_state( CREATE TABLE IF NOT EXISTS event_to_state_groups( event_id VARCHAR(150) NOT NULL, - state_group VARCHAR(150) NOT NULL, + state_group BIGINT NOT NULL, UNIQUE (event_id) ); From ffad75bd6284873c27efb2cfdfdcf9f909eb9db3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 17:00:50 +0100 Subject: [PATCH 102/173] Remove mysql/maria support --- ...ria.py => port_from_sqlite_to_postgres.py} | 58 ++++++++----------- synapse/app/homeserver.py | 9 +-- synapse/storage/engines/__init__.py | 2 - synapse/storage/engines/maria.py | 50 ---------------- .../schema/delta/16/unique_constraints.sql | 2 +- 5 files changed, 26 insertions(+), 95 deletions(-) rename scripts/{port_to_maria.py => port_from_sqlite_to_postgres.py} (85%) delete mode 100644 synapse/storage/engines/maria.py diff --git a/scripts/port_to_maria.py b/scripts/port_from_sqlite_to_postgres.py similarity index 85% rename from scripts/port_to_maria.py rename to scripts/port_from_sqlite_to_postgres.py index 0d7ba9235..4b3fd9e52 100644 --- a/scripts/port_to_maria.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -26,7 +26,7 @@ import types import yaml -logger = logging.getLogger("port_to_maria") +logger = logging.getLogger("port_from_sqlite_to_postgres") BINARY_COLUMNS = { @@ -159,10 +159,10 @@ def chunks(n): @defer.inlineCallbacks -def handle_table(table, sqlite_store, mysql_store): +def handle_table(table, sqlite_store, postgres_store): if table in APPEND_ONLY_TABLES: # It's safe to just carry on inserting. - next_chunk = yield mysql_store._simple_select_one_onecol( + next_chunk = yield postgres_store._simple_select_one_onecol( table="port_from_sqlite3", keyvalues={"table_name": table}, retcol="rowid", @@ -170,7 +170,7 @@ def handle_table(table, sqlite_store, mysql_store): ) if next_chunk is None: - yield mysql_store._simple_insert( + yield postgres_store._simple_insert( table="port_from_sqlite3", values={"table_name": table, "rowid": 0} ) @@ -183,13 +183,13 @@ def handle_table(table, sqlite_store, mysql_store): (table,) ) txn.execute("TRUNCATE %s CASCADE" % (table,)) - mysql_store._simple_insert_txn( + postgres_store._simple_insert_txn( txn, table="port_from_sqlite3", values={"table_name": table, "rowid": 0} ) - yield mysql_store.runInteraction( + yield postgres_store.runInteraction( "delete_non_append_only", delete_all ) @@ -237,7 +237,7 @@ def handle_table(table, sqlite_store, mysql_store): for i, row in enumerate(rows): rows[i] = tuple( - mysql_store.database_engine.encode_parameter( + postgres_store.database_engine.encode_parameter( conv(j, col) ) for j, col in enumerate(row) @@ -245,16 +245,16 @@ def handle_table(table, sqlite_store, mysql_store): ) def ins(txn): - mysql_store.insert_many_txn(txn, table, headers[1:], rows) + postgres_store.insert_many_txn(txn, table, headers[1:], rows) - mysql_store._simple_update_one_txn( + postgres_store._simple_update_one_txn( txn, table="port_from_sqlite3", keyvalues={"table_name": table}, updatevalues={"rowid": next_chunk}, ) - yield mysql_store.runInteraction("insert_many", ins) + yield postgres_store.runInteraction("insert_many", ins) else: return @@ -273,30 +273,30 @@ def setup_db(db_config, database_engine): @defer.inlineCallbacks -def main(sqlite_config, mysql_config): +def main(sqlite_config, postgress_config): try: sqlite_db_pool = adbapi.ConnectionPool( sqlite_config["name"], **sqlite_config["args"] ) - mysql_db_pool = adbapi.ConnectionPool( - mysql_config["name"], - **mysql_config["args"] + postgres_db_pool = adbapi.ConnectionPool( + postgress_config["name"], + **postgress_config["args"] ) sqlite_engine = create_engine("sqlite3") - mysql_engine = create_engine("psycopg2") + postgres_engine = create_engine("psycopg2") sqlite_store = Store(sqlite_db_pool, sqlite_engine) - mysql_store = Store(mysql_db_pool, mysql_engine) + postgres_store = Store(postgres_db_pool, postgres_engine) - # Step 1. Set up mysql database. + # Step 1. Set up databases. logger.info("Preparing sqlite database...") setup_db(sqlite_config, sqlite_engine) - logger.info("Preparing mysql database...") - setup_db(mysql_config, mysql_engine) + logger.info("Preparing postgres database...") + setup_db(postgress_config, postgres_engine) # Step 2. Get tables. logger.info("Fetching tables...") @@ -319,7 +319,7 @@ def main(sqlite_config, mysql_config): ) try: - yield mysql_store.runInteraction( + yield postgres_store.runInteraction( "create_port_table", create_port_table ) except Exception as e: @@ -328,7 +328,7 @@ def main(sqlite_config, mysql_config): # Process tables. yield defer.gatherResults( [ - handle_table(table, sqlite_store, mysql_store) + handle_table(table, sqlite_store, postgres_store) for table in tables if table not in ["schema_version", "applied_schema_deltas"] and not table.startswith("sqlite_") @@ -336,10 +336,6 @@ def main(sqlite_config, mysql_config): consumeErrors=True, ) - # for table in ["current_state_events"]: # tables: - # if table not in ["schema_version", "applied_schema_deltas"]: - # if not table.startswith("sqlite_"): - # yield handle_table(table, sqlite_store, mysql_store) except: logger.exception("") finally: @@ -350,7 +346,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--sqlite-database") parser.add_argument( - "--mysql-config", type=argparse.FileType('r'), + "--postgres-config", type=argparse.FileType('r'), ) args = parser.parse_args() @@ -366,18 +362,12 @@ if __name__ == "__main__": }, } - mysql_config = yaml.safe_load(args.mysql_config) - # mysql_config["args"].update({ - # "sql_mode": "TRADITIONAL", - # "charset": "utf8mb4", - # "use_unicode": True, - # "collation": "utf8mb4_bin", - # }) + postgres_config = yaml.safe_load(args.postgres_config) reactor.callWhenRunning( main, sqlite_config=sqlite_config, - mysql_config=mysql_config, + postgres_config=postgres_config, ) reactor.run() diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f8a33120b..93500dd79 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -366,14 +366,7 @@ def setup(config_options): } name = db_config.get("name", None) - if name in ["MySQLdb", "mysql.connector"]: - db_config.setdefault("args", {}).update({ - "sql_mode": "TRADITIONAL", - "charset": "utf8mb4", - "use_unicode": True, - "collation": "utf8mb4_bin", - }) - elif name == "psycopg2": + if name == "psycopg2": pass elif name == "sqlite3": db_config.setdefault("args", {}).update({ diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index 548d4e1b4..eb76df7f0 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .maria import MariaEngine from .postgres import PostgresEngine from .sqlite3 import Sqlite3Engine @@ -22,7 +21,6 @@ import importlib SUPPORTED_MODULE = { "sqlite3": Sqlite3Engine, - "mysql.connector": MariaEngine, "psycopg2": PostgresEngine, } diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py deleted file mode 100644 index 90165f684..000000000 --- a/synapse/storage/engines/maria.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from synapse.storage import prepare_database - -import types - - -class MariaEngine(object): - def __init__(self, database_module): - self.module = database_module - - def convert_param_style(self, sql): - return sql.replace("?", "%s") - - def encode_parameter(self, param): - if isinstance(param, types.BufferType): - return bytes(param) - return param - - def on_new_connection(self, db_conn): - pass - - def prepare_database(self, db_conn): - cur = db_conn.cursor() - cur.execute( - "ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_bin" - ) - db_conn.commit() - prepare_database(db_conn, self) - - def is_deadlock(self, error): - if isinstance(error, self.module.DatabaseError): - return error.sqlstate == "40001" and error.errno == 1213 - return False - - def load_unicode(self, v): - return bytes(v).decode("UTF8") diff --git a/synapse/storage/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/delta/16/unique_constraints.sql index f9fbb6b44..3604ea842 100644 --- a/synapse/storage/schema/delta/16/unique_constraints.sql +++ b/synapse/storage/schema/delta/16/unique_constraints.sql @@ -1,5 +1,5 @@ --- We can use SQLite features here, since mysql support was only added in v16 +-- We can use SQLite features here, since other db support was only added in v16 -- DELETE FROM current_state_events WHERE rowid not in ( From 766bd8e88077cbeabffc353d9735a3af190abe61 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 15 Apr 2015 17:14:25 +0100 Subject: [PATCH 103/173] Dummy login so we can do the first POST request to get login flows without it just succeeding --- synapse/api/constants.py | 1 + synapse/handlers/auth.py | 6 ++++++ synapse/handlers/identity.py | 6 +++--- synapse/rest/client/v2_alpha/register.py | 18 ++++++++++++++---- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index d29c2dde0..d8a18ee87 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -59,6 +59,7 @@ class LoginType(object): EMAIL_URL = u"m.login.email.url" EMAIL_IDENTITY = u"m.login.email.identity" RECAPTCHA = u"m.login.recaptcha" + DUMMY = u"m.login.dummy" # Only for C/S API v1 APPLICATION_SERVICE = u"m.login.application_service" diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 2cc54707a..87866f298 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -42,6 +42,7 @@ class AuthHandler(BaseHandler): LoginType.PASSWORD: self._check_password_auth, LoginType.RECAPTCHA: self._check_recaptcha, LoginType.EMAIL_IDENTITY: self._check_email_identity, + LoginType.DUMMY: self._check_dummy_auth, } self.sessions = {} @@ -202,6 +203,11 @@ class AuthHandler(BaseHandler): defer.returnValue(threepid) + @defer.inlineCallbacks + def _check_dummy_auth(self, authdict, _): + yield run_on_reactor() + defer.returnValue(True) + def _get_params_recaptcha(self): return {"public_key": self.hs.config.recaptcha_public_key} diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 671d366e4..19896ce90 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -42,8 +42,8 @@ class IdentityHandler(BaseHandler): # each request http_client = SimpleHttpClient(self.hs) # XXX: make this configurable! - #trustedIdServers = ['matrix.org', 'localhost:8090'] - trustedIdServers = ['matrix.org'] + trustedIdServers = ['matrix.org', 'localhost:8090'] + #trustedIdServers = ['matrix.org'] if not creds['idServer'] in trustedIdServers: logger.warn('%s is not a trusted ID server: rejecting 3pid ' + 'credentials', creds['idServer']) @@ -52,7 +52,7 @@ class IdentityHandler(BaseHandler): data = {} try: data = yield http_client.get_json( - "https://%s%s" % ( + "http://%s%s" % ( creds['idServer'], "/_matrix/identity/api/v1/3pid/getValidated3pid" ), diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index d7a20fc96..ee99b74fd 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -63,6 +63,17 @@ class RegisterRestServlet(RestServlet): if 'access_token' in request.args: service = yield self.auth.get_appservice_by_req(request) + if self.hs.config.enable_registration_captcha: + flows = [ + [LoginType.RECAPTCHA], + [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA] + ] + else: + flows = [ + [LoginType.DUMMY], + [LoginType.EMAIL_IDENTITY] + ] + if service: is_application_server = True elif 'mac' in body: @@ -74,10 +85,9 @@ class RegisterRestServlet(RestServlet): ) is_using_shared_secret = True else: - authed, result, params = yield self.auth_handler.check_auth([ - [LoginType.RECAPTCHA], - [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA], - ], body, self.hs.get_ip_from_request(request)) + authed, result, params = yield self.auth_handler.check_auth( + flows, body, self.hs.get_ip_from_request(request) + ) if not authed: defer.returnValue((401, result)) From bc9e69e16099142acdb06c3ea7c7ef9aec9e4ec9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 16 Apr 2015 11:01:09 +0100 Subject: [PATCH 104/173] Move encoding and decoding of JSON into storage layer --- synapse/push/pusherpool.py | 8 +------- synapse/storage/pusher.py | 26 ++++++-------------------- 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 90babd722..7f3dd50b4 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -19,10 +19,7 @@ from twisted.internet import defer from httppusher import HttpPusher from synapse.push import PusherConfigException -from syutil.jsonutil import encode_canonical_json - import logging -import simplejson as json logger = logging.getLogger(__name__) @@ -52,8 +49,6 @@ class PusherPool: @defer.inlineCallbacks def start(self): pushers = yield self.store.get_all_pushers() - for p in pushers: - p['data'] = json.loads(p['data']) self._start_pushers(pushers) @defer.inlineCallbacks @@ -98,7 +93,7 @@ class PusherPool: pushkey=pushkey, pushkey_ts=self.hs.get_clock().time_msec(), lang=lang, - data=encode_canonical_json(data).decode("UTF-8"), + data=data, ) self._refresh_pusher((app_id, pushkey)) @@ -129,7 +124,6 @@ class PusherPool: p = yield self.store.get_pushers_by_app_id_and_pushkey( app_id_pushkey ) - p['data'] = json.loads(p['data']) self._start_pushers([p]) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 80c3ba154..5c1c3d32e 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -20,7 +20,10 @@ from twisted.internet import defer from synapse.api.errors import StoreError +from syutil.jsonutil import encode_canonical_json + import logging +import simplejson as json logger = logging.getLogger(__name__) @@ -52,7 +55,7 @@ class PusherStore(SQLBaseStore): "device_display_name": r[6], "pushkey": r[7], "pushkey_ts": r[8], - "data": r[9], + "data": json.loads(r[9]), "last_token": r[10], "last_success": r[11], "failing_since": r[12] @@ -84,7 +87,7 @@ class PusherStore(SQLBaseStore): "device_display_name": r[6], "pushkey": r[7], "pushkey_ts": r[8], - "data": r[9], + "data": json.loads(r[9]), "last_token": r[10], "last_success": r[11], "failing_since": r[12] @@ -114,7 +117,7 @@ class PusherStore(SQLBaseStore): device_display_name=device_display_name, ts=pushkey_ts, lang=lang, - data=data + data=encode_canonical_json(data).decode("UTF-8"), ), insertion_values=dict( id=next_id, @@ -165,20 +168,3 @@ class PusherStore(SQLBaseStore): class PushersTable(Table): table_name = "pushers" - fields = [ - "id", - "user_name", - "kind", - "profile_tag", - "app_id", - "app_display_name", - "device_display_name", - "pushkey", - "pushkey_ts", - "data", - "last_token", - "last_success", - "failing_since" - ] - - EntryType = collections.namedtuple("PusherEntry", fields) From b8092fbc82edb3c7d8aa09f0756fa853ad6a6ad8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 16 Apr 2015 11:17:52 +0100 Subject: [PATCH 105/173] Go back to storing JSON in TEXT --- synapse/storage/_base.py | 3 --- synapse/storage/appservice.py | 4 +--- synapse/storage/engines/postgres.py | 3 --- synapse/storage/engines/sqlite3.py | 5 ----- synapse/storage/events.py | 16 ++++++++-------- synapse/storage/profile.py | 5 +---- synapse/storage/registration.py | 5 ----- .../storage/schema/delta/15/appservice_txns.sql | 2 +- synapse/storage/schema/full_schemas/11/im.sql | 8 ++++---- .../full_schemas/16/application_services.sql | 2 +- synapse/storage/schema/full_schemas/16/im.sql | 8 ++++---- 11 files changed, 20 insertions(+), 41 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 5ec1d2613..f5952d1fc 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -818,9 +818,6 @@ class SQLBaseStore(object): internal_metadata, js, redacted, rejected_reason = res - internal_metadata = self.database_engine.load_unicode(internal_metadata) - js = self.database_engine.load_unicode(js) - start_time = update_counter("select_event", start_time) result = self._get_event_from_row_txn( diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 40e05b363..63d1af4e8 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -366,9 +366,7 @@ class ApplicationServiceTransactionStore(SQLBaseStore): new_txn_id = max(highest_txn_id, last_txn_id) + 1 # Insert new txn into txn table - event_ids = buffer( - json.dumps([e.event_id for e in events]).encode("utf8") - ) + event_ids = json.dumps([e.event_id for e in events]) txn.execute( "INSERT INTO application_services_txns(as_id, txn_id, event_ids) " "VALUES(?,?,?)", diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 457c1f70a..6f75245fa 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -39,6 +39,3 @@ class PostgresEngine(object): if isinstance(error, self.module.DatabaseError): return error.pgcode in ["40001", "40P01"] return False - - def load_unicode(self, v): - return bytes(v).decode("UTF8") diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 389df35eb..dd0d8e0e0 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -37,8 +37,3 @@ class Sqlite3Engine(object): def is_deadlock(self, error): return False - - def load_unicode(self, v): - if isinstance(v, types.UnicodeType): - return v - return bytes(v).decode("UTF8") diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 0373d152b..7dbf7a396 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -168,7 +168,7 @@ class EventsStore(SQLBaseStore): metadata_json = encode_canonical_json( event.internal_metadata.get_dict() - ) + ).decode("UTF-8") # If we have already persisted this event, we don't need to do any # more processing. @@ -184,7 +184,7 @@ class EventsStore(SQLBaseStore): ) txn.execute( sql, - (buffer(metadata_json), event.event_id,) + (metadata_json, event.event_id,) ) sql = ( @@ -229,14 +229,14 @@ class EventsStore(SQLBaseStore): values={ "event_id": event.event_id, "room_id": event.room_id, - "internal_metadata": buffer(metadata_json), - "json": buffer(encode_canonical_json(event_dict)), + "internal_metadata": metadata_json, + "json": encode_canonical_json(event_dict).decode("UTF-8"), }, ) - content = buffer(encode_canonical_json( + content = encode_canonical_json( event.content - )) + ).decode("UTF-8") vals = { "topological_ordering": event.depth, @@ -261,9 +261,9 @@ class EventsStore(SQLBaseStore): ] } - vals["unrecognized_keys"] = buffer(encode_canonical_json( + vals["unrecognized_keys"] = encode_canonical_json( unrec - )) + ).decode("UTF-8") sql = ( "INSERT INTO events" diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py index e33963d0b..047698aa1 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py @@ -35,16 +35,13 @@ class ProfileStore(SQLBaseStore): desc="get_profile_displayname", ) - if name: - name = self.database_engine.load_unicode(name) - defer.returnValue(name) def set_profile_displayname(self, user_localpart, new_displayname): return self._simple_update_one( table="profiles", keyvalues={"user_id": user_localpart}, - updatevalues={"displayname": new_displayname.encode("utf8")}, + updatevalues={"displayname": new_displayname}, desc="set_profile_displayname", ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 8a63fe469..2a5c5080e 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -98,11 +98,6 @@ class RegistrationStore(SQLBaseStore): allow_none=True, ) - if user_info: - user_info["password_hash"] = self.database_engine.load_unicode( - user_info["password_hash"] - ) - defer.returnValue(user_info) @cached() diff --git a/synapse/storage/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/delta/15/appservice_txns.sql index ddea8fc69..1c3324f41 100644 --- a/synapse/storage/schema/delta/15/appservice_txns.sql +++ b/synapse/storage/schema/delta/15/appservice_txns.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS application_services_state( CREATE TABLE IF NOT EXISTS application_services_txns( as_id VARCHAR(150) NOT NULL, txn_id INTEGER NOT NULL, - event_ids LONGBLOB NOT NULL, + event_ids TEXT NOT NULL, UNIQUE(as_id, txn_id) ); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index e9e09214d..addbec588 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS events( event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - content bytea NOT NULL, - unrecognized_keys bytea, + content TEXT NOT NULL, + unrecognized_keys TEXT, processed BOOL NOT NULL, outlier BOOL NOT NULL, depth BIGINT DEFAULT 0 NOT NULL, @@ -35,8 +35,8 @@ CREATE INDEX events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - internal_metadata bytea NOT NULL, - json bytea NOT NULL, + internal_metadata TEXT NOT NULL, + json TEXT NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql index f08c5bcf7..5d63d57d5 100644 --- a/synapse/storage/schema/full_schemas/16/application_services.sql +++ b/synapse/storage/schema/full_schemas/16/application_services.sql @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS application_services_state( CREATE TABLE IF NOT EXISTS application_services_txns( as_id VARCHAR(150) NOT NULL, txn_id INTEGER NOT NULL, - event_ids bytea NOT NULL, + event_ids TEXT NOT NULL, UNIQUE(as_id, txn_id) ); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index 17e4c949b..5b4b49448 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS events( event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - content bytea NOT NULL, - unrecognized_keys bytea, + content TEXT NOT NULL, + unrecognized_keys TEXT, processed BOOL NOT NULL, outlier BOOL NOT NULL, depth BIGINT DEFAULT 0 NOT NULL, @@ -39,8 +39,8 @@ CREATE INDEX events_order_room ON events ( CREATE TABLE IF NOT EXISTS event_json( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - internal_metadata bytea NOT NULL, - json bytea NOT NULL, + internal_metadata TEXT NOT NULL, + json TEXT NOT NULL, UNIQUE (event_id) ); From e1c0970c116fe3700fc80401dc50aeb9d52c45a6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 16 Apr 2015 11:18:45 +0100 Subject: [PATCH 106/173] PEP8 --- synapse/handlers/_base.py | 1 - synapse/storage/engines/sqlite3.py | 2 -- synapse/storage/pusher.py | 3 --- 3 files changed, 6 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index e4471dd9b..dffb033fb 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -16,7 +16,6 @@ from twisted.internet import defer from synapse.api.errors import LimitExceededError, SynapseError -from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.api.constants import Membership, EventTypes from synapse.types import UserID diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index dd0d8e0e0..72c11df46 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -15,8 +15,6 @@ from synapse.storage import prepare_database, prepare_sqlite3_database -import types - class Sqlite3Engine(object): def __init__(self, database_module): diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 5c1c3d32e..a44bccdca 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections - from ._base import SQLBaseStore, Table from twisted.internet import defer @@ -167,4 +165,3 @@ class PusherStore(SQLBaseStore): class PushersTable(Table): table_name = "pushers" - From ea1776f556edaf6ca483bc5faed5e9d244aa1a15 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 16 Apr 2015 19:56:44 +0100 Subject: [PATCH 107/173] Return user ID in use error straight away --- synapse/handlers/auth.py | 2 + synapse/handlers/identity.py | 25 +++++- synapse/handlers/register.py | 102 +++++++++-------------- synapse/rest/client/v2_alpha/register.py | 25 +++++- 4 files changed, 88 insertions(+), 66 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 87866f298..1f927e67a 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -201,6 +201,8 @@ class AuthHandler(BaseHandler): logger.debug("Getting validated threepid. threepidcreds: %r" % (threepidCreds,)) threepid = yield identity_handler.threepid_from_creds(threepidCreds) + threepid['threepidCreds'] = authdict['threepidCreds'] + defer.returnValue(threepid) @defer.inlineCallbacks diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 19896ce90..cb5e1e80a 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -63,4 +63,27 @@ class IdentityHandler(BaseHandler): if 'medium' in data: defer.returnValue(data) - defer.returnValue(None) \ No newline at end of file + defer.returnValue(None) + + @defer.inlineCallbacks + def bind_threepid(self, creds, mxid): + yield run_on_reactor() + logger.debug("binding threepid %r to %s", creds, mxid) + http_client = SimpleHttpClient(self.hs) + data = None + try: + data = yield http_client.post_urlencoded_get_json( + # XXX: Change when ID servers are all HTTPS + "http://%s%s" % ( + creds['idServer'], "/_matrix/identity/api/v1/3pid/bind" + ), + { + 'sid': creds['sid'], + 'clientSecret': creds['clientSecret'], + 'mxid': mxid, + } + ) + logger.debug("bound threepid %r to %s", creds, mxid) + except CodeMessageException as e: + data = json.loads(e.msg) + defer.returnValue(data) \ No newline at end of file diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 6759a8c58..541b1019d 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -44,6 +44,36 @@ class RegistrationHandler(BaseHandler): self.distributor = hs.get_distributor() self.distributor.declare("registered_user") + @defer.inlineCallbacks + def check_username(self, localpart): + yield run_on_reactor() + + print "checking username %s" % (localpart) + + if urllib.quote(localpart) != localpart: + raise SynapseError( + 400, + "User ID must only contain characters which do not" + " require URL encoding." + ) + + user = UserID(localpart, self.hs.hostname) + user_id = user.to_string() + + yield self.check_user_id_is_valid(user_id) + + print "is valid" + + u = yield self.store.get_user_by_id(user_id) + print "user is: " + print u + if u: + raise SynapseError( + 400, + "User ID already taken.", + errcode=Codes.USER_IN_USE, + ) + @defer.inlineCallbacks def register(self, localpart=None, password=None): """Registers a new client on the server. @@ -64,18 +94,11 @@ class RegistrationHandler(BaseHandler): password_hash = bcrypt.hashpw(password, bcrypt.gensalt()) if localpart: - if localpart and urllib.quote(localpart) != localpart: - raise SynapseError( - 400, - "User ID must only contain characters which do not" - " require URL encoding." - ) + self.check_username(localpart) user = UserID(localpart, self.hs.hostname) user_id = user.to_string() - yield self.check_user_id_is_valid(user_id) - token = self._generate_token(user_id) yield self.store.register( user_id=user_id, @@ -190,7 +213,8 @@ class RegistrationHandler(BaseHandler): logger.info("validating theeepidcred sid %s on id server %s", c['sid'], c['idServer']) try: - threepid = yield self._threepid_from_creds(c) + identity_handler = self.hs.get_handlers().identity_handler + threepid = yield identity_handler.threepid_from_creds(c) except: logger.exception("Couldn't validate 3pid") raise RegistrationError(400, "Couldn't validate 3pid") @@ -202,12 +226,16 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def bind_emails(self, user_id, threepidCreds): - """Links emails with a user ID and informs an identity server.""" + """Links emails with a user ID and informs an identity server. + + Used only by c/s api v1 + """ # Now we have a matrix ID, bind it to the threepids we were given for c in threepidCreds: + identity_handler = self.hs.get_handlers().identity_handler # XXX: This should be a deferred list, shouldn't it? - yield self._bind_threepid(c, user_id) + yield identity_handler.bind_threepid(c, user_id) @defer.inlineCallbacks def check_user_id_is_valid(self, user_id): @@ -234,58 +262,6 @@ class RegistrationHandler(BaseHandler): def _generate_user_id(self): return "-" + stringutils.random_string(18) - @defer.inlineCallbacks - def _threepid_from_creds(self, creds): - # TODO: get this from the homeserver rather than creating a new one for - # each request - http_client = SimpleHttpClient(self.hs) - # XXX: make this configurable! - trustedIdServers = ['matrix.org:8090', 'matrix.org'] - if not creds['idServer'] in trustedIdServers: - logger.warn('%s is not a trusted ID server: rejecting 3pid ' + - 'credentials', creds['idServer']) - defer.returnValue(None) - - data = {} - try: - data = yield http_client.get_json( - # XXX: This should be HTTPS - "http://%s%s" % ( - creds['idServer'], - "/_matrix/identity/api/v1/3pid/getValidated3pid" - ), - {'sid': creds['sid'], 'clientSecret': creds['clientSecret']} - ) - except CodeMessageException as e: - data = json.loads(e.msg) - - if 'medium' in data: - defer.returnValue(data) - defer.returnValue(None) - - @defer.inlineCallbacks - def _bind_threepid(self, creds, mxid): - yield - logger.debug("binding threepid") - http_client = SimpleHttpClient(self.hs) - data = None - try: - data = yield http_client.post_urlencoded_get_json( - # XXX: Change when ID servers are all HTTPS - "http://%s%s" % ( - creds['idServer'], "/_matrix/identity/api/v1/3pid/bind" - ), - { - 'sid': creds['sid'], - 'clientSecret': creds['clientSecret'], - 'mxid': mxid, - } - ) - logger.debug("bound threepid") - except CodeMessageException as e: - data = json.loads(e.msg) - defer.returnValue(data) - @defer.inlineCallbacks def _validate_captcha(self, ip_addr, private_key, challenge, response): """Validates the captcha provided. diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index ee99b74fd..a5fec45dc 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -49,12 +49,20 @@ class RegisterRestServlet(RestServlet): self.auth = hs.get_auth() self.auth_handler = hs.get_handlers().auth_handler self.registration_handler = hs.get_handlers().registration_handler + self.identity_handler = hs.get_handlers().identity_handler @defer.inlineCallbacks def on_POST(self, request): yield run_on_reactor() body = parse_request_allow_empty(request) + if 'password' not in body: + raise SynapseError(400, "", Codes.MISSING_PARAM) + + if 'username' in body: + desired_username = body['username'] + print "username in body" + yield self.registration_handler.check_username(desired_username) is_using_shared_secret = False is_application_server = False @@ -100,15 +108,28 @@ class RegisterRestServlet(RestServlet): if not can_register: raise SynapseError(403, "Registration has been disabled") - if 'username' not in params or 'password' not in params: + if 'password' not in params: raise SynapseError(400, "", Codes.MISSING_PARAM) - desired_username = params['username'] + desired_username = params['username'] if 'username' in params else None new_password = params['password'] (user_id, token) = yield self.registration_handler.register( localpart=desired_username, password=new_password ) + + if 'bind_email' in params and params['bind_email']: + logger.info("bind_email specified: binding") + + emailThreepid = result[LoginType.EMAIL_IDENTITY] + threepidCreds = emailThreepid['threepidCreds'] + logger.debug("Binding emails %s to %s" % ( + emailThreepid, user_id + )) + yield self.identity_handler.bind_threepid(threepidCreds, user_id) + else: + logger.info("bind_email not specified: not binding email") + result = { "user_id": user_id, "access_token": token, From 4cd5fb13a31a4da1d7d8feb06d211a2f7842f5ad Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 16 Apr 2015 20:03:13 +0100 Subject: [PATCH 108/173] Oops, left debugging in. --- synapse/handlers/register.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 541b1019d..25b1db62e 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -48,8 +48,6 @@ class RegistrationHandler(BaseHandler): def check_username(self, localpart): yield run_on_reactor() - print "checking username %s" % (localpart) - if urllib.quote(localpart) != localpart: raise SynapseError( 400, @@ -62,11 +60,7 @@ class RegistrationHandler(BaseHandler): yield self.check_user_id_is_valid(user_id) - print "is valid" - u = yield self.store.get_user_by_id(user_id) - print "user is: " - print u if u: raise SynapseError( 400, From 0ec346d942450ac4a29c74d115428fd856af94b2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 11:10:20 +0100 Subject: [PATCH 109/173] Add unique index to room_aliases, remove duplicates on upgrade. Convert some columns back to TEXT from bytea --- synapse/storage/schema/delta/16/room_alias_index.sql | 3 +++ synapse/storage/schema/delta/16/unique_constraints.sql | 8 ++++++++ synapse/storage/schema/full_schemas/11/transactions.sql | 2 +- synapse/storage/schema/full_schemas/11/users.sql | 2 +- synapse/storage/schema/full_schemas/16/room_aliases.sql | 4 ++++ synapse/storage/schema/full_schemas/16/transactions.sql | 2 +- synapse/storage/schema/full_schemas/16/users.sql | 2 +- 7 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 synapse/storage/schema/delta/16/room_alias_index.sql diff --git a/synapse/storage/schema/delta/16/room_alias_index.sql b/synapse/storage/schema/delta/16/room_alias_index.sql new file mode 100644 index 000000000..f82486132 --- /dev/null +++ b/synapse/storage/schema/delta/16/room_alias_index.sql @@ -0,0 +1,3 @@ + +CREATE INDEX IF NOT EXISTS room_aliases_id ON room_aliases(room_id); +CREATE INDEX IF NOT EXISTS room_alias_servers_alias ON room_alias_servers(room_alias); diff --git a/synapse/storage/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/delta/16/unique_constraints.sql index 3604ea842..fecf11118 100644 --- a/synapse/storage/schema/delta/16/unique_constraints.sql +++ b/synapse/storage/schema/delta/16/unique_constraints.sql @@ -70,3 +70,11 @@ DROP INDEX IF EXISTS presence_list_observers; CREATE UNIQUE INDEX presence_list_observers ON presence_list( user_id, observed_user_id ); + +-- +DELETE FROM room_aliases WHERE rowid not in ( + SELECT MIN(rowid) FROM room_aliases GROUP BY room_alias +); + +DROP INDEX IF EXISTS room_aliases_id; +CREATE INDEX room_aliases_id ON room_aliases(room_id); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index f01d9eba3..524a69692 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, - response_json bytea, + response_json TEXT, ts BIGINT ); diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 2edc13c3b..48a6aecfe 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS user_ips ( access_token VARCHAR(150) NOT NULL, device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, - user_agent VARCHAR(150) NOT NULL, + user_agent TEXT NOT NULL, last_seen BIGINT NOT NULL, UNIQUE (user, access_token, ip, user_agent) ); diff --git a/synapse/storage/schema/full_schemas/16/room_aliases.sql b/synapse/storage/schema/full_schemas/16/room_aliases.sql index 952cae35b..2c0853a2a 100644 --- a/synapse/storage/schema/full_schemas/16/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/16/room_aliases.sql @@ -19,7 +19,11 @@ CREATE TABLE IF NOT EXISTS room_aliases( UNIQUE (room_alias) ); +CREATE INDEX room_aliases_id ON room_aliases(room_id); + CREATE TABLE IF NOT EXISTS room_alias_servers( room_alias VARCHAR(150) NOT NULL, server VARCHAR(150) NOT NULL ); + +CREATE INDEX room_alias_servers_alias ON room_alias_servers(room_alias); diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql index bc6406493..ed431bd3a 100644 --- a/synapse/storage/schema/full_schemas/16/transactions.sql +++ b/synapse/storage/schema/full_schemas/16/transactions.sql @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, - response_json bytea, + response_json TEXT, ts BIGINT ); diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql index 006b249fc..033e3244b 100644 --- a/synapse/storage/schema/full_schemas/16/users.sql +++ b/synapse/storage/schema/full_schemas/16/users.sql @@ -34,7 +34,7 @@ CREATE TABLE IF NOT EXISTS user_ips ( access_token VARCHAR(150) NOT NULL, device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, - user_agent VARCHAR(150) NOT NULL, + user_agent TEXT NOT NULL, last_seen BIGINT NOT NULL ); From 427bcb76085c005069f8163e035ebce395a14b2a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 11:13:05 +0100 Subject: [PATCH 110/173] Fix port script after storage changes. Add very simple (off by default) curses UI to see progress. --- scripts/port_from_sqlite_to_postgres.py | 632 ++++++++++++++++-------- 1 file changed, 435 insertions(+), 197 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 4b3fd9e52..fc1603c1c 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -20,38 +20,17 @@ from synapse.storage._base import LoggingTransaction, SQLBaseStore from synapse.storage.engines import create_engine import argparse -import itertools +import curses import logging -import types +import sys +import time +import traceback import yaml logger = logging.getLogger("port_from_sqlite_to_postgres") -BINARY_COLUMNS = { - "event_content_hashes": ["hash"], - "event_reference_hashes": ["hash"], - "event_signatures": ["signature"], - "event_edge_hashes": ["hash"], - "events": ["content", "unrecognized_keys"], - "event_json": ["internal_metadata", "json"], - "application_services_txns": ["event_ids"], - "received_transactions": ["response_json"], - "sent_transactions": ["response_json"], - "server_tls_certificates": ["tls_certificate"], - "server_signature_keys": ["verify_key"], - "pushers": ["pushkey", "data"], - "user_filters": ["filter_json"], -} - -UNICODE_COLUMNS = { - "events": ["content", "unrecognized_keys"], - "event_json": ["internal_metadata", "json"], - "users": ["password_hash"], -} - - BOOLEAN_COLUMNS = { "events": ["processed", "outlier"], "rooms": ["is_public"], @@ -91,7 +70,15 @@ APPEND_ONLY_TABLES = [ ] +end_error_exec_info = None + + class Store(object): + """This object is used to pull out some of the convenience API from the + Storage layer. + + *All* database interactions should go through this object. + """ def __init__(self, db_pool, engine): self.db_pool = db_pool self.database_engine = engine @@ -130,11 +117,14 @@ class Store(object): continue raise except Exception as e: - logger.debug("[TXN FAIL] {%s}", desc, e) + logger.debug("[TXN FAIL] {%s} %s", desc, e) raise return self.db_pool.runWithConnection(r) + def execute(self, f): + return self.runInteraction(f.__name__, f) + def insert_many_txn(self, txn, table, headers, rows): sql = "INSERT INTO %s (%s) VALUES (%s)" % ( table, @@ -152,205 +142,435 @@ class Store(object): raise +class Progress(object): + """Used to report progress of the port + """ + def __init__(self): + self.tables = {} -def chunks(n): - for i in itertools.count(0, n): - yield range(i, i+n) + self.start_time = int(time.time()) + + def add_table(self, table, cur, size): + self.tables[table] = { + "start": cur, + "num_done": cur, + "total": size, + "perc": int(cur * 100 / size), + } + + def update(self, table, num_done): + data = self.tables[table] + data["num_done"] = num_done + data["perc"] = int(num_done * 100 / data["total"]) + + def done(self): + pass -@defer.inlineCallbacks -def handle_table(table, sqlite_store, postgres_store): - if table in APPEND_ONLY_TABLES: - # It's safe to just carry on inserting. - next_chunk = yield postgres_store._simple_select_one_onecol( - table="port_from_sqlite3", - keyvalues={"table_name": table}, - retcol="rowid", - allow_none=True, - ) +class CursesProgress(Progress): + """Reports progress to a curses window + """ + def __init__(self, stdscr): + self.stdscr = stdscr - if next_chunk is None: - yield postgres_store._simple_insert( - table="port_from_sqlite3", - values={"table_name": table, "rowid": 0} - ) + curses.use_default_colors() + curses.curs_set(0) - next_chunk = 0 - else: - def delete_all(txn): - txn.execute( - "DELETE FROM port_from_sqlite3 WHERE table_name = %s", - (table,) - ) - txn.execute("TRUNCATE %s CASCADE" % (table,)) - postgres_store._simple_insert_txn( - txn, - table="port_from_sqlite3", - values={"table_name": table, "rowid": 0} - ) + curses.init_pair(1, curses.COLOR_RED, -1) + curses.init_pair(2, curses.COLOR_GREEN, -1) - yield postgres_store.runInteraction( - "delete_non_append_only", delete_all - ) + self.last_update = 0 - next_chunk = 0 + self.finished = False - logger.info("next_chunk for %s: %d", table, next_chunk) + super(CursesProgress, self).__init__() - N = 5000 + def update(self, table, num_done): + super(CursesProgress, self).update(table, num_done) - select = "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,) + self.render() - uni_col_names = UNICODE_COLUMNS.get(table, []) - bool_col_names = BOOLEAN_COLUMNS.get(table, []) - bin_col_names = BINARY_COLUMNS.get(table, []) + def render(self, force=False): + now = time.time() - while True: - def r(txn): - txn.execute(select, (next_chunk, N,)) - rows = txn.fetchall() - headers = [column[0] for column in txn.description] - - return headers, rows - - headers, rows = yield sqlite_store.runInteraction("select", r) - - logger.info("Got %d rows for %s", len(rows), table) - - if rows: - uni_cols = [i for i, h in enumerate(headers) if h in uni_col_names] - bool_cols = [i for i, h in enumerate(headers) if h in bool_col_names] - bin_cols = [i for i, h in enumerate(headers) if h in bin_col_names] - next_chunk = rows[-1][0] + 1 - - def conv(j, col): - if j in uni_cols: - col = sqlite_store.database_engine.load_unicode(col) - if j in bool_cols: - return bool(col) - - if j in bin_cols: - if isinstance(col, types.UnicodeType): - col = buffer(col.encode("utf8")) - - return col - - for i, row in enumerate(rows): - rows[i] = tuple( - postgres_store.database_engine.encode_parameter( - conv(j, col) - ) - for j, col in enumerate(row) - if j > 0 - ) - - def ins(txn): - postgres_store.insert_many_txn(txn, table, headers[1:], rows) - - postgres_store._simple_update_one_txn( - txn, - table="port_from_sqlite3", - keyvalues={"table_name": table}, - updatevalues={"rowid": next_chunk}, - ) - - yield postgres_store.runInteraction("insert_many", ins) - else: + if not force and now - self.last_update < 0.2: + # reactor.callLater(1, self.render) return + self.stdscr.clear() -def setup_db(db_config, database_engine): - db_conn = database_engine.module.connect( - **{ - k: v for k, v in db_config.get("args", {}).items() - if not k.startswith("cp_") - } - ) + rows, cols = self.stdscr.getmaxyx() - database_engine.prepare_database(db_conn) + duration = int(now) - int(self.start_time) - db_conn.commit() + minutes, seconds = divmod(duration, 60) + duration_str = '%02dm %02ds' % (minutes, seconds,) - -@defer.inlineCallbacks -def main(sqlite_config, postgress_config): - try: - sqlite_db_pool = adbapi.ConnectionPool( - sqlite_config["name"], - **sqlite_config["args"] - ) - - postgres_db_pool = adbapi.ConnectionPool( - postgress_config["name"], - **postgress_config["args"] - ) - - sqlite_engine = create_engine("sqlite3") - postgres_engine = create_engine("psycopg2") - - sqlite_store = Store(sqlite_db_pool, sqlite_engine) - postgres_store = Store(postgres_db_pool, postgres_engine) - - # Step 1. Set up databases. - logger.info("Preparing sqlite database...") - setup_db(sqlite_config, sqlite_engine) - - logger.info("Preparing postgres database...") - setup_db(postgress_config, postgres_engine) - - # Step 2. Get tables. - logger.info("Fetching tables...") - tables = yield sqlite_store._simple_select_onecol( - table="sqlite_master", - keyvalues={ - "type": "table", - }, - retcol="name", - ) - - logger.info("Found %d tables", len(tables)) - - def create_port_table(txn): - txn.execute( - "CREATE TABLE port_from_sqlite3 (" - " table_name varchar(100) NOT NULL UNIQUE," - " rowid bigint NOT NULL" - ")" + if self.finished: + status = "Time spent: %s (Done!)" % (duration_str,) + else: + min_perc = min( + (v["num_done"] - v["start"]) * 100. / (v["total"] - v["start"]) + if v["total"] - v["start"] else 100 + for v in self.tables.values() + ) + if min_perc > 0: + est_remaining = (int(now) - self.start_time) * 100 / min_perc + est_remaining_str = '%02dm %02ds remaining' % divmod(est_remaining, 60) + else: + est_remaining_str = "Unknown" + status = ( + "Time spent: %s (est. remaining: %s)" + % (duration_str, est_remaining_str,) ) + self.stdscr.addstr( + 0, 0, + status, + curses.A_BOLD, + ) + + max_len = max([len(t) for t in self.tables.keys()]) + + left_margin = 5 + middle_space = 1 + + items = self.tables.items() + items.sort( + key=lambda i: (i[1]["perc"], i[0]), + ) + + for i, (table, data) in enumerate(items): + if i + 2 >= rows: + break + + perc = data["perc"] + + color = curses.color_pair(2) if perc == 100 else curses.color_pair(1) + + self.stdscr.addstr( + i+2, left_margin + max_len - len(table), + table, + curses.A_BOLD | color, + ) + + size = 20 + + progress = "[%s%s]" % ( + "#" * int(perc*size/100), + " " * (size - int(perc*size/100)), + ) + + self.stdscr.addstr( + i+2, left_margin + max_len + middle_space, + "%s %3d%% (%d/%d)" % (progress, perc, data["num_done"], data["total"]), + ) + + if self.finished: + self.stdscr.addstr( + self.rows-1, 0 , + "Press any key to exit...", + ) + + self.stdscr.refresh() + self.last_update = time.time() + + def done(self): + self.finished = True + self.render(True) + self.stdscr.getch() + + def on_prepare_sqlite(self): + self.stdscr.clear() + self.stdscr.addstr( + 0, 0, + "Preparing SQLite database...", + curses.A_BOLD, + ) + self.stdscr.refresh() + + def on_prepare_postgres(self): + self.stdscr.clear() + self.stdscr.addstr( + 0, 0, + "Preparing PostgreSQL database...", + curses.A_BOLD, + ) + self.stdscr.refresh() + + def fetching_tables(self): + self.stdscr.clear() + self.stdscr.addstr( + 0, 0, + "Fetching tables...", + curses.A_BOLD, + ) + self.stdscr.refresh() + + def preparing_tables(self): + self.stdscr.clear() + self.stdscr.addstr( + 0, 0, + "Preparing tables...", + curses.A_BOLD, + ) + self.stdscr.refresh() + + +class TerminalProgress(Progress): + """Just prints progress to the terminal + """ + def update(self, table, num_done): + super(TerminalProgress, self).update(table, num_done) + + data = self.tables[table] + + print "%s: %d%% (%d/%d)" % ( + table, data["perc"], + data["num_done"], data["total"], + ) + + def on_prepare_sqlite(self): + print "Preparing SQLite database..." + + def on_prepare_postgres(self): + print "Preparing PostgreSQL database..." + + def fetching_tables(self): + print "Fetching tables..." + + def preparing_tables(self): + print "Preparing tables..." + + +class Porter(object): + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + @defer.inlineCallbacks + def handle_table(self, table): + if table in APPEND_ONLY_TABLES: + # It's safe to just carry on inserting. + next_chunk = yield self.postgres_store._simple_select_one_onecol( + table="port_from_sqlite3", + keyvalues={"table_name": table}, + retcol="rowid", + allow_none=True, + ) + + if next_chunk is None: + yield self.postgres_store._simple_insert( + table="port_from_sqlite3", + values={"table_name": table, "rowid": 1} + ) + + next_chunk = 1 + else: + def delete_all(txn): + txn.execute( + "DELETE FROM port_from_sqlite3 WHERE table_name = %s", + (table,) + ) + txn.execute("TRUNCATE %s CASCADE" % (table,)) + self.postgres_store._simple_insert_txn( + txn, + table="port_from_sqlite3", + values={"table_name": table, "rowid": 0} + ) + yield self.postgres_store.execute(delete_all) + + next_chunk = 1 + + def get_table_size(txn): + txn.execute("SELECT count(*) FROM %s" % (table,)) + size, = txn.fetchone() + return int(size) + + table_size = yield self.sqlite_store.execute(get_table_size) + postgres_size = yield self.postgres_store.execute(get_table_size) + + if not table_size: + return + + self.progress.add_table(table, postgres_size, table_size) + + select = ( + "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" + % (table,) + ) + + bool_col_names = BOOLEAN_COLUMNS.get(table, []) + + while True: + def r(txn): + txn.execute(select, (next_chunk, self.batch_size,)) + rows = txn.fetchall() + headers = [column[0] for column in txn.description] + + return headers, rows + + headers, rows = yield self.sqlite_store.runInteraction("select", r) + + if rows: + bool_cols = [ + i for i, h in enumerate(headers) if h in bool_col_names + ] + next_chunk = rows[-1][0] + 1 + + def conv(j, col): + if j in bool_cols: + return bool(col) + return col + + for i, row in enumerate(rows): + rows[i] = tuple( + self.postgres_store.database_engine.encode_parameter( + conv(j, col) + ) + for j, col in enumerate(row) + if j > 0 + ) + + def insert(txn): + self.postgres_store.insert_many_txn( + txn, table, headers[1:], rows + ) + + self.postgres_store._simple_update_one_txn( + txn, + table="port_from_sqlite3", + keyvalues={"table_name": table}, + updatevalues={"rowid": next_chunk}, + ) + + yield self.postgres_store.execute(insert) + + postgres_size += len(rows) + + self.progress.update(table, postgres_size) + else: + return + + def setup_db(self, db_config, database_engine): + db_conn = database_engine.module.connect( + **{ + k: v for k, v in db_config.get("args", {}).items() + if not k.startswith("cp_") + } + ) + + database_engine.prepare_database(db_conn) + + db_conn.commit() + + @defer.inlineCallbacks + def run(self): try: - yield postgres_store.runInteraction( - "create_port_table", create_port_table + sqlite_db_pool = adbapi.ConnectionPool( + self.sqlite_config["name"], + **self.sqlite_config["args"] ) - except Exception as e: - logger.info("Failed to create port table: %s", e) - # Process tables. - yield defer.gatherResults( - [ - handle_table(table, sqlite_store, postgres_store) - for table in tables - if table not in ["schema_version", "applied_schema_deltas"] - and not table.startswith("sqlite_") - ], - consumeErrors=True, - ) + postgres_db_pool = adbapi.ConnectionPool( + self.postgres_config["name"], + **self.postgres_config["args"] + ) - except: - logger.exception("") - finally: - reactor.stop() + sqlite_engine = create_engine("sqlite3") + postgres_engine = create_engine("psycopg2") + + self.sqlite_store = Store(sqlite_db_pool, sqlite_engine) + self.postgres_store = Store(postgres_db_pool, postgres_engine) + + # Step 1. Set up databases. + self.progress.on_prepare_sqlite() + self.setup_db(sqlite_config, sqlite_engine) + + self.progress.on_prepare_postgres() + self.setup_db(postgres_config, postgres_engine) + + # Step 2. Get tables. + self.progress.fetching_tables() + sqlite_tables = yield self.sqlite_store._simple_select_onecol( + table="sqlite_master", + keyvalues={ + "type": "table", + }, + retcol="name", + ) + + postgres_tables = yield self.postgres_store._simple_select_onecol( + table="information_schema.tables", + keyvalues={ + "table_schema": "public", + }, + retcol="distinct table_name", + ) + + tables = set(sqlite_tables) & set(postgres_tables) + + self.progress.preparing_tables() + + logger.info("Found %d tables", len(tables)) + + def create_port_table(txn): + txn.execute( + "CREATE TABLE port_from_sqlite3 (" + " table_name varchar(100) NOT NULL UNIQUE," + " rowid bigint NOT NULL" + ")" + ) + + try: + yield self.postgres_store.runInteraction( + "create_port_table", create_port_table + ) + except Exception as e: + logger.info("Failed to create port table: %s", e) + + # Process tables. + yield defer.gatherResults( + [ + self.handle_table(table) + for table in tables + if table not in ["schema_version", "applied_schema_deltas"] + and not table.startswith("sqlite_") + ], + consumeErrors=True, + ) + + self.progress.done() + except: + global end_error_exec_info + end_error_exec_info = sys.exc_info() + logger.exception("") + finally: + reactor.stop() if __name__ == "__main__": parser = argparse.ArgumentParser() + parser.add_argument("-v", action='store_true') + parser.add_argument("--curses", action='store_true') parser.add_argument("--sqlite-database") parser.add_argument( "--postgres-config", type=argparse.FileType('r'), ) + parser.add_argument("--batch-size", type=int, default=1000) + args = parser.parse_args() - logging.basicConfig(level=logging.INFO) + + + logging_config = { + "level": logging.DEBUG if args.v else logging.INFO, + "format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s" + } + + if args.curses: + logging_config["filename"] = "port-synapse.log" + + logging.basicConfig(**logging_config) sqlite_config = { "name": "sqlite3", @@ -364,10 +584,28 @@ if __name__ == "__main__": postgres_config = yaml.safe_load(args.postgres_config) - reactor.callWhenRunning( - main, - sqlite_config=sqlite_config, - postgres_config=postgres_config, - ) + def start(stdscr=None): + if stdscr: + progress = CursesProgress(stdscr) + else: + progress = TerminalProgress() - reactor.run() + porter = Porter( + sqlite_config=sqlite_config, + postgres_config=postgres_config, + progress=progress, + batch_size=args.batch_size, + ) + + reactor.callWhenRunning(porter.run) + + reactor.run() + + if args.curses: + curses.wrapper(start) + else: + start() + + if end_error_exec_info: + exc_type, exc_value, exc_traceback = end_error_exec_info + traceback.print_exception(exc_type, exc_value, exc_traceback) From dfc46c6220b7439262de6fc7034e7a6df99d4e40 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 12:46:29 +0100 Subject: [PATCH 111/173] PEP8 --- scripts/port_from_sqlite_to_postgres.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index fc1603c1c..f4b6ed068 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -268,7 +268,7 @@ class CursesProgress(Progress): if self.finished: self.stdscr.addstr( - self.rows-1, 0 , + self.rows-1, 0, "Press any key to exit...", ) @@ -561,7 +561,6 @@ if __name__ == "__main__": args = parser.parse_args() - logging_config = { "level": logging.DEBUG if args.v else logging.INFO, "format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s" From 83b554437ec9810dd09de992c728c2a2f01aa0e1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Apr 2015 12:57:25 +0100 Subject: [PATCH 112/173] Need to yield the username check, otherwise very very weird things happen. --- synapse/handlers/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 25b1db62e..d4483c3a1 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -88,7 +88,7 @@ class RegistrationHandler(BaseHandler): password_hash = bcrypt.hashpw(password, bcrypt.gensalt()) if localpart: - self.check_username(localpart) + yield self.check_username(localpart) user = UserID(localpart, self.hs.hostname) user_id = user.to_string() From 94e1e58b4de00ea92fa60d6fc7970dafa9ebce2a Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Apr 2015 13:44:12 +0100 Subject: [PATCH 113/173] password -> account servlet and add start of an 'add 3pid' endpoint --- synapse/rest/client/v2_alpha/__init__.py | 4 ++-- .../client/v2_alpha/{password.py => account.py} | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) rename synapse/rest/client/v2_alpha/{password.py => account.py} (86%) diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py index 86e4bc729..28d95b272 100644 --- a/synapse/rest/client/v2_alpha/__init__.py +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -16,7 +16,7 @@ from . import ( sync, filter, - password, + account, register, auth ) @@ -35,6 +35,6 @@ class ClientV2AlphaRestResource(JsonResource): def register_servlets(client_resource, hs): sync.register_servlets(hs, client_resource) filter.register_servlets(hs, client_resource) - password.register_servlets(hs, client_resource) + account.register_servlets(hs, client_resource) register.register_servlets(hs, client_resource) auth.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/password.py b/synapse/rest/client/v2_alpha/account.py similarity index 86% rename from synapse/rest/client/v2_alpha/password.py rename to synapse/rest/client/v2_alpha/account.py index cb0c8cfb5..0f3b12880 100644 --- a/synapse/rest/client/v2_alpha/password.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -75,5 +75,20 @@ class PasswordRestServlet(RestServlet): return 200, {} +class AddThreepidRestServlet(RestServlet): + PATTERN = client_v2_pattern("/account/3pid") + + @defer.inlineCallbacks + def on_POST(self, request): + body = parse_json_dict_from_request(request) + + if 'threePidCreds' not in body: + raise SynapseError(400, "Missing param", Codes.MISSING_PARAM) + + auth_user, client = yield self.auth.get_user_by_req(request) + + + + def register_servlets(hs, http_server): PasswordRestServlet(hs).register(http_server) From bf5e54f25537e293b78b26d0b8109c1ee43eee06 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Apr 2015 13:44:55 +0100 Subject: [PATCH 114/173] Register the 3pid servlet --- synapse/rest/client/v2_alpha/account.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 0f3b12880..ca92a6a4c 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -75,7 +75,7 @@ class PasswordRestServlet(RestServlet): return 200, {} -class AddThreepidRestServlet(RestServlet): +class ThreepidRestServlet(RestServlet): PATTERN = client_v2_pattern("/account/3pid") @defer.inlineCallbacks @@ -92,3 +92,5 @@ class AddThreepidRestServlet(RestServlet): def register_servlets(hs, http_server): PasswordRestServlet(hs).register(http_server) + ThreepidRestServlet(hs).register(http_server) + ThreepidRestServlet(hs).register(http_server) From 0b1a8500a297ea0f988c3fd04403163d265239a1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Apr 2015 13:53:54 +0100 Subject: [PATCH 115/173] just the once would probably be fine --- synapse/rest/client/v2_alpha/account.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index ca92a6a4c..750d826f9 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -93,4 +93,3 @@ class ThreepidRestServlet(RestServlet): def register_servlets(hs, http_server): PasswordRestServlet(hs).register(http_server) ThreepidRestServlet(hs).register(http_server) - ThreepidRestServlet(hs).register(http_server) From 5e7a90316d43f0dc42c5b18e505ce9b432e15461 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 16:08:18 +0100 Subject: [PATCH 116/173] Update --database-path metavar to SQLITE_DATABASE_PATH --- synapse/config/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/database.py b/synapse/config/database.py index 8dc9873f8..f3d0898c0 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -37,7 +37,7 @@ class DatabaseConfig(Config): db_group = parser.add_argument_group("database") db_group.add_argument( "-d", "--database-path", default="homeserver.db", - help="The database name." + metavar="SQLITE_DATABASE_PATH", help="The database name." ) db_group.add_argument( "--event-cache-size", default="100K", From e3417bbbe0d40d17eec9f07981359d081c9b1f9d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 16:13:14 +0100 Subject: [PATCH 117/173] Revert needless change to storage.profile --- synapse/storage/profile.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py index 047698aa1..a6e52cb24 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer - from ._base import SQLBaseStore @@ -26,17 +24,14 @@ class ProfileStore(SQLBaseStore): desc="create_profile", ) - @defer.inlineCallbacks def get_profile_displayname(self, user_localpart): - name = yield self._simple_select_one_onecol( + return self._simple_select_one_onecol( table="profiles", keyvalues={"user_id": user_localpart}, retcol="displayname", desc="get_profile_displayname", ) - defer.returnValue(name) - def set_profile_displayname(self, user_localpart, new_displayname): return self._simple_update_one( table="profiles", From 865398b4a953109015a7fa595634070b39e68a6d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 16:14:51 +0100 Subject: [PATCH 118/173] Revert needless change to rest.profile --- synapse/rest/client/v1/profile.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index 7387b4adb..1e77eb49c 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -19,13 +19,9 @@ from twisted.internet import defer from .base import ClientV1RestServlet, client_path_pattern from synapse.types import UserID -import logging import simplejson as json -logger = logging.getLogger(__name__) - - class ProfileDisplaynameRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/profile/(?P[^/]*)/displayname") @@ -51,8 +47,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): defer.returnValue((400, "Unable to parse name")) yield self.handlers.profile_handler.set_displayname( - user, auth_user, new_name - ) + user, auth_user, new_name) defer.returnValue((200, {})) From f96ab9d18dcebf995700f096792101a490b3a9b8 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Apr 2015 16:44:49 +0100 Subject: [PATCH 119/173] make add3pid servlet work --- synapse/handlers/login.py | 7 +++++ synapse/rest/client/v2_alpha/account.py | 38 ++++++++++++++++++++++++ synapse/rest/client/v2_alpha/register.py | 36 +++++++++++++++------- synapse/storage/registration.py | 11 +++++++ 4 files changed, 81 insertions(+), 11 deletions(-) diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 04f6dbb95..5c39356d7 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -74,3 +74,10 @@ class LoginHandler(BaseHandler): user_id, token_id ) yield self.store.flush_user(user_id) + + @defer.inlineCallbacks + def add_threepid(self, user_id, medium, address, validated_at): + yield self.store.user_add_threepid( + user_id, medium, address, validated_at, + self.hs.get_clock().time_msec() + ) \ No newline at end of file diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 750d826f9..6045b016e 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.api.constants import LoginType from synapse.api.errors import LoginError, SynapseError, Codes from synapse.http.servlet import RestServlet +from synapse.util.async import run_on_reactor from ._base import client_v2_pattern, parse_json_dict_from_request @@ -39,6 +40,8 @@ class PasswordRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request): + yield run_on_reactor() + body = parse_json_dict_from_request(request) authed, result, params = yield self.auth_handler.check_auth([ @@ -78,16 +81,51 @@ class PasswordRestServlet(RestServlet): class ThreepidRestServlet(RestServlet): PATTERN = client_v2_pattern("/account/3pid") + def __init__(self, hs): + super(ThreepidRestServlet, self).__init__() + self.hs = hs + self.login_handler = hs.get_handlers().login_handler + self.identity_handler = hs.get_handlers().identity_handler + self.auth = hs.get_auth() + @defer.inlineCallbacks def on_POST(self, request): + yield run_on_reactor() + body = parse_json_dict_from_request(request) if 'threePidCreds' not in body: raise SynapseError(400, "Missing param", Codes.MISSING_PARAM) + threePidCreds = body['threePidCreds'] auth_user, client = yield self.auth.get_user_by_req(request) + threepid = yield self.identity_handler.threepid_from_creds(threePidCreds) + if not threepid: + raise SynapseError(400, "Failed to auth 3pid") + + for reqd in ['medium', 'address', 'validatedAt']: + if reqd not in threepid: + logger.warn("Couldn't add 3pid: invalid response from ID sevrer") + raise SynapseError(500, "Invalid response from ID Server") + + yield self.login_handler.add_threepid( + auth_user.to_string(), + threepid['medium'], + threepid['address'], + threepid['validatedAt'], + ) + + if 'bind' in body and body['bind']: + logger.debug("Binding emails %s to %s" % ( + threepid, auth_user.to_string() + )) + yield self.identity_handler.bind_threepid( + threePidCreds, auth_user.to_string() + ) + + defer.returnValue((200, {})) def register_servlets(hs, http_server): diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index a5fec45dc..e93897e28 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -50,6 +50,7 @@ class RegisterRestServlet(RestServlet): self.auth_handler = hs.get_handlers().auth_handler self.registration_handler = hs.get_handlers().registration_handler self.identity_handler = hs.get_handlers().identity_handler + self.login_handler = hs.get_handlers().login_handler @defer.inlineCallbacks def on_POST(self, request): @@ -61,7 +62,6 @@ class RegisterRestServlet(RestServlet): if 'username' in body: desired_username = body['username'] - print "username in body" yield self.registration_handler.check_username(desired_username) is_using_shared_secret = False @@ -118,17 +118,31 @@ class RegisterRestServlet(RestServlet): password=new_password ) - if 'bind_email' in params and params['bind_email']: - logger.info("bind_email specified: binding") + if LoginType.EMAIL_IDENTITY in result: + threepid = result[LoginType.EMAIL_IDENTITY] - emailThreepid = result[LoginType.EMAIL_IDENTITY] - threepidCreds = emailThreepid['threepidCreds'] - logger.debug("Binding emails %s to %s" % ( - emailThreepid, user_id - )) - yield self.identity_handler.bind_threepid(threepidCreds, user_id) - else: - logger.info("bind_email not specified: not binding email") + for reqd in ['medium', 'address', 'validatedAt']: + if reqd not in threepid: + logger.info("Can't add incomplete 3pid") + else: + yield self.login_handler.add_threepid( + user_id, + threepid['medium'], + threepid['address'], + threepid['validatedAt'], + ) + + if 'bind_email' in params and params['bind_email']: + logger.info("bind_email specified: binding") + + emailThreepid = result[LoginType.EMAIL_IDENTITY] + threepidCreds = emailThreepid['threepidCreds'] + logger.debug("Binding emails %s to %s" % ( + emailThreepid, user_id + )) + yield self.identity_handler.bind_threepid(threepidCreds, user_id) + else: + logger.info("bind_email not specified: not binding email") result = { "user_id": user_id, diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index f61d8fdb6..4bc01f3cc 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -175,3 +175,14 @@ class RegistrationStore(SQLBaseStore): return rows[0] return None + + @defer.inlineCallbacks + def user_add_threepid(self, user_id, medium, address, validated_at, added_at): + yield self._simple_upsert("user_threepids", { + "user": user_id, + "medium": medium, + "address": address, + }, { + "validated_at": validated_at, + "added_at": added_at, + }) \ No newline at end of file From 4eea5cf6c2a301938466828b02557d8500197bb3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Apr 2015 16:46:45 +0100 Subject: [PATCH 120/173] pep8 --- synapse/handlers/identity.py | 6 +++--- synapse/handlers/login.py | 2 +- synapse/handlers/register.py | 5 +---- synapse/storage/registration.py | 2 +- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index cb5e1e80a..5c7263591 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -42,8 +42,8 @@ class IdentityHandler(BaseHandler): # each request http_client = SimpleHttpClient(self.hs) # XXX: make this configurable! - trustedIdServers = ['matrix.org', 'localhost:8090'] - #trustedIdServers = ['matrix.org'] + # trustedIdServers = ['matrix.org', 'localhost:8090'] + trustedIdServers = ['matrix.org'] if not creds['idServer'] in trustedIdServers: logger.warn('%s is not a trusted ID server: rejecting 3pid ' + 'credentials', creds['idServer']) @@ -86,4 +86,4 @@ class IdentityHandler(BaseHandler): logger.debug("bound threepid %r to %s", creds, mxid) except CodeMessageException as e: data = json.loads(e.msg) - defer.returnValue(data) \ No newline at end of file + defer.returnValue(data) diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 5c39356d7..f7f369834 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -80,4 +80,4 @@ class LoginHandler(BaseHandler): yield self.store.user_add_threepid( user_id, medium, address, validated_at, self.hs.get_clock().time_msec() - ) \ No newline at end of file + ) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index d4483c3a1..7b68585a1 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -18,18 +18,15 @@ from twisted.internet import defer from synapse.types import UserID from synapse.api.errors import ( - AuthError, Codes, SynapseError, RegistrationError, InvalidCaptchaError, - CodeMessageException + AuthError, Codes, SynapseError, RegistrationError, InvalidCaptchaError ) from ._base import BaseHandler import synapse.util.stringutils as stringutils from synapse.util.async import run_on_reactor -from synapse.http.client import SimpleHttpClient from synapse.http.client import CaptchaServerHttpClient import base64 import bcrypt -import json import logging import urllib diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 4bc01f3cc..8f62e5c6f 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -185,4 +185,4 @@ class RegistrationStore(SQLBaseStore): }, { "validated_at": validated_at, "added_at": added_at, - }) \ No newline at end of file + }) From 117f35ac4ac4f8d344ae1efbc629a3f8bc25f459 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Apr 2015 17:20:18 +0100 Subject: [PATCH 121/173] Add endpoint to get threepids from server --- synapse/rest/client/v2_alpha/account.py | 12 ++++++++++++ synapse/storage/registration.py | 11 +++++++++++ 2 files changed, 23 insertions(+) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 6045b016e..5ac3ac0f7 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -88,6 +88,18 @@ class ThreepidRestServlet(RestServlet): self.identity_handler = hs.get_handlers().identity_handler self.auth = hs.get_auth() + @defer.inlineCallbacks + def on_GET(self, request): + yield run_on_reactor() + + auth_user, _ = yield self.auth.get_user_by_req(request) + + threepids = yield self.hs.get_datastore().user_get_threepids( + auth_user.to_string() + ) + + defer.returnValue((200, {'threepids': threepids})) + @defer.inlineCallbacks def on_POST(self, request): yield run_on_reactor() diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 8f62e5c6f..08d60f081 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -186,3 +186,14 @@ class RegistrationStore(SQLBaseStore): "validated_at": validated_at, "added_at": added_at, }) + + @defer.inlineCallbacks + def user_get_threepids(self, user_id): + ret = yield self._simple_select_list( + "user_threepids", { + "user": user_id + }, + ['medium', 'address', 'validated_at', 'added_at'], + 'user_get_threepids' + ) + defer.returnValue(ret) \ No newline at end of file From 8db6832db8a8ad1a68ff6781b90f3e2cb1a72fc0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Apr 2015 19:53:47 +0100 Subject: [PATCH 122/173] Password reset, finally. --- synapse/handlers/auth.py | 8 +++++++- synapse/rest/client/v2_alpha/account.py | 21 ++++++++++++++++----- synapse/storage/registration.py | 16 +++++++++++++++- 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 1f927e67a..7b0ab4829 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -195,12 +195,18 @@ class AuthHandler(BaseHandler): def _check_email_identity(self, authdict, _): yield run_on_reactor() + if 'threepidCreds' not in authdict: + raise LoginError(400, "Missing threepidCreds", Codes.MISSING_PARAM) + threepidCreds = authdict['threepidCreds'] identity_handler = self.hs.get_handlers().identity_handler - logger.debug("Getting validated threepid. threepidcreds: %r" % (threepidCreds,)) + logger.info("Getting validated threepid. threepidcreds: %r" % (threepidCreds,)) threepid = yield identity_handler.threepid_from_creds(threepidCreds) + if not threepid: + raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) + threepid['threepidCreds'] = authdict['threepidCreds'] defer.returnValue(threepid) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 5ac3ac0f7..e33607b79 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -45,31 +45,42 @@ class PasswordRestServlet(RestServlet): body = parse_json_dict_from_request(request) authed, result, params = yield self.auth_handler.check_auth([ - [LoginType.PASSWORD] + [LoginType.PASSWORD], + [LoginType.EMAIL_IDENTITY] ], body) if not authed: defer.returnValue((401, result)) - auth_user = None + user_id = None if LoginType.PASSWORD in result: # if using password, they should also be logged in auth_user, client = yield self.auth.get_user_by_req(request) if auth_user.to_string() != result[LoginType.PASSWORD]: raise LoginError(400, "", Codes.UNKNOWN) + user_id = auth_user.to_string() + elif LoginType.EMAIL_IDENTITY in result: + threepid = result[LoginType.EMAIL_IDENTITY] + if 'medium' not in threepid or 'address' not in threepid: + raise SynapseError(500, "Malformed threepid") + # if using email, we must know about the email they're authing with! + threepid_user = yield self.hs.get_datastore().get_user_by_threepid( + threepid['medium'], threepid['address'] + ) + if not threepid_user: + raise SynapseError(404, "Email address not found", Codes.NOT_FOUND) + user_id = threepid_user else: logger.error("Auth succeeded but no known type!", result.keys()) raise SynapseError(500, "", Codes.UNKNOWN) - user_id = auth_user.to_string() - if 'new_password' not in params: raise SynapseError(400, "", Codes.MISSING_PARAM) new_password = params['new_password'] yield self.login_handler.set_password( - user_id, new_password, client.token_id + user_id, new_password, None ) defer.returnValue((200, {})) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 08d60f081..ab4385602 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -196,4 +196,18 @@ class RegistrationStore(SQLBaseStore): ['medium', 'address', 'validated_at', 'added_at'], 'user_get_threepids' ) - defer.returnValue(ret) \ No newline at end of file + defer.returnValue(ret) + + @defer.inlineCallbacks + def get_user_by_threepid(self, medium, address): + ret = yield self._simple_select_one( + "user_threepids", + { + "medium": medium, + "address": address + }, + ['user'], True, 'get_user_by_threepid' + ) + if ret: + defer.returnValue(ret['user']) + defer.returnValue(None) \ No newline at end of file From 91c8f828e13c71fb4d5124191740b0c9ac8a4bc2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Apr 2015 19:56:04 +0100 Subject: [PATCH 123/173] pep8 --- synapse/storage/registration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index ab4385602..f85cbb0d9 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -210,4 +210,4 @@ class RegistrationStore(SQLBaseStore): ) if ret: defer.returnValue(ret['user']) - defer.returnValue(None) \ No newline at end of file + defer.returnValue(None) From bc41f0398f081177ebc8036280f935d8bcb7bca0 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 21 Apr 2015 20:56:08 +0100 Subject: [PATCH 124/173] Initial implementation of an 'invite' power_level --- synapse/api/auth.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 9a5058a36..84a46385b 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -232,6 +232,13 @@ class Auth(object): elif target_in_room: # the target is already in the room. raise AuthError(403, "%s is already in the room." % target_user_id) + else: + invite_level = self._get_named_level(auth_events, "invite", 0) + + if user_level < invite_level: + raise AuthError( + 403, "You cannot invite user %s." % target_user_id + ) elif Membership.JOIN == membership: # Joins are valid iff caller == target and they were: # invited: They are accepting the invitation From 48b6ee2b67ad4f8bce9774267f863e45192f67d4 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 21 Apr 2015 21:07:35 +0100 Subject: [PATCH 125/173] Create an 'invite' powerlevel when making new rooms --- synapse/handlers/room.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index f9fc4a9c9..1226b23bc 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -213,7 +213,8 @@ class RoomCreationHandler(BaseHandler): "state_default": 50, "ban": 50, "kick": 50, - "redact": 50 + "redact": 50, + "invite": 0, }, ) From 2808c040ef4070e6a91e2b210383393cc54f66b5 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 21 Apr 2015 21:13:14 +0100 Subject: [PATCH 126/173] Also remember to check 'invite' level for changes --- synapse/api/auth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 84a46385b..d8b761479 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -592,6 +592,7 @@ class Auth(object): ("ban", []), ("redact", []), ("kick", []), + ("invite", []), ] old_list = current_state.content.get("users") From 2e0d9219b9585801ac8a8b5f3911107643ea3519 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 23 Apr 2015 11:45:29 +0100 Subject: [PATCH 127/173] Remove now-redundant email config --- synapse/config/email.py | 42 ------------------------------------ synapse/config/homeserver.py | 3 +-- 2 files changed, 1 insertion(+), 44 deletions(-) delete mode 100644 synapse/config/email.py diff --git a/synapse/config/email.py b/synapse/config/email.py deleted file mode 100644 index f0854f8c3..000000000 --- a/synapse/config/email.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ._base import Config - - -class EmailConfig(Config): - - def __init__(self, args): - super(EmailConfig, self).__init__(args) - self.email_from_address = args.email_from_address - self.email_smtp_server = args.email_smtp_server - - @classmethod - def add_arguments(cls, parser): - super(EmailConfig, cls).add_arguments(parser) - email_group = parser.add_argument_group("email") - email_group.add_argument( - "--email-from-address", - default="FROM@EXAMPLE.COM", - help="The address to send emails from (e.g. for password resets)." - ) - email_group.add_argument( - "--email-smtp-server", - default="", - help=( - "The SMTP server to send emails from (e.g. for password" - " resets)." - ) - ) diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 3edfadb98..efbdd93c2 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -20,7 +20,6 @@ from .database import DatabaseConfig from .ratelimiting import RatelimitConfig from .repository import ContentRepositoryConfig from .captcha import CaptchaConfig -from .email import EmailConfig from .voip import VoipConfig from .registration import RegistrationConfig from .metrics import MetricsConfig @@ -29,7 +28,7 @@ from .appservice import AppServiceConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, - EmailConfig, VoipConfig, RegistrationConfig, + VoipConfig, RegistrationConfig, MetricsConfig, AppServiceConfig,): pass From a2c10d37d7052a1ab6cf7188c3b4d763850e1561 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 23 Apr 2015 13:23:44 +0100 Subject: [PATCH 128/173] Add an error code to 'missing token' response. --- synapse/api/auth.py | 3 ++- synapse/api/errors.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 3d2b45d21..11f76c06f 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -372,7 +372,8 @@ class Auth(object): defer.returnValue((user, ClientInfo(device_id, token_id))) except KeyError: raise AuthError( - self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token." + self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token.", + errcode=Codes.MISSING_TOKEN ) @defer.inlineCallbacks diff --git a/synapse/api/errors.py b/synapse/api/errors.py index eddd88977..109547b3c 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -31,6 +31,7 @@ class Codes(object): BAD_PAGINATION = "M_BAD_PAGINATION" UNKNOWN = "M_UNKNOWN" NOT_FOUND = "M_NOT_FOUND" + MISSING_TOKEN = "M_MISSING_TOKEN" UNKNOWN_TOKEN = "M_UNKNOWN_TOKEN" LIMIT_EXCEEDED = "M_LIMIT_EXCEEDED" CAPTCHA_NEEDED = "M_CAPTCHA_NEEDED" From 0eb61a3d16bffa83b0963418fa17a8cf6c760631 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 23 Apr 2015 14:44:12 +0100 Subject: [PATCH 129/173] Remove ultimately unused feature of saving params from the first call in the session: it's probably too open to abuse. --- synapse/handlers/auth.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 7b0ab4829..ac07add2f 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -78,8 +78,16 @@ class AuthHandler(BaseHandler): sess = self._get_session_info(sid) if len(clientdict) > 0: - sess['clientdict'] = clientdict - self._save_session(sess) + # This was designed to allow the client to omit the parameters + # and just supply the session in subsequent calls so it split + # auth between devices by just sharing the session, (eg. so you + # could continue registration from your phone having clicked the + # email auth link on there). It's probably too open to abuse + # because it lets unauthenticated clients store arbitrary objects + # on a home server. + #sess['clientdict'] = clientdict + #self._save_session(sess) + pass elif 'clientdict' in sess: clientdict = sess['clientdict'] From 03eb4adc6ead31b69af6a87b8d05ae7e0e965fd0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 23 Apr 2015 18:20:17 +0100 Subject: [PATCH 130/173] Dedicated error code for failed 3pid auth verification --- synapse/api/errors.py | 1 + synapse/rest/client/v2_alpha/account.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 109547b3c..e8b9ee533 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -39,6 +39,7 @@ class Codes(object): MISSING_PARAM = "M_MISSING_PARAM", TOO_LARGE = "M_TOO_LARGE", EXCLUSIVE = "M_EXCLUSIVE" + THREEPID_AUTH_FAILED = "M_THREEPID_AUTH_FAILED" class CodeMessageException(RuntimeError): diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index e33607b79..4d199bbbb 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -126,7 +126,9 @@ class ThreepidRestServlet(RestServlet): threepid = yield self.identity_handler.threepid_from_creds(threePidCreds) if not threepid: - raise SynapseError(400, "Failed to auth 3pid") + raise SynapseError( + 400, "Failed to auth 3pid", Codes.THREEPID_AUTH_FAILED + ) for reqd in ['medium', 'address', 'validatedAt']: if reqd not in threepid: From f7a79a37beb6bbb217b53a1d8d93a33cf577e6ac Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 09:42:37 +0100 Subject: [PATCH 131/173] pep8 --- synapse/handlers/auth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index ac07add2f..34d7080fa 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -85,8 +85,8 @@ class AuthHandler(BaseHandler): # email auth link on there). It's probably too open to abuse # because it lets unauthenticated clients store arbitrary objects # on a home server. - #sess['clientdict'] = clientdict - #self._save_session(sess) + # sess['clientdict'] = clientdict + # self._save_session(sess) pass elif 'clientdict' in sess: clientdict = sess['clientdict'] From a21861962608726a5fe443762421c80119517778 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 11:27:38 +0100 Subject: [PATCH 132/173] Use underscores instead of camelcase for id server stuff --- synapse/handlers/auth.py | 12 ++++++------ synapse/handlers/identity.py | 12 ++++++------ synapse/rest/client/v2_alpha/register.py | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 34d7080fa..ef3219b38 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -203,19 +203,19 @@ class AuthHandler(BaseHandler): def _check_email_identity(self, authdict, _): yield run_on_reactor() - if 'threepidCreds' not in authdict: - raise LoginError(400, "Missing threepidCreds", Codes.MISSING_PARAM) + if 'threepid_creds' not in authdict: + raise LoginError(400, "Missing threepid_creds", Codes.MISSING_PARAM) - threepidCreds = authdict['threepidCreds'] + threepid_creds = authdict['threepid_creds'] identity_handler = self.hs.get_handlers().identity_handler - logger.info("Getting validated threepid. threepidcreds: %r" % (threepidCreds,)) - threepid = yield identity_handler.threepid_from_creds(threepidCreds) + logger.info("Getting validated threepid. threepidcreds: %r" % (threepid_creds,)) + threepid = yield identity_handler.threepid_from_creds(threepid_creds) if not threepid: raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) - threepid['threepidCreds'] = authdict['threepidCreds'] + threepid['threepid_creds'] = authdict['threepid_creds'] defer.returnValue(threepid) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 5c7263591..3ddd834c6 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -44,19 +44,19 @@ class IdentityHandler(BaseHandler): # XXX: make this configurable! # trustedIdServers = ['matrix.org', 'localhost:8090'] trustedIdServers = ['matrix.org'] - if not creds['idServer'] in trustedIdServers: + if not creds['id_server'] in trustedIdServers: logger.warn('%s is not a trusted ID server: rejecting 3pid ' + - 'credentials', creds['idServer']) + 'credentials', creds['id_server']) defer.returnValue(None) data = {} try: data = yield http_client.get_json( "http://%s%s" % ( - creds['idServer'], + creds['id_server'], "/_matrix/identity/api/v1/3pid/getValidated3pid" ), - {'sid': creds['sid'], 'clientSecret': creds['clientSecret']} + {'sid': creds['sid'], 'client_secret': creds['client_secret']} ) except CodeMessageException as e: data = json.loads(e.msg) @@ -75,11 +75,11 @@ class IdentityHandler(BaseHandler): data = yield http_client.post_urlencoded_get_json( # XXX: Change when ID servers are all HTTPS "http://%s%s" % ( - creds['idServer'], "/_matrix/identity/api/v1/3pid/bind" + creds['id_server'], "/_matrix/identity/api/v1/3pid/bind" ), { 'sid': creds['sid'], - 'clientSecret': creds['clientSecret'], + 'client_secret': creds['client_secret'], 'mxid': mxid, } ) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index e93897e28..dd176c7e7 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -136,11 +136,11 @@ class RegisterRestServlet(RestServlet): logger.info("bind_email specified: binding") emailThreepid = result[LoginType.EMAIL_IDENTITY] - threepidCreds = emailThreepid['threepidCreds'] + threepid_creds = emailThreepid['threepid_creds'] logger.debug("Binding emails %s to %s" % ( emailThreepid, user_id )) - yield self.identity_handler.bind_threepid(threepidCreds, user_id) + yield self.identity_handler.bind_threepid(threepid_creds, user_id) else: logger.info("bind_email not specified: not binding email") From 7ac8a60c6fdd1555cc86203d7c78415d9d236661 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 11:44:27 +0100 Subject: [PATCH 133/173] More underscores --- synapse/rest/client/v2_alpha/account.py | 4 ++-- synapse/rest/client/v2_alpha/register.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 4d199bbbb..394e6b380 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -130,7 +130,7 @@ class ThreepidRestServlet(RestServlet): 400, "Failed to auth 3pid", Codes.THREEPID_AUTH_FAILED ) - for reqd in ['medium', 'address', 'validatedAt']: + for reqd in ['medium', 'address', 'validated_at']: if reqd not in threepid: logger.warn("Couldn't add 3pid: invalid response from ID sevrer") raise SynapseError(500, "Invalid response from ID Server") @@ -139,7 +139,7 @@ class ThreepidRestServlet(RestServlet): auth_user.to_string(), threepid['medium'], threepid['address'], - threepid['validatedAt'], + threepid['validated_at'], ) if 'bind' in body and body['bind']: diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index dd176c7e7..3640fb4a2 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -121,7 +121,7 @@ class RegisterRestServlet(RestServlet): if LoginType.EMAIL_IDENTITY in result: threepid = result[LoginType.EMAIL_IDENTITY] - for reqd in ['medium', 'address', 'validatedAt']: + for reqd in ['medium', 'address', 'validated_at']: if reqd not in threepid: logger.info("Can't add incomplete 3pid") else: @@ -129,7 +129,7 @@ class RegisterRestServlet(RestServlet): user_id, threepid['medium'], threepid['address'], - threepid['validatedAt'], + threepid['validated_at'], ) if 'bind_email' in params and params['bind_email']: From 1bac74b9aea46f9e46152955ecf06d8cc7eacdd3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 14:48:49 +0100 Subject: [PATCH 134/173] Change to https for ID server communication --- synapse/handlers/identity.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 3ddd834c6..ad8246b58 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -52,7 +52,7 @@ class IdentityHandler(BaseHandler): data = {} try: data = yield http_client.get_json( - "http://%s%s" % ( + "https://%s%s" % ( creds['id_server'], "/_matrix/identity/api/v1/3pid/getValidated3pid" ), @@ -73,8 +73,7 @@ class IdentityHandler(BaseHandler): data = None try: data = yield http_client.post_urlencoded_get_json( - # XXX: Change when ID servers are all HTTPS - "http://%s%s" % ( + "https://%s%s" % ( creds['id_server'], "/_matrix/identity/api/v1/3pid/bind" ), { From a7b51f4539af90460d47efe0bae2692de285cd26 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 18:11:21 +0100 Subject: [PATCH 135/173] Check users in our table aren't on a different domain to the one we're configured with to try & fix SYN-266 --- synapse/app/homeserver.py | 19 +++++++++++++++++++ synapse/storage/registration.py | 18 ++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 27e53a9e5..5f6cf4dee 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -26,6 +26,7 @@ from synapse.server import HomeServer from synapse.python_dependencies import check_requirements from twisted.internet import reactor +from twisted.internet import defer from twisted.application import service from twisted.enterprise import adbapi from twisted.web.resource import Resource @@ -241,6 +242,22 @@ class SynapseHomeServer(HomeServer): ) logger.info("Metrics now running on 127.0.0.1 port %d", config.metrics_port) + @defer.inlineCallbacks + def post_startup_check(self): + all_users_native = yield self.get_datastore().all_users_on_domain( + self.hostname + ) + if not all_users_native: + sys.stderr.write( + "\n" + "******************************************************\n" + "Found users in database not native to %s!\n" + "You cannot changed a synapse server_name after it's been configured\n" + "******************************************************\n" + "\n" + ) + reactor.stop() + def get_version_string(): try: @@ -399,6 +416,8 @@ def setup(config_options): hs.get_datastore().start_profiling() hs.get_replication_layer().start_get_pdu_cache() + reactor.callWhenRunning(hs.post_startup_check) + return hs diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index f24154f14..c2efc3fd3 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -144,3 +144,21 @@ class RegistrationStore(SQLBaseStore): return rows[0] raise StoreError(404, "Token not found.") + + @defer.inlineCallbacks + def all_users_on_domain(self, domain): + res = yield self.runInteraction( + "all_users_on_domain", + self._all_users_on_domain_txn, + domain + ) + defer.returnValue(res) + + def _all_users_on_domain_txn(self, txn, domain): + sql = "SELECT COUNT(*) FROM users WHERE name NOT LIKE ?" + pat = "%:"+domain + cursor = txn.execute(sql, (pat,)) + num_not_matching = cursor.fetchall()[0][0] + if num_not_matching == 0: + return True + return False \ No newline at end of file From 2f475bd5d5cd9426a38d748238a214aaf77be4a2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 18:15:07 +0100 Subject: [PATCH 136/173] pep8 --- synapse/storage/registration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index c2efc3fd3..697d48712 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -156,9 +156,9 @@ class RegistrationStore(SQLBaseStore): def _all_users_on_domain_txn(self, txn, domain): sql = "SELECT COUNT(*) FROM users WHERE name NOT LIKE ?" - pat = "%:"+domain + pat = "%:" + domain cursor = txn.execute(sql, (pat,)) num_not_matching = cursor.fetchall()[0][0] if num_not_matching == 0: return True - return False \ No newline at end of file + return False From f8152f2708cc0c476f5e1ec028a63ca632927eff Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 27 Apr 2015 10:16:26 +0100 Subject: [PATCH 137/173] rename db method to be more informative --- synapse/app/homeserver.py | 2 +- synapse/storage/registration.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 5f6cf4dee..8da1a4baf 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -244,7 +244,7 @@ class SynapseHomeServer(HomeServer): @defer.inlineCallbacks def post_startup_check(self): - all_users_native = yield self.get_datastore().all_users_on_domain( + all_users_native = yield self.get_datastore().are_all_users_on_domain( self.hostname ) if not all_users_native: diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 697d48712..65ae58a39 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -146,15 +146,15 @@ class RegistrationStore(SQLBaseStore): raise StoreError(404, "Token not found.") @defer.inlineCallbacks - def all_users_on_domain(self, domain): + def are_all_users_on_domain(self, domain): res = yield self.runInteraction( - "all_users_on_domain", - self._all_users_on_domain_txn, + "are_all_users_on_domain", + self._are_all_users_on_domain_txn, domain ) defer.returnValue(res) - def _all_users_on_domain_txn(self, txn, domain): + def _are_all_users_on_domain_txn(self, txn, domain): sql = "SELECT COUNT(*) FROM users WHERE name NOT LIKE ?" pat = "%:" + domain cursor = txn.execute(sql, (pat,)) From b02e1006b9d7282cdc9983d52ac478d4670a8361 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 27 Apr 2015 11:46:00 +0100 Subject: [PATCH 138/173] Run database check before daemonizing, at the cost of database hygiene. --- synapse/app/homeserver.py | 17 ++++++++--------- synapse/storage/__init__.py | 10 ++++++++++ synapse/storage/registration.py | 18 ------------------ 3 files changed, 18 insertions(+), 27 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 8da1a4baf..8a00b21aa 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -18,7 +18,8 @@ import sys sys.dont_write_bytecode = True from synapse.storage import ( - prepare_database, prepare_sqlite3_database, UpgradeDatabaseException, + prepare_database, prepare_sqlite3_database, are_all_users_on_domain, + UpgradeDatabaseException, ) from synapse.server import HomeServer @@ -242,10 +243,9 @@ class SynapseHomeServer(HomeServer): ) logger.info("Metrics now running on 127.0.0.1 port %d", config.metrics_port) - @defer.inlineCallbacks - def post_startup_check(self): - all_users_native = yield self.get_datastore().are_all_users_on_domain( - self.hostname + def run_startup_checks(self, db_conn): + all_users_native = are_all_users_on_domain( + db_conn, self.hostname ) if not all_users_native: sys.stderr.write( @@ -254,9 +254,9 @@ class SynapseHomeServer(HomeServer): "Found users in database not native to %s!\n" "You cannot changed a synapse server_name after it's been configured\n" "******************************************************\n" - "\n" + "\n" % (self.hostname,) ) - reactor.stop() + sys.exit(1) def get_version_string(): @@ -392,6 +392,7 @@ def setup(config_options): with sqlite3.connect(db_name) as db_conn: prepare_sqlite3_database(db_conn) prepare_database(db_conn) + hs.run_startup_checks(db_conn) except UpgradeDatabaseException: sys.stderr.write( "\nFailed to upgrade database.\n" @@ -416,8 +417,6 @@ def setup(config_options): hs.get_datastore().start_profiling() hs.get_replication_layer().start_get_pdu_cache() - reactor.callWhenRunning(hs.post_startup_check) - return hs diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index f4dec7039..0c4744368 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -421,3 +421,13 @@ def prepare_sqlite3_database(db_conn): " VALUES (?,?)", (row[0], False) ) + + +def are_all_users_on_domain(txn, domain): + sql = "SELECT COUNT(*) FROM users WHERE name NOT LIKE ?" + pat = "%:" + domain + cursor = txn.execute(sql, (pat,)) + num_not_matching = cursor.fetchall()[0][0] + if num_not_matching == 0: + return True + return False \ No newline at end of file diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 65ae58a39..f24154f14 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -144,21 +144,3 @@ class RegistrationStore(SQLBaseStore): return rows[0] raise StoreError(404, "Token not found.") - - @defer.inlineCallbacks - def are_all_users_on_domain(self, domain): - res = yield self.runInteraction( - "are_all_users_on_domain", - self._are_all_users_on_domain_txn, - domain - ) - defer.returnValue(res) - - def _are_all_users_on_domain_txn(self, txn, domain): - sql = "SELECT COUNT(*) FROM users WHERE name NOT LIKE ?" - pat = "%:" + domain - cursor = txn.execute(sql, (pat,)) - num_not_matching = cursor.fetchall()[0][0] - if num_not_matching == 0: - return True - return False From df7591479115b6ea73f774d2dcd2b071c92d7a37 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 27 Apr 2015 11:48:33 +0100 Subject: [PATCH 139/173] pep8 --- synapse/app/homeserver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 8a00b21aa..541059b20 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -27,7 +27,6 @@ from synapse.server import HomeServer from synapse.python_dependencies import check_requirements from twisted.internet import reactor -from twisted.internet import defer from twisted.application import service from twisted.enterprise import adbapi from twisted.web.resource import Resource From 8a5be236e0113125ab2fa9a5fb2f950b546acea9 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 27 Apr 2015 11:49:18 +0100 Subject: [PATCH 140/173] pep8 --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 0c4744368..9b6471aec 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -430,4 +430,4 @@ def are_all_users_on_domain(txn, domain): num_not_matching = cursor.fetchall()[0][0] if num_not_matching == 0: return True - return False \ No newline at end of file + return False From f1acb9fd40475563acbb01758b25099d2095a524 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 27 Apr 2015 11:56:34 +0100 Subject: [PATCH 141/173] logging args --- synapse/rest/client/v2_alpha/account.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 394e6b380..bd57d11dc 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -143,9 +143,9 @@ class ThreepidRestServlet(RestServlet): ) if 'bind' in body and body['bind']: - logger.debug("Binding emails %s to %s" % ( + logger.debug("Binding emails %s to %s", threepid, auth_user.to_string() - )) + ) yield self.identity_handler.bind_threepid( threePidCreds, auth_user.to_string() ) From 03c4f0ed67fe9e2aab78fc55ff10d554c0cd7317 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 27 Apr 2015 12:36:59 +0100 Subject: [PATCH 142/173] pep8 --- synapse/rest/client/v2_alpha/account.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index bd57d11dc..3e522ad39 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -143,7 +143,8 @@ class ThreepidRestServlet(RestServlet): ) if 'bind' in body and body['bind']: - logger.debug("Binding emails %s to %s", + logger.debug( + "Binding emails %s to %s", threepid, auth_user.to_string() ) yield self.identity_handler.bind_threepid( From e4c4664d73abd260264f51a0645281d21afc358e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 12:40:49 +0100 Subject: [PATCH 143/173] Handle the fact that postgres databases can be restarted from under us --- synapse/storage/_base.py | 28 +++++++++++++++++++++++++++- synapse/storage/engines/postgres.py | 3 +++ synapse/storage/engines/sqlite3.py | 3 +++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index f5952d1fc..874d41447 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -300,6 +300,9 @@ class SQLBaseStore(object): def inner_func(conn, *args, **kwargs): with LoggingContext("runInteraction") as context: + if self.database_engine.is_connection_closed(conn): + conn.reconnect() + current_context.copy_to(context) start = time.time() * 1000 txn_id = self._TXN_ID @@ -322,12 +325,35 @@ class SQLBaseStore(object): LoggingTransaction(txn, name, self.database_engine), *args, **kwargs ) + except self.database_engine.module.OperationalError as e: + # This can happen if the database disappears mid + # transaction. + logger.warn( + "[TXN OPERROR] {%s} %s %d/%d", + name, e, i, N + ) + if i < N: + i += 1 + try: + conn.rollback() + except self.database_engine.module.Error as e1: + logger.warn( + "[TXN EROLL] {%s} %s", + name, e1, + ) + continue except self.database_engine.module.DatabaseError as e: if self.database_engine.is_deadlock(e): logger.warn("[TXN DEADLOCK] {%s} %d/%d", name, i, N) if i < N: i += 1 - conn.rollback() + try: + conn.rollback() + except self.database_engine.module.Error as e1: + logger.warn( + "[TXN EROLL] {%s} %s", + name, e1, + ) continue raise except Exception as e: diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 6f75245fa..00dbae7b6 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -39,3 +39,6 @@ class PostgresEngine(object): if isinstance(error, self.module.DatabaseError): return error.pgcode in ["40001", "40P01"] return False + + def is_connection_closed(self, conn): + return bool(conn) diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 72c11df46..39828a597 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -35,3 +35,6 @@ class Sqlite3Engine(object): def is_deadlock(self, error): return False + + def is_connection_closed(self, conn): + return False From 2732be83d9e883184f4a783fb7ba15487f30c20d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 13:22:30 +0100 Subject: [PATCH 144/173] Shuffle operations so that locking upsert happens last in the txn. This ensures the lock is held for the least amount of time possible. --- synapse/storage/_base.py | 3 + synapse/storage/engines/postgres.py | 3 + synapse/storage/engines/sqlite3.py | 3 + synapse/storage/events.py | 88 ++++++++++++++--------------- synapse/storage/transactions.py | 21 ++----- 5 files changed, 59 insertions(+), 59 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 874d41447..6017c2a6e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -466,6 +466,9 @@ class SQLBaseStore(object): ) def _simple_upsert_txn(self, txn, table, keyvalues, values, insertion_values={}): + # We need to lock the table :( + self.database_engine.lock_table(txn, table) + # Try to update sql = "UPDATE %s SET %s WHERE %s" % ( table, diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 00dbae7b6..b8cca9b18 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -42,3 +42,6 @@ class PostgresEngine(object): def is_connection_closed(self, conn): return bool(conn) + + def lock_table(self, txn, table): + txn.execute("LOCK TABLE %s in EXCLUSIVE MODE" % (table,)) diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 39828a597..f62d5d120 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -38,3 +38,6 @@ class Sqlite3Engine(object): def is_connection_closed(self, conn): return False + + def lock_table(self, txn, table): + return diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 7dbf7a396..a3c260ddc 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -283,50 +283,6 @@ class EventsStore(SQLBaseStore): if context.rejected: self._store_rejections_txn(txn, event.event_id, context.rejected) - if event.is_state(): - vals = { - "event_id": event.event_id, - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - } - - # TODO: How does this work with backfilling? - if hasattr(event, "replaces_state"): - vals["prev_state"] = event.replaces_state - - self._simple_insert_txn( - txn, - "state_events", - vals, - ) - - if is_new_state and not context.rejected: - self._simple_upsert_txn( - txn, - "current_state_events", - keyvalues={ - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - }, - values={ - "event_id": event.event_id, - } - ) - - for e_id, h in event.prev_state: - self._simple_insert_txn( - txn, - table="event_edges", - values={ - "event_id": event.event_id, - "prev_event_id": e_id, - "room_id": event.room_id, - "is_state": True, - }, - ) - for hash_alg, hash_base64 in event.hashes.items(): hash_bytes = decode_base64(hash_base64) self._store_event_content_hash_txn( @@ -356,6 +312,50 @@ class EventsStore(SQLBaseStore): txn, event.event_id, ref_alg, ref_hash_bytes ) + if event.is_state(): + vals = { + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + } + + # TODO: How does this work with backfilling? + if hasattr(event, "replaces_state"): + vals["prev_state"] = event.replaces_state + + self._simple_insert_txn( + txn, + "state_events", + vals, + ) + + for e_id, h in event.prev_state: + self._simple_insert_txn( + txn, + table="event_edges", + values={ + "event_id": event.event_id, + "prev_event_id": e_id, + "room_id": event.room_id, + "is_state": True, + }, + ) + + if is_new_state and not context.rejected: + self._simple_upsert_txn( + txn, + "current_state_events", + keyvalues={ + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + }, + values={ + "event_id": event.event_id, + } + ) + def _store_redaction(self, txn, event): # invalidate the cache for the redacted event self._invalidate_get_event_cache(event.redacts) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 7e3add528..89dd7d894 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -76,25 +76,16 @@ class TransactionStore(SQLBaseStore): response_json (str) """ - return self.runInteraction( - "set_received_txn_response", - self._set_received_txn_response, - transaction_id, origin, code, response_dict - ) - - def _set_received_txn_response(self, txn, transaction_id, origin, code, - response_json): - self._simple_upsert_txn( - txn, + return self._simple_insert( table=ReceivedTransactionsTable.table_name, - keyvalues={ + values={ "transaction_id": transaction_id, "origin": origin, - }, - values={ "response_code": code, - "response_json": response_json, - } + "response_json": response_dict, + }, + or_ignore=True, + desc="set_received_txn_response", ) def prep_send_transaction(self, transaction_id, destination, From 412ece18e7edb87053a3684e49d5dd485f88a65d Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 27 Apr 2015 14:08:45 +0100 Subject: [PATCH 145/173] Add commentage. --- synapse/handlers/auth.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index ef3219b38..2e8009d3c 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -121,6 +121,10 @@ class AuthHandler(BaseHandler): @defer.inlineCallbacks def add_oob_auth(self, stagetype, authdict, clientip): + """ + Adds the result of out-of-band authentication into an existing auth + session. Currently used for adding the result of fallback auth. + """ if stagetype not in self.checkers: raise LoginError(400, "", Codes.MISSING_PARAM) if 'session' not in authdict: From 56f518d279b642efce92e172b463232937d50b8c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 14:53:35 +0100 Subject: [PATCH 146/173] Add docs on how to use synapse with psycopg2 --- docs/postgres.rst | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 docs/postgres.rst diff --git a/docs/postgres.rst b/docs/postgres.rst new file mode 100644 index 000000000..5bb45d3ed --- /dev/null +++ b/docs/postgres.rst @@ -0,0 +1,34 @@ +Using Postgres +-------------- + +Set up client +============= +We need to have installed the postgres python connector ``psycopg2``. In the +virtual env:: + + sudo apt-get install libpq-dev + pip install psycopg2 + + +Synapse config +============== + +Add the following line to your config file:: + + database_config: + +Where ```` is the file name that points to a yaml file of the +following form:: + + name: psycopg2 + args: + user: + password: + database: + host: + cp_min: 5 + cp_max: 10 + +All key, values in ``args`` are passed to the ``psycopg2.connect(..)`` +function, except keys beginning with ``cp_``, which are consumed by the twisted +adbapi connection pool. From 8558e1ec7338a1f60342024736614dec71d104ce Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 15:19:44 +0100 Subject: [PATCH 147/173] Make get_max_token into inlineCallbacks so that the lock works. --- synapse/storage/util/id_generators.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index e5dec1c94..9d461d5e9 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -105,21 +105,22 @@ class StreamIdGenerator(object): return manager() + @defer.inlineCallbacks def get_max_token(self, store): """Returns the maximum stream id such that all stream ids less than or equal to it have been successfully persisted. """ with self._lock: if self._unfinished_ids: - return self._unfinished_ids[0] - 1 + defer.returnValue(self._unfinished_ids[0] - 1) if not self._current_max: - return store.runInteraction( + yield store.runInteraction( "_compute_current_max", self._compute_current_max, ) - return self._current_max + defer.returnValue(self._current_max) def _compute_current_max(self, txn): txn.execute("SELECT MAX(stream_ordering) FROM events") From 416a3e6c4f40c94cf859a07dbc4341c907aac091 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 15:44:30 +0100 Subject: [PATCH 148/173] Ensure check_same_thread is enabled for sqlite3 --- synapse/app/homeserver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 93500dd79..3709cd7bf 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -372,6 +372,7 @@ def setup(config_options): db_config.setdefault("args", {}).update({ "cp_min": 1, "cp_max": 1, + "check_same_thread": False, }) else: raise RuntimeError("Unsupported database type '%s'" % (name,)) From 1ef66cc3bd541ee1e4a017cfdd008eacaec5bcf8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 15:57:43 +0100 Subject: [PATCH 149/173] Move database configuration into config module --- synapse/app/homeserver.py | 35 ++++------------------------------- synapse/config/database.py | 23 +++++++++++++++++++++-- 2 files changed, 25 insertions(+), 33 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 3709cd7bf..f29f9d702 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -350,42 +350,15 @@ def setup(config_options): tls_context_factory = context_factory.ServerContextFactory(config) - if config.database_config: - with open(config.database_config, 'r') as f: - db_config = yaml.safe_load(f) - else: - db_config = { - "name": "sqlite3", - "args": { - "database": config.database_path, - }, - } - - db_config = { - k: v for k, v in db_config.items() - } - - name = db_config.get("name", None) - if name == "psycopg2": - pass - elif name == "sqlite3": - db_config.setdefault("args", {}).update({ - "cp_min": 1, - "cp_max": 1, - "check_same_thread": False, - }) - else: - raise RuntimeError("Unsupported database type '%s'" % (name,)) - - database_engine = create_engine(name) - db_config["args"]["cp_openfun"] = database_engine.on_new_connection + database_engine = create_engine(config.database_config["name"]) + config.database_config["args"]["cp_openfun"] = database_engine.on_new_connection hs = SynapseHomeServer( config.server_name, domain_with_port=domain_with_port, upload_dir=os.path.abspath("uploads"), db_name=config.database_path, - db_config=db_config, + db_config=config.database_config, tls_context_factory=tls_context_factory, config=config, content_addr=config.content_addr, @@ -404,7 +377,7 @@ def setup(config_options): try: db_conn = database_engine.module.connect( **{ - k: v for k, v in db_config.get("args", {}).items() + k: v for k, v in config.database_config.get("args", {}).items() if not k.startswith("cp_") } ) diff --git a/synapse/config/database.py b/synapse/config/database.py index f3d0898c0..190d119df 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -15,6 +15,7 @@ from ._base import Config import os +import yaml class DatabaseConfig(Config): @@ -27,9 +28,27 @@ class DatabaseConfig(Config): self.event_cache_size = self.parse_size(args.event_cache_size) if args.database_config: - self.database_config = self.abspath(args.database_config) + with open(args.database_config) as f: + self.database_config = yaml.safe_load(f) else: - self.database_config = None + self.database_config = { + "name": "sqlite3", + "args": { + "database": self.database_path, + }, + } + + name = self.database_config.get("name", None) + if name == "psycopg2": + pass + elif name == "sqlite3": + self.database_config.setdefault("args", {}).update({ + "cp_min": 1, + "cp_max": 1, + "check_same_thread": False, + }) + else: + raise RuntimeError("Unsupported database type '%s'" % (name,)) @classmethod def add_arguments(cls, parser): From 42b7139deced5d9614bcb487625f4ce5d99e560b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 15:59:56 +0100 Subject: [PATCH 150/173] Remove unused import --- synapse/app/homeserver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f29f9d702..110566e33 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -56,7 +56,6 @@ import os import re import resource import subprocess -import yaml logger = logging.getLogger("synapse.app.homeserver") From 4e49f52375acb705b32937115210d055b0e1ce38 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 17:36:37 +0100 Subject: [PATCH 151/173] Don't port over all of the sent_transactions table --- scripts/port_from_sqlite_to_postgres.py | 162 ++++++++++++++++++------ 1 file changed, 122 insertions(+), 40 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index f4b6ed068..19e35bf80 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -122,8 +122,8 @@ class Store(object): return self.db_pool.runWithConnection(r) - def execute(self, f): - return self.runInteraction(f.__name__, f) + def execute(self, f, *args, **kwargs): + return self.runInteraction(f.__name__, f, *args, **kwargs) def insert_many_txn(self, txn, table, headers, rows): sql = "INSERT INTO %s (%s) VALUES (%s)" % ( @@ -347,9 +347,118 @@ class Porter(object): def __init__(self, **kwargs): self.__dict__.update(kwargs) + def convert_rows(self, table, headers, rows): + bool_col_names = BOOLEAN_COLUMNS.get(table, []) + + bool_cols = [ + i for i, h in enumerate(headers) if h in bool_col_names + ] + + def conv(j, col): + if j in bool_cols: + return bool(col) + return col + + for i, row in enumerate(rows): + rows[i] = tuple( + self.postgres_store.database_engine.encode_parameter( + conv(j, col) + ) + for j, col in enumerate(row) + if j > 0 + ) + @defer.inlineCallbacks def handle_table(self, table): - if table in APPEND_ONLY_TABLES: + def delete_all(txn): + txn.execute( + "DELETE FROM port_from_sqlite3 WHERE table_name = %s", + (table,) + ) + txn.execute("TRUNCATE %s CASCADE" % (table,)) + + def get_table_size(txn): + txn.execute("SELECT count(*) FROM %s" % (table,)) + size, = txn.fetchone() + return int(size) + + if table == "sent_transactions": + # This is a big table, and we really only need some of the recent + # data + yield self.postgres_store.execute(delete_all) + + # Only save things from the last day + yesterday = 1429114568820 #int(time.time()*1000) - 86400000 + + # And save the max transaction id from each destination + select = ( + "SELECT rowid, * FROM sent_transactions WHERE rowid IN (" + "SELECT max(rowid) FROM sent_transactions" + " GROUP BY destination" + ")" + ) + + def r(txn): + txn.execute(select) + rows = txn.fetchall() + headers = [column[0] for column in txn.description] + + ts_ind = headers.index('ts') + + return headers, [r for r in rows if r[ts_ind] < yesterday] + + headers, rows = yield self.sqlite_store.runInteraction( + "select", r, + ) + + self.convert_rows(table, headers, rows) + + inserted_rows = len(rows) + max_inserted_rowid = max(r[0] for r in rows) + + def insert(txn): + self.postgres_store.insert_many_txn( + txn, table, headers[1:], rows + ) + + yield self.postgres_store.execute(insert) + + def get_start_id(txn): + txn.execute( + "SELECT rowid FROM sent_transactions WHERE ts >= ?" + " ORDER BY rowid ASC LIMIT 1", + (yesterday,) + ) + + rows = txn.fetchall() + if rows: + return rows[0][0] + else: + return 1 + + next_chunk = yield self.sqlite_store.execute(get_start_id) + next_chunk = max(max_inserted_rowid + 1, next_chunk) + + yield self.postgres_store._simple_insert( + table="port_from_sqlite3", + values={"table_name": table, "rowid": next_chunk} + ) + + def get_sent_table_size(txn): + txn.execute( + "SELECT count(*) FROM sent_transactions" + " WHERE ts >= ?", + (yesterday,) + ) + size, = txn.fetchone() + return int(size) + + table_size = yield self.sqlite_store.execute( + get_sent_table_size + ) + + table_size += inserted_rows + elif table in APPEND_ONLY_TABLES: # It's safe to just carry on inserting. next_chunk = yield self.postgres_store._simple_select_one_onecol( table="port_from_sqlite3", @@ -365,28 +474,18 @@ class Porter(object): ) next_chunk = 1 - else: - def delete_all(txn): - txn.execute( - "DELETE FROM port_from_sqlite3 WHERE table_name = %s", - (table,) - ) - txn.execute("TRUNCATE %s CASCADE" % (table,)) - self.postgres_store._simple_insert_txn( - txn, - table="port_from_sqlite3", - values={"table_name": table, "rowid": 0} - ) - yield self.postgres_store.execute(delete_all) + table_size = yield self.sqlite_store.execute(get_table_size) + else: + yield self.postgres_store.execute(delete_all) + self.postgres_store._simple_insert( + table="port_from_sqlite3", + values={"table_name": table, "rowid": 0} + ) + + table_size = yield self.sqlite_store.execute(get_table_size) next_chunk = 1 - def get_table_size(txn): - txn.execute("SELECT count(*) FROM %s" % (table,)) - size, = txn.fetchone() - return int(size) - - table_size = yield self.sqlite_store.execute(get_table_size) postgres_size = yield self.postgres_store.execute(get_table_size) if not table_size: @@ -399,8 +498,6 @@ class Porter(object): % (table,) ) - bool_col_names = BOOLEAN_COLUMNS.get(table, []) - while True: def r(txn): txn.execute(select, (next_chunk, self.batch_size,)) @@ -412,24 +509,9 @@ class Porter(object): headers, rows = yield self.sqlite_store.runInteraction("select", r) if rows: - bool_cols = [ - i for i, h in enumerate(headers) if h in bool_col_names - ] next_chunk = rows[-1][0] + 1 - def conv(j, col): - if j in bool_cols: - return bool(col) - return col - - for i, row in enumerate(rows): - rows[i] = tuple( - self.postgres_store.database_engine.encode_parameter( - conv(j, col) - ) - for j, col in enumerate(row) - if j > 0 - ) + self.convert_rows(table, headers, rows) def insert(txn): self.postgres_store.insert_many_txn( From 40cbd6b6ee809c23750ea818ebd668be78bbf328 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 17:53:15 +0100 Subject: [PATCH 152/173] Shuffle progress stuff --- scripts/port_from_sqlite_to_postgres.py | 375 +++++++++++------------- 1 file changed, 174 insertions(+), 201 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 19e35bf80..845e3fcf3 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -142,207 +142,6 @@ class Store(object): raise -class Progress(object): - """Used to report progress of the port - """ - def __init__(self): - self.tables = {} - - self.start_time = int(time.time()) - - def add_table(self, table, cur, size): - self.tables[table] = { - "start": cur, - "num_done": cur, - "total": size, - "perc": int(cur * 100 / size), - } - - def update(self, table, num_done): - data = self.tables[table] - data["num_done"] = num_done - data["perc"] = int(num_done * 100 / data["total"]) - - def done(self): - pass - - -class CursesProgress(Progress): - """Reports progress to a curses window - """ - def __init__(self, stdscr): - self.stdscr = stdscr - - curses.use_default_colors() - curses.curs_set(0) - - curses.init_pair(1, curses.COLOR_RED, -1) - curses.init_pair(2, curses.COLOR_GREEN, -1) - - self.last_update = 0 - - self.finished = False - - super(CursesProgress, self).__init__() - - def update(self, table, num_done): - super(CursesProgress, self).update(table, num_done) - - self.render() - - def render(self, force=False): - now = time.time() - - if not force and now - self.last_update < 0.2: - # reactor.callLater(1, self.render) - return - - self.stdscr.clear() - - rows, cols = self.stdscr.getmaxyx() - - duration = int(now) - int(self.start_time) - - minutes, seconds = divmod(duration, 60) - duration_str = '%02dm %02ds' % (minutes, seconds,) - - if self.finished: - status = "Time spent: %s (Done!)" % (duration_str,) - else: - min_perc = min( - (v["num_done"] - v["start"]) * 100. / (v["total"] - v["start"]) - if v["total"] - v["start"] else 100 - for v in self.tables.values() - ) - if min_perc > 0: - est_remaining = (int(now) - self.start_time) * 100 / min_perc - est_remaining_str = '%02dm %02ds remaining' % divmod(est_remaining, 60) - else: - est_remaining_str = "Unknown" - status = ( - "Time spent: %s (est. remaining: %s)" - % (duration_str, est_remaining_str,) - ) - - self.stdscr.addstr( - 0, 0, - status, - curses.A_BOLD, - ) - - max_len = max([len(t) for t in self.tables.keys()]) - - left_margin = 5 - middle_space = 1 - - items = self.tables.items() - items.sort( - key=lambda i: (i[1]["perc"], i[0]), - ) - - for i, (table, data) in enumerate(items): - if i + 2 >= rows: - break - - perc = data["perc"] - - color = curses.color_pair(2) if perc == 100 else curses.color_pair(1) - - self.stdscr.addstr( - i+2, left_margin + max_len - len(table), - table, - curses.A_BOLD | color, - ) - - size = 20 - - progress = "[%s%s]" % ( - "#" * int(perc*size/100), - " " * (size - int(perc*size/100)), - ) - - self.stdscr.addstr( - i+2, left_margin + max_len + middle_space, - "%s %3d%% (%d/%d)" % (progress, perc, data["num_done"], data["total"]), - ) - - if self.finished: - self.stdscr.addstr( - self.rows-1, 0, - "Press any key to exit...", - ) - - self.stdscr.refresh() - self.last_update = time.time() - - def done(self): - self.finished = True - self.render(True) - self.stdscr.getch() - - def on_prepare_sqlite(self): - self.stdscr.clear() - self.stdscr.addstr( - 0, 0, - "Preparing SQLite database...", - curses.A_BOLD, - ) - self.stdscr.refresh() - - def on_prepare_postgres(self): - self.stdscr.clear() - self.stdscr.addstr( - 0, 0, - "Preparing PostgreSQL database...", - curses.A_BOLD, - ) - self.stdscr.refresh() - - def fetching_tables(self): - self.stdscr.clear() - self.stdscr.addstr( - 0, 0, - "Fetching tables...", - curses.A_BOLD, - ) - self.stdscr.refresh() - - def preparing_tables(self): - self.stdscr.clear() - self.stdscr.addstr( - 0, 0, - "Preparing tables...", - curses.A_BOLD, - ) - self.stdscr.refresh() - - -class TerminalProgress(Progress): - """Just prints progress to the terminal - """ - def update(self, table, num_done): - super(TerminalProgress, self).update(table, num_done) - - data = self.tables[table] - - print "%s: %d%% (%d/%d)" % ( - table, data["perc"], - data["num_done"], data["total"], - ) - - def on_prepare_sqlite(self): - print "Preparing SQLite database..." - - def on_prepare_postgres(self): - print "Preparing PostgreSQL database..." - - def fetching_tables(self): - print "Fetching tables..." - - def preparing_tables(self): - print "Preparing tables..." - - class Porter(object): def __init__(self, **kwargs): self.__dict__.update(kwargs) @@ -630,6 +429,180 @@ class Porter(object): reactor.stop() +############################################## +###### The following is simply UI stuff ###### +############################################## + + +class Progress(object): + """Used to report progress of the port + """ + def __init__(self): + self.tables = {} + + self.start_time = int(time.time()) + + def add_table(self, table, cur, size): + self.tables[table] = { + "start": cur, + "num_done": cur, + "total": size, + "perc": int(cur * 100 / size), + } + + def update(self, table, num_done): + data = self.tables[table] + data["num_done"] = num_done + data["perc"] = int(num_done * 100 / data["total"]) + + def done(self): + pass + + +class CursesProgress(Progress): + """Reports progress to a curses window + """ + def __init__(self, stdscr): + self.stdscr = stdscr + + curses.use_default_colors() + curses.curs_set(0) + + curses.init_pair(1, curses.COLOR_RED, -1) + curses.init_pair(2, curses.COLOR_GREEN, -1) + + self.last_update = 0 + + self.finished = False + + super(CursesProgress, self).__init__() + + def update(self, table, num_done): + super(CursesProgress, self).update(table, num_done) + + self.render() + + def render(self, force=False): + now = time.time() + + if not force and now - self.last_update < 0.2: + # reactor.callLater(1, self.render) + return + + self.stdscr.clear() + + rows, cols = self.stdscr.getmaxyx() + + duration = int(now) - int(self.start_time) + + minutes, seconds = divmod(duration, 60) + duration_str = '%02dm %02ds' % (minutes, seconds,) + + if self.finished: + status = "Time spent: %s (Done!)" % (duration_str,) + else: + min_perc = min( + (v["num_done"] - v["start"]) * 100. / (v["total"] - v["start"]) + if v["total"] - v["start"] else 100 + for v in self.tables.values() + ) + if min_perc > 0: + est_remaining = (int(now) - self.start_time) * 100 / min_perc + est_remaining_str = '%02dm %02ds remaining' % divmod(est_remaining, 60) + else: + est_remaining_str = "Unknown" + status = ( + "Time spent: %s (est. remaining: %s)" + % (duration_str, est_remaining_str,) + ) + + self.stdscr.addstr( + 0, 0, + status, + curses.A_BOLD, + ) + + max_len = max([len(t) for t in self.tables.keys()]) + + left_margin = 5 + middle_space = 1 + + items = self.tables.items() + items.sort( + key=lambda i: (i[1]["perc"], i[0]), + ) + + for i, (table, data) in enumerate(items): + if i + 2 >= rows: + break + + perc = data["perc"] + + color = curses.color_pair(2) if perc == 100 else curses.color_pair(1) + + self.stdscr.addstr( + i+2, left_margin + max_len - len(table), + table, + curses.A_BOLD | color, + ) + + size = 20 + + progress = "[%s%s]" % ( + "#" * int(perc*size/100), + " " * (size - int(perc*size/100)), + ) + + self.stdscr.addstr( + i+2, left_margin + max_len + middle_space, + "%s %3d%% (%d/%d)" % (progress, perc, data["num_done"], data["total"]), + ) + + if self.finished: + self.stdscr.addstr( + self.rows-1, 0, + "Press any key to exit...", + ) + + self.stdscr.refresh() + self.last_update = time.time() + + def done(self): + self.finished = True + self.render(True) + self.stdscr.getch() + + def set_state(self, state): + self.stdscr.clear() + self.stdscr.addstr( + 0, 0, + state + "...", + curses.A_BOLD, + ) + self.stdscr.refresh() + + +class TerminalProgress(Progress): + """Just prints progress to the terminal + """ + def update(self, table, num_done): + super(TerminalProgress, self).update(table, num_done) + + data = self.tables[table] + + print "%s: %d%% (%d/%d)" % ( + table, data["perc"], + data["num_done"], data["total"], + ) + + def set_state(self, state): + print state + "..." + + +############################################## +############################################## + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("-v", action='store_true') From 5b8b1a43bd392559b1960c4a79f5dcdde0d37f23 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 17:53:40 +0100 Subject: [PATCH 153/173] Split setuping up and processing of tables --- scripts/port_from_sqlite_to_postgres.py | 31 ++++++++++++++++++------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 845e3fcf3..1e7ac072f 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -168,7 +168,7 @@ class Porter(object): ) @defer.inlineCallbacks - def handle_table(self, table): + def setup_table(self, table): def delete_all(txn): txn.execute( "DELETE FROM port_from_sqlite3 WHERE table_name = %s", @@ -287,6 +287,10 @@ class Porter(object): postgres_size = yield self.postgres_store.execute(get_table_size) + defer.returnValue((table, postgres_size, table_size, next_chunk)) + + @defer.inlineCallbacks + def handle_table(self, table, postgres_size, table_size, next_chunk): if not table_size: return @@ -364,14 +368,14 @@ class Porter(object): self.postgres_store = Store(postgres_db_pool, postgres_engine) # Step 1. Set up databases. - self.progress.on_prepare_sqlite() + self.progress.set_state("Preparing SQLite3") self.setup_db(sqlite_config, sqlite_engine) - self.progress.on_prepare_postgres() + self.progress.set_state("Preparing PostgreSQL") self.setup_db(postgres_config, postgres_engine) # Step 2. Get tables. - self.progress.fetching_tables() + self.progress.set_state("Fetching tables") sqlite_tables = yield self.sqlite_store._simple_select_onecol( table="sqlite_master", keyvalues={ @@ -390,7 +394,7 @@ class Porter(object): tables = set(sqlite_tables) & set(postgres_tables) - self.progress.preparing_tables() + self.progress.set_state("Creating tables") logger.info("Found %d tables", len(tables)) @@ -409,10 +413,12 @@ class Porter(object): except Exception as e: logger.info("Failed to create port table: %s", e) - # Process tables. - yield defer.gatherResults( + self.progress.set_state("Preparing tables") + + # Set up tables. + setup_res = yield defer.gatherResults( [ - self.handle_table(table) + self.setup_table(table) for table in tables if table not in ["schema_version", "applied_schema_deltas"] and not table.startswith("sqlite_") @@ -420,6 +426,15 @@ class Porter(object): consumeErrors=True, ) + # Process tables. + yield defer.gatherResults( + [ + self.handle_table(*res) + for res in setup_res + ], + consumeErrors=True, + ) + self.progress.done() except: global end_error_exec_info From 0bc71103e16a774b53383365d461c7705e313a5b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 28 Apr 2015 10:17:10 +0100 Subject: [PATCH 154/173] Output vim style mode markers into the yaml config file --- synapse/config/_base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 87cdbf1d3..6017cb633 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -147,9 +147,10 @@ class Config(object): and value is not None): config[key] = value with open(config_args.config_path, "w") as config_file: - # TODO(paul) it would be lovely if we wrote out vim- and emacs- - # style mode markers into the file, to hint to people that - # this is a YAML file. + # TODO(mark/paul) We might want to output emacs-style mode + # markers as well as vim-style mode markers into the file, + # to further hint to people this is a YAML file. + config_file.write("# vim:ft=yaml\n") yaml.dump(config, config_file, default_flow_style=False) print ( "A config file has been generated in %s for server name" From 1ccaea5b92e476ebb38a246ea933258a7199bbae Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 10:34:06 +0100 Subject: [PATCH 155/173] Typo in port script --- scripts/port_from_sqlite_to_postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 1e7ac072f..56e149c47 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -575,7 +575,7 @@ class CursesProgress(Progress): if self.finished: self.stdscr.addstr( - self.rows-1, 0, + rows-1, 0, "Press any key to exit...", ) From ce8b0b2868af32fac46a8414393760d326b764b7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 10:45:05 +0100 Subject: [PATCH 156/173] Remove accidentally committed debug hardcode hack --- scripts/port_from_sqlite_to_postgres.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 56e149c47..596be75c4 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -184,10 +184,11 @@ class Porter(object): if table == "sent_transactions": # This is a big table, and we really only need some of the recent # data + yield self.postgres_store.execute(delete_all) # Only save things from the last day - yesterday = 1429114568820 #int(time.time()*1000) - 86400000 + yesterday = int(time.time()*1000) - 86400000 # And save the max transaction id from each destination select = ( From 4a13ae72019655ad0531f93af18382c196fb362d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 11:16:44 +0100 Subject: [PATCH 157/173] Correctly handle total/remaining counts in the presence of sent_transasctions table --- scripts/port_from_sqlite_to_postgres.py | 292 ++++++++++++++---------- 1 file changed, 168 insertions(+), 124 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 596be75c4..1e52d82fe 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -125,6 +125,12 @@ class Store(object): def execute(self, f, *args, **kwargs): return self.runInteraction(f.__name__, f, *args, **kwargs) + def execute_sql(self, sql, *args): + def r(txn): + txn.execute(sql, args) + return txn.fetchall() + return self.runInteraction("execute_sql", r) + def insert_many_txn(self, txn, table, headers, rows): sql = "INSERT INTO %s (%s) VALUES (%s)" % ( table, @@ -146,119 +152,9 @@ class Porter(object): def __init__(self, **kwargs): self.__dict__.update(kwargs) - def convert_rows(self, table, headers, rows): - bool_col_names = BOOLEAN_COLUMNS.get(table, []) - - bool_cols = [ - i for i, h in enumerate(headers) if h in bool_col_names - ] - - def conv(j, col): - if j in bool_cols: - return bool(col) - return col - - for i, row in enumerate(rows): - rows[i] = tuple( - self.postgres_store.database_engine.encode_parameter( - conv(j, col) - ) - for j, col in enumerate(row) - if j > 0 - ) - @defer.inlineCallbacks def setup_table(self, table): - def delete_all(txn): - txn.execute( - "DELETE FROM port_from_sqlite3 WHERE table_name = %s", - (table,) - ) - txn.execute("TRUNCATE %s CASCADE" % (table,)) - - def get_table_size(txn): - txn.execute("SELECT count(*) FROM %s" % (table,)) - size, = txn.fetchone() - return int(size) - - if table == "sent_transactions": - # This is a big table, and we really only need some of the recent - # data - - yield self.postgres_store.execute(delete_all) - - # Only save things from the last day - yesterday = int(time.time()*1000) - 86400000 - - # And save the max transaction id from each destination - select = ( - "SELECT rowid, * FROM sent_transactions WHERE rowid IN (" - "SELECT max(rowid) FROM sent_transactions" - " GROUP BY destination" - ")" - ) - - def r(txn): - txn.execute(select) - rows = txn.fetchall() - headers = [column[0] for column in txn.description] - - ts_ind = headers.index('ts') - - return headers, [r for r in rows if r[ts_ind] < yesterday] - - headers, rows = yield self.sqlite_store.runInteraction( - "select", r, - ) - - self.convert_rows(table, headers, rows) - - inserted_rows = len(rows) - max_inserted_rowid = max(r[0] for r in rows) - - def insert(txn): - self.postgres_store.insert_many_txn( - txn, table, headers[1:], rows - ) - - yield self.postgres_store.execute(insert) - - def get_start_id(txn): - txn.execute( - "SELECT rowid FROM sent_transactions WHERE ts >= ?" - " ORDER BY rowid ASC LIMIT 1", - (yesterday,) - ) - - rows = txn.fetchall() - if rows: - return rows[0][0] - else: - return 1 - - next_chunk = yield self.sqlite_store.execute(get_start_id) - next_chunk = max(max_inserted_rowid + 1, next_chunk) - - yield self.postgres_store._simple_insert( - table="port_from_sqlite3", - values={"table_name": table, "rowid": next_chunk} - ) - - def get_sent_table_size(txn): - txn.execute( - "SELECT count(*) FROM sent_transactions" - " WHERE ts >= ?", - (yesterday,) - ) - size, = txn.fetchone() - return int(size) - - table_size = yield self.sqlite_store.execute( - get_sent_table_size - ) - - table_size += inserted_rows - elif table in APPEND_ONLY_TABLES: + if table in APPEND_ONLY_TABLES: # It's safe to just carry on inserting. next_chunk = yield self.postgres_store._simple_select_one_onecol( table="port_from_sqlite3", @@ -267,28 +163,47 @@ class Porter(object): allow_none=True, ) + total_to_port = None if next_chunk is None: - yield self.postgres_store._simple_insert( - table="port_from_sqlite3", - values={"table_name": table, "rowid": 1} + if table == "sent_transactions": + next_chunk, already_ported, total_to_port = ( + yield self._setup_sent_transactions() + ) + else: + yield self.postgres_store._simple_insert( + table="port_from_sqlite3", + values={"table_name": table, "rowid": 1} + ) + + next_chunk = 1 + already_ported = 0 + + if total_to_port is None: + already_ported, total_to_port = yield self._get_total_count_to_port( + table, next_chunk ) - - next_chunk = 1 - - table_size = yield self.sqlite_store.execute(get_table_size) else: + def delete_all(txn): + txn.execute( + "DELETE FROM port_from_sqlite3 WHERE table_name = %s", + (table,) + ) + txn.execute("TRUNCATE %s CASCADE" % (table,)) + yield self.postgres_store.execute(delete_all) - self.postgres_store._simple_insert( + + yield self.postgres_store._simple_insert( table="port_from_sqlite3", values={"table_name": table, "rowid": 0} ) - table_size = yield self.sqlite_store.execute(get_table_size) next_chunk = 1 - postgres_size = yield self.postgres_store.execute(get_table_size) + already_ported, total_to_port = yield self._get_total_count_to_port( + table, next_chunk + ) - defer.returnValue((table, postgres_size, table_size, next_chunk)) + defer.returnValue((table, already_ported, total_to_port, next_chunk)) @defer.inlineCallbacks def handle_table(self, table, postgres_size, table_size, next_chunk): @@ -315,7 +230,7 @@ class Porter(object): if rows: next_chunk = rows[-1][0] + 1 - self.convert_rows(table, headers, rows) + self._convert_rows(table, headers, rows) def insert(txn): self.postgres_store.insert_many_txn( @@ -414,7 +329,7 @@ class Porter(object): except Exception as e: logger.info("Failed to create port table: %s", e) - self.progress.set_state("Preparing tables") + self.progress.set_state("Setting up") # Set up tables. setup_res = yield defer.gatherResults( @@ -444,6 +359,135 @@ class Porter(object): finally: reactor.stop() + def _convert_rows(self, table, headers, rows): + bool_col_names = BOOLEAN_COLUMNS.get(table, []) + + bool_cols = [ + i for i, h in enumerate(headers) if h in bool_col_names + ] + + def conv(j, col): + if j in bool_cols: + return bool(col) + return col + + for i, row in enumerate(rows): + rows[i] = tuple( + self.postgres_store.database_engine.encode_parameter( + conv(j, col) + ) + for j, col in enumerate(row) + if j > 0 + ) + + @defer.inlineCallbacks + def _setup_sent_transactions(self): + # Only save things from the last day + yesterday = int(time.time()*1000) - 86400000 + + # And save the max transaction id from each destination + select = ( + "SELECT rowid, * FROM sent_transactions WHERE rowid IN (" + "SELECT max(rowid) FROM sent_transactions" + " GROUP BY destination" + ")" + ) + + def r(txn): + txn.execute(select) + rows = txn.fetchall() + headers = [column[0] for column in txn.description] + + ts_ind = headers.index('ts') + + return headers, [r for r in rows if r[ts_ind] < yesterday] + + headers, rows = yield self.sqlite_store.runInteraction( + "select", r, + ) + + self._convert_rows("sent_transactions", headers, rows) + + inserted_rows = len(rows) + max_inserted_rowid = max(r[0] for r in rows) + + def insert(txn): + self.postgres_store.insert_many_txn( + txn, "sent_transactions", headers[1:], rows + ) + + yield self.postgres_store.execute(insert) + + def get_start_id(txn): + txn.execute( + "SELECT rowid FROM sent_transactions WHERE ts >= ?" + " ORDER BY rowid ASC LIMIT 1", + (yesterday,) + ) + + rows = txn.fetchall() + if rows: + return rows[0][0] + else: + return 1 + + next_chunk = yield self.sqlite_store.execute(get_start_id) + next_chunk = max(max_inserted_rowid + 1, next_chunk) + + yield self.postgres_store._simple_insert( + table="port_from_sqlite3", + values={"table_name": "sent_transactions", "rowid": next_chunk} + ) + + def get_sent_table_size(txn): + txn.execute( + "SELECT count(*) FROM sent_transactions" + " WHERE ts >= ?", + (yesterday,) + ) + size, = txn.fetchone() + return int(size) + + remaining_count = yield self.sqlite_store.execute( + get_sent_table_size + ) + + total_count = remaining_count + inserted_rows + + defer.returnValue((next_chunk, remaining_count, total_count)) + + @defer.inlineCallbacks + def _get_remaining_count_to_port(self, table, next_chunk): + rows = yield self.sqlite_store.execute_sql( + "SELECT count(*) FROM %s WHERE rowid >= ?" % (table,), + next_chunk, + ) + + defer.returnValue(rows[0][0]) + + @defer.inlineCallbacks + def _get_already_ported_count(self, table): + rows = yield self.postgres_store.execute_sql( + "SELECT count(*) FROM %s" % (table,), + ) + + defer.returnValue(rows[0][0]) + + @defer.inlineCallbacks + def _get_total_count_to_port(self, table, next_chunk): + remaining, done = yield defer.gatherResults( + [ + self._get_remaining_count_to_port(table, next_chunk), + self._get_already_ported_count(table), + ], + consumeErrors=True, + ) + + remaining = int(remaining) if remaining else 0 + done = int(done) if done else 0 + + defer.returnValue((done, remaining + done)) + ############################################## ###### The following is simply UI stuff ###### From af27b84ff769ceb9fe1aaa10c9435586be4c6867 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 12:40:04 +0100 Subject: [PATCH 158/173] Correctly handle total/remaining counts in the presence of sent_transasctions table --- scripts/port_from_sqlite_to_postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 1e52d82fe..a40a93ca7 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -454,7 +454,7 @@ class Porter(object): total_count = remaining_count + inserted_rows - defer.returnValue((next_chunk, remaining_count, total_count)) + defer.returnValue((next_chunk, inserted_rows, total_count)) @defer.inlineCallbacks def _get_remaining_count_to_port(self, table, next_chunk): From 18f82477011ee9279bde8ad995d654c116d3f652 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 12:41:33 +0100 Subject: [PATCH 159/173] Use TEXT instead of VARCHAR(n), since PostgreSQL treats them the same except for a limit --- synapse/storage/schema/delta/12/v12.sql | 18 +++--- synapse/storage/schema/delta/13/v13.sql | 10 +-- synapse/storage/schema/delta/14/v14.sql | 4 +- .../schema/delta/15/appservice_txns.sql | 4 +- synapse/storage/schema/delta/16/users.sql | 20 +++--- .../schema/full_schemas/11/event_edges.sql | 34 +++++----- .../full_schemas/11/event_signatures.sql | 20 +++--- synapse/storage/schema/full_schemas/11/im.sql | 64 +++++++++---------- .../storage/schema/full_schemas/11/keys.sql | 12 ++-- .../full_schemas/11/media_repository.sql | 34 +++++----- .../schema/full_schemas/11/presence.sql | 12 ++-- .../schema/full_schemas/11/profiles.sql | 6 +- .../schema/full_schemas/11/redactions.sql | 4 +- .../schema/full_schemas/11/room_aliases.sql | 8 +-- .../storage/schema/full_schemas/11/state.sql | 14 ++-- .../schema/full_schemas/11/transactions.sql | 16 ++--- .../storage/schema/full_schemas/11/users.sql | 18 +++--- .../full_schemas/16/application_services.sql | 14 ++-- .../schema/full_schemas/16/event_edges.sql | 34 +++++----- .../full_schemas/16/event_signatures.sql | 20 +++--- synapse/storage/schema/full_schemas/16/im.sql | 64 +++++++++---------- .../storage/schema/full_schemas/16/keys.sql | 12 ++-- .../full_schemas/16/media_repository.sql | 34 +++++----- .../schema/full_schemas/16/presence.sql | 12 ++-- .../schema/full_schemas/16/profiles.sql | 6 +- .../storage/schema/full_schemas/16/push.sql | 22 +++---- .../schema/full_schemas/16/redactions.sql | 4 +- .../schema/full_schemas/16/room_aliases.sql | 8 +-- .../storage/schema/full_schemas/16/state.sql | 14 ++-- .../schema/full_schemas/16/transactions.sql | 16 ++--- .../storage/schema/full_schemas/16/users.sql | 18 +++--- synapse/storage/schema/schema_version.sql | 2 +- 32 files changed, 289 insertions(+), 289 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index a246943f5..878c36260 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -14,16 +14,16 @@ */ CREATE TABLE IF NOT EXISTS rejections( - event_id VARCHAR(150) NOT NULL, - reason VARCHAR(150) NOT NULL, - last_check VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + reason TEXT NOT NULL, + last_check TEXT NOT NULL, UNIQUE (event_id) ); -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name VARCHAR(150) NOT NULL, + user_name TEXT NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, app_id VARCHAR(64) NOT NULL, @@ -41,19 +41,19 @@ CREATE TABLE IF NOT EXISTS pushers ( CREATE TABLE IF NOT EXISTS push_rules ( id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name VARCHAR(150) NOT NULL, - rule_id VARCHAR(150) NOT NULL, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, priority_class TINYINT NOT NULL, priority INTEGER NOT NULL DEFAULT 0, - conditions VARCHAR(150) NOT NULL, - actions VARCHAR(150) NOT NULL, + conditions TEXT NOT NULL, + actions TEXT NOT NULL, UNIQUE(user_name, rule_id) ); CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( - user_id VARCHAR(150), + user_id TEXT, filter_id BIGINT UNSIGNED, filter_json LONGBLOB ); diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index d1da2b48e..326592401 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -15,10 +15,10 @@ CREATE TABLE IF NOT EXISTS application_services( id INTEGER PRIMARY KEY AUTOINCREMENT, - url VARCHAR(150), - token VARCHAR(150), - hs_token VARCHAR(150), - sender VARCHAR(150), + url TEXT, + token TEXT, + hs_token TEXT, + sender TEXT, UNIQUE(token) ); @@ -26,6 +26,6 @@ CREATE TABLE IF NOT EXISTS application_services_regex( id INTEGER PRIMARY KEY AUTOINCREMENT, as_id BIGINT UNSIGNED NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ - regex VARCHAR(150), + regex TEXT, FOREIGN KEY(as_id) REFERENCES application_services(id) ); diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index e5fedc585..1d09ad7a1 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS push_rules_enable ( id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name VARCHAR(150) NOT NULL, - rule_id VARCHAR(150) NOT NULL, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, enabled TINYINT, UNIQUE(user_name, rule_id) ); diff --git a/synapse/storage/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/delta/15/appservice_txns.sql index 1c3324f41..db2e72039 100644 --- a/synapse/storage/schema/delta/15/appservice_txns.sql +++ b/synapse/storage/schema/delta/15/appservice_txns.sql @@ -14,13 +14,13 @@ */ CREATE TABLE IF NOT EXISTS application_services_state( - as_id VARCHAR(150) PRIMARY KEY, + as_id TEXT PRIMARY KEY, state VARCHAR(5), last_txn INTEGER ); CREATE TABLE IF NOT EXISTS application_services_txns( - as_id VARCHAR(150) NOT NULL, + as_id TEXT NOT NULL, txn_id INTEGER NOT NULL, event_ids TEXT NOT NULL, UNIQUE(as_id, txn_id) diff --git a/synapse/storage/schema/delta/16/users.sql b/synapse/storage/schema/delta/16/users.sql index db27bdca0..cd0709250 100644 --- a/synapse/storage/schema/delta/16/users.sql +++ b/synapse/storage/schema/delta/16/users.sql @@ -2,9 +2,9 @@ -- MUST BE DONE BEFORE REMOVING ID COLUMN FROM USERS TABLE BELOW CREATE TABLE IF NOT EXISTS new_access_tokens( id BIGINT UNSIGNED PRIMARY KEY, - user_id VARCHAR(150) NOT NULL, - device_id VARCHAR(150), - token VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, + device_id TEXT, + token TEXT NOT NULL, last_used BIGINT UNSIGNED, UNIQUE(token) ); @@ -20,8 +20,8 @@ ALTER TABLE new_access_tokens RENAME TO access_tokens; -- Remove ID column from `users` table CREATE TABLE IF NOT EXISTS new_users( - name VARCHAR(150), - password_hash VARCHAR(150), + name TEXT, + password_hash TEXT, creation_ts BIGINT UNSIGNED, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) @@ -36,11 +36,11 @@ ALTER TABLE new_users RENAME TO users; -- Remove UNIQUE constraint from `user_ips` table CREATE TABLE IF NOT EXISTS new_user_ips ( - user_id VARCHAR(150) NOT NULL, - access_token VARCHAR(150) NOT NULL, - device_id VARCHAR(150), - ip VARCHAR(150) NOT NULL, - user_agent VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, + access_token TEXT NOT NULL, + device_id TEXT, + ip TEXT NOT NULL, + user_agent TEXT NOT NULL, last_seen BIGINT UNSIGNED NOT NULL ); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 05d0874f0..f7020f779 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS event_forward_extremities( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (event_id, room_id) ); @@ -24,8 +24,8 @@ CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_backward_extremities( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (event_id, room_id) ); @@ -34,9 +34,9 @@ CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_edges( - event_id VARCHAR(150) NOT NULL, - prev_event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + prev_event_id TEXT NOT NULL, + room_id TEXT NOT NULL, is_state BOOL NOT NULL, UNIQUE (event_id, prev_event_id, room_id, is_state) ); @@ -46,7 +46,7 @@ CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id); CREATE TABLE IF NOT EXISTS room_depth( - room_id VARCHAR(150) NOT NULL, + room_id TEXT NOT NULL, min_depth INTEGER NOT NULL, UNIQUE (room_id) ); @@ -55,8 +55,8 @@ CREATE INDEX room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( - event_id VARCHAR(150) NOT NULL, - destination VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + destination TEXT NOT NULL, delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ); @@ -65,10 +65,10 @@ CREATE INDEX event_destinations_id ON event_destinations(event_id); CREATE TABLE IF NOT EXISTS state_forward_extremities( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, UNIQUE (event_id, room_id) ); @@ -79,9 +79,9 @@ CREATE INDEX st_extrem_id ON state_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_auth( - event_id VARCHAR(150) NOT NULL, - auth_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + auth_id TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (event_id, auth_id, room_id) ); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index 429182736..636b2d335 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS event_content_hashes ( - event_id VARCHAR(150), - algorithm VARCHAR(150), + event_id TEXT, + algorithm TEXT, hash bytea, UNIQUE (event_id, algorithm) ); @@ -24,8 +24,8 @@ CREATE INDEX event_content_hashes_id ON event_content_hashes(event_id); CREATE TABLE IF NOT EXISTS event_reference_hashes ( - event_id VARCHAR(150), - algorithm VARCHAR(150), + event_id TEXT, + algorithm TEXT, hash bytea, UNIQUE (event_id, algorithm) ); @@ -34,9 +34,9 @@ CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id); CREATE TABLE IF NOT EXISTS event_signatures ( - event_id VARCHAR(150), - signature_name VARCHAR(150), - key_id VARCHAR(150), + event_id TEXT, + signature_name TEXT, + key_id TEXT, signature bytea, UNIQUE (event_id, signature_name, key_id) ); @@ -45,9 +45,9 @@ CREATE INDEX event_signatures_id ON event_signatures(event_id); CREATE TABLE IF NOT EXISTS event_edge_hashes( - event_id VARCHAR(150), - prev_event_id VARCHAR(150), - algorithm VARCHAR(150), + event_id TEXT, + prev_event_id TEXT, + algorithm TEXT, hash bytea, UNIQUE (event_id, prev_event_id, algorithm) ); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index addbec588..1901654ac 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -16,9 +16,9 @@ CREATE TABLE IF NOT EXISTS events( stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, topological_ordering BIGINT NOT NULL, - event_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + type TEXT NOT NULL, + room_id TEXT NOT NULL, content TEXT NOT NULL, unrecognized_keys TEXT, processed BOOL NOT NULL, @@ -33,8 +33,8 @@ CREATE INDEX events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, internal_metadata TEXT NOT NULL, json TEXT NOT NULL, UNIQUE (event_id) @@ -44,11 +44,11 @@ CREATE INDEX event_json_room_id ON event_json(room_id); CREATE TABLE IF NOT EXISTS state_events( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, - prev_state VARCHAR(150), + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + prev_state TEXT, UNIQUE (event_id) ); @@ -58,10 +58,10 @@ CREATE INDEX state_events_state_key ON state_events (state_key); CREATE TABLE IF NOT EXISTS current_state_events( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, UNIQUE (room_id, type, state_key) ); @@ -71,11 +71,11 @@ CREATE INDEX current_state_events_type ON current_state_events (type); CREATE INDEX current_state_events_state_key ON current_state_events (state_key); CREATE TABLE IF NOT EXISTS room_memberships( - event_id VARCHAR(150) NOT NULL, - user_id VARCHAR(150) NOT NULL, - sender VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - membership VARCHAR(150) NOT NULL + event_id TEXT NOT NULL, + user_id TEXT NOT NULL, + sender TEXT NOT NULL, + room_id TEXT NOT NULL, + membership TEXT NOT NULL ); CREATE INDEX room_memberships_event_id ON room_memberships (event_id); @@ -83,16 +83,16 @@ CREATE INDEX room_memberships_room_id ON room_memberships (room_id); CREATE INDEX room_memberships_user_id ON room_memberships (user_id); CREATE TABLE IF NOT EXISTS feedback( - event_id VARCHAR(150) NOT NULL, - feedback_type VARCHAR(150), - target_event_id VARCHAR(150), - sender VARCHAR(150), - room_id VARCHAR(150) + event_id TEXT NOT NULL, + feedback_type TEXT, + target_event_id TEXT, + sender TEXT, + room_id TEXT ); CREATE TABLE IF NOT EXISTS topics( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, topic TEXT NOT NULL ); @@ -100,8 +100,8 @@ CREATE INDEX topics_event_id ON topics(event_id); CREATE INDEX topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, name TEXT NOT NULL ); @@ -109,14 +109,14 @@ CREATE INDEX room_names_event_id ON room_names(event_id); CREATE INDEX room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( - room_id VARCHAR(150) PRIMARY KEY NOT NULL, + room_id TEXT PRIMARY KEY NOT NULL, is_public BOOL, - creator VARCHAR(150) + creator TEXT ); CREATE TABLE IF NOT EXISTS room_hosts( - room_id VARCHAR(150) NOT NULL, - host VARCHAR(150) NOT NULL, + room_id TEXT NOT NULL, + host TEXT NOT NULL, UNIQUE (room_id, host) ); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index 459b51042..afc142045 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -13,18 +13,18 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS server_tls_certificates( - server_name VARCHAR(150), -- Server name. - fingerprint VARCHAR(150), -- Certificate fingerprint. - from_server VARCHAR(150), -- Which key server the certificate was fetched from. + server_name TEXT, -- Server name. + fingerprint TEXT, -- Certificate fingerprint. + from_server TEXT, -- Which key server the certificate was fetched from. ts_added_ms BIGINT, -- When the certifcate was added. tls_certificate bytea, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); CREATE TABLE IF NOT EXISTS server_signature_keys( - server_name VARCHAR(150), -- Server name. - key_id VARCHAR(150), -- Key version. - from_server VARCHAR(150), -- Which key server the key was fetched form. + server_name TEXT, -- Server name. + key_id TEXT, -- Key version. + from_server TEXT, -- Which key server the key was fetched form. ts_added_ms BIGINT, -- When the key was added. verify_key bytea, -- NACL verification key. UNIQUE (server_name, key_id) diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index 6e0ee0db4..e927e581d 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -14,21 +14,21 @@ */ CREATE TABLE IF NOT EXISTS local_media_repository ( - media_id VARCHAR(150), -- The id used to refer to the media. - media_type VARCHAR(150), -- The MIME-type of the media. + media_id TEXT, -- The id used to refer to the media. + media_type TEXT, -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. created_ts BIGINT, -- When the content was uploaded in ms. - upload_name VARCHAR(150), -- The name the media was uploaded with. - user_id VARCHAR(150), -- The user who uploaded the file. + upload_name TEXT, -- The name the media was uploaded with. + user_id TEXT, -- The user who uploaded the file. UNIQUE (media_id) ); CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( - media_id VARCHAR(150), -- The id used to refer to the media. + media_id TEXT, -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. - thumbnail_method VARCHAR(150), -- The method used to make the thumbnail. + thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_method TEXT, -- The method used to make the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type @@ -39,25 +39,25 @@ CREATE INDEX local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); CREATE TABLE IF NOT EXISTS remote_media_cache ( - media_origin VARCHAR(150), -- The remote HS the media came from. - media_id VARCHAR(150), -- The id used to refer to the media on that server. - media_type VARCHAR(150), -- The MIME-type of the media. + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media on that server. + media_type TEXT, -- The MIME-type of the media. created_ts BIGINT, -- When the content was uploaded in ms. - upload_name VARCHAR(150), -- The name the media was uploaded with. + upload_name TEXT, -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. - filesystem_id VARCHAR(150), -- The name used to store the media on disk. + filesystem_id TEXT, -- The name used to store the media on disk. UNIQUE (media_origin, media_id) ); CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( - media_origin VARCHAR(150), -- The remote HS the media came from. - media_id VARCHAR(150), -- The id used to refer to the media. + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_method VARCHAR(150), -- The method used to make the thumbnail - thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. + thumbnail_method TEXT, -- The method used to make the thumbnail + thumbnail_type TEXT, -- The MIME-type of the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - filesystem_id VARCHAR(150), -- The name used to store the media on disk. + filesystem_id TEXT, -- The name used to store the media on disk. UNIQUE ( media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index fce324b89..d8d82e9fe 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -13,23 +13,23 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS presence( - user_id VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, state VARCHAR(20), - status_msg VARCHAR(150), + status_msg TEXT, mtime BIGINT -- miliseconds since last state change ); -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state CREATE TABLE IF NOT EXISTS presence_allow_inbound( - observed_user_id VARCHAR(150) NOT NULL, - observer_user_id VARCHAR(150) NOT NULL -- a UserID, + observed_user_id TEXT NOT NULL, + observer_user_id TEXT NOT NULL -- a UserID, ); -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? CREATE TABLE IF NOT EXISTS presence_list( - user_id VARCHAR(150) NOT NULL, - observed_user_id VARCHAR(150) NOT NULL, -- a UserID, + user_id TEXT NOT NULL, + observed_user_id TEXT NOT NULL, -- a UserID, accepted BOOLEAN NOT NULL ); diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index ffe75edf9..26e420443 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -13,7 +13,7 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS profiles( - user_id VARCHAR(150) NOT NULL, - displayname VARCHAR(150), - avatar_url VARCHAR(150) + user_id TEXT NOT NULL, + displayname TEXT, + avatar_url TEXT ); diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index 492fd2203..69621955d 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS redactions ( - event_id VARCHAR(150) NOT NULL, - redacts VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + redacts TEXT NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index 622691322..5027b1e3f 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -14,11 +14,11 @@ */ CREATE TABLE IF NOT EXISTS room_aliases( - room_alias VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL + room_alias TEXT NOT NULL, + room_id TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS room_alias_servers( - room_alias VARCHAR(150) NOT NULL, - server VARCHAR(150) NOT NULL + room_alias TEXT NOT NULL, + server TEXT NOT NULL ); diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index 62c20819f..ffd164ab7 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -15,20 +15,20 @@ CREATE TABLE IF NOT EXISTS state_groups( id INTEGER PRIMARY KEY, - room_id VARCHAR(150) NOT NULL, - event_id VARCHAR(150) NOT NULL + room_id TEXT NOT NULL, + event_id TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS state_groups_state( state_group INTEGER NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, - event_id VARCHAR(150) NOT NULL + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + event_id TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS event_to_state_groups( - event_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, state_group INTEGER NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 524a69692..cc5b54f5a 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -14,8 +14,8 @@ */ -- Stores what transaction ids we have received and what our response was CREATE TABLE IF NOT EXISTS received_transactions( - transaction_id VARCHAR(150), - origin VARCHAR(150), + transaction_id TEXT, + origin TEXT, ts BIGINT, response_code INTEGER, response_json bytea, @@ -30,8 +30,8 @@ CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_ref -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( id INTEGER PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering - transaction_id VARCHAR(150), - destination VARCHAR(150), + transaction_id TEXT, + destination TEXT, response_code INTEGER DEFAULT 0, response_json TEXT, ts BIGINT @@ -47,9 +47,9 @@ CREATE INDEX sent_transaction_sent ON sent_transactions(response_code); -- For sent transactions only. CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( transaction_id INTEGER, - destination VARCHAR(150), - pdu_id VARCHAR(150), - pdu_origin VARCHAR(150) + destination TEXT, + pdu_id TEXT, + pdu_origin TEXT ); CREATE INDEX transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination); @@ -57,7 +57,7 @@ CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health CREATE TABLE IF NOT EXISTS destinations( - destination VARCHAR(150) PRIMARY KEY, + destination TEXT PRIMARY KEY, retry_last_ts BIGINT, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 48a6aecfe..eec3da3c3 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS users( id INTEGER PRIMARY KEY AUTOINCREMENT, - name VARCHAR(150), - password_hash VARCHAR(150), + name TEXT, + password_hash TEXT, creation_ts BIGINT, admin SMALLINT DEFAULT 0 NOT NULL, UNIQUE(name) @@ -23,18 +23,18 @@ CREATE TABLE IF NOT EXISTS users( CREATE TABLE IF NOT EXISTS access_tokens( id INTEGER PRIMARY KEY AUTOINCREMENT, - user_id VARCHAR(150) NOT NULL, - device_id VARCHAR(150), - token VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, + device_id TEXT, + token TEXT NOT NULL, last_used BIGINT, UNIQUE(token) ); CREATE TABLE IF NOT EXISTS user_ips ( - user VARCHAR(150) NOT NULL, - access_token VARCHAR(150) NOT NULL, - device_id VARCHAR(150), - ip VARCHAR(150) NOT NULL, + user TEXT NOT NULL, + access_token TEXT NOT NULL, + device_id TEXT, + ip TEXT NOT NULL, user_agent TEXT NOT NULL, last_seen BIGINT NOT NULL, UNIQUE (user, access_token, ip, user_agent) diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql index 5d63d57d5..d382d63fb 100644 --- a/synapse/storage/schema/full_schemas/16/application_services.sql +++ b/synapse/storage/schema/full_schemas/16/application_services.sql @@ -15,10 +15,10 @@ CREATE TABLE IF NOT EXISTS application_services( id BIGINT PRIMARY KEY, - url VARCHAR(150), - token VARCHAR(150), - hs_token VARCHAR(150), - sender VARCHAR(150), + url TEXT, + token TEXT, + hs_token TEXT, + sender TEXT, UNIQUE(token) ); @@ -26,18 +26,18 @@ CREATE TABLE IF NOT EXISTS application_services_regex( id BIGINT PRIMARY KEY, as_id BIGINT NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ - regex VARCHAR(150), + regex TEXT, FOREIGN KEY(as_id) REFERENCES application_services(id) ); CREATE TABLE IF NOT EXISTS application_services_state( - as_id VARCHAR(150) PRIMARY KEY, + as_id TEXT PRIMARY KEY, state VARCHAR(5), last_txn INTEGER ); CREATE TABLE IF NOT EXISTS application_services_txns( - as_id VARCHAR(150) NOT NULL, + as_id TEXT NOT NULL, txn_id INTEGER NOT NULL, event_ids TEXT NOT NULL, UNIQUE(as_id, txn_id) diff --git a/synapse/storage/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/full_schemas/16/event_edges.sql index 05d0874f0..f7020f779 100644 --- a/synapse/storage/schema/full_schemas/16/event_edges.sql +++ b/synapse/storage/schema/full_schemas/16/event_edges.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS event_forward_extremities( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (event_id, room_id) ); @@ -24,8 +24,8 @@ CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_backward_extremities( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (event_id, room_id) ); @@ -34,9 +34,9 @@ CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_edges( - event_id VARCHAR(150) NOT NULL, - prev_event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + prev_event_id TEXT NOT NULL, + room_id TEXT NOT NULL, is_state BOOL NOT NULL, UNIQUE (event_id, prev_event_id, room_id, is_state) ); @@ -46,7 +46,7 @@ CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id); CREATE TABLE IF NOT EXISTS room_depth( - room_id VARCHAR(150) NOT NULL, + room_id TEXT NOT NULL, min_depth INTEGER NOT NULL, UNIQUE (room_id) ); @@ -55,8 +55,8 @@ CREATE INDEX room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( - event_id VARCHAR(150) NOT NULL, - destination VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + destination TEXT NOT NULL, delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ); @@ -65,10 +65,10 @@ CREATE INDEX event_destinations_id ON event_destinations(event_id); CREATE TABLE IF NOT EXISTS state_forward_extremities( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, UNIQUE (event_id, room_id) ); @@ -79,9 +79,9 @@ CREATE INDEX st_extrem_id ON state_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_auth( - event_id VARCHAR(150) NOT NULL, - auth_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + auth_id TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (event_id, auth_id, room_id) ); diff --git a/synapse/storage/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/full_schemas/16/event_signatures.sql index 429182736..636b2d335 100644 --- a/synapse/storage/schema/full_schemas/16/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/16/event_signatures.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS event_content_hashes ( - event_id VARCHAR(150), - algorithm VARCHAR(150), + event_id TEXT, + algorithm TEXT, hash bytea, UNIQUE (event_id, algorithm) ); @@ -24,8 +24,8 @@ CREATE INDEX event_content_hashes_id ON event_content_hashes(event_id); CREATE TABLE IF NOT EXISTS event_reference_hashes ( - event_id VARCHAR(150), - algorithm VARCHAR(150), + event_id TEXT, + algorithm TEXT, hash bytea, UNIQUE (event_id, algorithm) ); @@ -34,9 +34,9 @@ CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id); CREATE TABLE IF NOT EXISTS event_signatures ( - event_id VARCHAR(150), - signature_name VARCHAR(150), - key_id VARCHAR(150), + event_id TEXT, + signature_name TEXT, + key_id TEXT, signature bytea, UNIQUE (event_id, signature_name, key_id) ); @@ -45,9 +45,9 @@ CREATE INDEX event_signatures_id ON event_signatures(event_id); CREATE TABLE IF NOT EXISTS event_edge_hashes( - event_id VARCHAR(150), - prev_event_id VARCHAR(150), - algorithm VARCHAR(150), + event_id TEXT, + prev_event_id TEXT, + algorithm TEXT, hash bytea, UNIQUE (event_id, prev_event_id, algorithm) ); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index 5b4b49448..576653a3c 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -16,9 +16,9 @@ CREATE TABLE IF NOT EXISTS events( stream_ordering INTEGER PRIMARY KEY, topological_ordering BIGINT NOT NULL, - event_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + type TEXT NOT NULL, + room_id TEXT NOT NULL, content TEXT NOT NULL, unrecognized_keys TEXT, processed BOOL NOT NULL, @@ -37,8 +37,8 @@ CREATE INDEX events_order_room ON events ( CREATE TABLE IF NOT EXISTS event_json( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, internal_metadata TEXT NOT NULL, json TEXT NOT NULL, UNIQUE (event_id) @@ -48,11 +48,11 @@ CREATE INDEX event_json_room_id ON event_json(room_id); CREATE TABLE IF NOT EXISTS state_events( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, - prev_state VARCHAR(150), + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + prev_state TEXT, UNIQUE (event_id) ); @@ -62,10 +62,10 @@ CREATE INDEX state_events_state_key ON state_events (state_key); CREATE TABLE IF NOT EXISTS current_state_events( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, UNIQUE (event_id), UNIQUE (room_id, type, state_key) ); @@ -75,11 +75,11 @@ CREATE INDEX current_state_events_type ON current_state_events (type); CREATE INDEX current_state_events_state_key ON current_state_events (state_key); CREATE TABLE IF NOT EXISTS room_memberships( - event_id VARCHAR(150) NOT NULL, - user_id VARCHAR(150) NOT NULL, - sender VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - membership VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + user_id TEXT NOT NULL, + sender TEXT NOT NULL, + room_id TEXT NOT NULL, + membership TEXT NOT NULL, UNIQUE (event_id) ); @@ -87,17 +87,17 @@ CREATE INDEX room_memberships_room_id ON room_memberships (room_id); CREATE INDEX room_memberships_user_id ON room_memberships (user_id); CREATE TABLE IF NOT EXISTS feedback( - event_id VARCHAR(150) NOT NULL, - feedback_type VARCHAR(150), - target_event_id VARCHAR(150), - sender VARCHAR(150), - room_id VARCHAR(150), + event_id TEXT NOT NULL, + feedback_type TEXT, + target_event_id TEXT, + sender TEXT, + room_id TEXT, UNIQUE (event_id) ); CREATE TABLE IF NOT EXISTS topics( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, topic TEXT NOT NULL, UNIQUE (event_id) ); @@ -105,8 +105,8 @@ CREATE TABLE IF NOT EXISTS topics( CREATE INDEX topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, name TEXT NOT NULL, UNIQUE (event_id) ); @@ -114,14 +114,14 @@ CREATE TABLE IF NOT EXISTS room_names( CREATE INDEX room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( - room_id VARCHAR(150) PRIMARY KEY NOT NULL, + room_id TEXT PRIMARY KEY NOT NULL, is_public BOOL, - creator VARCHAR(150) + creator TEXT ); CREATE TABLE IF NOT EXISTS room_hosts( - room_id VARCHAR(150) NOT NULL, - host VARCHAR(150) NOT NULL, + room_id TEXT NOT NULL, + host TEXT NOT NULL, UNIQUE (room_id, host) ); diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql index 459b51042..afc142045 100644 --- a/synapse/storage/schema/full_schemas/16/keys.sql +++ b/synapse/storage/schema/full_schemas/16/keys.sql @@ -13,18 +13,18 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS server_tls_certificates( - server_name VARCHAR(150), -- Server name. - fingerprint VARCHAR(150), -- Certificate fingerprint. - from_server VARCHAR(150), -- Which key server the certificate was fetched from. + server_name TEXT, -- Server name. + fingerprint TEXT, -- Certificate fingerprint. + from_server TEXT, -- Which key server the certificate was fetched from. ts_added_ms BIGINT, -- When the certifcate was added. tls_certificate bytea, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); CREATE TABLE IF NOT EXISTS server_signature_keys( - server_name VARCHAR(150), -- Server name. - key_id VARCHAR(150), -- Key version. - from_server VARCHAR(150), -- Which key server the key was fetched form. + server_name TEXT, -- Server name. + key_id TEXT, -- Key version. + from_server TEXT, -- Which key server the key was fetched form. ts_added_ms BIGINT, -- When the key was added. verify_key bytea, -- NACL verification key. UNIQUE (server_name, key_id) diff --git a/synapse/storage/schema/full_schemas/16/media_repository.sql b/synapse/storage/schema/full_schemas/16/media_repository.sql index 0e819fca3..dacbda40c 100644 --- a/synapse/storage/schema/full_schemas/16/media_repository.sql +++ b/synapse/storage/schema/full_schemas/16/media_repository.sql @@ -14,21 +14,21 @@ */ CREATE TABLE IF NOT EXISTS local_media_repository ( - media_id VARCHAR(150), -- The id used to refer to the media. - media_type VARCHAR(150), -- The MIME-type of the media. + media_id TEXT, -- The id used to refer to the media. + media_type TEXT, -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. created_ts BIGINT, -- When the content was uploaded in ms. - upload_name VARCHAR(150), -- The name the media was uploaded with. - user_id VARCHAR(150), -- The user who uploaded the file. + upload_name TEXT, -- The name the media was uploaded with. + user_id TEXT, -- The user who uploaded the file. UNIQUE (media_id) ); CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( - media_id VARCHAR(150), -- The id used to refer to the media. + media_id TEXT, -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. - thumbnail_method VARCHAR(150), -- The method used to make the thumbnail. + thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_method TEXT, -- The method used to make the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type @@ -39,25 +39,25 @@ CREATE INDEX local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); CREATE TABLE IF NOT EXISTS remote_media_cache ( - media_origin VARCHAR(150), -- The remote HS the media came from. - media_id VARCHAR(150), -- The id used to refer to the media on that server. - media_type VARCHAR(150), -- The MIME-type of the media. + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media on that server. + media_type TEXT, -- The MIME-type of the media. created_ts BIGINT, -- When the content was uploaded in ms. - upload_name VARCHAR(150), -- The name the media was uploaded with. + upload_name TEXT, -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. - filesystem_id VARCHAR(150), -- The name used to store the media on disk. + filesystem_id TEXT, -- The name used to store the media on disk. UNIQUE (media_origin, media_id) ); CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( - media_origin VARCHAR(150), -- The remote HS the media came from. - media_id VARCHAR(150), -- The id used to refer to the media. + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_method VARCHAR(150), -- The method used to make the thumbnail - thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. + thumbnail_method TEXT, -- The method used to make the thumbnail + thumbnail_type TEXT, -- The MIME-type of the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - filesystem_id VARCHAR(150), -- The name used to store the media on disk. + filesystem_id TEXT, -- The name used to store the media on disk. UNIQUE ( media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql index 9c41be296..80088413b 100644 --- a/synapse/storage/schema/full_schemas/16/presence.sql +++ b/synapse/storage/schema/full_schemas/16/presence.sql @@ -13,9 +13,9 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS presence( - user_id VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, state VARCHAR(20), - status_msg VARCHAR(150), + status_msg TEXT, mtime BIGINT, -- miliseconds since last state change UNIQUE (user_id) ); @@ -23,16 +23,16 @@ CREATE TABLE IF NOT EXISTS presence( -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state CREATE TABLE IF NOT EXISTS presence_allow_inbound( - observed_user_id VARCHAR(150) NOT NULL, - observer_user_id VARCHAR(150) NOT NULL, -- a UserID, + observed_user_id TEXT NOT NULL, + observer_user_id TEXT NOT NULL, -- a UserID, UNIQUE (observed_user_id, observer_user_id) ); -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? CREATE TABLE IF NOT EXISTS presence_list( - user_id VARCHAR(150) NOT NULL, - observed_user_id VARCHAR(150) NOT NULL, -- a UserID, + user_id TEXT NOT NULL, + observed_user_id TEXT NOT NULL, -- a UserID, accepted BOOLEAN NOT NULL, UNIQUE (user_id, observed_user_id) ); diff --git a/synapse/storage/schema/full_schemas/16/profiles.sql b/synapse/storage/schema/full_schemas/16/profiles.sql index 21c58a99b..934be8652 100644 --- a/synapse/storage/schema/full_schemas/16/profiles.sql +++ b/synapse/storage/schema/full_schemas/16/profiles.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS profiles( - user_id VARCHAR(150) NOT NULL, - displayname VARCHAR(150), - avatar_url VARCHAR(150), + user_id TEXT NOT NULL, + displayname TEXT, + avatar_url TEXT, UNIQUE(user_id) ); diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql index 5c0c7bc20..db6e05cbd 100644 --- a/synapse/storage/schema/full_schemas/16/push.sql +++ b/synapse/storage/schema/full_schemas/16/push.sql @@ -14,16 +14,16 @@ */ CREATE TABLE IF NOT EXISTS rejections( - event_id VARCHAR(150) NOT NULL, - reason VARCHAR(150) NOT NULL, - last_check VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + reason TEXT NOT NULL, + last_check TEXT NOT NULL, UNIQUE (event_id) ); -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( id BIGINT PRIMARY KEY, - user_name VARCHAR(150) NOT NULL, + user_name TEXT NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, app_id VARCHAR(64) NOT NULL, @@ -41,19 +41,19 @@ CREATE TABLE IF NOT EXISTS pushers ( CREATE TABLE IF NOT EXISTS push_rules ( id BIGINT PRIMARY KEY, - user_name VARCHAR(150) NOT NULL, - rule_id VARCHAR(150) NOT NULL, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, priority_class SMALLINT NOT NULL, priority INTEGER NOT NULL DEFAULT 0, - conditions VARCHAR(150) NOT NULL, - actions VARCHAR(150) NOT NULL, + conditions TEXT NOT NULL, + actions TEXT NOT NULL, UNIQUE(user_name, rule_id) ); CREATE INDEX push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( - user_id VARCHAR(150), + user_id TEXT, filter_id BIGINT, filter_json bytea ); @@ -64,8 +64,8 @@ CREATE INDEX user_filters_by_user_id_filter_id ON user_filters( CREATE TABLE IF NOT EXISTS push_rules_enable ( id BIGINT PRIMARY KEY, - user_name VARCHAR(150) NOT NULL, - rule_id VARCHAR(150) NOT NULL, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, enabled SMALLINT, UNIQUE(user_name, rule_id) ); diff --git a/synapse/storage/schema/full_schemas/16/redactions.sql b/synapse/storage/schema/full_schemas/16/redactions.sql index 492fd2203..69621955d 100644 --- a/synapse/storage/schema/full_schemas/16/redactions.sql +++ b/synapse/storage/schema/full_schemas/16/redactions.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS redactions ( - event_id VARCHAR(150) NOT NULL, - redacts VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + redacts TEXT NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/16/room_aliases.sql b/synapse/storage/schema/full_schemas/16/room_aliases.sql index 2c0853a2a..412bb97fa 100644 --- a/synapse/storage/schema/full_schemas/16/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/16/room_aliases.sql @@ -14,16 +14,16 @@ */ CREATE TABLE IF NOT EXISTS room_aliases( - room_alias VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + room_alias TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (room_alias) ); CREATE INDEX room_aliases_id ON room_aliases(room_id); CREATE TABLE IF NOT EXISTS room_alias_servers( - room_alias VARCHAR(150) NOT NULL, - server VARCHAR(150) NOT NULL + room_alias TEXT NOT NULL, + server TEXT NOT NULL ); CREATE INDEX room_alias_servers_alias ON room_alias_servers(room_alias); diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql index b0cd5ee75..705cac6ce 100644 --- a/synapse/storage/schema/full_schemas/16/state.sql +++ b/synapse/storage/schema/full_schemas/16/state.sql @@ -15,20 +15,20 @@ CREATE TABLE IF NOT EXISTS state_groups( id BIGINT PRIMARY KEY, - room_id VARCHAR(150) NOT NULL, - event_id VARCHAR(150) NOT NULL + room_id TEXT NOT NULL, + event_id TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS state_groups_state( state_group BIGINT NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, - event_id VARCHAR(150) NOT NULL + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + event_id TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS event_to_state_groups( - event_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, state_group BIGINT NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql index ed431bd3a..1ab77cdb6 100644 --- a/synapse/storage/schema/full_schemas/16/transactions.sql +++ b/synapse/storage/schema/full_schemas/16/transactions.sql @@ -14,8 +14,8 @@ */ -- Stores what transaction ids we have received and what our response was CREATE TABLE IF NOT EXISTS received_transactions( - transaction_id VARCHAR(150), - origin VARCHAR(150), + transaction_id TEXT, + origin TEXT, ts BIGINT, response_code INTEGER, response_json bytea, @@ -30,8 +30,8 @@ CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_ref -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering - transaction_id VARCHAR(150), - destination VARCHAR(150), + transaction_id TEXT, + destination TEXT, response_code INTEGER DEFAULT 0, response_json TEXT, ts BIGINT @@ -47,9 +47,9 @@ CREATE INDEX sent_transaction_sent ON sent_transactions(response_code); -- For sent transactions only. CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( transaction_id INTEGER, - destination VARCHAR(150), - pdu_id VARCHAR(150), - pdu_origin VARCHAR(150), + destination TEXT, + pdu_id TEXT, + pdu_origin TEXT, UNIQUE (transaction_id, destination) ); @@ -57,7 +57,7 @@ CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health CREATE TABLE IF NOT EXISTS destinations( - destination VARCHAR(150) PRIMARY KEY, + destination TEXT PRIMARY KEY, retry_last_ts BIGINT, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql index 033e3244b..d2fa3122d 100644 --- a/synapse/storage/schema/full_schemas/16/users.sql +++ b/synapse/storage/schema/full_schemas/16/users.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS users( - name VARCHAR(150), - password_hash VARCHAR(150), + name TEXT, + password_hash TEXT, creation_ts BIGINT, admin SMALLINT DEFAULT 0 NOT NULL, UNIQUE(name) @@ -22,18 +22,18 @@ CREATE TABLE IF NOT EXISTS users( CREATE TABLE IF NOT EXISTS access_tokens( id BIGINT PRIMARY KEY, - user_id VARCHAR(150) NOT NULL, - device_id VARCHAR(150), - token VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, + device_id TEXT, + token TEXT NOT NULL, last_used BIGINT, UNIQUE(token) ); CREATE TABLE IF NOT EXISTS user_ips ( - user_id VARCHAR(150) NOT NULL, - access_token VARCHAR(150) NOT NULL, - device_id VARCHAR(150), - ip VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, + access_token TEXT NOT NULL, + device_id TEXT, + ip TEXT NOT NULL, user_agent TEXT NOT NULL, last_seen BIGINT NOT NULL ); diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql index d9494611e..d682608aa 100644 --- a/synapse/storage/schema/schema_version.sql +++ b/synapse/storage/schema/schema_version.sql @@ -22,6 +22,6 @@ CREATE TABLE IF NOT EXISTS schema_version( CREATE TABLE IF NOT EXISTS applied_schema_deltas( version INTEGER NOT NULL, - file VARCHAR(150) NOT NULL, + file TEXT NOT NULL, UNIQUE(version, file) ); From a1d4813a54ad711d6db7da51f301412e3f139346 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 12:55:29 +0100 Subject: [PATCH 160/173] Quickly fix dodgy est. time remaining --- scripts/port_from_sqlite_to_postgres.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index a40a93ca7..b4ee860a9 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -535,11 +535,20 @@ class CursesProgress(Progress): self.finished = False + self.total_processed = 0 + self.total_remaining = 0 + super(CursesProgress, self).__init__() def update(self, table, num_done): super(CursesProgress, self).update(table, num_done) + self.total_processed = 0 + self.total_remaining = 0 + for table, data in self.tables.items(): + self.total_processed += data["num_done"] - data["start"] + self.total_remaining += data["total"] - data["num_done"] + self.render() def render(self, force=False): @@ -561,13 +570,11 @@ class CursesProgress(Progress): if self.finished: status = "Time spent: %s (Done!)" % (duration_str,) else: - min_perc = min( - (v["num_done"] - v["start"]) * 100. / (v["total"] - v["start"]) - if v["total"] - v["start"] else 100 - for v in self.tables.values() - ) - if min_perc > 0: - est_remaining = (int(now) - self.start_time) * 100 / min_perc + + if self.total_processed > 0: + left = float(self.total_remaining) / self.total_processed + + est_remaining = (int(now) - self.start_time) * left est_remaining_str = '%02dm %02ds remaining' % divmod(est_remaining, 60) else: est_remaining_str = "Unknown" From 073b891ec13e8964759d6f0498cf385fe77729ee Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 13:44:23 +0100 Subject: [PATCH 161/173] Remove unused imports --- synapse/app/homeserver.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 43ce12af7..694a0125a 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -19,8 +19,7 @@ sys.dont_write_bytecode = True from synapse.storage.engines import create_engine from synapse.storage import ( - prepare_database, prepare_sqlite3_database, are_all_users_on_domain, - UpgradeDatabaseException, + are_all_users_on_domain, UpgradeDatabaseException, ) from synapse.server import HomeServer From d98edb548af8833a7c44eded610a528e3d0515c6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 17:20:32 +0100 Subject: [PATCH 162/173] Ensure the serial returned by presence is always an integer --- synapse/handlers/presence.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 571eacd34..774df46ab 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -859,7 +859,7 @@ class PresenceEventSource(object): presence = self.hs.get_handlers().presence_handler cachemap = presence._user_cachemap clock = self.clock - latest_serial = None + latest_serial = 0 updates = [] # TODO(paul): use a DeferredList ? How to limit concurrency. @@ -872,8 +872,7 @@ class PresenceEventSource(object): if not (yield self.is_visible(observer_user, observed_user)): continue - if latest_serial is None or cached.serial > latest_serial: - latest_serial = cached.serial + latest_serial = max(cached.serial, latest_serial) updates.append(cached.make_event(user=observed_user, clock=clock)) # TODO(paul): limit @@ -882,6 +881,7 @@ class PresenceEventSource(object): if serial < from_key: break + latest_serial = max(cached.serial, serial) for u in user_ids: updates.append({ "type": "m.presence", From 0126ef7f3c1304bd920260db369628d39b4badd3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 17:23:53 +0100 Subject: [PATCH 163/173] Fix typo --- synapse/handlers/presence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 774df46ab..47cfe62c8 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -881,7 +881,7 @@ class PresenceEventSource(object): if serial < from_key: break - latest_serial = max(cached.serial, serial) + latest_serial = max(latest_serial, serial) for u in user_ids: updates.append({ "type": "m.presence", From 1783c7ca920cdacb22fa8536af03f42557af9d41 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 17:24:24 +0100 Subject: [PATCH 164/173] Ensure we never miss any presence updates --- synapse/handlers/presence.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 47cfe62c8..42cd52890 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -858,6 +858,9 @@ class PresenceEventSource(object): presence = self.hs.get_handlers().presence_handler cachemap = presence._user_cachemap + + max_serial = presence._user_cachemap_latest_serial + clock = self.clock latest_serial = 0 @@ -866,7 +869,7 @@ class PresenceEventSource(object): for observed_user in cachemap.keys(): cached = cachemap[observed_user] - if cached.serial <= from_key: + if cached.serial <= from_key or cached.serial > max_serial: continue if not (yield self.is_visible(observer_user, observed_user)): @@ -881,6 +884,9 @@ class PresenceEventSource(object): if serial < from_key: break + if serial > max_serial: + continue + latest_serial = max(latest_serial, serial) for u in user_ids: updates.append({ From f41a9a1ffcb652c045dea5f4108984d287c6de17 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 17:42:36 +0100 Subject: [PATCH 165/173] Add better help to scripts/port_from_sqlite_to_postgres.py --- scripts/port_from_sqlite_to_postgres.py | 32 +++++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index b4ee860a9..3296f1f54 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -671,15 +671,30 @@ class TerminalProgress(Progress): if __name__ == "__main__": - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser( + description="A script to port an existing synapse SQLite database to" + " a new PostgreSQL database." + ) parser.add_argument("-v", action='store_true') - parser.add_argument("--curses", action='store_true') - parser.add_argument("--sqlite-database") parser.add_argument( - "--postgres-config", type=argparse.FileType('r'), + "--sqlite-database", required=True, + help="The snapshot of the SQLite database file. This must not be" + " currently used by a running synapse server" + ) + parser.add_argument( + "--postgres-config", type=argparse.FileType('r'), required=True, + help="The database config file for the PostgreSQL database" + ) + parser.add_argument( + "--curses", action='store_true', + help="display a curses based progress UI" ) - parser.add_argument("--batch-size", type=int, default=1000) + parser.add_argument( + "--batch-size", type=int, default=1000, + help="The number of rows to select from the SQLite table each" + " iteration [default=1000]", + ) args = parser.parse_args() @@ -705,6 +720,13 @@ if __name__ == "__main__": postgres_config = yaml.safe_load(args.postgres_config) + if "name" not in postgres_config: + sys.stderr.write("Malformed database config: no 'name'") + sys.exit(2) + if postgres_config["name"] != "psycopg2": + sys.stderr.write("Database must use 'psycopg2' connector.") + sys.exit(3) + def start(stdscr=None): if stdscr: progress = CursesProgress(stdscr) From 3151afee9e3213ba439f701e9b968a32ec41054d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 17:59:27 +0100 Subject: [PATCH 166/173] Update docs/postgres.rst to explain port script usage --- docs/postgres.rst | 51 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/docs/postgres.rst b/docs/postgres.rst index 5bb45d3ed..bbf1b12a8 100644 --- a/docs/postgres.rst +++ b/docs/postgres.rst @@ -32,3 +32,54 @@ following form:: All key, values in ``args`` are passed to the ``psycopg2.connect(..)`` function, except keys beginning with ``cp_``, which are consumed by the twisted adbapi connection pool. + + +Porting from SQLite +=================== + +The script ``port_from_sqlite_to_postgres.py`` allows porting an existing +synapse server backed by SQLite to using PostgreSQL. This is done in as a two +phase process: + +1. Copy the existing SQLite database to a separate location (while the server + is down) and running the port script against that offline database. +2. Shut down the server. Rerun the port script to port any data that has come + in since taking the first snapshot. Restart server against the PostgrSQL + database. + +The port script is designed to be run repeatedly against newer snapshots of the +SQLite database file. This makes it safe to repeat step 1 if there was a delay +between taking the previous snapshot and ready to do step 2. + +It is safe to at any time kill the port script and restart it. + +Using the port script +~~~~~~~~~~~~~~~~~~~~~ + +Firstly, shut down the currently running synapse server and copy its database +file to another location. Once the copy is complete, restart synapse. + +Assuming your database config file (as described in the section *Synapse +config*) is named ``database_config.yaml`` and the SQLite snapshot is at +``homeserver.db.snapshot`` then simply run:: + + python scripts/port_from_sqlite_to_postgres.py \ + --sqlite-database homeserver.db.snapshot \ + --postgres-config database_config.yaml + +The flag ``--curses`` displays a coloured curses progress UI. + +If the script took a long time to complete, or time has otherwise passed since +the original snapshot was taken, repeat the previous steps with a newer +snapshot. + +To complete the conversion shut down the synapse server and run the port +script one last time, e.g. if the SQLite database is at ``homeserver.db`` run: + + python scripts/port_from_sqlite_to_postgres.py \ + --sqlite-database homeserver.db \ + --postgres-config database_config.yaml + +Once that has completed, change the synapse config to point at the PostgreSQL +database configuration file and restart synapse. Synapse should now be running +against PostgreSQL. From cc52f02d74e5eee4eb64ff16fb0dfaa7d4d7d38e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 18:09:20 +0100 Subject: [PATCH 167/173] Fix rst --- docs/postgres.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/postgres.rst b/docs/postgres.rst index bbf1b12a8..ca72131c6 100644 --- a/docs/postgres.rst +++ b/docs/postgres.rst @@ -74,7 +74,8 @@ the original snapshot was taken, repeat the previous steps with a newer snapshot. To complete the conversion shut down the synapse server and run the port -script one last time, e.g. if the SQLite database is at ``homeserver.db`` run: +script one last time, e.g. if the SQLite database is at ``homeserver.db`` +run:: python scripts/port_from_sqlite_to_postgres.py \ --sqlite-database homeserver.db \ From cc5d68f4c4d35c7b81876c8e9a7b9b2047142a27 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 29 Apr 2015 00:00:24 +0100 Subject: [PATCH 168/173] general clean up. s/alpha/beta/g. add intelfx's AUR package for Arch. s/the homeserver/Synapse/g. move installation & running sections closer together. --- README.rst | 134 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 88 insertions(+), 46 deletions(-) diff --git a/README.rst b/README.rst index 714ad6443..463b0c6d7 100644 --- a/README.rst +++ b/README.rst @@ -20,7 +20,7 @@ The overall architecture is:: https://somewhere.org/_matrix https://elsewhere.net/_matrix ``#matrix:matrix.org`` is the official support room for Matrix, and can be -accessed by the web client at http://matrix.org/alpha or via an IRC bridge at +accessed by the web client at http://matrix.org/beta or via an IRC bridge at irc://irc.freenode.net/matrix. Synapse is currently in rapid development, but as of version 0.5 we believe it @@ -69,21 +69,27 @@ Synapse ships with two basic demo Matrix clients: webclient (a basic group chat web client demo implemented in AngularJS) and cmdclient (a basic Python command line utility which lets you easily see what the JSON APIs are up to). -Meanwhile, iOS and Android SDKs and clients are currently in development and available from: +Meanwhile, iOS and Android SDKs and clients are available from: - https://github.com/matrix-org/matrix-ios-sdk +- https://github.com/matrix-org/matrix-ios-kit +- https://github.com/matrix-org/matrix-ios-console - https://github.com/matrix-org/matrix-android-sdk -We'd like to invite you to join #matrix:matrix.org (via http://matrix.org/alpha), run a homeserver, take a look at the Matrix spec at -http://matrix.org/docs/spec, experiment with the APIs and the demo -clients, and report any bugs via http://matrix.org/jira. +We'd like to invite you to join #matrix:matrix.org (via +https://matrix.org/beta), run a homeserver, take a look at the Matrix spec at +https://matrix.org/docs/spec and API docs at https://matrix.org/docs/api, +experiment with the APIs and the demo clients, and report any bugs via +https://matrix.org/jira. Thanks for using Matrix! [1] End-to-end encryption is currently in development -Homeserver Installation -======================= +Synapse Installation +==================== + +Synapse is the reference python/twisted Matrix homeserver implementation. System requirements: - POSIX-compliant system (tested on Linux & OS X) @@ -152,36 +158,51 @@ you can use the command line to register new users:: For reliable VoIP calls to be routed via this homeserver, you MUST configure a TURN server. See docs/turn-howto.rst for details. -Troubleshooting Installation ----------------------------- +Using PostgreSQL +================ -Synapse requires pip 1.7 or later, so if your OS provides too old a version and -you get errors about ``error: no such option: --process-dependency-links`` you -may need to manually upgrade it:: +As of Synapse 0.9, `PostgreSQL `_ is supported as an +alternative to the `SQLite `_ database that Synapse has +traditionally used for convenience and simplicity. - $ sudo pip install --upgrade pip +The advantages of Postgres include: -If pip crashes mid-installation for reason (e.g. lost terminal), pip may -refuse to run until you remove the temporary installation directory it -created. To reset the installation:: + * significant performance improvements due to the superior threading and + caching model, smarter query optimiser + * allowing the DB to be run on separate hardware + * allowing basic active/backup high-availability with a "hot spare" synapse + pointing at the same DB master, as well as enabling DB replication in + synapse itself. + +The only disadvantage is that the code is relatively new as of April 2015 and +may have a few regressions relative to SQLite. - $ rm -rf /tmp/pip_install_matrix +For information on how to install and use PostgreSQL, please see +`docs/postgres.rst `_. -pip seems to leak *lots* of memory during installation. For instance, a Linux -host with 512MB of RAM may run out of memory whilst installing Twisted. If this -happens, you will have to individually install the dependencies which are -failing, e.g.:: +Running Synapse +=============== - $ pip install twisted +To actually run your new homeserver, pick a working directory for Synapse to run +(e.g. ``~/.synapse``), and:: -On OSX, if you encounter clang: error: unknown argument: '-mno-fused-madd' you -will need to export CFLAGS=-Qunused-arguments. + $ cd ~/.synapse + $ source ./bin/activate + $ synctl start + +Platform Specific Instructions +============================== ArchLinux --------- -Installation on ArchLinux may encounter a few hiccups as Arch defaults to -python 3, but synapse currently assumes python 2.7 by default. +The quickest way to get up and running with ArchLinux is probably with Ivan +Shapovalov's AUR package from +https://aur.archlinux.org/packages/matrix-synapse/, which should pull in all +the necessary dependencies. + +Alternatively, to install using pip a few changes may be needed as ArchLinux +defaults to python 3, but synapse currently assumes python 2.7 by default: pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):: @@ -201,7 +222,7 @@ installing under virtualenv):: $ sudo pip2.7 uninstall py-bcrypt $ sudo pip2.7 install py-bcrypt -During setup of homeserver you need to call python2.7 directly again:: +During setup of Synapse you need to call python2.7 directly again:: $ cd ~/.synapse $ python2.7 -m synapse.app.homeserver \ @@ -242,15 +263,33 @@ Troubleshooting: you do, you may need to create a symlink to ``libsodium.a`` so ``ld`` can find it: ``ln -s /usr/local/lib/libsodium.a /usr/lib/libsodium.a`` -Running Your Homeserver -======================= +Troubleshooting +=============== -To actually run your new homeserver, pick a working directory for Synapse to run -(e.g. ``~/.synapse``), and:: +Troubleshooting Installation +---------------------------- - $ cd ~/.synapse - $ source ./bin/activate - $ synctl start +Synapse requires pip 1.7 or later, so if your OS provides too old a version and +you get errors about ``error: no such option: --process-dependency-links`` you +may need to manually upgrade it:: + + $ sudo pip install --upgrade pip + +If pip crashes mid-installation for reason (e.g. lost terminal), pip may +refuse to run until you remove the temporary installation directory it +created. To reset the installation:: + + $ rm -rf /tmp/pip_install_matrix + +pip seems to leak *lots* of memory during installation. For instance, a Linux +host with 512MB of RAM may run out of memory whilst installing Twisted. If this +happens, you will have to individually install the dependencies which are +failing, e.g.:: + + $ pip install twisted + +On OSX, if you encounter clang: error: unknown argument: '-mno-fused-madd' you +will need to export CFLAGS=-Qunused-arguments. Troubleshooting Running ----------------------- @@ -271,7 +310,7 @@ fix try re-installing from PyPI or directly from $ pip install --user https://github.com/pyca/pynacl/tarball/master ArchLinux ---------- +~~~~~~~~~ If running `$ synctl start` fails with 'returned non-zero exit status 1', you will need to explicitly call Python2.7 - either running as:: @@ -280,16 +319,16 @@ you will need to explicitly call Python2.7 - either running as:: ...or by editing synctl with the correct python executable. -Homeserver Development -====================== +Synapse Development +=================== -To check out a homeserver for development, clone the git repo into a working +To check out a synapse for development, clone the git repo into a working directory of your choice:: $ git clone https://github.com/matrix-org/synapse.git $ cd synapse -The homeserver has a number of external dependencies, that are easiest +Synapse has a number of external dependencies, that are easiest to install using pip and a virtualenv:: $ virtualenv env @@ -300,7 +339,7 @@ to install using pip and a virtualenv:: This will run a process of downloading and installing all the needed dependencies into a virtual env. -Once this is done, you may wish to run the homeserver's unit tests, to +Once this is done, you may wish to run Synapse's unit tests, to check that everything is installed as it should be:: $ python setup.py test @@ -312,10 +351,10 @@ This should end with a 'PASSED' result:: PASSED (successes=143) -Upgrading an existing homeserver -================================ +Upgrading an existing Synapse +============================= -IMPORTANT: Before upgrading an existing homeserver to a new version, please +IMPORTANT: Before upgrading an existing synapse to a new version, please refer to UPGRADE.rst for any additional instructions. Otherwise, simply re-install the new codebase over the current one - e.g. @@ -376,8 +415,8 @@ SRV record, as that is the name other machines will expect it to have:: You may additionally want to pass one or more "-v" options, in order to increase the verbosity of logging output; at least for initial testing. -Running a Demo Federation of Homeservers ----------------------------------------- +Running a Demo Federation of Synapses +------------------------------------- If you want to get up and running quickly with a trio of homeservers in a private federation (``localhost:8080``, ``localhost:8081`` and @@ -412,7 +451,10 @@ account. Your name will take the form of:: Specify your desired localpart in the topmost box of the "Register for an account" form, and click the "Register" button. Hostnames can contain ports if required due to lack of SRV records (e.g. @matthew:localhost:8448 on an -internal synapse sandbox running on localhost) +internal synapse sandbox running on localhost). + +If registration fails, you may need to enable it in the homeserver (see +`Synapse Installation`_ above) Logging In To An Existing Account From e3005d3ddb62e7d2d7fe83e97d9ac04f5cfa5665 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 29 Apr 2015 00:14:29 +0100 Subject: [PATCH 169/173] mention silviof's dockerfile --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 463b0c6d7..b7a21558c 100644 --- a/README.rst +++ b/README.rst @@ -124,6 +124,9 @@ To install the synapse homeserver run:: This installs synapse, along with the libraries it uses, into a virtual environment under ``~/.synapse``. +Alternatively, Silvio Fricke has contributed a Dockerfile to automate the +above in Docker at https://registry.hub.docker.com/u/silviof/docker-matrix/. + To set up your homeserver, run (in your virtualenv, as before):: $ cd ~/.synapse From 68c06039461a342084787f77125d4c7898ac5899 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 29 Apr 2015 00:14:44 +0100 Subject: [PATCH 170/173] comment out ugly test logline --- synapse/config/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 63c8e3693..1a850f705 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -78,7 +78,7 @@ class LoggingConfig(Config): handler.addFilter(LoggingContextFilter(request="")) logger.addHandler(handler) - logger.info("Test") + #logger.info("Test") else: with open(self.log_config, 'r') as f: logging.config.dictConfig(yaml.load(f)) From 478e511db04548023f5a652e6ed7117ecd3d3cf5 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 29 Apr 2015 00:48:07 +0100 Subject: [PATCH 171/173] improve postgres blurb a bit --- docs/postgres.rst | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/docs/postgres.rst b/docs/postgres.rst index ca72131c6..a04ab12b1 100644 --- a/docs/postgres.rst +++ b/docs/postgres.rst @@ -3,7 +3,8 @@ Using Postgres Set up client ============= -We need to have installed the postgres python connector ``psycopg2``. In the + +Postgres support depends on the postgres python connector ``psycopg2``. In the virtual env:: sudo apt-get install libpq-dev @@ -13,7 +14,8 @@ virtual env:: Synapse config ============== -Add the following line to your config file:: +When you are ready to start using PostgreSQL, add the following line to your +config file:: database_config: @@ -37,6 +39,9 @@ adbapi connection pool. Porting from SQLite =================== +Overview +~~~~~~~~ + The script ``port_from_sqlite_to_postgres.py`` allows porting an existing synapse server backed by SQLite to using PostgreSQL. This is done in as a two phase process: @@ -44,12 +49,12 @@ phase process: 1. Copy the existing SQLite database to a separate location (while the server is down) and running the port script against that offline database. 2. Shut down the server. Rerun the port script to port any data that has come - in since taking the first snapshot. Restart server against the PostgrSQL + in since taking the first snapshot. Restart server against the PostgreSQL database. The port script is designed to be run repeatedly against newer snapshots of the SQLite database file. This makes it safe to repeat step 1 if there was a delay -between taking the previous snapshot and ready to do step 2. +between taking the previous snapshot and being ready to do step 2. It is safe to at any time kill the port script and restart it. @@ -57,7 +62,12 @@ Using the port script ~~~~~~~~~~~~~~~~~~~~~ Firstly, shut down the currently running synapse server and copy its database -file to another location. Once the copy is complete, restart synapse. +file (typically ``homeserver.db``) to another location. Once the copy is +complete, restart synapse. For instance:: + + ./synctl stop + cp homeserver.db homeserver.db.snapshot + ./synctl start Assuming your database config file (as described in the section *Synapse config*) is named ``database_config.yaml`` and the SQLite snapshot is at @@ -82,5 +92,6 @@ run:: --postgres-config database_config.yaml Once that has completed, change the synapse config to point at the PostgreSQL -database configuration file and restart synapse. Synapse should now be running -against PostgreSQL. +database configuration file using the ``database_config`` parameter (see +`Synapse Config`_) and restart synapse. Synapse should now be running against +PostgreSQL. From f4c9ebbc345d13c458f63ce2f946339af9eddd26 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 29 Apr 2015 11:07:13 +0100 Subject: [PATCH 172/173] Delete ugly commented out log line. --- synapse/config/logger.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 1a850f705..247b32481 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -78,7 +78,6 @@ class LoggingConfig(Config): handler.addFilter(LoggingContextFilter(request="")) logger.addHandler(handler) - #logger.info("Test") else: with open(self.log_config, 'r') as f: logging.config.dictConfig(yaml.load(f)) From 72443572bf5dd147f50b0168e1078d88476a3e9f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 11:50:18 +0100 Subject: [PATCH 173/173] Mention that postgres databases must have the correct charset encoding --- docs/postgres.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/postgres.rst b/docs/postgres.rst index a04ab12b1..d645e1d69 100644 --- a/docs/postgres.rst +++ b/docs/postgres.rst @@ -1,6 +1,23 @@ Using Postgres -------------- +Set up database +=============== + +The PostgreSQL database used *must* have the correct encoding set, otherwise +would not be able to store UTF8 strings. To create a database with the correct +encoding use, e.g.:: + + CREATE DATABASE synapse + ENCODING 'UTF8' + LC_COLLATE='C' + LC_CTYPE='C' + template=template0 + OWNER synapse_user; + +This would create an appropriate database named ``synapse`` owned by the +``synapse_user`` user (which must already exist). + Set up client =============