diff --git a/CHANGES.rst b/CHANGES.rst index 0d36e8eef..23bdac6a8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,12 @@ +Changes in synapse 0.6.0 (2014-12-16) +===================================== + + * Add new API for media upload and download that supports thumbnailing. + * Implement typing notifications. + * Fix bugs where we sent events with invalid signatures due to bugs where + we incorrectly persisted events. + * Improve performance of database queries involving retrieving events. + Changes in synapse 0.5.4a (2014-12-13) ====================================== diff --git a/UPGRADE.rst b/UPGRADE.rst index 5ebdd455c..a602a9f3e 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -1,3 +1,19 @@ +Upgrading to v0.6.0 +=================== + +This update includes a change to the database schema. To upgrade you first need +to upgrade the database by running:: + + python scripts/upgrade_db_to_v0.6.0.py + +Where `` is the location of the database, `` is the +server name as specified in the synapse configuration, and `` is +the location of the signing key as specified in the synapse configuration. + +This may take some time to complete. Failures of signatures and content hashes +can safely be ignored. + + Upgrading to v0.5.1 =================== diff --git a/VERSION b/VERSION index 64c6e2f46..a918a2aa1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5.4a +0.6.0 diff --git a/scripts/upgrade_db_to_v0.5.5.py b/scripts/upgrade_db_to_v0.6.0.py similarity index 74% rename from scripts/upgrade_db_to_v0.5.5.py rename to scripts/upgrade_db_to_v0.6.0.py index aa0d7667a..298e9c078 100644 --- a/scripts/upgrade_db_to_v0.5.5.py +++ b/scripts/upgrade_db_to_v0.6.0.py @@ -1,3 +1,5 @@ + +from synapse.storage import SCHEMA_VERSION, read_schema from synapse.storage._base import SQLBaseStore from synapse.storage.signatures import SignatureStore from synapse.storage.event_federation import EventFederationStore @@ -19,8 +21,9 @@ from syutil.crypto.signing_key import decode_verify_key_bytes from syutil.jsonutil import encode_canonical_json import argparse -import dns.resolver +# import dns.resolver import hashlib +import httplib import json import sqlite3 import syutil @@ -38,6 +41,8 @@ CREATE TABLE IF NOT EXISTS event_json( CREATE INDEX IF NOT EXISTS event_json_id ON event_json(event_id); CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); + +PRAGMA user_version = 10; """ @@ -142,50 +147,57 @@ class Store(object): store = Store() -def get_key(server_name): - print "Getting keys for: %s" % (server_name,) - targets = [] - if ":" in server_name: - target, port = server_name.split(":") - targets.append((target, int(port))) - return - try: - answers = dns.resolver.query("_matrix._tcp." + server_name, "SRV") - for srv in answers: - targets.append((srv.target, srv.port)) - except dns.resolver.NXDOMAIN: - targets.append((server_name, 8448)) - except: - print "Failed to lookup keys for %s" % (server_name,) - return {} - - for target, port in targets: - url = "https://%s:%i/_matrix/key/v1" % (target, port) - try: - keys = json.load(urllib2.urlopen(url, timeout=2)) - verify_keys = {} - for key_id, key_base64 in keys["verify_keys"].items(): - verify_key = decode_verify_key_bytes( - key_id, decode_base64(key_base64) - ) - verify_signed_json(keys, server_name, verify_key) - verify_keys[key_id] = verify_key - print "Got keys for: %s" % (server_name,) - return verify_keys - except urllib2.URLError: - pass - - print "Failed to get keys for %s" % (server_name,) - return {} +# def get_key(server_name): +# print "Getting keys for: %s" % (server_name,) +# targets = [] +# if ":" in server_name: +# target, port = server_name.split(":") +# targets.append((target, int(port))) +# try: +# answers = dns.resolver.query("_matrix._tcp." + server_name, "SRV") +# for srv in answers: +# targets.append((srv.target, srv.port)) +# except dns.resolver.NXDOMAIN: +# targets.append((server_name, 8448)) +# except: +# print "Failed to lookup keys for %s" % (server_name,) +# return {} +# +# for target, port in targets: +# url = "https://%s:%i/_matrix/key/v1" % (target, port) +# try: +# keys = json.load(urllib2.urlopen(url, timeout=2)) +# verify_keys = {} +# for key_id, key_base64 in keys["verify_keys"].items(): +# verify_key = decode_verify_key_bytes( +# key_id, decode_base64(key_base64) +# ) +# verify_signed_json(keys, server_name, verify_key) +# verify_keys[key_id] = verify_key +# print "Got keys for: %s" % (server_name,) +# return verify_keys +# except urllib2.URLError: +# pass +# except urllib2.HTTPError: +# pass +# except httplib.HTTPException: +# pass +# +# print "Failed to get keys for %s" % (server_name,) +# return {} def reinsert_events(cursor, server_name, signing_key): + print "Running delta: v10" + cursor.executescript(delta_sql) cursor.execute( "SELECT * FROM events ORDER BY rowid ASC" ) + print "Getting events..." + rows = store.cursor_to_dict(cursor) events = store._generate_event_json(cursor, rows) @@ -207,13 +219,20 @@ def reinsert_events(cursor, server_name, signing_key): } } + i = 0 + N = len(events) + for event in events: - for alg_name in event.hashes: - if check_event_content_hash(event, algorithms[alg_name]): - pass - else: - pass - print "FAIL content hash %s %s" % (alg_name, event.event_id, ) + if i % 100 == 0: + print "Processed: %d/%d events" % (i,N,) + i += 1 + + # for alg_name in event.hashes: + # if check_event_content_hash(event, algorithms[alg_name]): + # pass + # else: + # pass + # print "FAIL content hash %s %s" % (alg_name, event.event_id, ) have_own_correctly_signed = False for host, sigs in event.signatures.items(): @@ -221,7 +240,7 @@ def reinsert_events(cursor, server_name, signing_key): for key_id in sigs: if host not in server_keys: - server_keys[host] = get_key(host) + server_keys[host] = {} # get_key(host) if key_id in server_keys[host]: try: verify_signed_json( @@ -275,9 +294,25 @@ def reinsert_events(cursor, server_name, signing_key): def main(database, server_name, signing_key): conn = sqlite3.connect(database) cursor = conn.cursor() + + # Do other deltas: + cursor.execute("PRAGMA user_version") + row = cursor.fetchone() + + if row and row[0]: + user_version = row[0] + # Run every version since after the current version. + for v in range(user_version + 1, 10): + print "Running delta: %d" % (v,) + sql_script = read_schema("delta/v%d" % (v,)) + cursor.executescript(sql_script) + reinsert_events(cursor, server_name, signing_key) + conn.commit() + print "Success!" + if __name__ == "__main__": parser = argparse.ArgumentParser() diff --git a/synapse/__init__.py b/synapse/__init__.py index 7e49e1fd0..1cef40925 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.5.4a" +__version__ = "0.6.0" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index a6e29c086..140c99f18 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.storage import prepare_database +from synapse.storage import prepare_database, UpgradeDatabaseException from synapse.server import HomeServer @@ -228,8 +228,15 @@ def setup(): logger.info("Preparing database: %s...", db_name) - with sqlite3.connect(db_name) as db_conn: - prepare_database(db_conn) + try: + with sqlite3.connect(db_name) as db_conn: + prepare_database(db_conn) + except UpgradeDatabaseException: + sys.stderr.write( + "\nFailed to upgrade database.\n" + "Have you followed any instructions in UPGRADES.rst?\n" + ) + sys.exit(1) logger.info("Database prepared in %s.", db_name) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index b529d890b..49c0e9811 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -123,6 +123,11 @@ class MessageHandler(BaseHandler): self.validator.validate_new(builder) + self.ratelimit(builder.user_id) + # TODO(paul): Why does 'event' not have a 'user' object? + user = self.hs.parse_userid(builder.user_id) + assert self.hs.is_mine(user), "User must be our own: %s" % (user,) + if builder.type == EventTypes.Member: membership = builder.content.get("membership", None) if membership == Membership.JOIN: diff --git a/synapse/state.py b/synapse/state.py index 551058b51..15397e2f3 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -203,9 +203,10 @@ class StateHandler(object): } if event_type: - prev_states = conflicted_state.get( - (event_type, state_key), {} - ).keys() + prev_states_events = conflicted_state.get( + (event_type, state_key), [] + ) + prev_states = [s.event_id for s in prev_states_events] else: prev_states = [] diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index c051f33a5..60c2d6742 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -66,7 +66,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 9 +SCHEMA_VERSION = 10 class _RollbackButIsFineException(Exception): @@ -446,6 +446,14 @@ def read_schema(schema): return schema_file.read() +class PrepareDatabaseException(Exception): + pass + + +class UpgradeDatabaseException(PrepareDatabaseException): + pass + + def prepare_database(db_conn): """ Set up all the dbs. Since all the *.sql have IF NOT EXISTS, so we don't have to worry about overwriting existing content. @@ -470,6 +478,10 @@ def prepare_database(db_conn): # Run every version since after the current version. for v in range(user_version + 1, SCHEMA_VERSION + 1): + if v == 10: + raise UpgradeDatabaseException( + "No delta for version 10" + ) sql_script = read_schema("delta/v%d" % (v)) c.executescript(sql_script) diff --git a/synapse/storage/schema/delta/v9.sql b/synapse/storage/schema/delta/v9.sql index ad680c64d..0af29733a 100644 --- a/synapse/storage/schema/delta/v9.sql +++ b/synapse/storage/schema/delta/v9.sql @@ -20,4 +20,60 @@ CREATE TABLE IF NOT EXISTS destinations( retry_interval INTEGER ); -PRAGMA user_version = 9; \ No newline at end of file + +CREATE TABLE IF NOT EXISTS local_media_repository ( + media_id TEXT, -- The id used to refer to the media. + media_type TEXT, -- The MIME-type of the media. + media_length INTEGER, -- Length of the media in bytes. + created_ts INTEGER, -- When the content was uploaded in ms. + upload_name TEXT, -- The name the media was uploaded with. + user_id TEXT, -- The user who uploaded the file. + CONSTRAINT uniqueness UNIQUE (media_id) +); + +CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( + media_id TEXT, -- The id used to refer to the media. + thumbnail_width INTEGER, -- The width of the thumbnail in pixels. + thumbnail_height INTEGER, -- The height of the thumbnail in pixels. + thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_method TEXT, -- The method used to make the thumbnail. + thumbnail_length INTEGER, -- The length of the thumbnail in bytes. + CONSTRAINT uniqueness UNIQUE ( + media_id, thumbnail_width, thumbnail_height, thumbnail_type + ) +); + +CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id + ON local_media_repository_thumbnails (media_id); + +CREATE TABLE IF NOT EXISTS remote_media_cache ( + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media on that server. + media_type TEXT, -- The MIME-type of the media. + created_ts INTEGER, -- When the content was uploaded in ms. + upload_name TEXT, -- The name the media was uploaded with. + media_length INTEGER, -- Length of the media in bytes. + filesystem_id TEXT, -- The name used to store the media on disk. + CONSTRAINT uniqueness UNIQUE (media_origin, media_id) +); + +CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media. + thumbnail_width INTEGER, -- The width of the thumbnail in pixels. + thumbnail_height INTEGER, -- The height of the thumbnail in pixels. + thumbnail_method TEXT, -- The method used to make the thumbnail + thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_length INTEGER, -- The length of the thumbnail in bytes. + filesystem_id TEXT, -- The name used to store the media on disk. + CONSTRAINT uniqueness UNIQUE ( + media_origin, media_id, thumbnail_width, thumbnail_height, + thumbnail_type, thumbnail_type + ) +); + +CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id + ON local_media_repository_thumbnails (media_id); + + +PRAGMA user_version = 9;