From b994fb2b965debb59249696ab15e1fdabb5f1f2f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 6 May 2015 12:56:35 +0100 Subject: [PATCH 01/11] Don't read from the config file before checking it exists --- synapse/app/synctl.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py index 0a2b0d6fc..1f7d543c3 100755 --- a/synapse/app/synctl.py +++ b/synapse/app/synctl.py @@ -27,20 +27,21 @@ CONFIGFILE = "homeserver.yaml" GREEN = "\x1b[1;32m" NORMAL = "\x1b[m" +if not os.path.exists(CONFIGFILE): + sys.stderr.write( + "No config file found\n" + "To generate a config file, run '%s -c %s --generate-config" + " --server-name='\n" % ( + " ".join(SYNAPSE), CONFIGFILE + ) + ) + sys.exit(1) + CONFIG = yaml.load(open(CONFIGFILE)) PIDFILE = CONFIG["pid_file"] def start(): - if not os.path.exists(CONFIGFILE): - sys.stderr.write( - "No config file found\n" - "To generate a config file, run '%s -c %s --generate-config" - " --server-name='\n" % ( - " ".join(SYNAPSE), CONFIGFILE - ) - ) - sys.exit(1) print "Starting ...", args = SYNAPSE args.extend(["--daemonize", "-c", CONFIGFILE]) From 4a7a4a5b6cc6bf6201e996b5a6b1b82a4b874677 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 6 May 2015 17:08:00 +0100 Subject: [PATCH 02/11] Optional profiling using cProfile --- synapse/app/homeserver.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index d8d0df7e4..c22726519 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -496,11 +496,31 @@ class SynapseSite(Site): def run(hs): + PROFILE_SYNAPSE = False + if PROFILE_SYNAPSE: + def profile(func): + from cProfile import Profile + from threading import current_thread + + def profiled(*args, **kargs): + profile = Profile() + profile.enable() + func(*args, **kargs) + profile.disable() + ident = current_thread().ident + profile.dump_stats("/tmp/%s.%s.%i.pstat" % ( + hs.hostname, func.__name__, ident + )) + + return profiled + + from twisted.python.threadpool import ThreadPool + ThreadPool._worker = profile(ThreadPool._worker) + reactor.run = profile(reactor.run) def in_thread(): with LoggingContext("run"): change_resource_limit(hs.config.soft_file_limit) - reactor.run() if hs.config.daemonize: From b850c9fa04eca2dcc7118dfc20481617b84a623b Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 7 May 2015 09:33:30 +0100 Subject: [PATCH 03/11] Typo --- synapse/config/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index cd4bd28e8..447c6e312 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -156,7 +156,7 @@ class Config(object): if config_args.generate_config: server_name = config_args.server_name if not server_name: - print "Most specify a server_name to a generate config for." + print "Must specify a server_name to a generate config for." sys.exit(1) (config_path,) = config_args.config_path if not os.path.exists(config_dir_path): From ef8e8ebd917dd911ad6af4f71fc754b631ba4e49 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 7 May 2015 16:46:38 +0100 Subject: [PATCH 04/11] pynacl-0.3.0 was released so we can finally start using it directly from pypi --- synapse/handlers/register.py | 2 +- synapse/python_dependencies.py | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 7b68585a1..c0036e1f4 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -82,7 +82,7 @@ class RegistrationHandler(BaseHandler): yield run_on_reactor() password_hash = None if password: - password_hash = bcrypt.hashpw(password, bcrypt.gensalt()) + password_hash = bcrypt.hashpw(password, bcrypt.gensalt(1)) if localpart: yield self.check_username(localpart) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index df4b8f7f9..0d0db4d54 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -24,7 +24,7 @@ REQUIREMENTS = { "pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyyaml": ["yaml"], "pyasn1": ["pyasn1"], - "pynacl": ["nacl"], + "pynacl>=0.0.3": ["nacl>=0.0.3"], "daemonize": ["daemonize"], "py-bcrypt": ["bcrypt"], "frozendict>=0.4": ["frozendict"], @@ -50,11 +50,6 @@ def github_link(project, version, egg): return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg) DEPENDENCY_LINKS = [ - github_link( - project="pyca/pynacl", - version="d4d3175589b892f6ea7c22f466e0e223853516fa", - egg="pynacl-0.3.0", - ), github_link( project="matrix-org/syutil", version="v0.0.6", From dfa98f911be7de6b30a1e2adf2ba5c72f1030d61 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 7 May 2015 17:45:42 +0100 Subject: [PATCH 05/11] revert accidental bcrypt gensalt round reduction from loadtesting --- synapse/handlers/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index c0036e1f4..7b68585a1 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -82,7 +82,7 @@ class RegistrationHandler(BaseHandler): yield run_on_reactor() password_hash = None if password: - password_hash = bcrypt.hashpw(password, bcrypt.gensalt(1)) + password_hash = bcrypt.hashpw(password, bcrypt.gensalt()) if localpart: yield self.check_username(localpart) From 6538d445e885a8a163f5fcc792b11a9d9b890de3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 8 May 2015 15:55:11 +0100 Subject: [PATCH 06/11] Make the timestamps in server_keys_json bigints --- .../delta/18/server_keys_bigger_ints.sql | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 synapse/storage/schema/delta/18/server_keys_bigger_ints.sql diff --git a/synapse/storage/schema/delta/18/server_keys_bigger_ints.sql b/synapse/storage/schema/delta/18/server_keys_bigger_ints.sql new file mode 100644 index 000000000..ae866475a --- /dev/null +++ b/synapse/storage/schema/delta/18/server_keys_bigger_ints.sql @@ -0,0 +1,32 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +CREATE TABLE IF NOT EXISTS new_server_keys_json ( + server_name TEXT NOT NULL, -- Server name. + key_id TEXT NOT NULL, -- Requested key id. + from_server TEXT NOT NULL, -- Which server the keys were fetched from. + ts_added_ms BIGINT UNSIGNED NOT NULL, -- When the keys were fetched + ts_valid_until_ms BIGINT UNSIGNED NOT NULL, -- When this version of the keys exipires. + key_json bytea NOT NULL, -- JSON certificate for the remote server. + CONSTRAINT uniqueness UNIQUE (server_name, key_id, from_server) +); + +INSERT INTO new_server_keys_json + SELECT server_name, key_id, from_server,ts_added_ms, ts_valid_until_ms, key_json FROM server_keys_json ; + +DROP TABLE new_server_keys_json; + +ALTER TABLE new_server_keys_json RENAME TO server_keys_json; From 84e1cacea4403d9ffde1c9dcd36eb625a37113af Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 8 May 2015 15:58:06 +0100 Subject: [PATCH 07/11] Bump schema version --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 0cc14fb69..7cb91a0be 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -51,7 +51,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 17 +SCHEMA_VERSION = 18 dir_path = os.path.abspath(os.path.dirname(__file__)) From b0f71db3ffe3c65cd77248771ffb67774ecfadd5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 8 May 2015 15:59:51 +0100 Subject: [PATCH 08/11] Remove unsigned --- synapse/storage/schema/delta/18/server_keys_bigger_ints.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/schema/delta/18/server_keys_bigger_ints.sql b/synapse/storage/schema/delta/18/server_keys_bigger_ints.sql index ae866475a..d2c70d0aa 100644 --- a/synapse/storage/schema/delta/18/server_keys_bigger_ints.sql +++ b/synapse/storage/schema/delta/18/server_keys_bigger_ints.sql @@ -18,8 +18,8 @@ CREATE TABLE IF NOT EXISTS new_server_keys_json ( server_name TEXT NOT NULL, -- Server name. key_id TEXT NOT NULL, -- Requested key id. from_server TEXT NOT NULL, -- Which server the keys were fetched from. - ts_added_ms BIGINT UNSIGNED NOT NULL, -- When the keys were fetched - ts_valid_until_ms BIGINT UNSIGNED NOT NULL, -- When this version of the keys exipires. + ts_added_ms BIGINT NOT NULL, -- When the keys were fetched + ts_valid_until_ms BIGINT NOT NULL, -- When this version of the keys exipires. key_json bytea NOT NULL, -- JSON certificate for the remote server. CONSTRAINT uniqueness UNIQUE (server_name, key_id, from_server) ); From 9d36eb4eab036d9793132892b56c7fad4e585984 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 8 May 2015 16:01:55 +0100 Subject: [PATCH 09/11] Rename unique constraint --- synapse/storage/schema/delta/18/server_keys_bigger_ints.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/schema/delta/18/server_keys_bigger_ints.sql b/synapse/storage/schema/delta/18/server_keys_bigger_ints.sql index d2c70d0aa..566470ac6 100644 --- a/synapse/storage/schema/delta/18/server_keys_bigger_ints.sql +++ b/synapse/storage/schema/delta/18/server_keys_bigger_ints.sql @@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS new_server_keys_json ( ts_added_ms BIGINT NOT NULL, -- When the keys were fetched ts_valid_until_ms BIGINT NOT NULL, -- When this version of the keys exipires. key_json bytea NOT NULL, -- JSON certificate for the remote server. - CONSTRAINT uniqueness UNIQUE (server_name, key_id, from_server) + CONSTRAINT server_keys_json_uniqueness UNIQUE (server_name, key_id, from_server) ); INSERT INTO new_server_keys_json From 1c7912751ea53e115722bb81383734f06d2aa69d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 8 May 2015 16:04:32 +0100 Subject: [PATCH 10/11] Drop the old table not the new table --- synapse/storage/schema/delta/18/server_keys_bigger_ints.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/schema/delta/18/server_keys_bigger_ints.sql b/synapse/storage/schema/delta/18/server_keys_bigger_ints.sql index 566470ac6..c0b0fdfb6 100644 --- a/synapse/storage/schema/delta/18/server_keys_bigger_ints.sql +++ b/synapse/storage/schema/delta/18/server_keys_bigger_ints.sql @@ -27,6 +27,6 @@ CREATE TABLE IF NOT EXISTS new_server_keys_json ( INSERT INTO new_server_keys_json SELECT server_name, key_id, from_server,ts_added_ms, ts_valid_until_ms, key_json FROM server_keys_json ; -DROP TABLE new_server_keys_json; +DROP TABLE server_keys_json; ALTER TABLE new_server_keys_json RENAME TO server_keys_json; From 0cd1401f8ddaaef56301940b4d81c2b626380389 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 8 May 2015 16:11:51 +0100 Subject: [PATCH 11/11] Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 18b8ff775..bde6f7777 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.9.0-r1" +__version__ = "0.9.0-r2"