diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..f36f86fbb --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +Dockerfile +.travis.yml +.gitignore +demo/etc +tox.ini diff --git a/.gitignore b/.gitignore index c8901eb20..7940158c5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.pyc .*.swp +*~ .DS_Store _trial_temp/ @@ -13,6 +14,7 @@ docs/build/ cmdclient_config.json homeserver*.db homeserver*.log +homeserver*.log.* homeserver*.pid homeserver*.yaml @@ -32,6 +34,7 @@ demo/media_store.* demo/etc uploads +cache .idea/ media_store/ @@ -39,6 +42,7 @@ media_store/ *.tac build/ +venv/ localhost-800*/ static/client/register/register_config.js @@ -48,3 +52,4 @@ env/ *.config .vscode/ +.ropeproject/ diff --git a/.travis.yml b/.travis.yml index 3ce93cb43..e6ba6f475 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,22 @@ sudo: false language: python -python: 2.7 # tell travis to cache ~/.cache/pip cache: pip -env: - - TOX_ENV=packaging - - TOX_ENV=pep8 - - TOX_ENV=py27 +matrix: + include: + - python: 2.7 + env: TOX_ENV=packaging + + - python: 2.7 + env: TOX_ENV=pep8 + + - python: 2.7 + env: TOX_ENV=py27 + + - python: 3.6 + env: TOX_ENV=py36 install: - pip install tox diff --git a/AUTHORS.rst b/AUTHORS.rst index 3dcb1c2a8..e13ac5ad3 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -60,3 +60,6 @@ Niklas Riekenbrauck Christoph Witzany * Add LDAP support for authentication + +Pierre Jaury +* Docker packaging \ No newline at end of file diff --git a/CHANGES.rst b/CHANGES.rst index a7ed49e10..0569b581d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,11 +1,375 @@ -Unreleased -========== +Changes in synapse v0.30.0 (2018-05-24) +========================================== -synctl no longer starts the main synapse when using ``-a`` option with workers. -A new worker file should be added with ``worker_app: synapse.app.homeserver``. +'Server Notices' are a new feature introduced in Synapse 0.30. They provide a +channel whereby server administrators can send messages to users on the server. + +They are used as part of communication of the server policies (see ``docs/consent_tracking.md``), +however the intention is that they may also find a use for features such +as "Message of the day". + +This feature is specific to Synapse, but uses standard Matrix communication mechanisms, +so should work with any Matrix client. For more details see ``docs/server_notices.md`` + +Further Server Notices/Consent Tracking Support: + +* Allow overriding the server_notices user's avatar (PR #3273) +* Use the localpart in the consent uri (PR #3272) +* Support for putting %(consent_uri)s in messages (PR #3271) +* Block attempts to send server notices to remote users (PR #3270) +* Docs on consent bits (PR #3268) + + + +Changes in synapse v0.30.0-rc1 (2018-05-23) +========================================== + +Server Notices/Consent Tracking Support: + +* ConsentResource to gather policy consent from users (PR #3213) +* Move RoomCreationHandler out of synapse.handlers.Handlers (PR #3225) +* Infrastructure for a server notices room (PR #3232) +* Send users a server notice about consent (PR #3236) +* Reject attempts to send event before privacy consent is given (PR #3257) +* Add a 'has_consented' template var to consent forms (PR #3262) +* Fix dependency on jinja2 (PR #3263) + +Features: + +* Cohort analytics (PR #3163, #3241, #3251) +* Add lxml to docker image for web previews (PR #3239) Thanks to @ptman! +* Add in flight request metrics (PR #3252) + +Changes: + +* Remove unused `update_external_syncs` (PR #3233) +* Use stream rather depth ordering for push actions (PR #3212) +* Make purge_history operate on tokens (PR #3221) +* Don't support limitless pagination (PR #3265) + +Bug Fixes: + +* Fix logcontext resource usage tracking (PR #3258) +* Fix error in handling receipts (PR #3235) +* Stop the transaction cache caching failures (PR #3255) + + +Changes in synapse v0.29.1 (2018-05-17) +========================================== +Changes: + +* Update docker documentation (PR #3222) + +Changes in synapse v0.29.0 (2018-05-16) +=========================================== +Not changes since v0.29.0-rc1 + +Changes in synapse v0.29.0-rc1 (2018-05-14) +=========================================== + +Notable changes, a docker file for running Synapse (Thanks to @kaiyou!) and a +closed spec bug in the Client Server API. Additionally further prep for Python 3 +migration. + +Potentially breaking change: + +* Make Client-Server API return 401 for invalid token (PR #3161). + + This changes the Client-server spec to return a 401 error code instead of 403 + when the access token is unrecognised. This is the behaviour required by the + specification, but some clients may be relying on the old, incorrect + behaviour. + + Thanks to @NotAFile for fixing this. + +Features: + +* Add a Dockerfile for synapse (PR #2846) Thanks to @kaiyou! + +Changes - General: + +* nuke-room-from-db.sh: added postgresql option and help (PR #2337) Thanks to @rubo77! +* Part user from rooms on account deactivate (PR #3201) +* Make 'unexpected logging context' into warnings (PR #3007) +* Set Server header in SynapseRequest (PR #3208) +* remove duplicates from groups tables (PR #3129) +* Improve exception handling for background processes (PR #3138) +* Add missing consumeErrors to improve exception handling (PR #3139) +* reraise exceptions more carefully (PR #3142) +* Remove redundant call to preserve_fn (PR #3143) +* Trap exceptions thrown within run_in_background (PR #3144) + +Changes - Refactors: + +* Refactor /context to reuse pagination storage functions (PR #3193) +* Refactor recent events func to use pagination func (PR #3195) +* Refactor pagination DB API to return concrete type (PR #3196) +* Refactor get_recent_events_for_room return type (PR #3198) +* Refactor sync APIs to reuse pagination API (PR #3199) +* Remove unused code path from member change DB func (PR #3200) +* Refactor request handling wrappers (PR #3203) +* transaction_id, destination defined twice (PR #3209) Thanks to @damir-manapov! +* Refactor event storage to prepare for changes in state calculations (PR #3141) +* Set Server header in SynapseRequest (PR #3208) +* Use deferred.addTimeout instead of time_bound_deferred (PR #3127, #3178) +* Use run_in_background in preference to preserve_fn (PR #3140) + +Changes - Python 3 migration: + +* Construct HMAC as bytes on py3 (PR #3156) Thanks to @NotAFile! +* run config tests on py3 (PR #3159) Thanks to @NotAFile! +* Open certificate files as bytes (PR #3084) Thanks to @NotAFile! +* Open config file in non-bytes mode (PR #3085) Thanks to @NotAFile! +* Make event properties raise AttributeError instead (PR #3102) Thanks to @NotAFile! +* Use six.moves.urlparse (PR #3108) Thanks to @NotAFile! +* Add py3 tests to tox with folders that work (PR #3145) Thanks to @NotAFile! +* Don't yield in list comprehensions (PR #3150) Thanks to @NotAFile! +* Move more xrange to six (PR #3151) Thanks to @NotAFile! +* make imports local (PR #3152) Thanks to @NotAFile! +* move httplib import to six (PR #3153) Thanks to @NotAFile! +* Replace stringIO imports with six (PR #3154, #3168) Thanks to @NotAFile! +* more bytes strings (PR #3155) Thanks to @NotAFile! + +Bug Fixes: + +* synapse fails to start under Twisted >= 18.4 (PR #3157) +* Fix a class of logcontext leaks (PR #3170) +* Fix a couple of logcontext leaks in unit tests (PR #3172) +* Fix logcontext leak in media repo (PR #3174) +* Escape label values in prometheus metrics (PR #3175, #3186) +* Fix 'Unhandled Error' logs with Twisted 18.4 (PR #3182) Thanks to @Half-Shot! +* Fix logcontext leaks in rate limiter (PR #3183) +* notifications: Convert next_token to string according to the spec (PR #3190) Thanks to @mujx! +* nuke-room-from-db.sh: fix deletion from search table (PR #3194) Thanks to @rubo77! +* add guard for None on purge_history api (PR #3160) Thanks to @krombel! + +Changes in synapse v0.28.1 (2018-05-01) +======================================= + +SECURITY UPDATE + +* Clamp the allowed values of event depth received over federation to be + [0, 2^63 - 1]. This mitigates an attack where malicious events + injected with depth = 2^63 - 1 render rooms unusable. Depth is used to + determine the cosmetic ordering of events within a room, and so the ordering + of events in such a room will default to using stream_ordering rather than depth + (topological_ordering). + + This is a temporary solution to mitigate abuse in the wild, whilst a long term solution + is being implemented to improve how the depth parameter is used. + + Full details at + https://docs.google.com/document/d/1I3fi2S-XnpO45qrpCsowZv8P8dHcNZ4fsBsbOW7KABI + +* Pin Twisted to <18.4 until we stop using the private _OpenSSLECCurve API. + + +Changes in synapse v0.28.0 (2018-04-26) +======================================= + +Bug Fixes: + +* Fix quarantine media admin API and search reindex (PR #3130) +* Fix media admin APIs (PR #3134) + + +Changes in synapse v0.28.0-rc1 (2018-04-24) +=========================================== + +Minor performance improvement to federation sending and bug fixes. + +(Note: This release does not include the delta state resolution implementation discussed in matrix live) + + +Features: + +* Add metrics for event processing lag (PR #3090) +* Add metrics for ResponseCache (PR #3092) + +Changes: + +* Synapse on PyPy (PR #2760) Thanks to @Valodim! +* move handling of auto_join_rooms to RegisterHandler (PR #2996) Thanks to @krombel! +* Improve handling of SRV records for federation connections (PR #3016) Thanks to @silkeh! +* Document the behaviour of ResponseCache (PR #3059) +* Preparation for py3 (PR #3061, #3073, #3074, #3075, #3103, #3104, #3106, #3107, #3109, #3110) Thanks to @NotAFile! +* update prometheus dashboard to use new metric names (PR #3069) Thanks to @krombel! +* use python3-compatible prints (PR #3074) Thanks to @NotAFile! +* Send federation events concurrently (PR #3078) +* Limit concurrent event sends for a room (PR #3079) +* Improve R30 stat definition (PR #3086) +* Send events to ASes concurrently (PR #3088) +* Refactor ResponseCache usage (PR #3093) +* Clarify that SRV may not point to a CNAME (PR #3100) Thanks to @silkeh! +* Use str(e) instead of e.message (PR #3103) Thanks to @NotAFile! +* Use six.itervalues in some places (PR #3106) Thanks to @NotAFile! +* Refactor store.have_events (PR #3117) + +Bug Fixes: + +* Return 401 for invalid access_token on logout (PR #2938) Thanks to @dklug! +* Return a 404 rather than a 500 on rejoining empty rooms (PR #3080) +* fix federation_domain_whitelist (PR #3099) +* Avoid creating events with huge numbers of prev_events (PR #3113) +* Reject events which have lots of prev_events (PR #3118) + + +Changes in synapse v0.27.4 (2018-04-13) +====================================== + +Changes: + +* Update canonicaljson dependency (#3095) + + +Changes in synapse v0.27.3 (2018-04-11) +====================================== + +Bug fixes: + +* URL quote path segments over federation (#3082) + +Changes in synapse v0.27.3-rc2 (2018-04-09) +========================================== + +v0.27.3-rc1 used a stale version of the develop branch so the changelog overstates +the functionality. v0.27.3-rc2 is up to date, rc1 should be ignored. + +Changes in synapse v0.27.3-rc1 (2018-04-09) +======================================= + +Notable changes include API support for joinability of groups. Also new metrics +and phone home stats. Phone home stats include better visibility of system usage +so we can tweak synpase to work better for all users rather than our own experience +with matrix.org. Also, recording 'r30' stat which is the measure we use to track +overal growth of the Matrix ecosystem. It is defined as:- + +Counts the number of native 30 day retained users, defined as:- + * Users who have created their accounts more than 30 days + * Where last seen at most 30 days ago + * Where account creation and last_seen are > 30 days" + + +Features: + +* Add joinability for groups (PR #3045) +* Implement group join API (PR #3046) +* Add counter metrics for calculating state delta (PR #3033) +* R30 stats (PR #3041) +* Measure time it takes to calculate state group ID (PR #3043) +* Add basic performance statistics to phone home (PR #3044) +* Add response size metrics (PR #3071) +* phone home cache size configurations (PR #3063) + +Changes: + +* Add a blurb explaining the main synapse worker (PR #2886) Thanks to @turt2live! +* Replace old style error catching with 'as' keyword (PR #3000) Thanks to @NotAFile! +* Use .iter* to avoid copies in StateHandler (PR #3006) +* Linearize calls to _generate_user_id (PR #3029) +* Remove last usage of ujson (PR #3030) +* Use simplejson throughout (PR #3048) +* Use static JSONEncoders (PR #3049) +* Remove uses of events.content (PR #3060) +* Improve database cache performance (PR #3068) + +Bug fixes: + +* Add room_id to the response of `rooms/{roomId}/join` (PR #2986) Thanks to @jplatte! +* Fix replication after switch to simplejson (PR #3015) +* 404 correctly on missing paths via NoResource (PR #3022) +* Fix error when claiming e2e keys from offline servers (PR #3034) +* fix tests/storage/test_user_directory.py (PR #3042) +* use PUT instead of POST for federating groups/m.join_policy (PR #3070) Thanks to @krombel! +* postgres port script: fix state_groups_pkey error (PR #3072) + + +Changes in synapse v0.27.2 (2018-03-26) +======================================= + +Bug fixes: + +* Fix bug which broke TCP replication between workers (PR #3015) + + +Changes in synapse v0.27.1 (2018-03-26) +======================================= + +Meta release as v0.27.0 temporarily pointed to the wrong commit + + +Changes in synapse v0.27.0 (2018-03-26) +======================================= + +No changes since v0.27.0-rc2 + + +Changes in synapse v0.27.0-rc2 (2018-03-19) +=========================================== + +Pulls in v0.26.1 + +Bug fixes: + +* Fix bug introduced in v0.27.0-rc1 that causes much increased memory usage in state cache (PR #3005) + + +Changes in synapse v0.26.1 (2018-03-15) +======================================= + +Bug fixes: + +* Fix bug where an invalid event caused server to stop functioning correctly, + due to parsing and serializing bugs in ujson library (PR #3008) + + +Changes in synapse v0.27.0-rc1 (2018-03-14) +=========================================== + +The common case for running Synapse is not to run separate workers, but for those that do, be aware that synctl no longer starts the main synapse when using ``-a`` option with workers. A new worker file should be added with ``worker_app: synapse.app.homeserver``. This release also begins the process of renaming a number of the metrics reported to prometheus. See `docs/metrics-howto.rst `_. +Note that the v0.28.0 release will remove the deprecated metric names. + +Features: + +* Add ability for ASes to override message send time (PR #2754) +* Add support for custom storage providers for media repository (PR #2867, #2777, #2783, #2789, #2791, #2804, #2812, #2814, #2857, #2868, #2767) +* Add purge API features, see `docs/admin_api/purge_history_api.rst `_ for full details (PR #2858, #2867, #2882, #2946, #2962, #2943) +* Add support for whitelisting 3PIDs that users can register. (PR #2813) +* Add ``/room/{id}/event/{id}`` API (PR #2766) +* Add an admin API to get all the media in a room (PR #2818) Thanks to @turt2live! +* Add ``federation_domain_whitelist`` option (PR #2820, #2821) + + +Changes: + +* Continue to factor out processing from main process and into worker processes. See updated `docs/workers.rst `_ (PR #2892 - #2904, #2913, #2920 - #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856, #2976 - #2984, #2987 - #2989, #2991 - #2993, #2995, #2784) +* Ensure state cache is used when persisting events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849) +* Change the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh! +* No longer require a specific version of saml2 (PR #2695) Thanks to @okurz! +* Remove ``verbosity``/``log_file`` from generated config (PR #2755) +* Add and improve metrics and logging (PR #2770, #2778, #2785, #2786, #2787, #2793, #2794, #2795, #2809, #2810, #2833, #2834, #2844, #2965, #2927, #2975, #2790, #2796, #2838) +* When using synctl with workers, don't start the main synapse automatically (PR #2774) +* Minor performance improvements (PR #2773, #2792) +* Use a connection pool for non-federation outbound connections (PR #2817) +* Make it possible to run unit tests against postgres (PR #2829) +* Update pynacl dependency to 1.2.1 or higher (PR #2888) Thanks to @bachp! +* Remove ability for AS users to call /events and /sync (PR #2948) +* Use bcrypt.checkpw (PR #2949) Thanks to @krombel! + +Bug fixes: + +* Fix broken ``ldap_config`` config option (PR #2683) Thanks to @seckrv! +* Fix error message when user is not allowed to unban (PR #2761) Thanks to @turt2live! +* Fix publicised groups GET API (singular) over federation (PR #2772) +* Fix user directory when using ``user_directory_search_all_users`` config option (PR #2803, #2831) +* Fix error on ``/publicRooms`` when no rooms exist (PR #2827) +* Fix bug in quarantine_media (PR #2837) +* Fix url_previews when no Content-Type is returned from URL (PR #2845) +* Fix rare race in sync API when joining room (PR #2944) +* Fix slow event search, switch back from GIST to GIN indexes (PR #2769, #2848) Changes in synapse v0.26.0 (2018-01-05) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 2a88647ca..c6ee16efc 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -30,8 +30,12 @@ use github's pull request workflow to review the contribution, and either ask you to make any refinements needed or merge it and make them ourselves. The changes will then land on master when we next do a release. -We use Jenkins for continuous integration (http://matrix.org/jenkins), and -typically all pull requests get automatically tested Jenkins: if your change breaks the build, Jenkins will yell about it in #matrix-dev:matrix.org so please lurk there and keep an eye open. +We use `Jenkins `_ and +`Travis `_ for continuous +integration. All pull requests to synapse get automatically tested by Travis; +the Jenkins builds require an adminstrator to start them. If your change +breaks the build, this will be shown in github, so please keep an eye on the +pull request for feedback. Code style ~~~~~~~~~~ @@ -115,4 +119,4 @@ can't be accepted. Git makes this trivial - just use the -s flag when you do Conclusion ~~~~~~~~~~ -That's it! Matrix is a very open and collaborative project as you might expect given our obsession with open communication. If we're going to successfully matrix together all the fragmented communication technologies out there we are reliant on contributions and collaboration from the community to do so. So please get involved - and we hope you have as much fun hacking on Matrix as we do! \ No newline at end of file +That's it! Matrix is a very open and collaborative project as you might expect given our obsession with open communication. If we're going to successfully matrix together all the fragmented communication technologies out there we are reliant on contributions and collaboration from the community to do so. So please get involved - and we hope you have as much fun hacking on Matrix as we do! diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..565341fee --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM docker.io/python:2-alpine3.7 + +RUN apk add --no-cache --virtual .nacl_deps su-exec build-base libffi-dev zlib-dev libressl-dev libjpeg-turbo-dev linux-headers postgresql-dev libxslt-dev + +COPY . /synapse + +# A wheel cache may be provided in ./cache for faster build +RUN cd /synapse \ + && pip install --upgrade pip setuptools psycopg2 lxml \ + && mkdir -p /synapse/cache \ + && pip install -f /synapse/cache --upgrade --process-dependency-links . \ + && mv /synapse/contrib/docker/start.py /synapse/contrib/docker/conf / \ + && rm -rf setup.py setup.cfg synapse + +VOLUME ["/data"] + +EXPOSE 8008/tcp 8448/tcp + +ENTRYPOINT ["/start.py"] diff --git a/MANIFEST.in b/MANIFEST.in index afb60e12e..e2a6623a6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -25,6 +25,8 @@ recursive-include synapse/static *.js exclude jenkins.sh exclude jenkins*.sh exclude jenkins* +exclude Dockerfile +exclude .dockerignore recursive-exclude jenkins *.sh prune .github diff --git a/README.rst b/README.rst index 76fe2e413..4fe54b0c9 100644 --- a/README.rst +++ b/README.rst @@ -157,8 +157,9 @@ if you prefer. In case of problems, please see the _`Troubleshooting` section below. -Alternatively, Silvio Fricke has contributed a Dockerfile to automate the -above in Docker at https://registry.hub.docker.com/u/silviof/docker-matrix/. +There is an offical synapse image available at https://hub.docker.com/r/matrixdotorg/synapse/tags/ which can be used with the docker-compose file available at `contrib/docker`. Further information on this including configuration options is available in `contrib/docker/README.md`. + +Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a Dockerfile to automate a synapse server in a single Docker image, at https://hub.docker.com/r/avhost/docker-matrix/tags/ Also, Martin Giess has created an auto-deployment process with vagrant/ansible, tested with VirtualBox/AWS/DigitalOcean - see https://github.com/EMnify/matrix-synapse-auto-deploy @@ -354,6 +355,10 @@ https://matrix.org/docs/projects/try-matrix-now.html (or build your own with one Fedora ------ +Synapse is in the Fedora repositories as ``matrix-synapse``:: + + sudo dnf install matrix-synapse + Oleg Girko provides Fedora RPMs at https://obs.infoserver.lv/project/monitor/matrix-synapse @@ -610,6 +615,9 @@ should have the format ``_matrix._tcp. IN SRV 10 0 $ dig -t srv _matrix._tcp.example.com _matrix._tcp.example.com. 3600 IN SRV 10 0 8448 synapse.example.com. +Note that the server hostname cannot be an alias (CNAME record): it has to point +directly to the server hosting the synapse instance. + You can then configure your homeserver to use ```` as the domain in its user-ids, by setting ``server_name``:: @@ -890,6 +898,17 @@ This should end with a 'PASSED' result:: PASSED (successes=143) +Running the Integration Tests +============================= + +Synapse is accompanied by `SyTest `_, +a Matrix homeserver integration testing suite, which uses HTTP requests to +access the API as a Matrix client would. It is able to run Synapse directly from +the source tree, so installation of the server is not required. + +Testing with SyTest is recommended for verifying that changes related to the +Client-Server API are functioning correctly. See the `installation instructions +`_ for details. Building Internal API Documentation =================================== diff --git a/UPGRADE.rst b/UPGRADE.rst index 2efe7ea60..f6bb1070b 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -48,6 +48,18 @@ returned by the Client-Server API: # configured on port 443. curl -kv https:///_matrix/client/versions 2>&1 | grep "Server:" +Upgrading to $NEXT_VERSION +==================== + +This release expands the anonymous usage stats sent if the opt-in +``report_stats`` configuration is set to ``true``. We now capture RSS memory +and cpu use at a very coarse level. This requires administrators to install +the optional ``psutil`` python module. + +We would appreciate it if you could assist by ensuring this module is available +and ``report_stats`` is enabled. This will let us see if performance changes to +synapse are having an impact to the general community. + Upgrading to v0.15.0 ==================== diff --git a/contrib/README.rst b/contrib/README.rst new file mode 100644 index 000000000..c296c5562 --- /dev/null +++ b/contrib/README.rst @@ -0,0 +1,10 @@ +Community Contributions +======================= + +Everything in this directory are projects submitted by the community that may be useful +to others. As such, the project maintainers cannot guarantee support, stability +or backwards compatibility of these projects. + +Files in this directory should *not* be relied on directly, as they may not +continue to work or exist in future. If you wish to use any of these files then +they should be copied to avoid them breaking from underneath you. diff --git a/contrib/docker/README.md b/contrib/docker/README.md new file mode 100644 index 000000000..61592109c --- /dev/null +++ b/contrib/docker/README.md @@ -0,0 +1,153 @@ +# Synapse Docker + +The `matrixdotorg/synapse` Docker image will run Synapse as a single process. It does not provide a +database server or a TURN server, you should run these separately. + +If you run a Postgres server, you should simply include it in the same Compose +project or set the proper environment variables and the image will automatically +use that server. + +## Build + +Build the docker image with the `docker build` command from the root of the synapse repository. + +``` +docker build -t docker.io/matrixdotorg/synapse . +``` + +The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:` where `` is the same as the release tag in the synapse git repository. + +You may have a local Python wheel cache available, in which case copy the relevant packages in the ``cache/`` directory at the root of the project. + +## Run + +This image is designed to run either with an automatically generated configuration +file or with a custom configuration that requires manual edition. + +### Automated configuration + +It is recommended that you use Docker Compose to run your containers, including +this image and a Postgres server. A sample ``docker-compose.yml`` is provided, +including example labels for reverse proxying and other artifacts. + +Read the section about environment variables and set at least mandatory variables, +then run the server: + +``` +docker-compose up -d +``` + +If secrets are not specified in the environment variables, they will be generated +as part of the startup. Please ensure these secrets are kept between launches of the +Docker container, as their loss may require users to log in again. + +### Manual configuration + +A sample ``docker-compose.yml`` is provided, including example labels for +reverse proxying and other artifacts. The docker-compose file is an example, +please comment/uncomment sections that are not suitable for your usecase. + +Specify a ``SYNAPSE_CONFIG_PATH``, preferably to a persistent path, +to use manual configuration. To generate a fresh ``homeserver.yaml``, simply run: + +``` +docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host synapse generate +``` + +Then, customize your configuration and run the server: + +``` +docker-compose up -d +``` + +### Without Compose + +If you do not wish to use Compose, you may still run this image using plain +Docker commands. Note that the following is just a guideline and you may need +to add parameters to the docker run command to account for the network situation +with your postgres database. + +``` +docker run \ + -d \ + --name synapse \ + -v ${DATA_PATH}:/data \ + -e SYNAPSE_SERVER_NAME=my.matrix.host \ + -e SYNAPSE_REPORT_STATS=yes \ + docker.io/matrixdotorg/synapse:latest +``` + +## Volumes + +The image expects a single volume, located at ``/data``, that will hold: + +* temporary files during uploads; +* uploaded media and thumbnails; +* the SQLite database if you do not configure postgres; +* the appservices configuration. + +You are free to use separate volumes depending on storage endpoints at your +disposal. For instance, ``/data/media`` coud be stored on a large but low +performance hdd storage while other files could be stored on high performance +endpoints. + +In order to setup an application service, simply create an ``appservices`` +directory in the data volume and write the application service Yaml +configuration file there. Multiple application services are supported. + +## Environment + +Unless you specify a custom path for the configuration file, a very generic +file will be generated, based on the following environment settings. +These are a good starting point for setting up your own deployment. + +Global settings: + +* ``UID``, the user id Synapse will run as [default 991] +* ``GID``, the group id Synapse will run as [default 991] +* ``SYNAPSE_CONFIG_PATH``, path to a custom config file + +If ``SYNAPSE_CONFIG_PATH`` is set, you should generate a configuration file +then customize it manually. No other environment variable is required. + +Otherwise, a dynamic configuration file will be used. The following environment +variables are available for configuration: + +* ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname. +* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous + statistics reporting back to the Matrix project which helps us to get funding. +* ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if + you run your own TLS-capable reverse proxy). +* ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on + the Synapse instance. +* ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server. +* ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`]. +* ``SYNAPSE_CACHE_FACTOR``, the cache factor [default `0.5`]. +* ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public + key in order to enable recaptcha upon registration. +* ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private + key in order to enable recaptcha upon registration. +* ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN + uris to enable TURN for this homeserver. +* ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required. + +Shared secrets, that will be initialized to random values if not set: + +* ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if + registration is disable. +* ``SYNAPSE_MACAROON_SECRET_KEY`` secret for signing access tokens + to the server. + +Database specific values (will use SQLite if not set): + +* `POSTGRES_DB` - The database name for the synapse postgres database. [default: `synapse`] +* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`] +* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy. +* `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`] + +Mail server specific values (will not send emails if not set): + +* ``SYNAPSE_SMTP_HOST``, hostname to the mail server. +* ``SYNAPSE_SMTP_PORT``, TCP port for accessing the mail server [default ``25``]. +* ``SYNAPSE_SMTP_USER``, username for authenticating against the mail server if any. +* ``SYNAPSE_SMTP_PASSWORD``, password for authenticating against the mail server if any. diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml new file mode 100644 index 000000000..6bc25bb45 --- /dev/null +++ b/contrib/docker/conf/homeserver.yaml @@ -0,0 +1,219 @@ +# vim:ft=yaml + +## TLS ## + +tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt" +tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key" +tls_dh_params_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.dh" +no_tls: {{ "True" if SYNAPSE_NO_TLS else "False" }} +tls_fingerprints: [] + +## Server ## + +server_name: "{{ SYNAPSE_SERVER_NAME }}" +pid_file: /homeserver.pid +web_client: False +soft_file_limit: 0 + +## Ports ## + +listeners: + {% if not SYNAPSE_NO_TLS %} + - + port: 8448 + bind_addresses: ['0.0.0.0'] + type: http + tls: true + x_forwarded: false + resources: + - names: [client] + compress: true + - names: [federation] # Federation APIs + compress: false + {% endif %} + + - port: 8008 + tls: false + bind_addresses: ['0.0.0.0'] + type: http + x_forwarded: false + + resources: + - names: [client] + compress: true + - names: [federation] + compress: false + +## Database ## + +{% if POSTGRES_PASSWORD %} +database: + name: "psycopg2" + args: + user: "{{ POSTGRES_USER or "synapse" }}" + password: "{{ POSTGRES_PASSWORD }}" + database: "{{ POSTGRES_DB or "synapse" }}" + host: "{{ POSTGRES_HOST or "db" }}" + port: "{{ POSTGRES_PORT or "5432" }}" + cp_min: 5 + cp_max: 10 +{% else %} +database: + name: "sqlite3" + args: + database: "/data/homeserver.db" +{% endif %} + +## Performance ## + +event_cache_size: "{{ SYNAPSE_EVENT_CACHE_SIZE or "10K" }}" +verbose: 0 +log_file: "/data/homeserver.log" +log_config: "/compiled/log.config" + +## Ratelimiting ## + +rc_messages_per_second: 0.2 +rc_message_burst_count: 10.0 +federation_rc_window_size: 1000 +federation_rc_sleep_limit: 10 +federation_rc_sleep_delay: 500 +federation_rc_reject_limit: 50 +federation_rc_concurrent: 3 + +## Files ## + +media_store_path: "/data/media" +uploads_path: "/data/uploads" +max_upload_size: "10M" +max_image_pixels: "32M" +dynamic_thumbnails: false + +# List of thumbnail to precalculate when an image is uploaded. +thumbnail_sizes: +- width: 32 + height: 32 + method: crop +- width: 96 + height: 96 + method: crop +- width: 320 + height: 240 + method: scale +- width: 640 + height: 480 + method: scale +- width: 800 + height: 600 + method: scale + +url_preview_enabled: False +max_spider_size: "10M" + +## Captcha ## + +{% if SYNAPSE_RECAPTCHA_PUBLIC_KEY %} +recaptcha_public_key: "{{ SYNAPSE_RECAPTCHA_PUBLIC_KEY }}" +recaptcha_private_key: "{{ SYNAPSE_RECAPTCHA_PRIVATE_KEY }}" +enable_registration_captcha: True +recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" +{% else %} +recaptcha_public_key: "YOUR_PUBLIC_KEY" +recaptcha_private_key: "YOUR_PRIVATE_KEY" +enable_registration_captcha: False +recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" +{% endif %} + +## Turn ## + +{% if SYNAPSE_TURN_URIS %} +turn_uris: +{% for uri in SYNAPSE_TURN_URIS.split(',') %} - "{{ uri }}" +{% endfor %} +turn_shared_secret: "{{ SYNAPSE_TURN_SECRET }}" +turn_user_lifetime: "1h" +turn_allow_guests: True +{% else %} +turn_uris: [] +turn_shared_secret: "YOUR_SHARED_SECRET" +turn_user_lifetime: "1h" +turn_allow_guests: True +{% endif %} + +## Registration ## + +enable_registration: {{ "True" if SYNAPSE_ENABLE_REGISTRATION else "False" }} +registration_shared_secret: "{{ SYNAPSE_REGISTRATION_SHARED_SECRET }}" +bcrypt_rounds: 12 +allow_guest_access: {{ "True" if SYNAPSE_ALLOW_GUEST else "False" }} +enable_group_creation: true + +# The list of identity servers trusted to verify third party +# identifiers by this server. +trusted_third_party_id_servers: + - matrix.org + - vector.im + - riot.im + +## Metrics ### + +{% if SYNAPSE_REPORT_STATS.lower() == "yes" %} +enable_metrics: True +report_stats: True +{% else %} +enable_metrics: False +report_stats: False +{% endif %} + +## API Configuration ## + +room_invite_state_types: + - "m.room.join_rules" + - "m.room.canonical_alias" + - "m.room.avatar" + - "m.room.name" + +{% if SYNAPSE_APPSERVICES %} +app_service_config_files: +{% for appservice in SYNAPSE_APPSERVICES %} - "{{ appservice }}" +{% endfor %} +{% else %} +app_service_config_files: [] +{% endif %} + +macaroon_secret_key: "{{ SYNAPSE_MACAROON_SECRET_KEY }}" +expire_access_token: False + +## Signing Keys ## + +signing_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.signing.key" +old_signing_keys: {} +key_refresh_interval: "1d" # 1 Day. + +# The trusted servers to download signing keys from. +perspectives: + servers: + "matrix.org": + verify_keys: + "ed25519:auto": + key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw" + +password_config: + enabled: true + +{% if SYNAPSE_SMTP_HOST %} +email: + enable_notifs: false + smtp_host: "{{ SYNAPSE_SMTP_HOST }}" + smtp_port: {{ SYNAPSE_SMTP_PORT or "25" }} + smtp_user: "{{ SYNAPSE_SMTP_USER }}" + smtp_pass: "{{ SYNAPSE_SMTP_PASSWORD }}" + require_transport_security: False + notif_from: "{{ SYNAPSE_SMTP_FROM or "hostmaster@" + SYNAPSE_SERVER_NAME }}" + app_name: Matrix + template_dir: res/templates + notif_template_html: notif_mail.html + notif_template_text: notif_mail.txt + notif_for_new_users: True + riot_base_url: "https://{{ SYNAPSE_SERVER_NAME }}" +{% endif %} diff --git a/contrib/docker/conf/log.config b/contrib/docker/conf/log.config new file mode 100644 index 000000000..185199580 --- /dev/null +++ b/contrib/docker/conf/log.config @@ -0,0 +1,29 @@ +version: 1 + +formatters: + precise: + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s' + +filters: + context: + (): synapse.util.logcontext.LoggingContextFilter + request: "" + +handlers: + console: + class: logging.StreamHandler + formatter: precise + filters: [context] + +loggers: + synapse: + level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }} + + synapse.storage.SQL: + # beware: increasing this to DEBUG will make synapse log sensitive + # information such as access tokens. + level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }} + +root: + level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }} + handlers: [console] diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml new file mode 100644 index 000000000..0b531949e --- /dev/null +++ b/contrib/docker/docker-compose.yml @@ -0,0 +1,49 @@ +# This compose file is compatible with Compose itself, it might need some +# adjustments to run properly with stack. + +version: '3' + +services: + + synapse: + image: docker.io/matrixdotorg/synapse:latest + # Since snyapse does not retry to connect to the database, restart upon + # failure + restart: unless-stopped + # See the readme for a full documentation of the environment settings + environment: + - SYNAPSE_SERVER_NAME=my.matrix.host + - SYNAPSE_REPORT_STATS=no + - SYNAPSE_ENABLE_REGISTRATION=yes + - SYNAPSE_LOG_LEVEL=INFO + - POSTGRES_PASSWORD=changeme + volumes: + # You may either store all the files in a local folder + - ./files:/data + # .. or you may split this between different storage points + # - ./files:/data + # - /path/to/ssd:/data/uploads + # - /path/to/large_hdd:/data/media + depends_on: + - db + # In order to expose Synapse, remove one of the following, you might for + # instance expose the TLS port directly: + ports: + - 8448:8448/tcp + # ... or use a reverse proxy, here is an example for traefik: + labels: + - traefik.enable=true + - traefik.frontend.rule=Host:my.matrix.Host + - traefik.port=8448 + + db: + image: docker.io/postgres:10-alpine + # Change that password, of course! + environment: + - POSTGRES_USER=synapse + - POSTGRES_PASSWORD=changeme + volumes: + # You may store the database tables in a local folder.. + - ./schemas:/var/lib/postgresql/data + # .. or store them on some high performance storage for better results + # - /path/to/ssd/storage:/var/lib/postfesql/data diff --git a/contrib/docker/start.py b/contrib/docker/start.py new file mode 100755 index 000000000..90e8b9c51 --- /dev/null +++ b/contrib/docker/start.py @@ -0,0 +1,66 @@ +#!/usr/local/bin/python + +import jinja2 +import os +import sys +import subprocess +import glob + +# Utility functions +convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ)) + +def check_arguments(environ, args): + for argument in args: + if argument not in environ: + print("Environment variable %s is mandatory, exiting." % argument) + sys.exit(2) + +def generate_secrets(environ, secrets): + for name, secret in secrets.items(): + if secret not in environ: + filename = "/data/%s.%s.key" % (environ["SYNAPSE_SERVER_NAME"], name) + if os.path.exists(filename): + with open(filename) as handle: value = handle.read() + else: + print("Generating a random secret for {}".format(name)) + value = os.urandom(32).encode("hex") + with open(filename, "w") as handle: handle.write(value) + environ[secret] = value + +# Prepare the configuration +mode = sys.argv[1] if len(sys.argv) > 1 else None +environ = os.environ.copy() +ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991)) +args = ["python", "-m", "synapse.app.homeserver"] + +# In generate mode, generate a configuration, missing keys, then exit +if mode == "generate": + check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_CONFIG_PATH")) + args += [ + "--server-name", environ["SYNAPSE_SERVER_NAME"], + "--report-stats", environ["SYNAPSE_REPORT_STATS"], + "--config-path", environ["SYNAPSE_CONFIG_PATH"], + "--generate-config" + ] + os.execv("/usr/local/bin/python", args) + +# In normal mode, generate missing keys if any, then run synapse +else: + # Parse the configuration file + if "SYNAPSE_CONFIG_PATH" in environ: + args += ["--config-path", environ["SYNAPSE_CONFIG_PATH"]] + else: + check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS")) + generate_secrets(environ, { + "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET", + "macaroon": "SYNAPSE_MACAROON_SECRET_KEY" + }) + environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") + if not os.path.exists("/compiled"): os.mkdir("/compiled") + convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) + convert("/conf/log.config", "/compiled/log.config", environ) + subprocess.check_output(["chown", "-R", ownership, "/data"]) + args += ["--config-path", "/compiled/homeserver.yaml"] + # Generate missing keys and start synapse + subprocess.check_output(args + ["--generate-keys"]) + os.execv("/sbin/su-exec", ["su-exec", ownership] + args) diff --git a/contrib/graph/graph3.py b/contrib/graph/graph3.py index 88d92c89d..7d3b4d7eb 100644 --- a/contrib/graph/graph3.py +++ b/contrib/graph/graph3.py @@ -22,6 +22,8 @@ import argparse from synapse.events import FrozenEvent from synapse.util.frozenutils import unfreeze +from six import string_types + def make_graph(file_name, room_id, file_prefix, limit): print "Reading lines" @@ -58,7 +60,7 @@ def make_graph(file_name, room_id, file_prefix, limit): for key, value in unfreeze(event.get_dict()["content"]).items(): if value is None: value = "" - elif isinstance(value, basestring): + elif isinstance(value, string_types): pass else: value = json.dumps(value) diff --git a/contrib/prometheus/consoles/synapse.html b/contrib/prometheus/consoles/synapse.html index e23d8a1fc..69aa87f85 100644 --- a/contrib/prometheus/consoles/synapse.html +++ b/contrib/prometheus/consoles/synapse.html @@ -202,11 +202,11 @@ new PromConsole.Graph({

Requests

Requests by Servlet

-
+

 (without EventStreamRestServlet or SyncRestServlet)

-
+