Merge branch 'develop' of github.com:matrix-org/synapse into release-v0.11.0

MANIFEST.in

@@ -20,4 +20,6 @@ recursive-include synapse/static *.gif
 recursive-include synapse/static *.html
 recursive-include synapse/static *.js
 
+exclude jenkins.sh
+
 prune demo/etc

jenkins.sh

@@ -0,0 +1,39 @@
+#!/bin/bash -eu
+
+export PYTHONDONTWRITEBYTECODE=yep
+
+# Output test results as junit xml
+export TRIAL_FLAGS="--reporter=subunit"
+export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml"
+
+# Output coverage to coverage.xml
+export DUMP_COVERAGE_COMMAND="coverage xml -o coverage.xml"
+
+# Output flake8 violations to violations.flake8.log
+# Don't exit with non-0 status code on Jenkins,
+# so that the build steps continue and a later step can decided whether to
+# UNSTABLE or FAILURE this build.
+export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished with status code \$?"
+
+tox
+
+: ${GIT_BRANCH:="$(git rev-parse --abbrev-ref HEAD)"}
+
+set +u
+. .tox/py27/bin/activate
+set -u
+
+rm -rf sytest
+git clone https://github.com/matrix-org/sytest.git sytest
+cd sytest
+
+git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" ; git checkout develop)
+
+: ${PERL5LIB:=$WORKSPACE/perl5/lib/perl5}
+: ${PERL_MB_OPT:=--install_base=$WORKSPACE/perl5}
+: ${PERL_MM_OPT:=INSTALL_BASE=$WORKSPACE/perl5}
+export PERL5LIB PERL_MB_OPT PERL_MM_OPT
+
+./install-deps.pl
+
+./run-tests.pl -O tap --synapse-directory .. --all > results.tap

synapse/api/auth.py

@@ -587,7 +587,10 @@ class Auth(object):
     def _get_user_from_macaroon(self, macaroon_str):
         try:
             macaroon = pymacaroons.Macaroon.deserialize(macaroon_str)
-            self._validate_macaroon(macaroon)
+            self.validate_macaroon(
+                macaroon, "access",
+                [lambda c: c.startswith("time < ")]
+            )
 
             user_prefix = "user_id = "
             user = None
@@ -635,26 +638,25 @@ class Auth(object):
                 errcode=Codes.UNKNOWN_TOKEN
             )
 
-    def _validate_macaroon(self, macaroon):
+    def validate_macaroon(self, macaroon, type_string, additional_validation_functions):
         v = pymacaroons.Verifier()
         v.satisfy_exact("gen = 1")
-        v.satisfy_exact("type = access")
+        v.satisfy_exact("type = " + type_string)
         v.satisfy_general(lambda c: c.startswith("user_id = "))
-        v.satisfy_general(self._verify_expiry)
         v.satisfy_exact("guest = true")
+
+        for validation_function in additional_validation_functions:
+            v.satisfy_general(validation_function)
         v.verify(macaroon, self.hs.config.macaroon_secret_key)
 
         v = pymacaroons.Verifier()
         v.satisfy_general(self._verify_recognizes_caveats)
         v.verify(macaroon, self.hs.config.macaroon_secret_key)
 
-    def _verify_expiry(self, caveat):
+    def verify_expiry(self, caveat):
         prefix = "time < "
         if not caveat.startswith(prefix):
             return False
-        # TODO(daniel): Enable expiry check when clients actually know how to
-        # refresh tokens. (And remember to enable the tests)
-        return True
         expiry = int(caveat[len(prefix):])
         now = self.hs.get_clock().time_msec()
         return now < expiry

synapse/config/cas.py

@@ -27,10 +27,12 @@ class CasConfig(Config):
         if cas_config:
             self.cas_enabled = cas_config.get("enabled", True)
             self.cas_server_url = cas_config["server_url"]
+            self.cas_service_url = cas_config["service_url"]
             self.cas_required_attributes = cas_config.get("required_attributes", {})
         else:
             self.cas_enabled = False
             self.cas_server_url = None
+            self.cas_service_url = None
             self.cas_required_attributes = {}
 
     def default_config(self, config_dir_path, server_name, **kwargs):
@@ -39,6 +41,7 @@ class CasConfig(Config):
         #cas_config:
         #   enabled: true
         #   server_url: "https://cas-server.com"
+        #   service_url: "https://homesever.domain.com:8448"
         #   #required_attributes:
         #   #    name: value
         """

synapse/federation/federation_client.py

@@ -401,6 +401,12 @@ class FederationClient(FederationBase):
 
                 pdu_dict["content"].update(content)
 
+                # The protoevent received over the JSON wire may not have all
+                # the required fields. Lets just gloss over that because
+                # there's some we never care about
+                if "prev_state" not in pdu_dict:
+                    pdu_dict["prev_state"] = []
+
                 defer.returnValue(
                     (destination, self.event_from_pdu_json(pdu_dict))
                 )

synapse/handlers/auth.py

@@ -18,7 +18,7 @@ from twisted.internet import defer
 from ._base import BaseHandler
 from synapse.api.constants import LoginType
 from synapse.types import UserID
-from synapse.api.errors import LoginError, Codes
+from synapse.api.errors import AuthError, LoginError, Codes
 from synapse.util.async import run_on_reactor
 
 from twisted.web.client import PartialDownloadError
@@ -46,6 +46,7 @@ class AuthHandler(BaseHandler):
         }
         self.bcrypt_rounds = hs.config.bcrypt_rounds
         self.sessions = {}
+        self.INVALID_TOKEN_HTTP_STATUS = 401
 
     @defer.inlineCallbacks
     def check_auth(self, flows, clientdict, clientip):
@@ -297,10 +298,11 @@ class AuthHandler(BaseHandler):
         defer.returnValue((user_id, access_token, refresh_token))
 
     @defer.inlineCallbacks
-    def login_with_cas_user_id(self, user_id):
+    def get_login_tuple_for_user_id(self, user_id):
         """
-        Authenticates the user with the given user ID,
-        intended to have been captured from a CAS response
+        Gets login tuple for the user with the given user ID.
+        The user is assumed to have been authenticated by some other
+        machanism (e.g. CAS)
 
         Args:
             user_id (str): User ID
@@ -393,6 +395,23 @@ class AuthHandler(BaseHandler):
         ))
         return m.serialize()
 
+    def generate_short_term_login_token(self, user_id):
+        macaroon = self._generate_base_macaroon(user_id)
+        macaroon.add_first_party_caveat("type = login")
+        now = self.hs.get_clock().time_msec()
+        expiry = now + (2 * 60 * 1000)
+        macaroon.add_first_party_caveat("time < %d" % (expiry,))
+        return macaroon.serialize()
+
+    def validate_short_term_login_token_and_get_user_id(self, login_token):
+        try:
+            macaroon = pymacaroons.Macaroon.deserialize(login_token)
+            auth_api = self.hs.get_auth()
+            auth_api.validate_macaroon(macaroon, "login", [auth_api.verify_expiry])
+            return self._get_user_from_macaroon(macaroon)
+        except (pymacaroons.exceptions.MacaroonException, TypeError, ValueError):
+            raise AuthError(401, "Invalid token", errcode=Codes.UNKNOWN_TOKEN)
+
     def _generate_base_macaroon(self, user_id):
         macaroon = pymacaroons.Macaroon(
             location=self.hs.config.server_name,
@@ -402,6 +421,16 @@ class AuthHandler(BaseHandler):
         macaroon.add_first_party_caveat("user_id = %s" % (user_id,))
         return macaroon
 
+    def _get_user_from_macaroon(self, macaroon):
+        user_prefix = "user_id = "
+        for caveat in macaroon.caveats:
+            if caveat.caveat_id.startswith(user_prefix):
+                return caveat.caveat_id[len(user_prefix):]
+        raise AuthError(
+            self.INVALID_TOKEN_HTTP_STATUS, "No user_id found in token",
+            errcode=Codes.UNKNOWN_TOKEN
+        )
+
     @defer.inlineCallbacks
     def set_password(self, user_id, newpassword):
         password_hash = self.hash(newpassword)

synapse/rest/client/v1/login.py

@@ -22,6 +22,7 @@ from base import ClientV1RestServlet, client_path_pattern
 
 import simplejson as json
 import urllib
+import urlparse
 
 import logging
 from saml2 import BINDING_HTTP_POST
@@ -39,6 +40,7 @@ class LoginRestServlet(ClientV1RestServlet):
     PASS_TYPE = "m.login.password"
     SAML2_TYPE = "m.login.saml2"
     CAS_TYPE = "m.login.cas"
+    TOKEN_TYPE = "m.login.token"
 
     def __init__(self, hs):
         super(LoginRestServlet, self).__init__(hs)
@@ -58,6 +60,7 @@ class LoginRestServlet(ClientV1RestServlet):
             flows.append({"type": LoginRestServlet.CAS_TYPE})
         if self.password_enabled:
             flows.append({"type": LoginRestServlet.PASS_TYPE})
+        flows.append({"type": LoginRestServlet.TOKEN_TYPE})
         return (200, {"flows": flows})
 
     def on_OPTIONS(self, request):
@@ -83,6 +86,7 @@ class LoginRestServlet(ClientV1RestServlet):
                     "uri": "%s%s" % (self.idp_redirect_url, relay_state)
                 }
                 defer.returnValue((200, result))
+            # TODO Delete this after all CAS clients switch to token login instead
             elif self.cas_enabled and (login_submission["type"] ==
                                        LoginRestServlet.CAS_TYPE):
                 # TODO: get this from the homeserver rather than creating a new one for
@@ -96,6 +100,9 @@ class LoginRestServlet(ClientV1RestServlet):
                 body = yield http_client.get_raw(uri, args)
                 result = yield self.do_cas_login(body)
                 defer.returnValue(result)
+            elif login_submission["type"] == LoginRestServlet.TOKEN_TYPE:
+                result = yield self.do_token_login(login_submission)
+                defer.returnValue(result)
             else:
                 raise SynapseError(400, "Bad login type.")
         except KeyError:
@@ -131,6 +138,26 @@ class LoginRestServlet(ClientV1RestServlet):
 
         defer.returnValue((200, result))
 
+    @defer.inlineCallbacks
+    def do_token_login(self, login_submission):
+        token = login_submission['token']
+        auth_handler = self.handlers.auth_handler
+        user_id = (
+            yield auth_handler.validate_short_term_login_token_and_get_user_id(token)
+        )
+        user_id, access_token, refresh_token = (
+            yield auth_handler.get_login_tuple_for_user_id(user_id)
+        )
+        result = {
+            "user_id": user_id,  # may have changed
+            "access_token": access_token,
+            "refresh_token": refresh_token,
+            "home_server": self.hs.hostname,
+        }
+
+        defer.returnValue((200, result))
+
+    # TODO Delete this after all CAS clients switch to token login instead
     @defer.inlineCallbacks
     def do_cas_login(self, cas_response_body):
         user, attributes = self.parse_cas_response(cas_response_body)
@@ -152,7 +179,7 @@ class LoginRestServlet(ClientV1RestServlet):
         user_exists = yield auth_handler.does_user_exist(user_id)
         if user_exists:
             user_id, access_token, refresh_token = (
-                yield auth_handler.login_with_cas_user_id(user_id)
+                yield auth_handler.get_login_tuple_for_user_id(user_id)
             )
             result = {
                 "user_id": user_id,  # may have changed
@@ -173,6 +200,7 @@ class LoginRestServlet(ClientV1RestServlet):
 
         defer.returnValue((200, result))
 
+    # TODO Delete this after all CAS clients switch to token login instead
     def parse_cas_response(self, cas_response_body):
         root = ET.fromstring(cas_response_body)
         if not root.tag.endswith("serviceResponse"):
@@ -243,6 +271,7 @@ class SAML2RestServlet(ClientV1RestServlet):
         defer.returnValue((200, {"status": "not_authenticated"}))
 
 
+# TODO Delete this after all CAS clients switch to token login instead
 class CasRestServlet(ClientV1RestServlet):
     PATTERN = client_path_pattern("/login/cas")
 
@@ -254,6 +283,115 @@ class CasRestServlet(ClientV1RestServlet):
         return (200, {"serverUrl": self.cas_server_url})
 
 
+class CasRedirectServlet(ClientV1RestServlet):
+    PATTERN = client_path_pattern("/login/cas/redirect")
+
+    def __init__(self, hs):
+        super(CasRedirectServlet, self).__init__(hs)
+        self.cas_server_url = hs.config.cas_server_url
+        self.cas_service_url = hs.config.cas_service_url
+
+    def on_GET(self, request):
+        args = request.args
+        if "redirectUrl" not in args:
+            return (400, "Redirect URL not specified for CAS auth")
+        client_redirect_url_param = urllib.urlencode({
+            "redirectUrl": args["redirectUrl"][0]
+        })
+        hs_redirect_url = self.cas_service_url + "/_matrix/client/api/v1/login/cas/ticket"
+        service_param = urllib.urlencode({
+            "service": "%s?%s" % (hs_redirect_url, client_redirect_url_param)
+        })
+        request.redirect("%s?%s" % (self.cas_server_url, service_param))
+        request.finish()
+
+
+class CasTicketServlet(ClientV1RestServlet):
+    PATTERN = client_path_pattern("/login/cas/ticket")
+
+    def __init__(self, hs):
+        super(CasTicketServlet, self).__init__(hs)
+        self.cas_server_url = hs.config.cas_server_url
+        self.cas_service_url = hs.config.cas_service_url
+        self.cas_required_attributes = hs.config.cas_required_attributes
+
+    @defer.inlineCallbacks
+    def on_GET(self, request):
+        client_redirect_url = request.args["redirectUrl"][0]
+        http_client = self.hs.get_simple_http_client()
+        uri = self.cas_server_url + "/proxyValidate"
+        args = {
+            "ticket": request.args["ticket"],
+            "service": self.cas_service_url
+        }
+        body = yield http_client.get_raw(uri, args)
+        result = yield self.handle_cas_response(request, body, client_redirect_url)
+        defer.returnValue(result)
+
+    @defer.inlineCallbacks
+    def handle_cas_response(self, request, cas_response_body, client_redirect_url):
+        user, attributes = self.parse_cas_response(cas_response_body)
+
+        for required_attribute, required_value in self.cas_required_attributes.items():
+            # If required attribute was not in CAS Response - Forbidden
+            if required_attribute not in attributes:
+                raise LoginError(401, "Unauthorized", errcode=Codes.UNAUTHORIZED)
+
+            # Also need to check value
+            if required_value is not None:
+                actual_value = attributes[required_attribute]
+                # If required attribute value does not match expected - Forbidden
+                if required_value != actual_value:
+                    raise LoginError(401, "Unauthorized", errcode=Codes.UNAUTHORIZED)
+
+        user_id = UserID.create(user, self.hs.hostname).to_string()
+        auth_handler = self.handlers.auth_handler
+        user_exists = yield auth_handler.does_user_exist(user_id)
+        if not user_exists:
+            user_id, _ = (
+                yield self.handlers.registration_handler.register(localpart=user)
+            )
+
+        login_token = auth_handler.generate_short_term_login_token(user_id)
+        redirect_url = self.add_login_token_to_redirect_url(client_redirect_url,
+                                                            login_token)
+        request.redirect(redirect_url)
+        request.finish()
+
+    def add_login_token_to_redirect_url(self, url, token):
+        url_parts = list(urlparse.urlparse(url))
+        query = dict(urlparse.parse_qsl(url_parts[4]))
+        query.update({"loginToken": token})
+        url_parts[4] = urllib.urlencode(query)
+        return urlparse.urlunparse(url_parts)
+
+    def parse_cas_response(self, cas_response_body):
+        root = ET.fromstring(cas_response_body)
+        if not root.tag.endswith("serviceResponse"):
+            raise LoginError(401, "Invalid CAS response", errcode=Codes.UNAUTHORIZED)
+        if not root[0].tag.endswith("authenticationSuccess"):
+            raise LoginError(401, "Unsuccessful CAS response", errcode=Codes.UNAUTHORIZED)
+        for child in root[0]:
+            if child.tag.endswith("user"):
+                user = child.text
+            if child.tag.endswith("attributes"):
+                attributes = {}
+                for attribute in child:
+                    # ElementTree library expands the namespace in attribute tags
+                    # to the full URL of the namespace.
+                    # See (https://docs.python.org/2/library/xml.etree.elementtree.html)
+                    # We don't care about namespace here and it will always be encased in
+                    # curly braces, so we remove them.
+                    if "}" in attribute.tag:
+                        attributes[attribute.tag.split("}")[1]] = attribute.text
+                    else:
+                        attributes[attribute.tag] = attribute.text
+        if user is None or attributes is None:
+            raise LoginError(401, "Invalid CAS response", errcode=Codes.UNAUTHORIZED)
+
+        return (user, attributes)
+
+
 def _parse_json(request):
     try:
         content = json.loads(request.content.read())
@@ -269,5 +407,7 @@ def register_servlets(hs, http_server):
     if hs.config.saml2_enabled:
         SAML2RestServlet(hs).register(http_server)
     if hs.config.cas_enabled:
+        CasRedirectServlet(hs).register(http_server)
+        CasTicketServlet(hs).register(http_server)
         CasRestServlet(hs).register(http_server)
     # TODO PasswordResetRestServlet(hs).register(http_server)

synapse/rest/client/v1/room.py

@@ -17,7 +17,7 @@
 from twisted.internet import defer
 
 from base import ClientV1RestServlet, client_path_pattern
-from synapse.api.errors import SynapseError, Codes
+from synapse.api.errors import SynapseError, Codes, AuthError
 from synapse.streams.config import PaginationConfig
 from synapse.api.constants import EventTypes, Membership
 from synapse.types import UserID, RoomID, RoomAlias
@@ -453,7 +453,13 @@ class RoomMembershipRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_POST(self, request, room_id, membership_action, txn_id=None):
-        user, token_id, _ = yield self.auth.get_user_by_req(request)
+        user, token_id, is_guest = yield self.auth.get_user_by_req(
+            request,
+            allow_guest=True
+        )
+
+        if is_guest and membership_action not in {Membership.JOIN, Membership.LEAVE}:
+            raise AuthError(403, "Guest access not allowed")
 
         content = _parse_json(request)
 
@@ -486,16 +492,21 @@ class RoomMembershipRestServlet(ClientV1RestServlet):
 
         msg_handler = self.handlers.message_handler
 
+        content = {"membership": unicode(membership_action)}
+        if is_guest:
+            content["kind"] = "guest"
+
         yield msg_handler.create_and_send_event(
             {
                 "type": EventTypes.Member,
-                "content": {"membership": unicode(membership_action)},
+                "content": content,
                 "room_id": room_id,
                 "sender": user.to_string(),
                 "state_key": state_key,
             },
             token_id=token_id,
             txn_id=txn_id,
+            is_guest=is_guest,
         )
 
         defer.returnValue((200, {}))

synapse/static/client/login/js/login.js

@@ -17,12 +17,11 @@ var submitPassword = function(user, pwd) {
     }).error(errorFunc);
 };
 
-var submitCas = function(ticket, service) {
-    console.log("Logging in with cas...");
+var submitToken = function(loginToken) {
+    console.log("Logging in with login token...");
     var data = {
-        type: "m.login.cas",
-        ticket: ticket,
-        service: service,
+        type: "m.login.token",
+        token: loginToken
     };
     $.post(matrixLogin.endpoint, JSON.stringify(data), function(response) {
         show_login();
@@ -41,23 +40,10 @@ var errorFunc = function(err) {
     }
 };
 
-var getCasURL = function(cb) {
-    $.get(matrixLogin.endpoint + "/cas", function(response) {
-        var cas_url = response.serverUrl;
-
-        cb(cas_url);
-    }).error(errorFunc);
-};
-
-
 var gotoCas = function() {
-    getCasURL(function(cas_url) {
-        var this_page = window.location.origin + window.location.pathname;
-
-        var redirect_url = cas_url + "/login?service=" + encodeURIComponent(this_page);
-
-        window.location.replace(redirect_url);
-    });
+    var this_page = window.location.origin + window.location.pathname;
+    var redirect_url = matrixLogin.endpoint + "/cas/redirect?redirectUrl=" + encodeURIComponent(this_page);
+    window.location.replace(redirect_url);
 }
 
 var setFeedbackString = function(text) {
@@ -111,7 +97,7 @@ var fetch_info = function(cb) {
 
 matrixLogin.onLoad = function() {
     fetch_info(function() {
-        if (!try_cas()) {
+        if (!try_token()) {
             show_login();
         }
     });
@@ -148,20 +134,20 @@ var parseQsFromUrl = function(query) {
     return result;
 };
 
-var try_cas = function() {
+var try_token = function() {
     var pos = window.location.href.indexOf("?");
     if (pos == -1) {
         return false;
     }
     var qs = parseQsFromUrl(window.location.href.substr(pos+1));
 
-    var ticket = qs.ticket;
+    var loginToken = qs.loginToken;
 
-    if (!ticket) {
+    if (!loginToken) {
         return false;
     }
 
-    submitCas(ticket, location.origin);
+    submitToken(loginToken);
 
     return true;
 };

tox.ini

@@ -6,11 +6,13 @@ deps =
     coverage
     Twisted>=15.1
     mock
+    python-subunit
+    junitxml
 setenv =
     PYTHONDONTWRITEBYTECODE = no_byte_code
 commands =
-    coverage run --source=synapse {envbindir}/trial {posargs:tests}
-    coverage report -m
+    /bin/bash -c "coverage run --source=synapse {envbindir}/trial {env:TRIAL_FLAGS:} {posargs:tests} {env:TOXSUFFIX:}"
+    {env:DUMP_COVERAGE_COMMAND:coverage report -m}
 
 [testenv:packaging]
 deps =
@@ -23,4 +25,4 @@ skip_install = True
 basepython = python2.7
 deps =
     flake8
-commands = flake8 synapse
+commands = /bin/bash -c "flake8 synapse {env:PEP8SUFFIX:}"