Merge branch 'develop' into media_repository

CHANGES.rst

@@ -1,3 +1,39 @@
+Changes in synapse 0.5.4 (2014-12-03)
+=====================================
+
+ * Fix presence bug where some rooms did not display presence updates for
+   remote users.
+ * Do not log SQL timing log lines when started with "-v"
+ * Fix potential memory leak.
+
+Changes in synapse 0.5.3c (2014-12-02)
+======================================
+
+ * Change the default value for the `content_addr` option to use the HTTP
+   listener, as by default the HTTPS listener will be using a self-signed
+   certificate.
+
+Changes in synapse 0.5.3 (2014-11-27)
+=====================================
+
+ * Fix bug that caused joining a remote room to fail if a single event was not
+   signed correctly.
+ * Fix bug which caused servers to continuously try and fetch events from other
+   servers.
+
+Changes in synapse 0.5.2 (2014-11-26)
+=====================================
+
+Fix major bug that caused rooms to disappear from peoples initial sync.
+
+Changes in synapse 0.5.1 (2014-11-26)
+=====================================
+See UPGRADES.rst for specific instructions on how to upgrade.
+
+ * Fix bug where we served up an Event that did not match its signatures.
+ * Fix regression where we no longer correctly handled the case where a
+   homeserver receives an event for a room it doesn't recognise (but is in.)
+
 Changes in synapse 0.5.0 (2014-11-19)
 =====================================
 This release includes changes to the federation protocol and client-server API

README.rst

@@ -94,7 +94,8 @@ header files for python C extensions.
 Installing prerequisites on Ubuntu or Debian::
 
     $ sudo apt-get install build-essential python2.7-dev libffi-dev \
-                           python-pip python-setuptools
+                           python-pip python-setuptools sqlite3 \
+                           libssl-dev
 
 Installing prerequisites on Mac OS X::
 
@@ -129,6 +130,9 @@ failing, e.g.::
 
     $ pip install --user twisted
 
+On OSX, if you encounter clang: error: unknown argument: '-mno-fused-madd' you
+will need to export CFLAGS=-Qunused-arguments.
+
 Running Your Homeserver
 =======================
 

VERSION

@@ -1 +1 @@
-0.5.0
+0.5.4

contrib/jitsimeetbridge/jitsimeetbridge.py

@@ -0,0 +1,260 @@
+#!/usr/bin/env python
+
+"""
+This is an attempt at bridging matrix clients into a Jitis meet room via Matrix
+video call.  It uses hard-coded xml strings overg XMPP BOSH. It can display one
+of the streams from the Jitsi bridge until the second lot of SDP comes down and
+we set the remote SDP at which point the stream ends. Our video never gets to
+the bridge.
+
+Requires:
+npm install jquery jsdom 
+"""
+
+import gevent
+import grequests
+from BeautifulSoup import BeautifulSoup
+import json
+import urllib
+import subprocess
+import time
+
+#ACCESS_TOKEN="" #
+
+MATRIXBASE = 'https://matrix.org/_matrix/client/api/v1/'
+MYUSERNAME = '@davetest:matrix.org'
+
+HTTPBIND = 'https://meet.jit.si/http-bind'
+#HTTPBIND = 'https://jitsi.vuc.me/http-bind'
+#ROOMNAME = "matrix"
+ROOMNAME = "pibble"
+
+HOST="guest.jit.si"
+#HOST="jitsi.vuc.me"
+
+TURNSERVER="turn.guest.jit.si"
+#TURNSERVER="turn.jitsi.vuc.me"
+
+ROOMDOMAIN="meet.jit.si"
+#ROOMDOMAIN="conference.jitsi.vuc.me"
+
+class TrivialMatrixClient:
+	def __init__(self, access_token):
+		self.token = None
+		self.access_token = access_token
+
+	def getEvent(self):
+		while True:
+			url = MATRIXBASE+'events?access_token='+self.access_token+"&timeout=60000"
+			if self.token:
+				url += "&from="+self.token
+			req = grequests.get(url)
+			resps = grequests.map([req])
+			obj = json.loads(resps[0].content)
+			print "incoming from matrix",obj
+			if 'end' not in obj:
+				continue
+			self.token = obj['end']
+			if len(obj['chunk']):
+				return obj['chunk'][0]
+
+	def joinRoom(self, roomId):
+		url = MATRIXBASE+'rooms/'+roomId+'/join?access_token='+self.access_token
+		print url
+		headers={ 'Content-Type': 'application/json' }
+		req = grequests.post(url, headers=headers, data='{}')
+		resps = grequests.map([req])
+		obj = json.loads(resps[0].content)
+		print "response: ",obj
+
+	def sendEvent(self, roomId, evType, event):
+		url = MATRIXBASE+'rooms/'+roomId+'/send/'+evType+'?access_token='+self.access_token
+		print url
+		print json.dumps(event)
+		headers={ 'Content-Type': 'application/json' }
+		req = grequests.post(url, headers=headers, data=json.dumps(event))
+		resps = grequests.map([req])
+		obj = json.loads(resps[0].content)
+		print "response: ",obj
+
+
+
+xmppClients = {}
+
+
+def matrixLoop():
+	while True:
+		ev = matrixCli.getEvent()
+		print ev
+		if ev['type'] == 'm.room.member':
+			print 'membership event'
+			if ev['membership'] == 'invite' and ev['state_key'] == MYUSERNAME:
+				roomId = ev['room_id']
+				print "joining room %s" % (roomId)
+				matrixCli.joinRoom(roomId)
+		elif ev['type'] == 'm.room.message':
+			if ev['room_id'] in xmppClients:
+				print "already have a bridge for that user, ignoring"
+				continue
+			print "got message, connecting"
+			xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id'])
+			gevent.spawn(xmppClients[ev['room_id']].xmppLoop)
+		elif ev['type'] == 'm.call.invite':
+			print "Incoming call"
+			#sdp = ev['content']['offer']['sdp']
+			#print "sdp: %s" % (sdp)
+			#xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id'])
+			#gevent.spawn(xmppClients[ev['room_id']].xmppLoop)
+		elif ev['type'] == 'm.call.answer':
+			print "Call answered"
+			sdp = ev['content']['answer']['sdp']
+			if ev['room_id'] not in xmppClients:
+				print "We didn't have a call for that room"
+				continue
+			# should probably check call ID too
+			xmppCli = xmppClients[ev['room_id']]
+			xmppCli.sendAnswer(sdp)
+		elif ev['type'] == 'm.call.hangup':
+			if ev['room_id'] in xmppClients:
+				xmppClients[ev['room_id']].stop()
+				del xmppClients[ev['room_id']]
+			
+class TrivialXmppClient:
+	def __init__(self, matrixRoom, userId):
+		self.rid = 0
+		self.matrixRoom = matrixRoom
+		self.userId = userId
+		self.running = True
+
+	def stop(self):
+		self.running = False
+
+	def nextRid(self):
+		self.rid += 1
+		return '%d' % (self.rid)
+
+	def sendIq(self, xml):
+		fullXml = "<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s'>%s</body>" % (self.nextRid(), self.sid, xml)
+		#print "\t>>>%s" % (fullXml)
+		return self.xmppPoke(fullXml)
+		
+	def xmppPoke(self, xml):
+		headers = {'Content-Type': 'application/xml'}
+		req = grequests.post(HTTPBIND, verify=False, headers=headers, data=xml)
+		resps = grequests.map([req])
+		obj = BeautifulSoup(resps[0].content)
+		return obj
+
+	def sendAnswer(self, answer):
+		print "sdp from matrix client",answer
+		p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--sdp'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+		jingle, out_err = p.communicate(answer)
+		jingle = jingle % {
+			'tojid': self.callfrom,
+			'action': 'session-accept',
+			'initiator': self.callfrom,
+			'responder': self.jid,
+			'sid': self.callsid
+		}
+		print "answer jingle from sdp",jingle
+		res = self.sendIq(jingle)
+		print "reply from answer: ",res
+		
+		self.ssrcs = {}
+		jingleSoup = BeautifulSoup(jingle)
+		for cont in jingleSoup.iq.jingle.findAll('content'):
+			if cont.description:
+				self.ssrcs[cont['name']] = cont.description['ssrc']
+		print "my ssrcs:",self.ssrcs
+
+		gevent.joinall([
+ 		       gevent.spawn(self.advertiseSsrcs)
+		])
+		
+	def advertiseSsrcs(self):
+                time.sleep(7)
+		print "SSRC spammer started"
+		while self.running:
+			ssrcMsg = "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>" % { 'tojid': "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid), 'nick': self.userId, 'assrc': self.ssrcs['audio'], 'vssrc': self.ssrcs['video'] }
+			res = self.sendIq(ssrcMsg)
+			print "reply from ssrc announce: ",res
+			time.sleep(10)
+		
+		
+
+	def xmppLoop(self):
+		self.matrixCallId = time.time()
+		res = self.xmppPoke("<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' to='%s' xml:lang='en' wait='60' hold='1' content='text/xml; charset=utf-8' ver='1.6' xmpp:version='1.0' xmlns:xmpp='urn:xmpp:xbosh'/>" % (self.nextRid(), HOST))
+
+		print res
+		self.sid = res.body['sid']
+		print "sid %s" % (self.sid)
+
+		res = self.sendIq("<auth xmlns='urn:ietf:params:xml:ns:xmpp-sasl' mechanism='ANONYMOUS'/>")
+
+		res = self.xmppPoke("<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s' to='%s' xml:lang='en' xmpp:restart='true' xmlns:xmpp='urn:xmpp:xbosh'/>" % (self.nextRid(), self.sid, HOST))
+	
+		res = self.sendIq("<iq type='set' id='_bind_auth_2' xmlns='jabber:client'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'/></iq>")
+		print res
+
+		self.jid = res.body.iq.bind.jid.string
+		print "jid: %s" % (self.jid)
+		self.shortJid = self.jid.split('-')[0]
+
+		res = self.sendIq("<iq type='set' id='_session_auth_2' xmlns='jabber:client'><session xmlns='urn:ietf:params:xml:ns:xmpp-session'/></iq>")
+
+		#randomthing = res.body.iq['to']
+		#whatsitpart = randomthing.split('-')[0]
+
+		#print "other random bind thing: %s" % (randomthing)
+
+		# advertise preence to the jitsi room, with our nick
+		res = self.sendIq("<iq type='get' to='%s' xmlns='jabber:client' id='1:sendIQ'><services xmlns='urn:xmpp:extdisco:1'><service host='%s'/></services></iq><presence to='%s@%s/d98f6c40' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%s</nick></presence>" % (HOST, TURNSERVER, ROOMNAME, ROOMDOMAIN, self.userId))
+		self.muc = {'users': []}
+		for p in res.body.findAll('presence'):
+			u = {}
+			u['shortJid'] = p['from'].split('/')[1]
+			if p.c and p.c.nick:
+				u['nick'] = p.c.nick.string
+			self.muc['users'].append(u)
+		print "muc: ",self.muc
+
+		# wait for stuff
+		while True:
+			print "waiting..."
+			res = self.sendIq("")
+			print "got from stream: ",res
+			if res.body.iq:
+				jingles = res.body.iq.findAll('jingle')
+				if len(jingles):
+					self.callfrom = res.body.iq['from']
+					self.handleInvite(jingles[0])
+			elif 'type' in res.body and res.body['type'] == 'terminate':
+				self.running = False
+				del xmppClients[self.matrixRoom]
+            			return 
+
+	def handleInvite(self, jingle):
+		self.initiator = jingle['initiator']
+		self.callsid = jingle['sid']
+		p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--jingle'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+		print "raw jingle invite",str(jingle)
+		sdp, out_err = p.communicate(str(jingle))
+		print "transformed remote offer sdp",sdp
+		inviteEvent = {
+			'offer': {
+				'type': 'offer',
+				'sdp': sdp
+			},
+			'call_id': self.matrixCallId,
+			'version': 0,
+			'lifetime': 30000
+		}
+		matrixCli.sendEvent(self.matrixRoom, 'm.call.invite', inviteEvent)
+		
+matrixCli = TrivialMatrixClient(ACCESS_TOKEN)
+
+gevent.joinall([
+	gevent.spawn(matrixLoop)
+])
+

contrib/jitsimeetbridge/syweb-jitsi-conference.patch

@@ -0,0 +1,188 @@
+diff --git a/syweb/webclient/app/components/matrix/matrix-call.js b/syweb/webclient/app/components/matrix/matrix-call.js
+index 9fbfff0..dc68077 100644
+--- a/syweb/webclient/app/components/matrix/matrix-call.js
++++ b/syweb/webclient/app/components/matrix/matrix-call.js
+@@ -16,6 +16,45 @@ limitations under the License.
+ 
+ 'use strict';
+ 
++
++function sendKeyframe(pc) {
++    console.log('sendkeyframe', pc.iceConnectionState);
++    if (pc.iceConnectionState !== 'connected') return; // safe...
++    pc.setRemoteDescription(
++        pc.remoteDescription,
++        function () {
++            pc.createAnswer(
++                function (modifiedAnswer) {
++                    pc.setLocalDescription(
++                        modifiedAnswer,
++                        function () {
++                            // noop
++                        },
++                        function (error) {
++                            console.log('triggerKeyframe setLocalDescription failed', error);
++                            messageHandler.showError();
++                        }
++                    );
++                },
++                function (error) {
++                    console.log('triggerKeyframe createAnswer failed', error);
++                    messageHandler.showError();
++                }
++            );
++        },
++        function (error) {
++            console.log('triggerKeyframe setRemoteDescription failed', error);
++            messageHandler.showError();
++        }
++    );
++} 
++
++
++
++
++
++
++
+ var forAllVideoTracksOnStream = function(s, f) {
+     var tracks = s.getVideoTracks();
+     for (var i = 0; i < tracks.length; i++) {
+@@ -83,7 +122,7 @@ angular.module('MatrixCall', [])
+     }
+ 
+     // FIXME: we should prevent any calls from being placed or accepted before this has finished
+-    MatrixCall.getTurnServer();
++    //MatrixCall.getTurnServer();
+ 
+     MatrixCall.CALL_TIMEOUT = 60000;
+     MatrixCall.FALLBACK_STUN_SERVER = 'stun:stun.l.google.com:19302';
+@@ -132,6 +171,22 @@ angular.module('MatrixCall', [])
+         pc.onsignalingstatechange = function() { self.onSignallingStateChanged(); };
+         pc.onicecandidate = function(c) { self.gotLocalIceCandidate(c); };
+         pc.onaddstream = function(s) { self.onAddStream(s); };
++
++        var datachan = pc.createDataChannel('RTCDataChannel', {
++            reliable: false
++        });
++        console.log("data chan: "+datachan);
++        datachan.onopen = function() {
++            console.log("data channel open");
++        };
++        datachan.onmessage = function() {
++            console.log("data channel message");
++        };
++        pc.ondatachannel = function(event) {
++            console.log("have data channel");
++            event.channel.binaryType = 'blob';
++        };
++
+         return pc;
+     }
+ 
+@@ -200,6 +255,12 @@ angular.module('MatrixCall', [])
+         }, this.msg.lifetime - event.age);
+     };
+ 
++    MatrixCall.prototype.receivedInvite = function(event) {
++        console.log("Got second invite for call "+this.call_id);
++        this.peerConn.setRemoteDescription(new RTCSessionDescription(this.msg.offer), this.onSetRemoteDescriptionSuccess, this.onSetRemoteDescriptionError);
++    };
++    
++
+     // perverse as it may seem, sometimes we want to instantiate a call with a hangup message
+     // (because when getting the state of the room on load, events come in reverse order and
+     // we want to remember that a call has been hung up)
+@@ -349,7 +410,7 @@ angular.module('MatrixCall', [])
+             'mandatory': {
+                 'OfferToReceiveAudio': true,
+                 'OfferToReceiveVideo': this.type == 'video'
+-            },
++            }
+         };
+         this.peerConn.createAnswer(function(d) { self.createdAnswer(d); }, function(e) {}, constraints);
+         // This can't be in an apply() because it's called by a predecessor call under glare conditions :(
+@@ -359,8 +420,20 @@ angular.module('MatrixCall', [])
+     MatrixCall.prototype.gotLocalIceCandidate = function(event) {
+         if (event.candidate) {
+             console.log("Got local ICE "+event.candidate.sdpMid+" candidate: "+event.candidate.candidate);
+-            this.sendCandidate(event.candidate);
+-        }
++            //this.sendCandidate(event.candidate);
++        } else {
++            console.log("have all candidates, sending answer");
++            var content = {
++                version: 0,
++                call_id: this.call_id,
++                answer: this.peerConn.localDescription
++            };
++            this.sendEventWithRetry('m.call.answer', content);
++            var self = this;
++            $rootScope.$apply(function() {
++                self.state = 'connecting';
++            });
++	}
+     }
+ 
+     MatrixCall.prototype.gotRemoteIceCandidate = function(cand) {
+@@ -418,15 +491,6 @@ angular.module('MatrixCall', [])
+         console.log("Created answer: "+description);
+         var self = this;
+         this.peerConn.setLocalDescription(description, function() {
+-            var content = {
+-                version: 0,
+-                call_id: self.call_id,
+-                answer: self.peerConn.localDescription
+-            };
+-            self.sendEventWithRetry('m.call.answer', content);
+-            $rootScope.$apply(function() {
+-                self.state = 'connecting';
+-            });
+         }, function() { console.log("Error setting local description!"); } );
+     };
+ 
+@@ -448,6 +512,9 @@ angular.module('MatrixCall', [])
+             $rootScope.$apply(function() {
+                 self.state = 'connected';
+                 self.didConnect = true;
++		/*$timeout(function() {
++                    sendKeyframe(self.peerConn);
++                }, 1000);*/
+             });
+         } else if (this.peerConn.iceConnectionState == 'failed') {
+             this.hangup('ice_failed');
+@@ -518,6 +585,7 @@ angular.module('MatrixCall', [])
+ 
+     MatrixCall.prototype.onRemoteStreamEnded = function(event) {
+         console.log("Remote stream ended");
++        return;
+         var self = this;
+         $rootScope.$apply(function() {
+             self.state = 'ended';
+diff --git a/syweb/webclient/app/components/matrix/matrix-phone-service.js b/syweb/webclient/app/components/matrix/matrix-phone-service.js
+index 55dbbf5..272fa27 100644
+--- a/syweb/webclient/app/components/matrix/matrix-phone-service.js
++++ b/syweb/webclient/app/components/matrix/matrix-phone-service.js
+@@ -48,6 +48,13 @@ angular.module('matrixPhoneService', [])
+                 return;
+             }
+ 
++            // do we already have an entry for this call ID?
++            var existingEntry = matrixPhoneService.allCalls[msg.call_id];
++            if (existingEntry) {
++                 existingEntry.receivedInvite(msg);
++                 return;
++            }
++
+             var call = undefined;
+             if (!isLive) {
+                 // if this event wasn't live then this call may already be over
+@@ -108,7 +115,7 @@ angular.module('matrixPhoneService', [])
+                     call.hangup();
+                 }
+             } else {
+-                $rootScope.$broadcast(matrixPhoneService.INCOMING_CALL_EVENT, call);
++                 $rootScope.$broadcast(matrixPhoneService.INCOMING_CALL_EVENT, call);
+             }
+         } else if (event.type == 'm.call.answer') {
+             var call = matrixPhoneService.allCalls[msg.call_id];

contrib/jitsimeetbridge/unjingle/strophe.jingle.sdp.js

@@ -0,0 +1,712 @@
+/* jshint -W117 */
+// SDP STUFF
+function SDP(sdp) {
+    this.media = sdp.split('\r\nm=');
+    for (var i = 1; i < this.media.length; i++) {
+        this.media[i] = 'm=' + this.media[i];
+        if (i != this.media.length - 1) {
+            this.media[i] += '\r\n';
+        }
+    }
+    this.session = this.media.shift() + '\r\n';
+    this.raw = this.session + this.media.join('');
+}
+
+exports.SDP = SDP;
+
+var jsdom          = require("jsdom");
+var window = jsdom.jsdom().parentWindow;
+var $ = require('jquery')(window);
+
+var SDPUtil = require('./strophe.jingle.sdp.util.js').SDPUtil;
+
+/**
+ * Returns map of MediaChannel mapped per channel idx.
+ */
+SDP.prototype.getMediaSsrcMap = function() {
+    var self = this;
+    var media_ssrcs = {};
+    for (channelNum = 0; channelNum < self.media.length; channelNum++) {
+        modified = true;
+        tmp = SDPUtil.find_lines(self.media[channelNum], 'a=ssrc:');
+        var type = SDPUtil.parse_mid(SDPUtil.find_line(self.media[channelNum], 'a=mid:'));
+        var channel = new MediaChannel(channelNum, type);
+        media_ssrcs[channelNum] = channel;
+        tmp.forEach(function (line) {
+            var linessrc = line.substring(7).split(' ')[0];
+            // allocate new ChannelSsrc
+            if(!channel.ssrcs[linessrc]) {
+                channel.ssrcs[linessrc] = new ChannelSsrc(linessrc, type);
+            }
+            channel.ssrcs[linessrc].lines.push(line);
+        });
+        tmp = SDPUtil.find_lines(self.media[channelNum], 'a=ssrc-group:');
+        tmp.forEach(function(line){
+            var semantics = line.substr(0, idx).substr(13);
+            var ssrcs = line.substr(14 + semantics.length).split(' ');
+            if (ssrcs.length != 0) {
+                var ssrcGroup = new ChannelSsrcGroup(semantics, ssrcs);
+                channel.ssrcGroups.push(ssrcGroup);
+            }
+        });
+    }
+    return media_ssrcs;
+};
+/**
+ * Returns <tt>true</tt> if this SDP contains given SSRC.
+ * @param ssrc the ssrc to check.
+ * @returns {boolean} <tt>true</tt> if this SDP contains given SSRC.
+ */
+SDP.prototype.containsSSRC = function(ssrc) {
+    var channels = this.getMediaSsrcMap();
+    var contains = false;
+    Object.keys(channels).forEach(function(chNumber){
+        var channel = channels[chNumber];
+        //console.log("Check", channel, ssrc);
+        if(Object.keys(channel.ssrcs).indexOf(ssrc) != -1){
+            contains = true;
+        }
+    });
+    return contains;
+};
+
+/**
+ * Returns map of MediaChannel that contains only media not contained in <tt>otherSdp</tt>. Mapped by channel idx.
+ * @param otherSdp the other SDP to check ssrc with.
+ */
+SDP.prototype.getNewMedia = function(otherSdp) {
+
+    // this could be useful in Array.prototype.
+    function arrayEquals(array) {
+        // if the other array is a falsy value, return
+        if (!array)
+            return false;
+
+        // compare lengths - can save a lot of time
+        if (this.length != array.length)
+            return false;
+
+        for (var i = 0, l=this.length; i < l; i++) {
+            // Check if we have nested arrays
+            if (this[i] instanceof Array && array[i] instanceof Array) {
+                // recurse into the nested arrays
+                if (!this[i].equals(array[i]))
+                    return false;
+            }
+            else if (this[i] != array[i]) {
+                // Warning - two different object instances will never be equal: {x:20} != {x:20}
+                return false;
+            }
+        }
+        return true;
+    }
+
+    var myMedia = this.getMediaSsrcMap();
+    var othersMedia = otherSdp.getMediaSsrcMap();
+    var newMedia = {};
+    Object.keys(othersMedia).forEach(function(channelNum) {
+        var myChannel = myMedia[channelNum];
+        var othersChannel = othersMedia[channelNum];
+        if(!myChannel && othersChannel) {
+            // Add whole channel
+            newMedia[channelNum] = othersChannel;
+            return;
+        }
+        // Look for new ssrcs accross the channel
+        Object.keys(othersChannel.ssrcs).forEach(function(ssrc) {
+            if(Object.keys(myChannel.ssrcs).indexOf(ssrc) === -1) {
+                // Allocate channel if we've found ssrc that doesn't exist in our channel
+                if(!newMedia[channelNum]){
+                    newMedia[channelNum] = new MediaChannel(othersChannel.chNumber, othersChannel.mediaType);
+                }
+                newMedia[channelNum].ssrcs[ssrc] = othersChannel.ssrcs[ssrc];
+            }
+        });
+
+        // Look for new ssrc groups across the channels
+        othersChannel.ssrcGroups.forEach(function(otherSsrcGroup){
+
+            // try to match the other ssrc-group with an ssrc-group of ours
+            var matched = false;
+            for (var i = 0; i < myChannel.ssrcGroups.length; i++) {
+                var mySsrcGroup = myChannel.ssrcGroups[i];
+                if (otherSsrcGroup.semantics == mySsrcGroup.semantics
+                    && arrayEquals.apply(otherSsrcGroup.ssrcs, [mySsrcGroup.ssrcs])) {
+
+                    matched = true;
+                    break;
+                }
+            }
+
+            if (!matched) {
+                // Allocate channel if we've found an ssrc-group that doesn't
+                // exist in our channel
+
+                if(!newMedia[channelNum]){
+                    newMedia[channelNum] = new MediaChannel(othersChannel.chNumber, othersChannel.mediaType);
+                }
+                newMedia[channelNum].ssrcGroups.push(otherSsrcGroup);
+            }
+        });
+    });
+    return newMedia;
+};
+
+// remove iSAC and CN from SDP
+SDP.prototype.mangle = function () {
+    var i, j, mline, lines, rtpmap, newdesc;
+    for (i = 0; i < this.media.length; i++) {
+        lines = this.media[i].split('\r\n');
+        lines.pop(); // remove empty last element
+        mline = SDPUtil.parse_mline(lines.shift());
+        if (mline.media != 'audio')
+            continue;
+        newdesc = '';
+        mline.fmt.length = 0;
+        for (j = 0; j < lines.length; j++) {
+            if (lines[j].substr(0, 9) == 'a=rtpmap:') {
+                rtpmap = SDPUtil.parse_rtpmap(lines[j]);
+                if (rtpmap.name == 'CN' || rtpmap.name == 'ISAC')
+                    continue;
+                mline.fmt.push(rtpmap.id);
+                newdesc += lines[j] + '\r\n';
+            } else {
+                newdesc += lines[j] + '\r\n';
+            }
+        }
+        this.media[i] = SDPUtil.build_mline(mline) + '\r\n';
+        this.media[i] += newdesc;
+    }
+    this.raw = this.session + this.media.join('');
+};
+
+// remove lines matching prefix from session section
+SDP.prototype.removeSessionLines = function(prefix) {
+    var self = this;
+    var lines = SDPUtil.find_lines(this.session, prefix);
+    lines.forEach(function(line) {
+        self.session = self.session.replace(line + '\r\n', '');
+    });
+    this.raw = this.session + this.media.join('');
+    return lines;
+}
+// remove lines matching prefix from a media section specified by mediaindex
+// TODO: non-numeric mediaindex could match mid
+SDP.prototype.removeMediaLines = function(mediaindex, prefix) {
+    var self = this;
+    var lines = SDPUtil.find_lines(this.media[mediaindex], prefix);
+    lines.forEach(function(line) {
+        self.media[mediaindex] = self.media[mediaindex].replace(line + '\r\n', '');
+    });
+    this.raw = this.session + this.media.join('');
+    return lines;
+}
+
+// add content's to a jingle element
+SDP.prototype.toJingle = function (elem, thecreator) {
+    var i, j, k, mline, ssrc, rtpmap, tmp, line, lines;
+    var self = this;
+    // new bundle plan
+    if (SDPUtil.find_line(this.session, 'a=group:')) {
+        lines = SDPUtil.find_lines(this.session, 'a=group:');
+        for (i = 0; i < lines.length; i++) {
+            tmp = lines[i].split(' ');
+            var semantics = tmp.shift().substr(8);
+            elem.c('group', {xmlns: 'urn:xmpp:jingle:apps:grouping:0', semantics:semantics});
+            for (j = 0; j < tmp.length; j++) {
+                elem.c('content', {name: tmp[j]}).up();
+            }
+            elem.up();
+        }
+    }
+    // old bundle plan, to be removed
+    var bundle = [];
+    if (SDPUtil.find_line(this.session, 'a=group:BUNDLE')) {
+        bundle = SDPUtil.find_line(this.session, 'a=group:BUNDLE ').split(' ');
+        bundle.shift();
+    }
+    for (i = 0; i < this.media.length; i++) {
+        mline = SDPUtil.parse_mline(this.media[i].split('\r\n')[0]);
+        if (!(mline.media === 'audio' ||
+              mline.media === 'video' ||
+              mline.media === 'application'))
+        {
+            continue;
+        }
+        if (SDPUtil.find_line(this.media[i], 'a=ssrc:')) {
+            ssrc = SDPUtil.find_line(this.media[i], 'a=ssrc:').substring(7).split(' ')[0]; // take the first
+        } else {
+            ssrc = false;
+        }
+
+        elem.c('content', {creator: thecreator, name: mline.media});
+        if (SDPUtil.find_line(this.media[i], 'a=mid:')) {
+            // prefer identifier from a=mid if present
+            var mid = SDPUtil.parse_mid(SDPUtil.find_line(this.media[i], 'a=mid:'));
+            elem.attrs({ name: mid });
+
+            // old BUNDLE plan, to be removed
+            if (bundle.indexOf(mid) !== -1) {
+                elem.c('bundle', {xmlns: 'http://estos.de/ns/bundle'}).up();
+                bundle.splice(bundle.indexOf(mid), 1);
+            }
+        }
+
+        if (SDPUtil.find_line(this.media[i], 'a=rtpmap:').length)
+        {
+            elem.c('description',
+                {xmlns: 'urn:xmpp:jingle:apps:rtp:1',
+                    media: mline.media });
+            if (ssrc) {
+                elem.attrs({ssrc: ssrc});
+            }
+            for (j = 0; j < mline.fmt.length; j++) {
+                rtpmap = SDPUtil.find_line(this.media[i], 'a=rtpmap:' + mline.fmt[j]);
+                elem.c('payload-type', SDPUtil.parse_rtpmap(rtpmap));
+                // put any 'a=fmtp:' + mline.fmt[j] lines into <param name=foo value=bar/>
+                if (SDPUtil.find_line(this.media[i], 'a=fmtp:' + mline.fmt[j])) {
+                    tmp = SDPUtil.parse_fmtp(SDPUtil.find_line(this.media[i], 'a=fmtp:' + mline.fmt[j]));
+                    for (k = 0; k < tmp.length; k++) {
+                        elem.c('parameter', tmp[k]).up();
+                    }
+                }
+                this.RtcpFbToJingle(i, elem, mline.fmt[j]); // XEP-0293 -- map a=rtcp-fb
+
+                elem.up();
+            }
+            if (SDPUtil.find_line(this.media[i], 'a=crypto:', this.session)) {
+                elem.c('encryption', {required: 1});
+                var crypto = SDPUtil.find_lines(this.media[i], 'a=crypto:', this.session);
+                crypto.forEach(function(line) {
+                    elem.c('crypto', SDPUtil.parse_crypto(line)).up();
+                });
+                elem.up(); // end of encryption
+            }
+
+            if (ssrc) {
+                // new style mapping
+                elem.c('source', { ssrc: ssrc, xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0' });
+                // FIXME: group by ssrc and support multiple different ssrcs
+                var ssrclines = SDPUtil.find_lines(this.media[i], 'a=ssrc:');
+                ssrclines.forEach(function(line) {
+                    idx = line.indexOf(' ');
+                    var linessrc = line.substr(0, idx).substr(7);
+                    if (linessrc != ssrc) {
+                        elem.up();
+                        ssrc = linessrc;
+                        elem.c('source', { ssrc: ssrc, xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0' });
+                    }
+                    var kv = line.substr(idx + 1);
+                    elem.c('parameter');
+                    if (kv.indexOf(':') == -1) {
+                        elem.attrs({ name: kv });
+                    } else {
+                        elem.attrs({ name: kv.split(':', 2)[0] });
+                        elem.attrs({ value: kv.split(':', 2)[1] });
+                    }
+                    elem.up();
+                });
+                elem.up();
+
+                // old proprietary mapping, to be removed at some point
+                tmp = SDPUtil.parse_ssrc(this.media[i]);
+                tmp.xmlns = 'http://estos.de/ns/ssrc';
+                tmp.ssrc = ssrc;
+                elem.c('ssrc', tmp).up(); // ssrc is part of description
+
+                // XEP-0339 handle ssrc-group attributes
+                var ssrc_group_lines = SDPUtil.find_lines(this.media[i], 'a=ssrc-group:');
+                ssrc_group_lines.forEach(function(line) {
+                    idx = line.indexOf(' ');
+                    var semantics = line.substr(0, idx).substr(13);
+                    var ssrcs = line.substr(14 + semantics.length).split(' ');
+                    if (ssrcs.length != 0) {
+                        elem.c('ssrc-group', { semantics: semantics, xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0' });
+                        ssrcs.forEach(function(ssrc) {
+                            elem.c('source', { ssrc: ssrc })
+                                .up();
+                        });
+                        elem.up();
+                    }
+                });
+            }
+
+            if (SDPUtil.find_line(this.media[i], 'a=rtcp-mux')) {
+                elem.c('rtcp-mux').up();
+            }
+
+            // XEP-0293 -- map a=rtcp-fb:*
+            this.RtcpFbToJingle(i, elem, '*');
+
+            // XEP-0294
+            if (SDPUtil.find_line(this.media[i], 'a=extmap:')) {
+                lines = SDPUtil.find_lines(this.media[i], 'a=extmap:');
+                for (j = 0; j < lines.length; j++) {
+                    tmp = SDPUtil.parse_extmap(lines[j]);
+                    elem.c('rtp-hdrext', { xmlns: 'urn:xmpp:jingle:apps:rtp:rtp-hdrext:0',
+                        uri: tmp.uri,
+                        id: tmp.value });
+                    if (tmp.hasOwnProperty('direction')) {
+                        switch (tmp.direction) {
+                            case 'sendonly':
+                                elem.attrs({senders: 'responder'});
+                                break;
+                            case 'recvonly':
+                                elem.attrs({senders: 'initiator'});
+                                break;
+                            case 'sendrecv':
+                                elem.attrs({senders: 'both'});
+                                break;
+                            case 'inactive':
+                                elem.attrs({senders: 'none'});
+                                break;
+                        }
+                    }
+                    // TODO: handle params
+                    elem.up();
+                }
+            }
+            elem.up(); // end of description
+        }
+
+        // map ice-ufrag/pwd, dtls fingerprint, candidates
+        this.TransportToJingle(i, elem);
+
+        if (SDPUtil.find_line(this.media[i], 'a=sendrecv', this.session)) {
+            elem.attrs({senders: 'both'});
+        } else if (SDPUtil.find_line(this.media[i], 'a=sendonly', this.session)) {
+            elem.attrs({senders: 'initiator'});
+        } else if (SDPUtil.find_line(this.media[i], 'a=recvonly', this.session)) {
+            elem.attrs({senders: 'responder'});
+        } else if (SDPUtil.find_line(this.media[i], 'a=inactive', this.session)) {
+            elem.attrs({senders: 'none'});
+        }
+        if (mline.port == '0') {
+            // estos hack to reject an m-line
+            elem.attrs({senders: 'rejected'});
+        }
+        elem.up(); // end of content
+    }
+    elem.up();
+    return elem;
+};
+
+SDP.prototype.TransportToJingle = function (mediaindex, elem) {
+    var i = mediaindex;
+    var tmp;
+    var self = this;
+    elem.c('transport');
+
+    // XEP-0343 DTLS/SCTP
+    if (SDPUtil.find_line(this.media[mediaindex], 'a=sctpmap:').length)
+    {
+        var sctpmap = SDPUtil.find_line(
+            this.media[i], 'a=sctpmap:', self.session);
+        if (sctpmap)
+        {
+            var sctpAttrs = SDPUtil.parse_sctpmap(sctpmap);
+            elem.c('sctpmap',
+                {
+                    xmlns: 'urn:xmpp:jingle:transports:dtls-sctp:1',
+                    number: sctpAttrs[0], /* SCTP port */
+                    protocol: sctpAttrs[1], /* protocol */
+                });
+            // Optional stream count attribute
+            if (sctpAttrs.length > 2)
+                elem.attrs({ streams: sctpAttrs[2]});
+            elem.up();
+        }
+    }
+    // XEP-0320
+    var fingerprints = SDPUtil.find_lines(this.media[mediaindex], 'a=fingerprint:', this.session);
+    fingerprints.forEach(function(line) {
+        tmp = SDPUtil.parse_fingerprint(line);
+        tmp.xmlns = 'urn:xmpp:jingle:apps:dtls:0';
+        elem.c('fingerprint').t(tmp.fingerprint);
+        delete tmp.fingerprint;
+        line = SDPUtil.find_line(self.media[mediaindex], 'a=setup:', self.session);
+        if (line) {
+            tmp.setup = line.substr(8);
+        }
+        elem.attrs(tmp);
+        elem.up(); // end of fingerprint
+    });
+    tmp = SDPUtil.iceparams(this.media[mediaindex], this.session);
+    if (tmp) {
+        tmp.xmlns = 'urn:xmpp:jingle:transports:ice-udp:1';
+        elem.attrs(tmp);
+        // XEP-0176
+        if (SDPUtil.find_line(this.media[mediaindex], 'a=candidate:', this.session)) { // add any a=candidate lines
+            var lines = SDPUtil.find_lines(this.media[mediaindex], 'a=candidate:', this.session);
+            lines.forEach(function (line) {
+                elem.c('candidate', SDPUtil.candidateToJingle(line)).up();
+            });
+        }
+    }
+    elem.up(); // end of transport
+}
+
+SDP.prototype.RtcpFbToJingle = function (mediaindex, elem, payloadtype) { // XEP-0293
+    var lines = SDPUtil.find_lines(this.media[mediaindex], 'a=rtcp-fb:' + payloadtype);
+    lines.forEach(function (line) {
+        var tmp = SDPUtil.parse_rtcpfb(line);
+        if (tmp.type == 'trr-int') {
+            elem.c('rtcp-fb-trr-int', {xmlns: 'urn:xmpp:jingle:apps:rtp:rtcp-fb:0', value: tmp.params[0]});
+            elem.up();
+        } else {
+            elem.c('rtcp-fb', {xmlns: 'urn:xmpp:jingle:apps:rtp:rtcp-fb:0', type: tmp.type});
+            if (tmp.params.length > 0) {
+                elem.attrs({'subtype': tmp.params[0]});
+            }
+            elem.up();
+        }
+    });
+};
+
+SDP.prototype.RtcpFbFromJingle = function (elem, payloadtype) { // XEP-0293
+    var media = '';
+    var tmp = elem.find('>rtcp-fb-trr-int[xmlns="urn:xmpp:jingle:apps:rtp:rtcp-fb:0"]');
+    if (tmp.length) {
+        media += 'a=rtcp-fb:' + '*' + ' ' + 'trr-int' + ' ';
+        if (tmp.attr('value')) {
+            media += tmp.attr('value');
+        } else {
+            media += '0';
+        }
+        media += '\r\n';
+    }
+    tmp = elem.find('>rtcp-fb[xmlns="urn:xmpp:jingle:apps:rtp:rtcp-fb:0"]');
+    tmp.each(function () {
+        media += 'a=rtcp-fb:' + payloadtype + ' ' + $(this).attr('type');
+        if ($(this).attr('subtype')) {
+            media += ' ' + $(this).attr('subtype');
+        }
+        media += '\r\n';
+    });
+    return media;
+};
+
+// construct an SDP from a jingle stanza
+SDP.prototype.fromJingle = function (jingle) {
+    var self = this;
+    this.raw = 'v=0\r\n' +
+        'o=- ' + '1923518516' + ' 2 IN IP4 0.0.0.0\r\n' +// FIXME
+        's=-\r\n' +
+        't=0 0\r\n';
+    // http://tools.ietf.org/html/draft-ietf-mmusic-sdp-bundle-negotiation-04#section-8
+    if ($(jingle).find('>group[xmlns="urn:xmpp:jingle:apps:grouping:0"]').length) {
+        $(jingle).find('>group[xmlns="urn:xmpp:jingle:apps:grouping:0"]').each(function (idx, group) {
+            var contents = $(group).find('>content').map(function (idx, content) {
+                return content.getAttribute('name');
+            }).get();
+            if (contents.length > 0) {
+                self.raw += 'a=group:' + (group.getAttribute('semantics') || group.getAttribute('type')) + ' ' + contents.join(' ') + '\r\n';
+            }
+        });
+    } else if ($(jingle).find('>group[xmlns="urn:ietf:rfc:5888"]').length) {
+        // temporary namespace, not to be used. to be removed soon.
+        $(jingle).find('>group[xmlns="urn:ietf:rfc:5888"]').each(function (idx, group) {
+            var contents = $(group).find('>content').map(function (idx, content) {
+                return content.getAttribute('name');
+            }).get();
+            if (group.getAttribute('type') !== null && contents.length > 0) {
+                self.raw += 'a=group:' + group.getAttribute('type') + ' ' + contents.join(' ') + '\r\n';
+            }
+        });
+    } else {
+        // for backward compability, to be removed soon
+        // assume all contents are in the same bundle group, can be improved upon later
+        var bundle = $(jingle).find('>content').filter(function (idx, content) {
+            //elem.c('bundle', {xmlns:'http://estos.de/ns/bundle'});
+            return $(content).find('>bundle').length > 0;
+        }).map(function (idx, content) {
+                return content.getAttribute('name');
+            }).get();
+        if (bundle.length) {
+            this.raw += 'a=group:BUNDLE ' + bundle.join(' ') + '\r\n';
+        }
+    }
+
+    this.session = this.raw;
+    jingle.find('>content').each(function () {
+        var m = self.jingle2media($(this));
+        self.media.push(m);
+    });
+
+    // reconstruct msid-semantic -- apparently not necessary
+    /*
+     var msid = SDPUtil.parse_ssrc(this.raw);
+     if (msid.hasOwnProperty('mslabel')) {
+     this.session += "a=msid-semantic: WMS " + msid.mslabel + "\r\n";
+     }
+     */
+
+    this.raw = this.session + this.media.join('');
+};
+
+// translate a jingle content element into an an SDP media part
+SDP.prototype.jingle2media = function (content) {
+    var media = '',
+        desc = content.find('description'),
+        ssrc = desc.attr('ssrc'),
+        self = this,
+        tmp;
+    var sctp = content.find(
+        '>transport>sctpmap[xmlns="urn:xmpp:jingle:transports:dtls-sctp:1"]');
+
+    tmp = { media: desc.attr('media') };
+    tmp.port = '1';
+    if (content.attr('senders') == 'rejected') {
+        // estos hack to reject an m-line.
+        tmp.port = '0';
+    }
+    if (content.find('>transport>fingerprint').length || desc.find('encryption').length) {
+        if (sctp.length)
+            tmp.proto = 'DTLS/SCTP';
+        else
+            tmp.proto = 'RTP/SAVPF';
+    } else {
+        tmp.proto = 'RTP/AVPF';
+    }
+    if (!sctp.length)
+    {
+        tmp.fmt = desc.find('payload-type').map(
+            function () { return this.getAttribute('id'); }).get();
+        media += SDPUtil.build_mline(tmp) + '\r\n';
+    }
+    else
+    {
+        media += 'm=application 1 DTLS/SCTP ' + sctp.attr('number') + '\r\n';
+        media += 'a=sctpmap:' + sctp.attr('number') +
+            ' ' + sctp.attr('protocol');
+
+        var streamCount = sctp.attr('streams');
+        if (streamCount)
+            media += ' ' + streamCount + '\r\n';
+        else
+            media += '\r\n';
+    }
+
+    media += 'c=IN IP4 0.0.0.0\r\n';
+    if (!sctp.length)
+        media += 'a=rtcp:1 IN IP4 0.0.0.0\r\n';
+    //tmp = content.find('>transport[xmlns="urn:xmpp:jingle:transports:ice-udp:1"]');
+    tmp = content.find('>bundle>transport[xmlns="urn:xmpp:jingle:transports:ice-udp:1"]');
+    //console.log('transports: '+content.find('>transport[xmlns="urn:xmpp:jingle:transports:ice-udp:1"]').length);
+    //console.log('bundle.transports: '+content.find('>bundle>transport[xmlns="urn:xmpp:jingle:transports:ice-udp:1"]').length);
+    //console.log("tmp fingerprint: "+tmp.find('>fingerprint').innerHTML);
+    if (tmp.length) {
+        if (tmp.attr('ufrag')) {
+            media += SDPUtil.build_iceufrag(tmp.attr('ufrag')) + '\r\n';
+        }
+        if (tmp.attr('pwd')) {
+            media += SDPUtil.build_icepwd(tmp.attr('pwd')) + '\r\n';
+        }
+        tmp.find('>fingerprint').each(function () {
+            // FIXME: check namespace at some point
+            media += 'a=fingerprint:' + this.getAttribute('hash');
+            media += ' ' + $(this).text();
+            media += '\r\n';
+            //console.log("mline "+media);
+            if (this.getAttribute('setup')) {
+                media += 'a=setup:' + this.getAttribute('setup') + '\r\n';
+            }
+        });
+    }
+    switch (content.attr('senders')) {
+        case 'initiator':
+            media += 'a=sendonly\r\n';
+            break;
+        case 'responder':
+            media += 'a=recvonly\r\n';
+            break;
+        case 'none':
+            media += 'a=inactive\r\n';
+            break;
+        case 'both':
+            media += 'a=sendrecv\r\n';
+            break;
+    }
+    media += 'a=mid:' + content.attr('name') + '\r\n';
+    /*if (content.attr('name') == 'video') {
+        media += 'a=x-google-flag:conference' + '\r\n';
+    }*/
+
+    // <description><rtcp-mux/></description>
+    // see http://code.google.com/p/libjingle/issues/detail?id=309 -- no spec though
+    // and http://mail.jabber.org/pipermail/jingle/2011-December/001761.html
+    if (desc.find('rtcp-mux').length) {
+        media += 'a=rtcp-mux\r\n';
+    }
+
+    if (desc.find('encryption').length) {
+        desc.find('encryption>crypto').each(function () {
+            media += 'a=crypto:' + this.getAttribute('tag');
+            media += ' ' + this.getAttribute('crypto-suite');
+            media += ' ' + this.getAttribute('key-params');
+            if (this.getAttribute('session-params')) {
+                media += ' ' + this.getAttribute('session-params');
+            }
+            media += '\r\n';
+        });
+    }
+    desc.find('payload-type').each(function () {
+        media += SDPUtil.build_rtpmap(this) + '\r\n';
+        if ($(this).find('>parameter').length) {
+            media += 'a=fmtp:' + this.getAttribute('id') + ' ';
+            media += $(this).find('parameter').map(function () { return (this.getAttribute('name') ? (this.getAttribute('name') + '=') : '') + this.getAttribute('value'); }).get().join('; ');
+            media += '\r\n';
+        }
+        // xep-0293
+        media += self.RtcpFbFromJingle($(this), this.getAttribute('id'));
+    });
+
+    // xep-0293
+    media += self.RtcpFbFromJingle(desc, '*');
+
+    // xep-0294
+    tmp = desc.find('>rtp-hdrext[xmlns="urn:xmpp:jingle:apps:rtp:rtp-hdrext:0"]');
+    tmp.each(function () {
+        media += 'a=extmap:' + this.getAttribute('id') + ' ' + this.getAttribute('uri') + '\r\n';
+    });
+
+    content.find('>bundle>transport[xmlns="urn:xmpp:jingle:transports:ice-udp:1"]>candidate').each(function () {
+        media += SDPUtil.candidateFromJingle(this);
+    });
+
+    // XEP-0339 handle ssrc-group attributes
+    tmp = content.find('description>ssrc-group[xmlns="urn:xmpp:jingle:apps:rtp:ssma:0"]').each(function() {
+        var semantics = this.getAttribute('semantics');
+        var ssrcs = $(this).find('>source').map(function() {
+            return this.getAttribute('ssrc');
+        }).get();
+
+        if (ssrcs.length != 0) {
+            media += 'a=ssrc-group:' + semantics + ' ' + ssrcs.join(' ') + '\r\n';
+        }
+    });
+
+    tmp = content.find('description>source[xmlns="urn:xmpp:jingle:apps:rtp:ssma:0"]');
+    tmp.each(function () {
+        var ssrc = this.getAttribute('ssrc');
+        $(this).find('>parameter').each(function () {
+            media += 'a=ssrc:' + ssrc + ' ' + this.getAttribute('name');
+            if (this.getAttribute('value') && this.getAttribute('value').length)
+                media += ':' + this.getAttribute('value');
+            media += '\r\n';
+        });
+    });
+
+    if (tmp.length === 0) {
+        // fallback to proprietary mapping of a=ssrc lines
+        tmp = content.find('description>ssrc[xmlns="http://estos.de/ns/ssrc"]');
+        if (tmp.length) {
+            media += 'a=ssrc:' + ssrc + ' cname:' + tmp.attr('cname') + '\r\n';
+            media += 'a=ssrc:' + ssrc + ' msid:' + tmp.attr('msid') + '\r\n';
+            media += 'a=ssrc:' + ssrc + ' mslabel:' + tmp.attr('mslabel') + '\r\n';
+            media += 'a=ssrc:' + ssrc + ' label:' + tmp.attr('label') + '\r\n';
+        }
+    }
+    return media;
+};
+

contrib/jitsimeetbridge/unjingle/strophe.jingle.sdp.util.js

@@ -0,0 +1,408 @@
+/**
+ * Contains utility classes used in SDP class.
+ *
+ */
+
+/**
+ * Class holds a=ssrc lines and media type a=mid
+ * @param ssrc synchronization source identifier number(a=ssrc lines from SDP)
+ * @param type media type eg. "audio" or "video"(a=mid frm SDP)
+ * @constructor
+ */
+function ChannelSsrc(ssrc, type) {
+    this.ssrc = ssrc;
+    this.type = type;
+    this.lines = [];
+}
+
+/**
+ * Class holds a=ssrc-group: lines
+ * @param semantics
+ * @param ssrcs
+ * @constructor
+ */
+function ChannelSsrcGroup(semantics, ssrcs, line) {
+    this.semantics = semantics;
+    this.ssrcs = ssrcs;
+}
+
+/**
+ * Helper class represents media channel. Is a container for ChannelSsrc, holds channel idx and media type.
+ * @param channelNumber channel idx in SDP media array.
+ * @param mediaType media type(a=mid)
+ * @constructor
+ */
+function MediaChannel(channelNumber, mediaType) {
+    /**
+     * SDP channel number
+     * @type {*}
+     */
+    this.chNumber = channelNumber;
+    /**
+     * Channel media type(a=mid)
+     * @type {*}
+     */
+    this.mediaType = mediaType;
+    /**
+     * The maps of ssrc numbers to ChannelSsrc objects.
+     */
+    this.ssrcs = {};
+
+    /**
+     * The array of ChannelSsrcGroup objects.
+     * @type {Array}
+     */
+    this.ssrcGroups = [];
+}
+
+SDPUtil = {
+    iceparams: function (mediadesc, sessiondesc) {
+        var data = null;
+        if (SDPUtil.find_line(mediadesc, 'a=ice-ufrag:', sessiondesc) &&
+            SDPUtil.find_line(mediadesc, 'a=ice-pwd:', sessiondesc)) {
+            data = {
+                ufrag: SDPUtil.parse_iceufrag(SDPUtil.find_line(mediadesc, 'a=ice-ufrag:', sessiondesc)),
+                pwd: SDPUtil.parse_icepwd(SDPUtil.find_line(mediadesc, 'a=ice-pwd:', sessiondesc))
+            };
+        }
+        return data;
+    },
+    parse_iceufrag: function (line) {
+        return line.substring(12);
+    },
+    build_iceufrag: function (frag) {
+        return 'a=ice-ufrag:' + frag;
+    },
+    parse_icepwd: function (line) {
+        return line.substring(10);
+    },
+    build_icepwd: function (pwd) {
+        return 'a=ice-pwd:' + pwd;
+    },
+    parse_mid: function (line) {
+        return line.substring(6);
+    },
+    parse_mline: function (line) {
+        var parts = line.substring(2).split(' '),
+            data = {};
+        data.media = parts.shift();
+        data.port = parts.shift();
+        data.proto = parts.shift();
+        if (parts[parts.length - 1] === '') { // trailing whitespace
+            parts.pop();
+        }
+        data.fmt = parts;
+        return data;
+    },
+    build_mline: function (mline) {
+        return 'm=' + mline.media + ' ' + mline.port + ' ' + mline.proto + ' ' + mline.fmt.join(' ');
+    },
+    parse_rtpmap: function (line) {
+        var parts = line.substring(9).split(' '),
+            data = {};
+        data.id = parts.shift();
+        parts = parts[0].split('/');
+        data.name = parts.shift();
+        data.clockrate = parts.shift();
+        data.channels = parts.length ? parts.shift() : '1';
+        return data;
+    },
+    /**
+     * Parses SDP line "a=sctpmap:..." and extracts SCTP port from it.
+     * @param line eg. "a=sctpmap:5000 webrtc-datachannel"
+     * @returns [SCTP port number, protocol, streams]
+     */
+    parse_sctpmap: function (line)
+    {
+        var parts = line.substring(10).split(' ');
+        var sctpPort = parts[0];
+        var protocol = parts[1];
+        // Stream count is optional
+        var streamCount = parts.length > 2 ? parts[2] : null;
+        return [sctpPort, protocol, streamCount];// SCTP port
+    },
+    build_rtpmap: function (el) {
+        var line = 'a=rtpmap:' + el.getAttribute('id') + ' ' + el.getAttribute('name') + '/' + el.getAttribute('clockrate');
+        if (el.getAttribute('channels') && el.getAttribute('channels') != '1') {
+            line += '/' + el.getAttribute('channels');
+        }
+        return line;
+    },
+    parse_crypto: function (line) {
+        var parts = line.substring(9).split(' '),
+            data = {};
+        data.tag = parts.shift();
+        data['crypto-suite'] = parts.shift();
+        data['key-params'] = parts.shift();
+        if (parts.length) {
+            data['session-params'] = parts.join(' ');
+        }
+        return data;
+    },
+    parse_fingerprint: function (line) { // RFC 4572
+        var parts = line.substring(14).split(' '),
+            data = {};
+        data.hash = parts.shift();
+        data.fingerprint = parts.shift();
+        // TODO assert that fingerprint satisfies 2UHEX *(":" 2UHEX) ?
+        return data;
+    },
+    parse_fmtp: function (line) {
+        var parts = line.split(' '),
+            i, key, value,
+            data = [];
+        parts.shift();
+        parts = parts.join(' ').split(';');
+        for (i = 0; i < parts.length; i++) {
+            key = parts[i].split('=')[0];
+            while (key.length && key[0] == ' ') {
+                key = key.substring(1);
+            }
+            value = parts[i].split('=')[1];
+            if (key && value) {
+                data.push({name: key, value: value});
+            } else if (key) {
+                // rfc 4733 (DTMF) style stuff
+                data.push({name: '', value: key});
+            }
+        }
+        return data;
+    },
+    parse_icecandidate: function (line) {
+        var candidate = {},
+            elems = line.split(' ');
+        candidate.foundation = elems[0].substring(12);
+        candidate.component = elems[1];
+        candidate.protocol = elems[2].toLowerCase();
+        candidate.priority = elems[3];
+        candidate.ip = elems[4];
+        candidate.port = elems[5];
+        // elems[6] => "typ"
+        candidate.type = elems[7];
+        candidate.generation = 0; // default value, may be overwritten below
+        for (var i = 8; i < elems.length; i += 2) {
+            switch (elems[i]) {
+                case 'raddr':
+                    candidate['rel-addr'] = elems[i + 1];
+                    break;
+                case 'rport':
+                    candidate['rel-port'] = elems[i + 1];
+                    break;
+                case 'generation':
+                    candidate.generation = elems[i + 1];
+                    break;
+                case 'tcptype':
+                    candidate.tcptype = elems[i + 1];
+                    break;
+                default: // TODO
+                    console.log('parse_icecandidate not translating "' + elems[i] + '" = "' + elems[i + 1] + '"');
+            }
+        }
+        candidate.network = '1';
+        candidate.id = Math.random().toString(36).substr(2, 10); // not applicable to SDP -- FIXME: should be unique, not just random
+        return candidate;
+    },
+    build_icecandidate: function (cand) {
+        var line = ['a=candidate:' + cand.foundation, cand.component, cand.protocol, cand.priority, cand.ip, cand.port, 'typ', cand.type].join(' ');
+        line += ' ';
+        switch (cand.type) {
+            case 'srflx':
+            case 'prflx':
+            case 'relay':
+                if (cand.hasOwnAttribute('rel-addr') && cand.hasOwnAttribute('rel-port')) {
+                    line += 'raddr';
+                    line += ' ';
+                    line += cand['rel-addr'];
+                    line += ' ';
+                    line += 'rport';
+                    line += ' ';
+                    line += cand['rel-port'];
+                    line += ' ';
+                }
+                break;
+        }
+        if (cand.hasOwnAttribute('tcptype')) {
+            line += 'tcptype';
+            line += ' ';
+            line += cand.tcptype;
+            line += ' ';
+        }
+        line += 'generation';
+        line += ' ';
+        line += cand.hasOwnAttribute('generation') ? cand.generation : '0';
+        return line;
+    },
+    parse_ssrc: function (desc) {
+        // proprietary mapping of a=ssrc lines
+        // TODO: see "Jingle RTP Source Description" by Juberti and P. Thatcher on google docs
+        // and parse according to that
+        var lines = desc.split('\r\n'),
+            data = {};
+        for (var i = 0; i < lines.length; i++) {
+            if (lines[i].substring(0, 7) == 'a=ssrc:') {
+                var idx = lines[i].indexOf(' ');
+                data[lines[i].substr(idx + 1).split(':', 2)[0]] = lines[i].substr(idx + 1).split(':', 2)[1];
+            }
+        }
+        return data;
+    },
+    parse_rtcpfb: function (line) {
+        var parts = line.substr(10).split(' ');
+        var data = {};
+        data.pt = parts.shift();
+        data.type = parts.shift();
+        data.params = parts;
+        return data;
+    },
+    parse_extmap: function (line) {
+        var parts = line.substr(9).split(' ');
+        var data = {};
+        data.value = parts.shift();
+        if (data.value.indexOf('/') != -1) {
+            data.direction = data.value.substr(data.value.indexOf('/') + 1);
+            data.value = data.value.substr(0, data.value.indexOf('/'));
+        } else {
+            data.direction = 'both';
+        }
+        data.uri = parts.shift();
+        data.params = parts;
+        return data;
+    },
+    find_line: function (haystack, needle, sessionpart) {
+        var lines = haystack.split('\r\n');
+        for (var i = 0; i < lines.length; i++) {
+            if (lines[i].substring(0, needle.length) == needle) {
+                return lines[i];
+            }
+        }
+        if (!sessionpart) {
+            return false;
+        }
+        // search session part
+        lines = sessionpart.split('\r\n');
+        for (var j = 0; j < lines.length; j++) {
+            if (lines[j].substring(0, needle.length) == needle) {
+                return lines[j];
+            }
+        }
+        return false;
+    },
+    find_lines: function (haystack, needle, sessionpart) {
+        var lines = haystack.split('\r\n'),
+            needles = [];
+        for (var i = 0; i < lines.length; i++) {
+            if (lines[i].substring(0, needle.length) == needle)
+                needles.push(lines[i]);
+        }
+        if (needles.length || !sessionpart) {
+            return needles;
+        }
+        // search session part
+        lines = sessionpart.split('\r\n');
+        for (var j = 0; j < lines.length; j++) {
+            if (lines[j].substring(0, needle.length) == needle) {
+                needles.push(lines[j]);
+            }
+        }
+        return needles;
+    },
+    candidateToJingle: function (line) {
+        // a=candidate:2979166662 1 udp 2113937151 192.168.2.100 57698 typ host generation 0
+        //      <candidate component=... foundation=... generation=... id=... ip=... network=... port=... priority=... protocol=... type=.../>
+        if (line.indexOf('candidate:') === 0) {
+            line = 'a=' + line;
+        } else if (line.substring(0, 12) != 'a=candidate:') {
+            console.log('parseCandidate called with a line that is not a candidate line');
+            console.log(line);
+            return null;
+        }
+        if (line.substring(line.length - 2) == '\r\n') // chomp it
+            line = line.substring(0, line.length - 2);
+        var candidate = {},
+            elems = line.split(' '),
+            i;
+        if (elems[6] != 'typ') {
+            console.log('did not find typ in the right place');
+            console.log(line);
+            return null;
+        }
+        candidate.foundation = elems[0].substring(12);
+        candidate.component = elems[1];
+        candidate.protocol = elems[2].toLowerCase();
+        candidate.priority = elems[3];
+        candidate.ip = elems[4];
+        candidate.port = elems[5];
+        // elems[6] => "typ"
+        candidate.type = elems[7];
+
+        candidate.generation = '0'; // default, may be overwritten below
+        for (i = 8; i < elems.length; i += 2) {
+            switch (elems[i]) {
+                case 'raddr':
+                    candidate['rel-addr'] = elems[i + 1];
+                    break;
+                case 'rport':
+                    candidate['rel-port'] = elems[i + 1];
+                    break;
+                case 'generation':
+                    candidate.generation = elems[i + 1];
+                    break;
+                case 'tcptype':
+                    candidate.tcptype = elems[i + 1];
+                    break;
+                default: // TODO
+                    console.log('not translating "' + elems[i] + '" = "' + elems[i + 1] + '"');
+            }
+        }
+        candidate.network = '1';
+        candidate.id = Math.random().toString(36).substr(2, 10); // not applicable to SDP -- FIXME: should be unique, not just random
+        return candidate;
+    },
+    candidateFromJingle: function (cand) {
+        var line = 'a=candidate:';
+        line += cand.getAttribute('foundation');
+        line += ' ';
+        line += cand.getAttribute('component');
+        line += ' ';
+        line += cand.getAttribute('protocol'); //.toUpperCase(); // chrome M23 doesn't like this
+        line += ' ';
+        line += cand.getAttribute('priority');
+        line += ' ';
+        line += cand.getAttribute('ip');
+        line += ' ';
+        line += cand.getAttribute('port');
+        line += ' ';
+        line += 'typ';
+        line += ' ' + cand.getAttribute('type');
+        line += ' ';
+        switch (cand.getAttribute('type')) {
+            case 'srflx':
+            case 'prflx':
+            case 'relay':
+                if (cand.getAttribute('rel-addr') && cand.getAttribute('rel-port')) {
+                    line += 'raddr';
+                    line += ' ';
+                    line += cand.getAttribute('rel-addr');
+                    line += ' ';
+                    line += 'rport';
+                    line += ' ';
+                    line += cand.getAttribute('rel-port');
+                    line += ' ';
+                }
+                break;
+        }
+        if (cand.getAttribute('protocol').toLowerCase() == 'tcp') {
+            line += 'tcptype';
+            line += ' ';
+            line += cand.getAttribute('tcptype');
+            line += ' ';
+        }
+        line += 'generation';
+        line += ' ';
+        line += cand.getAttribute('generation') || '0';
+        return line + '\r\n';
+    }
+};
+
+exports.SDPUtil = SDPUtil;
+

contrib/jitsimeetbridge/unjingle/strophe/XMLHttpRequest.js

@@ -0,0 +1,254 @@
+/**
+ * Wrapper for built-in http.js to emulate the browser XMLHttpRequest object.
+ *
+ * This can be used with JS designed for browsers to improve reuse of code and
+ * allow the use of existing libraries.
+ *
+ * Usage: include("XMLHttpRequest.js") and use XMLHttpRequest per W3C specs.
+ *
+ * @todo SSL Support
+ * @author Dan DeFelippi <dan@driverdan.com>
+ * @license MIT
+ */
+
+var Url = require("url")
+	,sys = require("util");
+
+exports.XMLHttpRequest = function() {
+	/**
+	 * Private variables
+	 */
+	var self = this;
+	var http = require('http');
+	var https = require('https');
+
+	// Holds http.js objects
+	var client;
+	var request;
+	var response;
+	
+	// Request settings
+	var settings = {};
+	
+	// Set some default headers
+	var defaultHeaders = {
+		"User-Agent": "node.js",
+		"Accept": "*/*",
+	};
+	
+	var headers = defaultHeaders;
+	
+	/**
+	 * Constants
+	 */
+	this.UNSENT = 0;
+	this.OPENED = 1;
+	this.HEADERS_RECEIVED = 2;
+	this.LOADING = 3;
+	this.DONE = 4;
+
+	/**
+	 * Public vars
+	 */
+	// Current state
+	this.readyState = this.UNSENT;
+
+	// default ready state change handler in case one is not set or is set late
+	this.onreadystatechange = function() {};
+
+	// Result & response
+	this.responseText = "";
+	this.responseXML = "";
+	this.status = null;
+	this.statusText = null;
+		
+	/**
+	 * Open the connection. Currently supports local server requests.
+	 *
+	 * @param string method Connection method (eg GET, POST)
+	 * @param string url URL for the connection.
+	 * @param boolean async Asynchronous connection. Default is true.
+	 * @param string user Username for basic authentication (optional)
+	 * @param string password Password for basic authentication (optional)
+	 */
+	this.open = function(method, url, async, user, password) {
+		settings = {
+			"method": method,
+			"url": url,
+			"async": async || null,
+			"user": user || null,
+			"password": password || null
+		};
+		
+		this.abort();
+
+		setState(this.OPENED);
+	};
+	
+	/**
+	 * Sets a header for the request.
+	 *
+	 * @param string header Header name
+	 * @param string value Header value
+	 */
+	this.setRequestHeader = function(header, value) {
+		headers[header] = value;
+	};
+	
+	/**
+	 * Gets a header from the server response.
+	 *
+	 * @param string header Name of header to get.
+	 * @return string Text of the header or null if it doesn't exist.
+	 */
+	this.getResponseHeader = function(header) {
+		if (this.readyState > this.OPENED && response.headers[header]) {
+			return header + ": " + response.headers[header];
+		}
+		
+		return null;
+	};
+	
+	/**
+	 * Gets all the response headers.
+	 *
+	 * @return string 
+	 */
+	this.getAllResponseHeaders = function() {
+		if (this.readyState < this.HEADERS_RECEIVED) {
+			throw "INVALID_STATE_ERR: Headers have not been received.";
+		}
+		var result = "";
+		
+		for (var i in response.headers) {
+			result += i + ": " + response.headers[i] + "\r\n";
+		}
+		return result.substr(0, result.length - 2);
+	};
+
+	/**
+	 * Sends the request to the server.
+	 *
+	 * @param string data Optional data to send as request body.
+	 */
+	this.send = function(data) {
+		if (this.readyState != this.OPENED) {
+			throw "INVALID_STATE_ERR: connection must be opened before send() is called";
+		}
+		
+		var ssl = false;
+		var url = Url.parse(settings.url);
+		
+		// Determine the server
+		switch (url.protocol) {
+			case 'https:':
+				ssl = true;
+				// SSL & non-SSL both need host, no break here.
+			case 'http:':
+				var host = url.hostname;
+				break;
+			
+			case undefined:
+			case '':
+				var host = "localhost";
+				break;
+			
+			default:
+				throw "Protocol not supported.";
+		}
+
+		// Default to port 80. If accessing localhost on another port be sure
+		// to use http://localhost:port/path
+		var port = url.port || (ssl ? 443 : 80);
+		// Add query string if one is used
+		var uri = url.pathname + (url.search ? url.search : '');
+		
+		// Set the Host header or the server may reject the request
+		this.setRequestHeader("Host", host);
+		
+		// Set content length header
+		if (settings.method == "GET" || settings.method == "HEAD") {
+			data = null;
+		} else if (data) {
+			this.setRequestHeader("Content-Length", Buffer.byteLength(data));
+			
+			if (!headers["Content-Type"]) {
+				this.setRequestHeader("Content-Type", "text/plain;charset=UTF-8");
+			}
+		}
+
+		// Use the proper protocol
+		var doRequest = ssl ? https.request : http.request;
+
+		var options = {
+		    host: host,
+		    port: port,
+		    path: uri,
+		    method: settings.method,
+		    headers: headers, 
+                    agent: false
+		};
+		
+		var req = doRequest(options, function(res) {
+			response = res;
+			response.setEncoding("utf8");
+
+			setState(self.HEADERS_RECEIVED);
+			self.status = response.statusCode;
+
+			response.on('data', function(chunk) {
+				// Make sure there's some data
+				if (chunk) {
+					self.responseText += chunk;
+				}
+				setState(self.LOADING);
+			});
+
+			response.on('end', function() {
+				setState(self.DONE);
+			});
+
+			response.on('error', function() {
+				self.handleError(error);
+			});
+		}).on('error', function(error) {
+			self.handleError(error);
+		});
+
+		req.setHeader("Connection", "Close");
+
+		// Node 0.4 and later won't accept empty data. Make sure it's needed.
+		if (data) {
+			req.write(data);
+		}
+
+		req.end();
+	};
+
+	this.handleError = function(error) {
+		this.status = 503;
+		this.statusText = error;
+		this.responseText = error.stack;
+		setState(this.DONE);
+	};
+
+	/**
+	 * Aborts a request.
+	 */
+	this.abort = function() {
+		headers = defaultHeaders;
+		this.readyState = this.UNSENT;
+		this.responseText = "";
+		this.responseXML = "";
+	};
+	
+	/**
+	 * Changes readyState and calls onreadystatechange.
+	 *
+	 * @param int state New state
+	 */
+	var setState = function(state) {
+		self.readyState = state;
+		self.onreadystatechange();
+	}
+};

contrib/jitsimeetbridge/unjingle/strophe/base64.js

@@ -0,0 +1,83 @@
+// This code was written by Tyler Akins and has been placed in the
+// public domain.  It would be nice if you left this header intact.
+// Base64 code from Tyler Akins -- http://rumkin.com
+
+var Base64 = (function () {
+    var keyStr = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";
+
+    var obj = {
+        /**
+         * Encodes a string in base64
+         * @param {String} input The string to encode in base64.
+         */
+        encode: function (input) {
+            var output = "";
+            var chr1, chr2, chr3;
+            var enc1, enc2, enc3, enc4;
+            var i = 0;
+
+            do {
+                chr1 = input.charCodeAt(i++);
+                chr2 = input.charCodeAt(i++);
+                chr3 = input.charCodeAt(i++);
+
+                enc1 = chr1 >> 2;
+                enc2 = ((chr1 & 3) << 4) | (chr2 >> 4);
+                enc3 = ((chr2 & 15) << 2) | (chr3 >> 6);
+                enc4 = chr3 & 63;
+
+                if (isNaN(chr2)) {
+                    enc3 = enc4 = 64;
+                } else if (isNaN(chr3)) {
+                    enc4 = 64;
+                }
+
+                output = output + keyStr.charAt(enc1) + keyStr.charAt(enc2) +
+                    keyStr.charAt(enc3) + keyStr.charAt(enc4);
+            } while (i < input.length);
+
+            return output;
+        },
+
+        /**
+         * Decodes a base64 string.
+         * @param {String} input The string to decode.
+         */
+        decode: function (input) {
+            var output = "";
+            var chr1, chr2, chr3;
+            var enc1, enc2, enc3, enc4;
+            var i = 0;
+
+            // remove all characters that are not A-Z, a-z, 0-9, +, /, or =
+            input = input.replace(/[^A-Za-z0-9\+\/\=]/g, '');
+
+            do {
+                enc1 = keyStr.indexOf(input.charAt(i++));
+                enc2 = keyStr.indexOf(input.charAt(i++));
+                enc3 = keyStr.indexOf(input.charAt(i++));
+                enc4 = keyStr.indexOf(input.charAt(i++));
+
+                chr1 = (enc1 << 2) | (enc2 >> 4);
+                chr2 = ((enc2 & 15) << 4) | (enc3 >> 2);
+                chr3 = ((enc3 & 3) << 6) | enc4;
+
+                output = output + String.fromCharCode(chr1);
+
+                if (enc3 != 64) {
+                    output = output + String.fromCharCode(chr2);
+                }
+                if (enc4 != 64) {
+                    output = output + String.fromCharCode(chr3);
+                }
+            } while (i < input.length);
+
+            return output;
+        }
+    };
+
+    return obj;
+})();
+
+// Nodify
+exports.Base64 = Base64;

contrib/jitsimeetbridge/unjingle/strophe/md5.js

@@ -0,0 +1,279 @@
+/*
+ * A JavaScript implementation of the RSA Data Security, Inc. MD5 Message
+ * Digest Algorithm, as defined in RFC 1321.
+ * Version 2.1 Copyright (C) Paul Johnston 1999 - 2002.
+ * Other contributors: Greg Holt, Andrew Kepert, Ydnar, Lostinet
+ * Distributed under the BSD License
+ * See http://pajhome.org.uk/crypt/md5 for more info.
+ */
+
+var MD5 = (function () {
+    /*
+     * Configurable variables. You may need to tweak these to be compatible with
+     * the server-side, but the defaults work in most cases.
+     */
+    var hexcase = 0;  /* hex output format. 0 - lowercase; 1 - uppercase */
+    var b64pad  = ""; /* base-64 pad character. "=" for strict RFC compliance */
+    var chrsz   = 8;  /* bits per input character. 8 - ASCII; 16 - Unicode */
+
+    /*
+     * Add integers, wrapping at 2^32. This uses 16-bit operations internally
+     * to work around bugs in some JS interpreters.
+     */
+    var safe_add = function (x, y) {
+        var lsw = (x & 0xFFFF) + (y & 0xFFFF);
+        var msw = (x >> 16) + (y >> 16) + (lsw >> 16);
+        return (msw << 16) | (lsw & 0xFFFF);
+    };
+
+    /*
+     * Bitwise rotate a 32-bit number to the left.
+     */
+    var bit_rol = function (num, cnt) {
+        return (num << cnt) | (num >>> (32 - cnt));
+    };
+
+    /*
+     * Convert a string to an array of little-endian words
+     * If chrsz is ASCII, characters >255 have their hi-byte silently ignored.
+     */
+    var str2binl = function (str) {
+        var bin = [];
+        var mask = (1 << chrsz) - 1;
+        for(var i = 0; i < str.length * chrsz; i += chrsz)
+        {
+            bin[i>>5] |= (str.charCodeAt(i / chrsz) & mask) << (i%32);
+        }
+        return bin;
+    };
+
+    /*
+     * Convert an array of little-endian words to a string
+     */
+    var binl2str = function (bin) {
+        var str = "";
+        var mask = (1 << chrsz) - 1;
+        for(var i = 0; i < bin.length * 32; i += chrsz)
+        {
+            str += String.fromCharCode((bin[i>>5] >>> (i % 32)) & mask);
+        }
+        return str;
+    };
+
+    /*
+     * Convert an array of little-endian words to a hex string.
+     */
+    var binl2hex = function (binarray) {
+        var hex_tab = hexcase ? "0123456789ABCDEF" : "0123456789abcdef";
+        var str = "";
+        for(var i = 0; i < binarray.length * 4; i++)
+        {
+            str += hex_tab.charAt((binarray[i>>2] >> ((i%4)*8+4)) & 0xF) +
+                hex_tab.charAt((binarray[i>>2] >> ((i%4)*8  )) & 0xF);
+        }
+        return str;
+    };
+
+    /*
+     * Convert an array of little-endian words to a base-64 string
+     */
+    var binl2b64 = function (binarray) {
+        var tab = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+        var str = "";
+        var triplet, j;
+        for(var i = 0; i < binarray.length * 4; i += 3)
+        {
+            triplet = (((binarray[i   >> 2] >> 8 * ( i   %4)) & 0xFF) << 16) |
+                (((binarray[i+1 >> 2] >> 8 * ((i+1)%4)) & 0xFF) << 8 ) |
+                ((binarray[i+2 >> 2] >> 8 * ((i+2)%4)) & 0xFF);
+            for(j = 0; j < 4; j++)
+            {
+                if(i * 8 + j * 6 > binarray.length * 32) { str += b64pad; }
+                else { str += tab.charAt((triplet >> 6*(3-j)) & 0x3F); }
+            }
+        }
+        return str;
+    };
+
+    /*
+     * These functions implement the four basic operations the algorithm uses.
+     */
+    var md5_cmn = function (q, a, b, x, s, t) {
+        return safe_add(bit_rol(safe_add(safe_add(a, q),safe_add(x, t)), s),b);
+    };
+
+    var md5_ff = function (a, b, c, d, x, s, t) {
+        return md5_cmn((b & c) | ((~b) & d), a, b, x, s, t);
+    };
+
+    var md5_gg = function (a, b, c, d, x, s, t) {
+        return md5_cmn((b & d) | (c & (~d)), a, b, x, s, t);
+    };
+
+    var md5_hh = function (a, b, c, d, x, s, t) {
+        return md5_cmn(b ^ c ^ d, a, b, x, s, t);
+    };
+
+    var md5_ii = function (a, b, c, d, x, s, t) {
+        return md5_cmn(c ^ (b | (~d)), a, b, x, s, t);
+    };
+
+    /*
+     * Calculate the MD5 of an array of little-endian words, and a bit length
+     */
+    var core_md5 = function (x, len) {
+        /* append padding */
+        x[len >> 5] |= 0x80 << ((len) % 32);
+        x[(((len + 64) >>> 9) << 4) + 14] = len;
+
+        var a =  1732584193;
+        var b = -271733879;
+        var c = -1732584194;
+        var d =  271733878;
+
+        var olda, oldb, oldc, oldd;
+        for (var i = 0; i < x.length; i += 16)
+        {
+            olda = a;
+            oldb = b;
+            oldc = c;
+            oldd = d;
+
+            a = md5_ff(a, b, c, d, x[i+ 0], 7 , -680876936);
+            d = md5_ff(d, a, b, c, x[i+ 1], 12, -389564586);
+            c = md5_ff(c, d, a, b, x[i+ 2], 17,  606105819);
+            b = md5_ff(b, c, d, a, x[i+ 3], 22, -1044525330);
+            a = md5_ff(a, b, c, d, x[i+ 4], 7 , -176418897);
+            d = md5_ff(d, a, b, c, x[i+ 5], 12,  1200080426);
+            c = md5_ff(c, d, a, b, x[i+ 6], 17, -1473231341);
+            b = md5_ff(b, c, d, a, x[i+ 7], 22, -45705983);
+            a = md5_ff(a, b, c, d, x[i+ 8], 7 ,  1770035416);
+            d = md5_ff(d, a, b, c, x[i+ 9], 12, -1958414417);
+            c = md5_ff(c, d, a, b, x[i+10], 17, -42063);
+            b = md5_ff(b, c, d, a, x[i+11], 22, -1990404162);
+            a = md5_ff(a, b, c, d, x[i+12], 7 ,  1804603682);
+            d = md5_ff(d, a, b, c, x[i+13], 12, -40341101);
+            c = md5_ff(c, d, a, b, x[i+14], 17, -1502002290);
+            b = md5_ff(b, c, d, a, x[i+15], 22,  1236535329);
+
+            a = md5_gg(a, b, c, d, x[i+ 1], 5 , -165796510);
+            d = md5_gg(d, a, b, c, x[i+ 6], 9 , -1069501632);
+            c = md5_gg(c, d, a, b, x[i+11], 14,  643717713);
+            b = md5_gg(b, c, d, a, x[i+ 0], 20, -373897302);
+            a = md5_gg(a, b, c, d, x[i+ 5], 5 , -701558691);
+            d = md5_gg(d, a, b, c, x[i+10], 9 ,  38016083);
+            c = md5_gg(c, d, a, b, x[i+15], 14, -660478335);
+            b = md5_gg(b, c, d, a, x[i+ 4], 20, -405537848);
+            a = md5_gg(a, b, c, d, x[i+ 9], 5 ,  568446438);
+            d = md5_gg(d, a, b, c, x[i+14], 9 , -1019803690);
+            c = md5_gg(c, d, a, b, x[i+ 3], 14, -187363961);
+            b = md5_gg(b, c, d, a, x[i+ 8], 20,  1163531501);
+            a = md5_gg(a, b, c, d, x[i+13], 5 , -1444681467);
+            d = md5_gg(d, a, b, c, x[i+ 2], 9 , -51403784);
+            c = md5_gg(c, d, a, b, x[i+ 7], 14,  1735328473);
+            b = md5_gg(b, c, d, a, x[i+12], 20, -1926607734);
+
+            a = md5_hh(a, b, c, d, x[i+ 5], 4 , -378558);
+            d = md5_hh(d, a, b, c, x[i+ 8], 11, -2022574463);
+            c = md5_hh(c, d, a, b, x[i+11], 16,  1839030562);
+            b = md5_hh(b, c, d, a, x[i+14], 23, -35309556);
+            a = md5_hh(a, b, c, d, x[i+ 1], 4 , -1530992060);
+            d = md5_hh(d, a, b, c, x[i+ 4], 11,  1272893353);
+            c = md5_hh(c, d, a, b, x[i+ 7], 16, -155497632);
+            b = md5_hh(b, c, d, a, x[i+10], 23, -1094730640);
+            a = md5_hh(a, b, c, d, x[i+13], 4 ,  681279174);
+            d = md5_hh(d, a, b, c, x[i+ 0], 11, -358537222);
+            c = md5_hh(c, d, a, b, x[i+ 3], 16, -722521979);
+            b = md5_hh(b, c, d, a, x[i+ 6], 23,  76029189);
+            a = md5_hh(a, b, c, d, x[i+ 9], 4 , -640364487);
+            d = md5_hh(d, a, b, c, x[i+12], 11, -421815835);
+            c = md5_hh(c, d, a, b, x[i+15], 16,  530742520);
+            b = md5_hh(b, c, d, a, x[i+ 2], 23, -995338651);
+
+            a = md5_ii(a, b, c, d, x[i+ 0], 6 , -198630844);
+            d = md5_ii(d, a, b, c, x[i+ 7], 10,  1126891415);
+            c = md5_ii(c, d, a, b, x[i+14], 15, -1416354905);
+            b = md5_ii(b, c, d, a, x[i+ 5], 21, -57434055);
+            a = md5_ii(a, b, c, d, x[i+12], 6 ,  1700485571);
+            d = md5_ii(d, a, b, c, x[i+ 3], 10, -1894986606);
+            c = md5_ii(c, d, a, b, x[i+10], 15, -1051523);
+            b = md5_ii(b, c, d, a, x[i+ 1], 21, -2054922799);
+            a = md5_ii(a, b, c, d, x[i+ 8], 6 ,  1873313359);
+            d = md5_ii(d, a, b, c, x[i+15], 10, -30611744);
+            c = md5_ii(c, d, a, b, x[i+ 6], 15, -1560198380);
+            b = md5_ii(b, c, d, a, x[i+13], 21,  1309151649);
+            a = md5_ii(a, b, c, d, x[i+ 4], 6 , -145523070);
+            d = md5_ii(d, a, b, c, x[i+11], 10, -1120210379);
+            c = md5_ii(c, d, a, b, x[i+ 2], 15,  718787259);
+            b = md5_ii(b, c, d, a, x[i+ 9], 21, -343485551);
+
+            a = safe_add(a, olda);
+            b = safe_add(b, oldb);
+            c = safe_add(c, oldc);
+            d = safe_add(d, oldd);
+        }
+        return [a, b, c, d];
+    };
+
+
+    /*
+     * Calculate the HMAC-MD5, of a key and some data
+     */
+    var core_hmac_md5 = function (key, data) {
+        var bkey = str2binl(key);
+        if(bkey.length > 16) { bkey = core_md5(bkey, key.length * chrsz); }
+
+        var ipad = new Array(16), opad = new Array(16);
+        for(var i = 0; i < 16; i++)
+        {
+            ipad[i] = bkey[i] ^ 0x36363636;
+            opad[i] = bkey[i] ^ 0x5C5C5C5C;
+        }
+
+        var hash = core_md5(ipad.concat(str2binl(data)), 512 + data.length * chrsz);
+        return core_md5(opad.concat(hash), 512 + 128);
+    };
+
+    var obj = {
+        /*
+         * These are the functions you'll usually want to call.
+         * They take string arguments and return either hex or base-64 encoded
+         * strings.
+         */
+        hexdigest: function (s) {
+            return binl2hex(core_md5(str2binl(s), s.length * chrsz));
+        },
+
+        b64digest: function (s) {
+            return binl2b64(core_md5(str2binl(s), s.length * chrsz));
+        },
+
+        hash: function (s) {
+            return binl2str(core_md5(str2binl(s), s.length * chrsz));
+        },
+
+        hmac_hexdigest: function (key, data) {
+            return binl2hex(core_hmac_md5(key, data));
+        },
+
+        hmac_b64digest: function (key, data) {
+            return binl2b64(core_hmac_md5(key, data));
+        },
+
+        hmac_hash: function (key, data) {
+            return binl2str(core_hmac_md5(key, data));
+        },
+
+        /*
+         * Perform a simple self-test to see if the VM is working
+         */
+        test: function () {
+            return MD5.hexdigest("abc") === "900150983cd24fb0d6963f7d28e17f72";
+        }
+    };
+
+    return obj;
+})();
+
+// Nodify
+exports.MD5 = MD5;

contrib/jitsimeetbridge/unjingle/unjingle.js

@@ -0,0 +1,48 @@
+var strophe = require("./strophe/strophe.js").Strophe;
+
+var Strophe = strophe.Strophe;
+var $iq     = strophe.$iq;
+var $msg    = strophe.$msg;
+var $build  = strophe.$build;
+var $pres   = strophe.$pres;
+
+var jsdom = require("jsdom");
+var window = jsdom.jsdom().parentWindow;
+var $ = require('jquery')(window);
+
+var stropheJingle = require("./strophe.jingle.sdp.js");
+
+
+var input = '';
+
+process.stdin.on('readable', function() {
+  var chunk = process.stdin.read();
+  if (chunk !== null) {
+    input += chunk;
+  }
+});
+
+process.stdin.on('end', function() {
+	if (process.argv[2] == '--jingle') {
+		var elem = $(input);
+		// app does:
+		// sess.setRemoteDescription($(iq).find('>jingle'), 'offer');
+		//console.log(elem.find('>content'));
+		var sdp = new stropheJingle.SDP('');
+		sdp.fromJingle(elem);
+		console.log(sdp.raw);
+	} else if (process.argv[2] == '--sdp') {
+		var sdp = new stropheJingle.SDP(input);
+		var accept = $iq({to: '%(tojid)s',
+			type: 'set'})
+			.c('jingle', {xmlns: 'urn:xmpp:jingle:1',
+			    //action: 'session-accept',
+			    action: '%(action)s',
+			    initiator: '%(initiator)s',
+			    responder: '%(responder)s',
+			    sid: '%(sid)s' });
+		sdp.toJingle(accept, 'responder');
+		console.log(Strophe.serialize(accept));
+	}
+});
+

contrib/vertobot/.gitignore

@@ -0,0 +1,2 @@
+vucbot.yaml
+vertobot.yaml

contrib/vertobot/bot.pl

@@ -0,0 +1,300 @@
+#!/usr/bin/env perl 
+
+use strict;
+use warnings;
+use 5.010; # //
+use IO::Socket::SSL qw(SSL_VERIFY_NONE);
+use IO::Async::Loop;
+use Net::Async::WebSocket::Client;
+use Net::Async::Matrix 0.11_002;
+use JSON;
+use YAML;
+use Data::UUID;
+use Getopt::Long;
+use Data::Dumper;
+
+binmode STDOUT, ":encoding(UTF-8)";
+binmode STDERR, ":encoding(UTF-8)";
+
+my $loop = IO::Async::Loop->new;
+# Net::Async::HTTP + SSL + IO::Poll doesn't play well. See
+#   https://rt.cpan.org/Ticket/Display.html?id=93107
+ref $loop eq "IO::Async::Loop::Poll" and
+    warn "Using SSL with IO::Poll causes known memory-leaks!!\n";
+
+GetOptions(
+   'C|config=s' => \my $CONFIG,
+   'eval-from=s' => \my $EVAL_FROM,
+) or exit 1;
+
+if( defined $EVAL_FROM ) {
+    # An emergency 'eval() this file' hack
+    $SIG{HUP} = sub {
+        my $code = do {
+            open my $fh, "<", $EVAL_FROM or warn( "Cannot read - $!" ), return;
+            local $/; <$fh>
+        };
+
+        eval $code or warn "Cannot eval() - $@";
+    };
+}
+
+defined $CONFIG or die "Must supply --config\n";
+
+my %CONFIG = %{ YAML::LoadFile( $CONFIG ) };
+
+my %MATRIX_CONFIG = %{ $CONFIG{matrix} };
+# No harm in always applying this
+$MATRIX_CONFIG{SSL_verify_mode} = SSL_VERIFY_NONE;
+
+# Track every Room object, so we can ->leave them all on shutdown
+my %bot_matrix_rooms;
+
+my $bridgestate = {};
+my $roomid_by_callid = {};
+
+my $bot_verto = Net::Async::WebSocket::Client->new(
+    on_frame => sub {
+          my ( $self, $frame ) = @_;
+          warn "[Verto] receiving $frame";
+          on_verto_json($frame);
+    },
+);
+$loop->add( $bot_verto );
+
+my $sessid = lc new Data::UUID->create_str();
+
+my $bot_matrix = Net::Async::Matrix->new(
+    %MATRIX_CONFIG,
+    on_log => sub { warn "log: @_\n" },
+    on_invite => sub {
+        my ($matrix, $invite) = @_;
+        warn "[Matrix] invited to: " . $invite->{room_id} . " by " . $invite->{inviter} . "\n";
+        
+        $matrix->join_room( $invite->{room_id} )->get;
+    },
+    on_room_new => sub {
+        my ($matrix, $room) = @_;
+
+        warn "[Matrix] have a room ID: " . $room->room_id . "\n";
+
+        $bot_matrix_rooms{$room->room_id} = $room;
+        
+        # log in to verto on behalf of this room
+        $bridgestate->{$room->room_id}->{sessid} = $sessid;
+         
+        $room->configure(
+            on_message => \&on_room_message,
+        );
+        
+        my $f = send_verto_json_request("login", {
+                'login' => $CONFIG{'verto-dialog-params'}{'login'},
+                'passwd' => $CONFIG{'verto-config'}{'passwd'},
+                'sessid' => $sessid,
+            });
+        $matrix->adopt_future($f);
+        
+        # we deliberately don't paginate the room, as we only care about
+        # new calls
+    },
+    on_unknown_event => \&on_unknown_event,
+    on_error => sub {
+        print STDERR "Matrix failure: @_\n";
+    },
+);
+$loop->add( $bot_matrix );
+
+sub on_unknown_event
+{
+    my ($matrix, $event) = @_;
+    print Dumper($event);
+    
+    my $room_id = $event->{room_id};
+    my %dp = %{$CONFIG{'verto-dialog-params'}};
+    $dp{callID} = $bridgestate->{$room_id}->{callid};
+    
+    if ($event->{type} eq 'm.call.invite') {
+        $bridgestate->{$room_id}->{matrix_callid} = $event->{content}->{call_id};
+        $bridgestate->{$room_id}->{callid} = lc new Data::UUID->create_str();
+        $bridgestate->{$room_id}->{offer} = $event->{content}->{offer}->{sdp};
+        $bridgestate->{$room_id}->{gathered_candidates} = 0;
+        $roomid_by_callid->{ $bridgestate->{$room_id}->{callid} } = $room_id;
+        # no trickle ICE in verto apparently
+    }
+    elsif ($event->{type} eq 'm.call.candidates') {
+        # XXX: compare call IDs
+        if (!$bridgestate->{$room_id}->{gathered_candidates}) {
+            $bridgestate->{$room_id}->{gathered_candidates} = 1;
+            my $offer = $bridgestate->{$room_id}->{offer};
+            my $candidate_block = "";
+            foreach (@{$event->{content}->{candidates}}) {
+                $candidate_block .= "a=" . $_->{candidate} . "\r\n";
+            }
+            # XXX: collate using the right m= line - for now assume audio call
+            $offer =~ s/(a=rtcp.*[\r\n]+)/$1$candidate_block/;
+            
+            my $f = send_verto_json_request("verto.invite", {
+                "sdp" => $offer,
+                "dialogParams" => \%dp,
+                "sessid" => $bridgestate->{$room_id}->{sessid},
+            });
+            $matrix->adopt_future($f);
+        }
+        else {
+            # ignore them, as no trickle ICE, although we might as well
+            # batch them up
+            # foreach (@{$event->{content}->{candidates}}) {
+            #     push @{$bridgestate->{$room_id}->{candidates}}, $_;
+            # }
+        }
+    }
+    elsif ($event->{type} eq 'm.call.hangup') {
+        if ($bridgestate->{$room_id}->{matrix_callid} eq $event->{content}->{call_id}) {
+            my $f = send_verto_json_request("verto.bye", {
+                "dialogParams" => \%dp,
+                "sessid" => $bridgestate->{$room_id}->{sessid},
+            });
+            $matrix->adopt_future($f);
+        }
+        else {
+            warn "Ignoring unrecognised callid: ".$event->{content}->{call_id};
+        }
+    }
+    else {
+        warn "Unhandled event: $event->{type}";
+    }
+}
+
+sub on_room_message
+{
+    my ($room, $from, $content) = @_;
+    my $room_id = $room->room_id;
+    warn "[Matrix] in $room_id: $from: " . $content->{body} . "\n";    
+}
+
+my $verto_connecting = $loop->new_future;
+$bot_verto->connect(
+    %{ $CONFIG{"verto-bot"} },
+    on_connected => sub {
+        warn("[Verto] connected to websocket");
+        $verto_connecting->done($bot_verto) if not $verto_connecting->is_done;
+    },
+    on_connect_error => sub { die "Cannot connect to verto - $_[-1]" },
+    on_resolve_error => sub { die "Cannot resolve to verto - $_[-1]" },        
+);
+
+Future->needs_all(
+    $bot_matrix->login( %{ $CONFIG{"matrix-bot"} } )->then( sub {
+        $bot_matrix->start;
+    }),
+    
+    $verto_connecting,
+)->get;
+
+$loop->attach_signal(
+    PIPE => sub { warn "pipe\n" }
+);
+$loop->attach_signal(
+    INT => sub { $loop->stop },
+);
+$loop->attach_signal(
+    TERM => sub { $loop->stop },
+);
+
+eval {
+   $loop->run;
+} or my $e = $@;
+
+# When the bot gets shut down, have it leave the rooms so it's clear to observers
+# that it is no longer running.
+# if( $CONFIG{"leave-on-shutdown"} // 1 ) {
+#     print STDERR "Removing bot from Matrix rooms...\n";
+#     Future->wait_all( map { $_->leave->else_done() } values %bot_matrix_rooms )->get;
+# }
+# else {
+#     print STDERR "Leaving bot users in Matrix rooms.\n";
+# }
+
+die $e if $e;
+
+exit 0;
+
+{    
+    my $json_id;
+    my $requests;
+
+    sub send_verto_json_request
+    {
+        $json_id ||= 1;
+        
+        my ($method, $params) = @_;
+        my $json = {
+            jsonrpc => "2.0",
+            method  => $method,
+            params  => $params,
+            id      => $json_id,
+        };
+        my $text = JSON->new->encode( $json );
+        warn "[Verto] sending $text";
+        $bot_verto->send_frame ( $text );
+        my $request = $loop->new_future;
+        $requests->{$json_id} = $request;
+        $json_id++;
+        return $request;
+    }
+    
+    sub send_verto_json_response
+    {
+        my ($result, $id) = @_;
+        my $json = {
+            jsonrpc => "2.0",
+            result  => $result,
+            id      => $id,
+        };
+        my $text = JSON->new->encode( $json );
+        warn "[Verto] sending $text";
+        $bot_verto->send_frame ( $text );
+    }
+    
+    sub on_verto_json
+    {
+        my $json = JSON->new->decode( $_[0] );
+        if ($json->{method}) {
+            if (($json->{method} eq 'verto.answer' && $json->{params}->{sdp}) ||
+                $json->{method} eq 'verto.media') {
+                                
+                my $room_id = $roomid_by_callid->{$json->{params}->{callID}};
+                my $room = $bot_matrix_rooms{$room_id};
+
+                if ($json->{params}->{sdp}) {
+                    # HACK HACK HACK HACK
+                    $room->_do_POST_json( "/send/m.call.answer", {
+                        call_id => $bridgestate->{$room_id}->{matrix_callid},
+                        version => 0,
+                        answer  => {
+                            sdp => $json->{params}->{sdp},
+                            type => "answer",
+                        },
+                    })->then( sub {
+                        send_verto_json_response( {
+                            method => $json->{method},
+                        }, $json->{id});
+                    })->get;
+                }
+            }
+            else {
+                warn ("[Verto] unhandled method: " . $json->{method});
+                send_verto_json_response( {
+                    method => $json->{method},
+                }, $json->{id});
+            }
+        }
+        elsif ($json->{result}) {
+            $requests->{$json->{id}}->done($json->{result});
+        }
+        elsif ($json->{error}) {
+            $requests->{$json->{id}}->fail($json->{error}->{message}, $json->{error});
+        }
+    }
+}
+

contrib/vertobot/config.yaml

@@ -0,0 +1,29 @@
+# Generic Matrix connection params
+matrix:
+  server: 'matrix.org'
+  SSL: 1
+
+# Bot-user connection details
+matrix-bot:
+  user_id: '@vertobot:matrix.org'
+  password: ''
+
+verto-bot:
+  host: webrtc.freeswitch.org
+  service: 8081
+  url: "ws://webrtc.freeswitch.org:8081/"
+
+verto-config:
+  passwd: 1234
+  
+verto-dialog-params:
+  useVideo: false
+  useStereo: false
+  tag: "webcam"
+  login: "1008@webrtc.freeswitch.org"
+  destination_number: "9664"
+  caller_id_name: "FreeSWITCH User"
+  caller_id_number: "1008"
+  callID: ""
+  remote_caller_id_name: "Outbound Call"
+  remote_caller_id_number: "9664"

contrib/vertobot/cpanfile

@@ -0,0 +1,17 @@
+requires 'parent', 0;
+requires 'Future', '>= 0.29';
+requires 'Net::Async::Matrix', '>= 0.11_002';
+requires 'Net::Async::Matrix::Utils';
+requires 'Net::Async::WebSocket::Protocol', 0;
+requires 'Data::UUID', 0;
+requires 'IO::Async', '>= 0.63';
+requires 'IO::Async::SSL', 0;
+requires 'IO::Socket::SSL', 0;
+requires 'YAML', 0;
+requires 'JSON', 0;
+requires 'Getopt::Long', 0;
+
+on 'test' => sub {
+	requires 'Test::More', '>= 0.98';
+};
+

contrib/vertobot/verto-example.json

@@ -0,0 +1,207 @@
+# JSON is shown in *reverse* chronological order.
+# Send v. Receive is implicit.
+
+{
+    "jsonrpc": "2.0",
+    "id": 7,
+    "result": {
+        "callID": "12795aa6-2a8d-84ee-ce63-2e82ffe825ef",
+        "message": "CALL ENDED",
+        "causeCode": 16,
+        "cause": "NORMAL_CLEARING",
+        "sessid": "03a11060-3e14-23b6-c620-51b892c52983"
+    }
+}
+
+{
+    "jsonrpc": "2.0",
+    "method": "verto.bye",
+    "params": {
+        "dialogParams": {
+            "useVideo": false,
+            "useStereo": true,
+            "tag": "webcam",
+            "login": "1008@webrtc.freeswitch.org",
+            "destination_number": "9664",
+            "caller_id_name": "FreeSWITCH User",
+            "caller_id_number": "1008",
+            "callID": "12795aa6-2a8d-84ee-ce63-2e82ffe825ef",
+            "remote_caller_id_name": "Outbound Call",
+            "remote_caller_id_number": "9664"
+        },
+        "sessid": "03a11060-3e14-23b6-c620-51b892c52983"
+    },
+    "id": 7
+}
+
+{
+    "jsonrpc": "2.0",
+    "id": 6,
+    "result": {
+        "callID": "12795aa6-2a8d-84ee-ce63-2e82ffe825ef",
+        "action": "toggleHold",
+        "holdState": "active",
+        "sessid": "03a11060-3e14-23b6-c620-51b892c52983"
+    }
+}
+
+{
+    "jsonrpc": "2.0",
+    "method": "verto.modify",
+    "params": {
+        "action": "toggleHold",
+        "dialogParams": {
+            "useVideo": false,
+            "useStereo": true,
+            "tag": "webcam",
+            "login": "1008@webrtc.freeswitch.org",
+            "destination_number": "9664",
+            "caller_id_name": "FreeSWITCH User",
+            "caller_id_number": "1008",
+            "callID": "12795aa6-2a8d-84ee-ce63-2e82ffe825ef",
+            "remote_caller_id_name": "Outbound Call",
+            "remote_caller_id_number": "9664"
+        },
+        "sessid": "03a11060-3e14-23b6-c620-51b892c52983"
+    },
+    "id": 6
+}
+
+{
+    "jsonrpc": "2.0",
+    "id": 5,
+    "result": {
+        "callID": "12795aa6-2a8d-84ee-ce63-2e82ffe825ef",
+        "action": "toggleHold",
+        "holdState": "held",
+        "sessid": "03a11060-3e14-23b6-c620-51b892c52983"
+    }
+}
+
+{
+    "jsonrpc": "2.0",
+    "method": "verto.modify",
+    "params": {
+        "action": "toggleHold",
+        "dialogParams": {
+            "useVideo": false,
+            "useStereo": true,
+            "tag": "webcam",
+            "login": "1008@webrtc.freeswitch.org",
+            "destination_number": "9664",
+            "caller_id_name": "FreeSWITCH User",
+            "caller_id_number": "1008",
+            "callID": "12795aa6-2a8d-84ee-ce63-2e82ffe825ef",
+            "remote_caller_id_name": "Outbound Call",
+            "remote_caller_id_number": "9664"
+        },
+        "sessid": "03a11060-3e14-23b6-c620-51b892c52983"
+    },
+    "id": 5
+}
+
+{
+    "jsonrpc": "2.0",
+    "id": 349819,
+    "result": {
+        "method": "verto.answer"
+    }
+}
+
+{
+    "jsonrpc": "2.0",
+    "id": 349819,
+    "method": "verto.answer",
+    "params": {
+        "callID": "12795aa6-2a8d-84ee-ce63-2e82ffe825ef",
+        "sdp": "v=0\no=FreeSWITCH 1417101432 1417101433 IN IP4 209.105.235.10\ns=FreeSWITCH\nc=IN IP4 209.105.235.10\nt=0 0\na=msid-semantic: WMS jA3rmwLVwUq1iE6TYEYHeLk2YTUlh1Vq\nm=audio 30134 RTP/SAVPF 111 126\na=rtpmap:111 opus/48000/2\na=fmtp:111 minptime=10; stereo=1\na=rtpmap:126 telephone-event/8000\na=silenceSupp:off - - - -\na=ptime:20\na=sendrecv\na=fingerprint:sha-256 F8:72:18:E9:72:89:99:22:5B:F8:B6:C6:C6:0D:C5:9B:B2:FB:BC:CA:8D:AB:13:8A:66:E1:37:38:A0:16:AA:41\na=rtcp-mux\na=rtcp:30134 IN IP4 209.105.235.10\na=ssrc:210967934 cname:rOIEajpw4FocakWY\na=ssrc:210967934 msid:jA3rmwLVwUq1iE6TYEYHeLk2YTUlh1Vq a0\na=ssrc:210967934 mslabel:jA3rmwLVwUq1iE6TYEYHeLk2YTUlh1Vq\na=ssrc:210967934 label:jA3rmwLVwUq1iE6TYEYHeLk2YTUlh1Vqa0\na=ice-ufrag:OKwTmGLapwmxn7OF\na=ice-pwd:MmaMwq8rVmtWxfLbQ7U2Ew3T\na=candidate:2372654928 1 udp 659136 209.105.235.10 30134 typ host generation 0\n"
+    }
+}
+
+{
+    "jsonrpc": "2.0",
+    "id": 4,
+    "result": {
+        "message": "CALL CREATED",
+        "callID": "12795aa6-2a8d-84ee-ce63-2e82ffe825ef",
+        "sessid": "03a11060-3e14-23b6-c620-51b892c52983"
+    }
+}
+
+{
+    "jsonrpc": "2.0",
+    "method": "verto.invite",
+    "params": {
+        "sdp": "v=0\r\no=- 1381685806032722557 2 IN IP4 127.0.0.1\r\ns=-\r\nt=0 0\r\na=group:BUNDLE audio\r\na=msid-semantic: WMS 6OOMyGAyJakjwaOOBtV7WcBCCuIW6PpuXsNg\r\nm=audio 63088 RTP/SAVPF 111 103 104 0 8 106 105 13 126\r\nc=IN IP4 81.138.8.249\r\na=rtcp:63088 IN IP4 81.138.8.249\r\na=candidate:460398169 1 udp 2122260223 10.10.79.10 49945 typ host generation 0\r\na=candidate:460398169 2 udp 2122260223 10.10.79.10 49945 typ host generation 0\r\na=candidate:3460887983 1 udp 2122194687 192.168.1.64 63088 typ host generation 0\r\na=candidate:3460887983 2 udp 2122194687 192.168.1.64 63088 typ host generation 0\r\na=candidate:945327227 1 udp 1685987071 81.138.8.249 63088 typ srflx raddr 192.168.1.64 rport 63088 generation 0\r\na=candidate:945327227 2 udp 1685987071 81.138.8.249 63088 typ srflx raddr 192.168.1.64 rport 63088 generation 0\r\na=candidate:1441981097 1 tcp 1518280447 10.10.79.10 0 typ host tcptype active generation 0\r\na=candidate:1441981097 2 tcp 1518280447 10.10.79.10 0 typ host tcptype active generation 0\r\na=candidate:2160789855 1 tcp 1518214911 192.168.1.64 0 typ host tcptype active generation 0\r\na=candidate:2160789855 2 tcp 1518214911 192.168.1.64 0 typ host tcptype active generation 0\r\na=ice-ufrag:cP4qeRhn0LpcpA88\r\na=ice-pwd:fREmgSkXsDLGUUH1bwfrBQhW\r\na=ice-options:google-ice\r\na=fingerprint:sha-256 AF:35:64:1B:62:8A:EF:27:AE:2B:88:2E:FE:78:29:0B:08:DA:64:6C:DE:02:57:E3:EE:B1:D7:86:B8:36:8F:B0\r\na=setup:actpass\r\na=mid:audio\r\na=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\na=extmap:3 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\na=sendrecv\r\na=rtcp-mux\r\na=rtpmap:111 opus/48000/2\r\na=fmtp:111 minptime=10; stereo=1\r\na=rtpmap:103 ISAC/16000\r\na=rtpmap:104 ISAC/32000\r\na=rtpmap:0 PCMU/8000\r\na=rtpmap:8 PCMA/8000\r\na=rtpmap:106 CN/32000\r\na=rtpmap:105 CN/16000\r\na=rtpmap:13 CN/8000\r\na=rtpmap:126 telephone-event/8000\r\na=maxptime:60\r\na=ssrc:558827154 cname:vdKHBNqa17t2gmE3\r\na=ssrc:558827154 msid:6OOMyGAyJakjwaOOBtV7WcBCCuIW6PpuXsNg bf1303fb-9833-4d7d-b9e4-b32cfe04acc3\r\na=ssrc:558827154 mslabel:6OOMyGAyJakjwaOOBtV7WcBCCuIW6PpuXsNg\r\na=ssrc:558827154 label:bf1303fb-9833-4d7d-b9e4-b32cfe04acc3\r\n",
+        "dialogParams": {
+            "useVideo": false,
+            "useStereo": true,
+            "tag": "webcam",
+            "login": "1008@webrtc.freeswitch.org",
+            "destination_number": "9664",
+            "caller_id_name": "FreeSWITCH User",
+            "caller_id_number": "1008",
+            "callID": "12795aa6-2a8d-84ee-ce63-2e82ffe825ef",
+            "remote_caller_id_name": "Outbound Call",
+            "remote_caller_id_number": "9664"
+        },
+        "sessid": "03a11060-3e14-23b6-c620-51b892c52983"
+    },
+    "id": 4
+}
+
+{
+    "jsonrpc": "2.0",
+    "id": 3,
+    "result": {
+        "message": "logged in",
+        "sessid": "03a11060-3e14-23b6-c620-51b892c52983"
+    }
+}
+
+{
+    "jsonrpc": "2.0",
+    "id": 1,
+    "error": {
+        "code": -32000,
+        "message": "Authentication Required"
+    }
+}
+
+{
+    "jsonrpc": "2.0",
+    "method": "login",
+    "params": {
+        "login": "1008@webrtc.freeswitch.org",
+        "passwd": "1234",
+        "sessid": "03a11060-3e14-23b6-c620-51b892c52983"
+    },
+    "id": 3
+}
+
+{
+    "jsonrpc": "2.0",
+    "id": 2,
+    "error": {
+        "code": -32000,
+        "message": "Authentication Required"
+    }
+}
+
+{
+    "jsonrpc": "2.0",
+    "method": "login",
+    "params": {
+        "sessid": "03a11060-3e14-23b6-c620-51b892c52983"
+    },
+    "id": 1
+}
+
+{
+    "jsonrpc": "2.0",
+    "method": "login",
+    "params": {
+        "sessid": "03a11060-3e14-23b6-c620-51b892c52983"
+    },
+    "id": 2
+}

scripts/check_signature.py

@@ -23,7 +23,7 @@ def get_targets(server_name):
         for srv in answers:
             yield (srv.target, srv.port)
     except dns.resolver.NXDOMAIN:
-        yield (server_name, 8480)
+        yield (server_name, 8448)
 
 def get_server_keys(server_name, target, port):
     url = "https://%s:%i/_matrix/key/v1" % (target, port)

setup.py

@@ -32,7 +32,7 @@ setup(
     description="Reference Synapse Home Server",
     install_requires=[
         "syutil==0.0.2",
-        "matrix_angular_sdk==0.5.0",
+        "matrix_angular_sdk==0.5.3b",
         "Twisted>=14.0.0",
         "service_identity>=1.0.0",
         "pyopenssl>=0.14",
@@ -45,8 +45,8 @@ setup(
     ],
     dependency_links=[
         "https://github.com/matrix-org/syutil/tarball/v0.0.2#egg=syutil-0.0.2",
-        "https://github.com/pyca/pynacl/tarball/52dbe2dc33f1#egg=pynacl-0.3.0",
+        "https://github.com/pyca/pynacl/tarball/d4d3175589b892f6ea7c22f466e0e223853516fa#egg=pynacl-0.3.0",
-        "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.5.0/#egg=matrix_angular_sdk-0.5.0",
+        "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.5.3b/#egg=matrix_angular_sdk-0.5.3b",
     ],
     setup_requires=[
         "setuptools_trial",
@@ -60,6 +60,6 @@ setup(
     entry_points="""
     [console_scripts]
     synctl=synapse.app.synctl:main
-    synapse-homeserver=synapse.app.homeserver:run
+    synapse-homeserver=synapse.app.homeserver:main
     """
 )

synapse/__init__.py

@@ -16,4 +16,4 @@
 """ This is a reference implementation of a synapse home server.
 """
 
-__version__ = "0.5.0"
+__version__ = "0.5.4"

synapse/api/auth.py

@@ -38,27 +38,21 @@ class Auth(object):
         self.store = hs.get_datastore()
         self.state = hs.get_state_handler()
 
-    def check(self, event, raises=False):
+    def check(self, event, auth_events):
         """ Checks if this event is correctly authed.
 
         Returns:
             True if the auth checks pass.
-        Raises:
-            AuthError if there was a problem authorising this event. This will
-            be raised only if raises=True.
         """
         try:
-            if hasattr(event, "room_id"):
+            if not hasattr(event, "room_id"):
-                if event.old_state_events is None:
+                raise AuthError(500, "Event has no room_id: %s" % event)
+            if auth_events is None:
                 # Oh, we don't know what the state of the room was, so we
                 # are trusting that this is allowed (at least for now)
                 logger.warn("Trusting event: %s", event.event_id)
                 return True
 
-                if hasattr(event, "outlier") and event.outlier is True:
-                    # TODO (erikj): Auth for outliers is done differently.
-                    return True
-
             if event.type == RoomCreateEvent.TYPE:
                 # FIXME
                 return True
@@ -68,49 +62,42 @@ class Auth(object):
                 return True
 
             if event.type == RoomMemberEvent.TYPE:
-                    allowed = self.is_membership_change_allowed(event)
+                allowed = self.is_membership_change_allowed(
+                    event, auth_events
+                )
                 if allowed:
                     logger.debug("Allowing! %s", event)
                 else:
                     logger.debug("Denying! %s", event)
                 return allowed
 
-                self.check_event_sender_in_room(event)
+            self.check_event_sender_in_room(event, auth_events)
-                self._can_send_event(event)
+            self._can_send_event(event, auth_events)
 
             if event.type == RoomPowerLevelsEvent.TYPE:
-                    self._check_power_levels(event)
+                self._check_power_levels(event, auth_events)
 
             if event.type == RoomRedactionEvent.TYPE:
-                    self._check_redaction(event)
+                self._check_redaction(event, auth_events)
 
             logger.debug("Allowing! %s", event)
-                return True
-            else:
-                raise AuthError(500, "Unknown event: %s" % event)
         except AuthError as e:
             logger.info(
                 "Event auth check failed on event %s with msg: %s",
                 event, e.msg
             )
             logger.info("Denying! %s", event)
-            if raises:
             raise
 
-        return False
-
     @defer.inlineCallbacks
     def check_joined_room(self, room_id, user_id):
-        try:
+        member = yield self.state.get_current_state(
-            member = yield self.store.get_room_member(
             room_id=room_id,
-                user_id=user_id
+            event_type=RoomMemberEvent.TYPE,
+            state_key=user_id
         )
         self._check_joined_room(member, user_id, room_id)
         defer.returnValue(member)
-        except AttributeError:
-            pass
-        defer.returnValue(None)
 
     @defer.inlineCallbacks
     def check_host_in_room(self, room_id, host):
@@ -130,9 +117,9 @@ class Auth(object):
 
         defer.returnValue(False)
 
-    def check_event_sender_in_room(self, event):
+    def check_event_sender_in_room(self, event, auth_events):
         key = (RoomMemberEvent.TYPE, event.user_id, )
-        member_event = event.state_events.get(key)
+        member_event = auth_events.get(key)
 
         return self._check_joined_room(
             member_event,
@@ -147,14 +134,14 @@ class Auth(object):
             ))
 
     @log_function
-    def is_membership_change_allowed(self, event):
+    def is_membership_change_allowed(self, event, auth_events):
         membership = event.content["membership"]
 
         # Check if this is the room creator joining:
         if len(event.prev_events) == 1 and Membership.JOIN == membership:
             # Get room creation event:
             key = (RoomCreateEvent.TYPE, "", )
-            create = event.old_state_events.get(key)
+            create = auth_events.get(key)
             if create and event.prev_events[0][0] == create.event_id:
                 if create.content["creator"] == event.state_key:
                     return True
@@ -163,19 +150,19 @@ class Auth(object):
 
         # get info about the caller
         key = (RoomMemberEvent.TYPE, event.user_id, )
-        caller = event.old_state_events.get(key)
+        caller = auth_events.get(key)
 
         caller_in_room = caller and caller.membership == Membership.JOIN
         caller_invited = caller and caller.membership == Membership.INVITE
 
         # get info about the target
         key = (RoomMemberEvent.TYPE, target_user_id, )
-        target = event.old_state_events.get(key)
+        target = auth_events.get(key)
 
         target_in_room = target and target.membership == Membership.JOIN
 
         key = (RoomJoinRulesEvent.TYPE, "", )
-        join_rule_event = event.old_state_events.get(key)
+        join_rule_event = auth_events.get(key)
         if join_rule_event:
             join_rule = join_rule_event.content.get(
                 "join_rule", JoinRules.INVITE
@@ -186,11 +173,13 @@ class Auth(object):
         user_level = self._get_power_level_from_event_state(
             event,
             event.user_id,
+            auth_events,
         )
 
         ban_level, kick_level, redact_level = (
             self._get_ops_level_from_event_state(
-                event
+                event,
+                auth_events,
             )
         )
 
@@ -213,7 +202,10 @@ class Auth(object):
 
             # Invites are valid iff caller is in the room and target isn't.
             if not caller_in_room:  # caller isn't joined
-                raise AuthError(403, "You are not in room %s." % event.room_id)
+                raise AuthError(
+                    403,
+                    "%s not in room %s." % (event.user_id, event.room_id,)
+                )
             elif target_in_room:  # the target is already in the room.
                 raise AuthError(403, "%s is already in the room." %
                                      target_user_id)
@@ -236,7 +228,10 @@ class Auth(object):
             # TODO (erikj): Implement kicks.
 
             if not caller_in_room:  # trying to leave a room you aren't joined
-                raise AuthError(403, "You are not in room %s." % event.room_id)
+                raise AuthError(
+                    403,
+                    "%s not in room %s." % (target_user_id, event.room_id,)
+                )
             elif target_user_id != event.user_id:
                 if kick_level:
                     kick_level = int(kick_level)
@@ -260,9 +255,9 @@ class Auth(object):
 
         return True
 
-    def _get_power_level_from_event_state(self, event, user_id):
+    def _get_power_level_from_event_state(self, event, user_id, auth_events):
         key = (RoomPowerLevelsEvent.TYPE, "", )
-        power_level_event = event.old_state_events.get(key)
+        power_level_event = auth_events.get(key)
         level = None
         if power_level_event:
             level = power_level_event.content.get("users", {}).get(user_id)
@@ -270,16 +265,16 @@ class Auth(object):
                 level = power_level_event.content.get("users_default", 0)
         else:
             key = (RoomCreateEvent.TYPE, "", )
-            create_event = event.old_state_events.get(key)
+            create_event = auth_events.get(key)
             if (create_event is not None and
                     create_event.content["creator"] == user_id):
                 return 100
 
         return level
 
-    def _get_ops_level_from_event_state(self, event):
+    def _get_ops_level_from_event_state(self, event, auth_events):
         key = (RoomPowerLevelsEvent.TYPE, "", )
-        power_level_event = event.old_state_events.get(key)
+        power_level_event = auth_events.get(key)
 
         if power_level_event:
             return (
@@ -375,6 +370,11 @@ class Auth(object):
         key = (RoomMemberEvent.TYPE, event.user_id, )
         member_event = event.old_state_events.get(key)
 
+        key = (RoomCreateEvent.TYPE, "", )
+        create_event = event.old_state_events.get(key)
+        if create_event:
+            auth_events.append(create_event.event_id)
+
         if join_rule_event:
             join_rule = join_rule_event.content.get("join_rule")
             is_public = join_rule == JoinRules.PUBLIC if join_rule else False
@@ -406,9 +406,9 @@ class Auth(object):
         event.auth_events = zip(auth_events, hashes)
 
     @log_function
-    def _can_send_event(self, event):
+    def _can_send_event(self, event, auth_events):
         key = (RoomPowerLevelsEvent.TYPE, "", )
-        send_level_event = event.old_state_events.get(key)
+        send_level_event = auth_events.get(key)
         send_level = None
         if send_level_event:
             send_level = send_level_event.content.get("events", {}).get(
@@ -432,6 +432,7 @@ class Auth(object):
         user_level = self._get_power_level_from_event_state(
             event,
             event.user_id,
+            auth_events,
         )
 
         if user_level:
@@ -468,14 +469,16 @@ class Auth(object):
 
         return True
 
-    def _check_redaction(self, event):
+    def _check_redaction(self, event, auth_events):
         user_level = self._get_power_level_from_event_state(
             event,
             event.user_id,
+            auth_events,
         )
 
         _, _, redact_level = self._get_ops_level_from_event_state(
-            event
+            event,
+            auth_events,
         )
 
         if user_level < redact_level:
@@ -484,7 +487,7 @@ class Auth(object):
                 "You don't have permission to redact events"
             )
 
-    def _check_power_levels(self, event):
+    def _check_power_levels(self, event, auth_events):
         user_list = event.content.get("users", {})
         # Validate users
         for k, v in user_list.items():
@@ -499,7 +502,7 @@ class Auth(object):
                 raise SynapseError(400, "Not a valid power level: %s" % (v,))
 
         key = (event.type, event.state_key, )
-        current_state = event.old_state_events.get(key)
+        current_state = auth_events.get(key)
 
         if not current_state:
             return
@@ -507,6 +510,7 @@ class Auth(object):
         user_level = self._get_power_level_from_event_state(
             event,
             event.user_id,
+            auth_events,
         )
 
         # Check other levels:

synapse/api/events/__init__.py

@@ -125,6 +125,7 @@ class SynapseEvent(JsonEncodedObject):
         pdu_json.pop("outlier", None)
         pdu_json.pop("replaces_state", None)
         pdu_json.pop("redacted", None)
+        pdu_json.pop("prev_content", None)
         state_hash = pdu_json.pop("state_hash", None)
         if state_hash is not None:
             pdu_json.setdefault("unsigned", {})["state_hash"] = state_hash

synapse/config/logger.py

@@ -52,12 +52,18 @@ class LoggingConfig(Config):
         if self.log_config is None:
 
             level = logging.INFO
+            level_for_storage = logging.INFO
             if self.verbosity:
                 level = logging.DEBUG
+                if self.verbosity > 1:
+                    level_for_storage = logging.DEBUG
 
             # FIXME: we need a logging.WARN for a -q quiet option
             logger = logging.getLogger('')
             logger.setLevel(level)
+
+            logging.getLogger('synapse.storage').setLevel(level_for_storage)
+
             formatter = logging.Formatter(log_format)
             if self.log_file:
                 handler = logging.FileHandler(self.log_file)

synapse/config/server.py

@@ -35,8 +35,11 @@ class ServerConfig(Config):
         if not args.content_addr:
             host = args.server_name
             if ':' not in host:
-                host = "%s:%d" % (host, args.bind_port)
+                host = "%s:%d" % (host, args.unsecure_port)
-            args.content_addr = "https://%s" % (host,)
+            else:
+                host = host.split(':')[0]
+                host = "%s:%d" % (host, args.unsecure_port)
+            args.content_addr = "http://%s" % (host,)
 
         self.content_addr = args.content_addr
 

synapse/federation/replication.py

@@ -281,6 +281,22 @@ class ReplicationLayer(object):
 
         defer.returnValue(pdus)
 
+    @defer.inlineCallbacks
+    @log_function
+    def get_event_auth(self, destination, context, event_id):
+        res = yield self.transport_layer.get_event_auth(
+            destination, context, event_id,
+        )
+
+        auth_chain = [
+            self.event_from_pdu_json(p, outlier=True)
+            for p in res["auth_chain"]
+        ]
+
+        auth_chain.sort(key=lambda e: e.depth)
+
+        defer.returnValue(auth_chain)
+
     @defer.inlineCallbacks
     @log_function
     def on_backfill_request(self, origin, context, versions, limit):
@@ -481,11 +497,17 @@ class ReplicationLayer(object):
         # FIXME: We probably want to do something with the auth_chain given
         # to us
 
-        # auth_chain = [
+        auth_chain = [
-        #    Pdu(outlier=True, **p) for p in content.get("auth_chain", [])
+            self.event_from_pdu_json(p, outlier=True)
-        # ]
+            for p in content.get("auth_chain", [])
+        ]
 
-        defer.returnValue(state)
+        auth_chain.sort(key=lambda e: e.depth)
+
+        defer.returnValue({
+            "state": state,
+            "auth_chain": auth_chain,
+        })
 
     @defer.inlineCallbacks
     def send_invite(self, destination, context, event_id, pdu):
@@ -543,20 +565,34 @@ class ReplicationLayer(object):
         state = None
 
         # We need to make sure we have all the auth events.
-        for e_id, _ in pdu.auth_events:
+        # for e_id, _ in pdu.auth_events:
-            exists = yield self._get_persisted_pdu(
+        #     exists = yield self._get_persisted_pdu(
-                origin,
+        #         origin,
-                e_id,
+        #         e_id,
-                do_auth=False
+        #         do_auth=False
-            )
+        #     )
-
+        #
-            if not exists:
+        #     if not exists:
-                yield self.get_pdu(
+        #         try:
-                    origin,
+        #             logger.debug(
-                    event_id=e_id,
+        #                 "_handle_new_pdu fetch missing auth event %s from %s",
-                    outlier=True,
+        #                 e_id,
-                )
+        #                 origin,
-                logger.debug("Processed pdu %s", e_id)
+        #             )
+        #
+        #             yield self.get_pdu(
+        #                 origin,
+        #                 event_id=e_id,
+        #                 outlier=True,
+        #             )
+        #
+        #             logger.debug("Processed pdu %s", e_id)
+        #         except:
+        #             logger.warn(
+        #                 "Failed to get auth event %s from %s",
+        #                 e_id,
+        #                 origin
+        #             )
 
         # Get missing pdus if necessary.
         if not pdu.outlier:
@@ -565,6 +601,11 @@ class ReplicationLayer(object):
                 pdu.room_id
             )
 
+            logger.debug(
+                "_handle_new_pdu min_depth for %s: %d",
+                pdu.room_id, min_depth
+            )
+
             if min_depth and pdu.depth > min_depth:
                 for event_id, hashes in pdu.prev_events:
                     exists = yield self._get_persisted_pdu(
@@ -574,11 +615,14 @@ class ReplicationLayer(object):
                     )
 
                     if not exists:
-                        logger.debug("Requesting pdu %s", event_id)
+                        logger.debug(
+                            "_handle_new_pdu requesting pdu %s",
+                            event_id
+                        )
 
                         try:
                             yield self.get_pdu(
-                                pdu.origin,
+                                origin,
                                 event_id=event_id,
                             )
                             logger.debug("Processed pdu %s", event_id)
@@ -588,12 +632,17 @@ class ReplicationLayer(object):
             else:
                 # We need to get the state at this event, since we have reached
                 # a backward extremity edge.
+                logger.debug(
+                    "_handle_new_pdu getting state for %s",
+                    pdu.room_id
+                )
                 state = yield self.get_state_for_context(
                     origin, pdu.room_id, pdu.event_id,
                 )
 
         if not backfilled:
             ret = yield self.handler.on_receive_pdu(
+                origin,
                 pdu,
                 backfilled=backfilled,
                 state=state,
@@ -804,7 +853,10 @@ class _TransactionQueue(object):
 
                 # Ensures we don't continue until all callbacks on that
                 # deferred have fired
+                try:
                     yield deferred
+                except:
+                    pass
 
             logger.debug("TX [%s] Yielded to callbacks", destination)
 
@@ -816,6 +868,7 @@ class _TransactionQueue(object):
             logger.exception(e)
 
             for deferred in deferreds:
+                if not deferred.called:
                     deferred.errback(e)
 
         finally:

synapse/handlers/_base.py

@@ -78,7 +78,7 @@ class BaseHandler(object):
 
         if not suppress_auth:
             logger.debug("Authing...")
-            self.auth.check(event, raises=True)
+            self.auth.check(event, auth_events=event.old_state_events)
             logger.debug("Authed")
         else:
             logger.debug("Suppressed auth.")

synapse/handlers/events.py

@@ -53,8 +53,12 @@ class EventStreamHandler(BaseHandler):
             if auth_user not in self._streams_per_user:
                 self._streams_per_user[auth_user] = 0
                 if auth_user in self._stop_timer_per_user:
+                    try:
                         self.clock.cancel_call_later(
-                        self._stop_timer_per_user.pop(auth_user))
+                            self._stop_timer_per_user.pop(auth_user)
+                        )
+                    except:
+                        logger.exception("Failed to cancel event timer")
                 else:
                     yield self.distributor.fire(
                         "started_user_eventstream", auth_user
@@ -95,10 +99,12 @@ class EventStreamHandler(BaseHandler):
                     logger.debug(
                         "_later stopped_user_eventstream %s", auth_user
                     )
+
+                    self._stop_timer_per_user.pop(auth_user, None)
+
                     yield self.distributor.fire(
                         "stopped_user_eventstream", auth_user
                     )
-                    del self._stop_timer_per_user[auth_user]
 
                 logger.debug("Scheduling _later: for %s", auth_user)
                 self._stop_timer_per_user[auth_user] = (

synapse/handlers/federation.py

@@ -18,13 +18,16 @@
 from ._base import BaseHandler
 
 from synapse.api.events.utils import prune_event
-from synapse.api.errors import AuthError, FederationError, SynapseError
+from synapse.api.errors import (
-from synapse.api.events.room import RoomMemberEvent
+    AuthError, FederationError, SynapseError, StoreError,
+)
+from synapse.api.events.room import RoomMemberEvent, RoomCreateEvent
 from synapse.api.constants import Membership
 from synapse.util.logutils import log_function
 from synapse.util.async import run_on_reactor
 from synapse.crypto.event_signing import (
-    compute_event_signature, check_event_content_hash
+    compute_event_signature, check_event_content_hash,
+    add_hashes_and_signatures,
 )
 from syutil.jsonutil import encode_canonical_json
 
@@ -98,7 +101,7 @@ class FederationHandler(BaseHandler):
 
     @log_function
     @defer.inlineCallbacks
-    def on_receive_pdu(self, pdu, backfilled, state=None):
+    def on_receive_pdu(self, origin, pdu, backfilled, state=None):
         """ Called by the ReplicationLayer when we have a new pdu. We need to
         do auth checks and put it through the StateHandler.
         """
@@ -109,7 +112,7 @@ class FederationHandler(BaseHandler):
         # If we are currently in the process of joining this room, then we
         # queue up events for later processing.
         if event.room_id in self.room_queues:
-            self.room_queues[event.room_id].append(pdu)
+            self.room_queues[event.room_id].append((pdu, origin))
             return
 
         logger.debug("Processing event: %s", event.event_id)
@@ -141,15 +144,62 @@ class FederationHandler(BaseHandler):
             )
             event = redacted_event
 
-        is_new_state = yield self.state_handler.annotate_event_with_state(
-            event,
-            old_state=state
-        )
-
         logger.debug("Event: %s", event)
 
+        # FIXME (erikj): Awful hack to make the case where we are not currently
+        # in the room work
+        current_state = None
+        is_in_room = yield self.auth.check_host_in_room(
+            event.room_id,
+            self.server_name
+        )
+        if not is_in_room and not event.outlier:
+            logger.debug("Got event for room we're not in.")
+
+            replication_layer = self.replication_layer
+            auth_chain = yield replication_layer.get_event_auth(
+                origin,
+                context=event.room_id,
+                event_id=event.event_id,
+            )
+
+            for e in auth_chain:
+                e.outlier = True
                 try:
-            self.auth.check(event, raises=True)
+                    yield self._handle_new_event(e, fetch_missing=False)
+                except:
+                    logger.exception(
+                        "Failed to parse auth event %s",
+                        e.event_id,
+                    )
+
+            if not state:
+                state = yield replication_layer.get_state_for_context(
+                    origin,
+                    context=event.room_id,
+                    event_id=event.event_id,
+                )
+
+            current_state = state
+
+        if state:
+            for e in state:
+                e.outlier = True
+                try:
+                    yield self._handle_new_event(e)
+                except:
+                    logger.exception(
+                        "Failed to parse state event %s",
+                        e.event_id,
+                    )
+
+        try:
+            yield self._handle_new_event(
+                event,
+                state=state,
+                backfilled=backfilled,
+                current_state=current_state,
+            )
         except AuthError as e:
             raise FederationError(
                 "ERROR",
@@ -158,43 +208,17 @@ class FederationHandler(BaseHandler):
                 affected=event.event_id,
             )
 
-        is_new_state = is_new_state and not backfilled
-
-        # TODO: Implement something in federation that allows us to
-        # respond to PDU.
-
-        yield self.store.persist_event(
-            event,
-            backfilled,
-            is_new_state=is_new_state
-        )
-
         room = yield self.store.get_room(event.room_id)
 
         if not room:
-            # Huh, let's try and get the current state
-            try:
-                yield self.replication_layer.get_state_for_context(
-                    event.origin, event.room_id, event.event_id,
-                )
-
-                hosts = yield self.store.get_joined_hosts_for_room(
-                    event.room_id
-                )
-                if self.hs.hostname in hosts:
             try:
                 yield self.store.store_room(
                     room_id=event.room_id,
                     room_creator_user_id="",
                     is_public=False,
                 )
-                    except:
+            except StoreError:
-                        pass
+                logger.exception("Failed to store room.")
-            except:
-                logger.exception(
-                    "Failed to get current state for room %s",
-                    event.room_id
-                )
 
         if not backfilled:
             extra_users = []
@@ -255,11 +279,23 @@ class FederationHandler(BaseHandler):
             pdu=event
         )
 
+
+
         defer.returnValue(pdu)
 
     @defer.inlineCallbacks
     def on_event_auth(self, event_id):
         auth = yield self.store.get_auth_chain(event_id)
+
+        for event in auth:
+            event.signatures.update(
+                compute_event_signature(
+                    event,
+                    self.hs.hostname,
+                    self.hs.config.signing_key[0]
+                )
+            )
+
         defer.returnValue([e for e in auth])
 
     @log_function
@@ -276,6 +312,8 @@ class FederationHandler(BaseHandler):
         We suspend processing of any received events from this room until we
         have finished processing the join.
         """
+        logger.debug("Joining %s to %s", joinee, room_id)
+
         pdu = yield self.replication_layer.make_join(
             target_host,
             room_id,
@@ -298,19 +336,29 @@ class FederationHandler(BaseHandler):
 
         try:
             event.event_id = self.event_factory.create_event_id()
+            event.origin = self.hs.hostname
             event.content = content
 
-            state = yield self.replication_layer.send_join(
+            if not hasattr(event, "signatures"):
+                event.signatures = {}
+
+            add_hashes_and_signatures(
+                event,
+                self.hs.hostname,
+                self.hs.config.signing_key[0],
+            )
+
+            ret = yield self.replication_layer.send_join(
                 target_host,
                 event
             )
 
-            logger.debug("do_invite_join state: %s", state)
+            state = ret["state"]
+            auth_chain = ret["auth_chain"]
+            auth_chain.sort(key=lambda e: e.depth)
 
-            yield self.state_handler.annotate_event_with_state(
+            logger.debug("do_invite_join auth_chain: %s", auth_chain)
-                event,
+            logger.debug("do_invite_join state: %s", state)
-                old_state=state
-            )
 
             logger.debug("do_invite_join event: %s", event)
 
@@ -324,34 +372,50 @@ class FederationHandler(BaseHandler):
                 # FIXME
                 pass
 
+            for e in auth_chain:
+                e.outlier = True
+                try:
+                    yield self._handle_new_event(e, fetch_missing=False)
+                except:
+                    logger.exception(
+                        "Failed to parse auth event %s",
+                        e.event_id,
+                    )
+
             for e in state:
                 # FIXME: Auth these.
                 e.outlier = True
-
+                try:
-                yield self.state_handler.annotate_event_with_state(
+                    yield self._handle_new_event(
                         e,
+                        fetch_missing=True
+                    )
+                except:
+                    logger.exception(
+                        "Failed to parse state event %s",
+                        e.event_id,
                     )
 
-                yield self.store.persist_event(
+            yield self._handle_new_event(
-                    e,
-                    backfilled=False,
-                    is_new_state=True
-                )
-
-            yield self.store.persist_event(
                 event,
-                backfilled=False,
+                state=state,
-                is_new_state=True
+                current_state=state,
             )
+
+            yield self.notifier.on_new_room_event(
+                event, extra_users=[joinee]
+            )
+
+            logger.debug("Finished joining %s to %s", joinee, room_id)
         finally:
             room_queue = self.room_queues[room_id]
             del self.room_queues[room_id]
 
-            for p in room_queue:
+            for p, origin in room_queue:
                 try:
-                    yield self.on_receive_pdu(p, backfilled=False)
+                    self.on_receive_pdu(origin, p, backfilled=False)
                 except:
-                    pass
+                    logger.exception("Couldn't handle pdu")
 
         defer.returnValue(True)
 
@@ -375,7 +439,7 @@ class FederationHandler(BaseHandler):
 
         yield self.state_handler.annotate_event_with_state(event)
         yield self.auth.add_auth_events(event)
-        self.auth.check(event, raises=True)
+        self.auth.check(event, auth_events=event.old_state_events)
 
         pdu = event
 
@@ -391,17 +455,7 @@ class FederationHandler(BaseHandler):
 
         event.outlier = False
 
-        state_handler = self.state_handler
+        yield self._handle_new_event(event)
-        is_new_state = yield state_handler.annotate_event_with_state(event)
-        self.auth.check(event, raises=True)
-
-        # FIXME (erikj):  All this is duplicated above :(
-
-        yield self.store.persist_event(
-            event,
-            backfilled=False,
-            is_new_state=is_new_state
-        )
 
         extra_users = []
         if event.type == RoomMemberEvent.TYPE:
@@ -414,7 +468,7 @@ class FederationHandler(BaseHandler):
         )
 
         if event.type == RoomMemberEvent.TYPE:
-            if event.membership == Membership.JOIN:
+            if event.content["membership"] == Membership.JOIN:
                 user = self.hs.parse_userid(event.state_key)
                 yield self.distributor.fire(
                     "user_joined_room", user=user, room_id=event.room_id
@@ -508,7 +562,17 @@ class FederationHandler(BaseHandler):
                 else:
                     del results[(event.type, event.state_key)]
 
-            defer.returnValue(results.values())
+            res = results.values()
+            for event in res:
+                event.signatures.update(
+                    compute_event_signature(
+                        event,
+                        self.hs.hostname,
+                        self.hs.config.signing_key[0]
+                    )
+                )
+
+            defer.returnValue(res)
         else:
             defer.returnValue([])
 
@@ -541,6 +605,17 @@ class FederationHandler(BaseHandler):
         )
 
         if event:
+            # FIXME: This is a temporary work around where we occasionally
+            # return events slightly differently than when they were
+            # originally signed
+            event.signatures.update(
+                compute_event_signature(
+                    event,
+                    self.hs.hostname,
+                    self.hs.config.signing_key[0]
+                )
+            )
+
             if do_auth:
                 in_room = yield self.auth.check_host_in_room(
                     event.room_id,
@@ -565,3 +640,78 @@ class FederationHandler(BaseHandler):
         )
         while waiters:
             waiters.pop().callback(None)
+
+    @defer.inlineCallbacks
+    def _handle_new_event(self, event, state=None, backfilled=False,
+                          current_state=None, fetch_missing=True):
+        is_new_state = yield self.state_handler.annotate_event_with_state(
+            event,
+            old_state=state
+        )
+
+        if event.old_state_events:
+            known_ids = set(
+                [s.event_id for s in event.old_state_events.values()]
+            )
+            for e_id, _ in event.auth_events:
+                if e_id not in known_ids:
+                    e = yield self.store.get_event(
+                        e_id,
+                        allow_none=True,
+                    )
+
+                    if not e:
+                        # TODO: Do some conflict res to make sure that we're
+                        # not the ones who are wrong.
+                        logger.info(
+                            "Rejecting %s as %s not in %s",
+                            event.event_id, e_id, known_ids,
+                        )
+                        raise AuthError(403, "Auth events are stale")
+
+            auth_events = event.old_state_events
+        else:
+            # We need to get the auth events from somewhere.
+
+            # TODO: Don't just hit the DBs?
+
+            auth_events = {}
+            for e_id, _ in event.auth_events:
+                e = yield self.store.get_event(
+                    e_id,
+                    allow_none=True,
+                )
+
+                if not e:
+                    e = yield self.replication_layer.get_pdu(
+                        event.origin, e_id, outlier=True
+                    )
+
+                    if e and fetch_missing:
+                        try:
+                            yield self.on_receive_pdu(event.origin, e, False)
+                        except:
+                            logger.exception(
+                                "Failed to parse auth event %s",
+                                e_id,
+                            )
+
+                if not e:
+                    logger.warn("Can't find auth event %s.", e_id)
+
+                auth_events[(e.type, e.state_key)] = e
+
+            if event.type == RoomMemberEvent.TYPE and not event.auth_events:
+                if len(event.prev_events) == 1:
+                    c = yield self.store.get_event(event.prev_events[0][0])
+                    if c.type == RoomCreateEvent.TYPE:
+                        auth_events[(c.type, c.state_key)] = c
+
+        self.auth.check(event, auth_events=auth_events)
+
+        yield self.store.persist_event(
+            event,
+            backfilled=backfilled,
+            is_new_state=(is_new_state and not backfilled),
+            current_state=current_state,
+        )

synapse/handlers/message.py

@@ -306,7 +306,7 @@ class MessageHandler(BaseHandler):
         auth_user = self.hs.parse_userid(user_id)
 
         # TODO: These concurrently
-        state_tuples = yield self.store.get_current_state(room_id)
+        state_tuples = yield self.state_handler.get_current_state(room_id)
         state = [self.hs.serialize_event(x) for x in state_tuples]
 
         member_event = (yield self.store.get_room_member(

synapse/handlers/presence.py

@@ -651,12 +651,13 @@ class PresenceHandler(BaseHandler):
             logger.debug("Incoming presence update from %s", user)
 
             observers = set(self._remote_recvmap.get(user, set()))
+            if observers:
+                logger.debug(" | %d interested local observers %r", len(observers), observers)
 
             rm_handler = self.homeserver.get_handlers().room_member_handler
             room_ids = yield rm_handler.get_rooms_for_user(user)
-
+            if room_ids:
-            if not observers and not room_ids:
+                logger.debug(" | %d interested room IDs %r", len(room_ids), room_ids)
-                continue
 
             state = dict(push)
             del state["user_id"]
@@ -678,6 +679,10 @@ class PresenceHandler(BaseHandler):
             self._user_cachemap_latest_serial += 1
             statuscache.update(state, serial=self._user_cachemap_latest_serial)
 
+            if not observers and not room_ids:
+                logger.debug(" | no interested observers or room IDs")
+                continue
+
             self.push_update_to_clients(
                 observed_user=user,
                 users_to_push=observers,
@@ -799,6 +804,7 @@ class PresenceEventSource(object):
             )
 
     @defer.inlineCallbacks
+    @log_function
     def get_new_events_for_user(self, user, from_key, limit):
         from_key = int(from_key)
 
@@ -811,7 +817,8 @@ class PresenceEventSource(object):
         # TODO(paul): use a DeferredList ? How to limit concurrency.
         for observed_user in cachemap.keys():
             cached = cachemap[observed_user]
-            if not (from_key < cached.serial):
+
+            if cached.serial <= from_key:
                 continue
 
             if (yield self.is_visible(observer_user, observed_user)):

synapse/handlers/room.py

@@ -519,7 +519,11 @@ class RoomMemberHandler(BaseHandler):
             user_id=user.to_string(), membership_list=membership_list
         )
 
-        defer.returnValue([r.room_id for r in rooms])
+        # For some reason the list of events contains duplicates
+        # TODO(paul): work out why because I really don't think it should
+        room_ids = set(r.room_id for r in rooms)
+
+        defer.returnValue(room_ids)
 
     @defer.inlineCallbacks
     def _do_local_membership_update(self, event, membership, snapshot,

synapse/rest/room.py

@@ -148,7 +148,7 @@ class RoomStateEventRestServlet(RestServlet):
         content = _parse_json(request)
 
         event = self.event_factory.create_event(
-            etype=urllib.unquote(event_type),
+            etype=event_type,  # already urldecoded
             content=content,
             room_id=urllib.unquote(room_id),
             user_id=user.to_string(),
@@ -327,7 +327,9 @@ class RoomMessageListRestServlet(RestServlet):
     @defer.inlineCallbacks
     def on_GET(self, request, room_id):
         user = yield self.auth.get_user_by_req(request)
-        pagination_config = PaginationConfig.from_request(request)
+        pagination_config = PaginationConfig.from_request(request,
+            default_limit=10,
+        )
         with_feedback = "feedback" in request.args
         handler = self.handlers.message_handler
         msgs = yield handler.get_messages(

synapse/state.py

@@ -82,7 +82,7 @@ class StateHandler(object):
         if hasattr(event, "outlier") and event.outlier:
             event.state_group = None
             event.old_state_events = None
-            event.state_events = {}
+            event.state_events = None
             defer.returnValue(False)
             return
 

synapse/storage/__init__.py

@@ -69,7 +69,7 @@ SCHEMAS = [
 
 # Remember to update this number every time an incompatible change is made to
 # database schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 7
+SCHEMA_VERSION = 8
 
 
 class _RollbackButIsFineException(Exception):
@@ -97,7 +97,8 @@ class DataStore(RoomMemberStore, RoomStore,
 
     @defer.inlineCallbacks
     @log_function
-    def persist_event(self, event, backfilled=False, is_new_state=True):
+    def persist_event(self, event, backfilled=False, is_new_state=True,
+                      current_state=None):
         stream_ordering = None
         if backfilled:
             if not self.min_token_deferred.called:
@@ -113,6 +114,7 @@ class DataStore(RoomMemberStore, RoomStore,
                 backfilled=backfilled,
                 stream_ordering=stream_ordering,
                 is_new_state=is_new_state,
+                current_state=current_state,
             )
         except _RollbackButIsFineException:
             pass
@@ -141,7 +143,7 @@ class DataStore(RoomMemberStore, RoomStore,
 
     @log_function
     def _persist_event_txn(self, txn, event, backfilled, stream_ordering=None,
-                           is_new_state=True):
+                           is_new_state=True, current_state=None):
         if event.type == RoomMemberEvent.TYPE:
             self._store_room_member_txn(txn, event)
         elif event.type == FeedbackEvent.TYPE:
@@ -210,8 +212,27 @@ class DataStore(RoomMemberStore, RoomStore,
 
         self._store_state_groups_txn(txn, event)
 
+        if current_state:
+            txn.execute(
+                "DELETE FROM current_state_events WHERE room_id = ?",
+                (event.room_id,)
+            )
+
+            for s in current_state:
+                self._simple_insert_txn(
+                    txn,
+                    "current_state_events",
+                    {
+                        "event_id": s.event_id,
+                        "room_id": s.room_id,
+                        "type": s.type,
+                        "state_key": s.state_key,
+                    },
+                    or_replace=True,
+                )
+
         is_state = hasattr(event, "state_key") and event.state_key is not None
-        if is_new_state and is_state:
+        if is_state:
             vals = {
                 "event_id": event.event_id,
                 "room_id": event.room_id,
@@ -229,6 +250,7 @@ class DataStore(RoomMemberStore, RoomStore,
                 or_replace=True,
             )
 
+            if is_new_state:
                 self._simple_insert_txn(
                     txn,
                     "current_state_events",
@@ -316,7 +338,12 @@ class DataStore(RoomMemberStore, RoomStore,
             txn, event.event_id, ref_alg, ref_hash_bytes
         )
 
-        self._update_min_depth_for_room_txn(txn, event.room_id, event.depth)
+        if not outlier:
+            self._update_min_depth_for_room_txn(
+                txn,
+                event.room_id,
+                event.depth
+            )
 
     def _store_redaction(self, txn, event):
         txn.execute(

synapse/storage/roommember.py

@@ -218,7 +218,9 @@ class RoomMemberStore(SQLBaseStore):
             "ON m.event_id = c.event_id "
             "WHERE m.membership = 'join' "
             "AND (%(clause)s) "
-            "GROUP BY m.room_id HAVING COUNT(m.room_id) = ?"
+            # TODO(paul): We've got duplicate rows in the database somewhere
+            #   so we have to DISTINCT m.user_id here
+            "GROUP BY m.room_id HAVING COUNT(DISTINCT m.user_id) = ?"
         ) % {"clause": user_list_clause}
 
         args = list(user_id_list)

synapse/storage/schema/delta/v8.sql

@@ -0,0 +1,34 @@
+/* Copyright 2014 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ CREATE TABLE IF NOT EXISTS event_signatures_2 (
+    event_id TEXT,
+    signature_name TEXT,
+    key_id TEXT,
+    signature BLOB,
+    CONSTRAINT uniqueness UNIQUE (event_id, signature_name, key_id)
+);
+
+INSERT INTO event_signatures_2 (event_id, signature_name, key_id, signature)
+SELECT event_id, signature_name, key_id, signature FROM event_signatures;
+
+DROP TABLE event_signatures;
+ALTER TABLE event_signatures_2 RENAME TO event_signatures;
+
+CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures (
+    event_id
+);
+
+PRAGMA user_version = 8;

synapse/storage/schema/event_signatures.sql

@@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS event_signatures (
     signature_name TEXT,
     key_id TEXT,
     signature BLOB,
-    CONSTRAINT uniqueness UNIQUE (event_id, key_id)
+    CONSTRAINT uniqueness UNIQUE (event_id, signature_name, key_id)
 );
 
 CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures (

synapse/storage/state.py

@@ -87,7 +87,7 @@ class StateStore(SQLBaseStore):
         )
 
     def _store_state_groups_txn(self, txn, event):
-        if not event.state_events:
+        if event.state_events is None:
             return
 
         state_group = event.state_group

synapse/storage/stream.py

@@ -283,7 +283,7 @@ class StreamStore(SQLBaseStore):
 
         sql = (
             "SELECT *, (%(redacted)s) AS redacted FROM events "
-            "WHERE room_id = ? AND stream_ordering <= ? "
+            "WHERE room_id = ? AND stream_ordering <= ? AND outlier = 0 "
             "ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ? "
         ) % {
             "redacted": del_sql,

synapse/streams/config.py

@@ -47,7 +47,8 @@ class PaginationConfig(object):
         self.limit = int(limit) if limit is not None else None
 
     @classmethod
-    def from_request(cls, request, raise_invalid_params=True):
+    def from_request(cls, request, raise_invalid_params=True,
+                     default_limit=None):
         def get_param(name, default=None):
             lst = request.args.get(name, [])
             if len(lst) > 1:
@@ -84,6 +85,9 @@ class PaginationConfig(object):
         if limit is not None and not limit.isdigit():
             raise SynapseError(400, "'limit' parameter must be an integer.")
 
+        if limit is None:
+            limit = default_limit
+
         try:
             return PaginationConfig(from_tok, to_tok, direction, limit)
         except:

tests/handlers/test_federation.py

@@ -42,6 +42,7 @@ class FederationTestCase(unittest.TestCase):
 
         self.auth = NonCallableMock(spec_set=[
             "check",
+            "check_host_in_room",
         ])
 
         self.hostname = "test"
@@ -83,20 +84,25 @@ class FederationTestCase(unittest.TestCase):
             event_id="$a:b",
             user_id="@a:b",
             origin="b",
+            auth_events=[],
             hashes={"sha256":"AcLrgtUIqqwaGoHhrEvYG1YLDIsVPYJdSRGhkp3jJp8"},
         )
 
         self.datastore.persist_event.return_value = defer.succeed(None)
         self.datastore.get_room.return_value = defer.succeed(True)
+        self.auth.check_host_in_room.return_value = defer.succeed(True)
 
-        self.state_handler.annotate_event_with_state.return_value = (
+        def annotate(ev, old_state=None):
-            defer.succeed(False)
+            ev.old_state_events = []
+            return defer.succeed(False)
+        self.state_handler.annotate_event_with_state.side_effect = annotate
+
+        yield self.handlers.federation_handler.on_receive_pdu(
+            "fo", pdu, False
         )
 
-        yield self.handlers.federation_handler.on_receive_pdu(pdu, False)
-
         self.datastore.persist_event.assert_called_once_with(
-            ANY, False, is_new_state=False
+            ANY, is_new_state=False, backfilled=False, current_state=None
         )
 
         self.state_handler.annotate_event_with_state.assert_called_once_with(
@@ -104,7 +110,7 @@ class FederationTestCase(unittest.TestCase):
             old_state=None,
         )
 
-        self.auth.check.assert_called_once_with(ANY, raises=True)
+        self.auth.check.assert_called_once_with(ANY, auth_events={})
 
         self.notifier.on_new_room_event.assert_called_once_with(
             ANY,

tests/handlers/test_room.py

@@ -120,7 +120,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase):
 
         self.datastore.get_room_member.return_value = defer.succeed(None)
 
-        event.state_events = {
+        event.old_state_events = {
             (RoomMemberEvent.TYPE, "@alice:green"): self._create_member(
                 user_id="@alice:green",
                 room_id=room_id,
@@ -129,9 +129,11 @@ class RoomMemberHandlerTestCase(unittest.TestCase):
                 user_id="@bob:red",
                 room_id=room_id,
             ),
-            (RoomMemberEvent.TYPE, target_user_id): event,
         }
 
+        event.state_events = event.old_state_events
+        event.state_events[(RoomMemberEvent.TYPE, target_user_id)] = event
+
         # Actual invocation
         yield self.room_member_handler.change_membership(event)
 
@@ -187,6 +189,16 @@ class RoomMemberHandlerTestCase(unittest.TestCase):
             (RoomMemberEvent.TYPE, user_id): event,
         }
 
+        event.old_state_events = {
+            (RoomMemberEvent.TYPE, "@alice:green"): self._create_member(
+                user_id="@alice:green",
+                room_id=room_id,
+            ),
+        }
+
+        event.state_events = event.old_state_events
+        event.state_events[(RoomMemberEvent.TYPE, user_id)] = event
+
         # Actual invocation
         yield self.room_member_handler.change_membership(event)