Merge branch 'videocalls' into develop

Conflicts:
	webclient/room/room.html

webclient/app-controller.js

@@ -27,6 +27,12 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even
     // Check current URL to avoid to display the logout button on the login page
     $scope.location = $location.path();
 
+    // disable nganimate for the local and remote video elements because ngAnimate appears
+    // to be buggy and leaves animation classes on the video elements causing them to show
+    // when they should not (their animations are pure CSS3)
+    $animate.enabled(false, angular.element('#localVideo'));
+    $animate.enabled(false, angular.element('#remoteVideo'));
+    
     // Update the location state when the ng location changed
     $rootScope.$on('$routeChangeSuccess', function (event, current, previous) {
         $scope.location = $location.path();
@@ -93,7 +99,13 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even
     };
 
     $rootScope.$watch('currentCall', function(newVal, oldVal) {
-        if (!$rootScope.currentCall) return;
+        if (!$rootScope.currentCall) {
+            // This causes the still frame to be flushed out of the video elements,
+            // avoiding a flash of the last frame of the previous call when starting the next
+            angular.element('#localVideo')[0].load();
+            angular.element('#remoteVideo')[0].load();
+            return;
+        }
 
         var roomMembers = angular.copy($rootScope.events.rooms[$rootScope.currentCall.room_id].members);
         delete roomMembers[matrixService.config().user_id];
@@ -126,6 +138,7 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even
             angular.element('#ringAudio')[0].pause();
             angular.element('#ringbackAudio')[0].pause();
             angular.element('#callendAudio')[0].play();
+            $scope.videoMode = undefined;
         } else if (newVal == 'ended' && oldVal == 'invite_sent' && $rootScope.currentCall.hangupParty == 'remote') {
             angular.element('#ringAudio')[0].pause();
             angular.element('#ringbackAudio')[0].pause();
@@ -138,6 +151,20 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even
             angular.element('#ringbackAudio')[0].pause();
         } else if (oldVal == 'ringing') {
             angular.element('#ringAudio')[0].pause();
+        } else if (newVal == 'connected') {
+            $timeout(function() {
+                if ($scope.currentCall.type == 'video') $scope.videoMode = 'large';
+            }, 500);
+        }
+
+        if ($rootScope.currentCall && $rootScope.currentCall.type == 'video' && $rootScope.currentCall.state != 'connected') {
+            $scope.videoMode = 'mini';
+        }
+    });
+    $rootScope.$watch('currentCall.type', function(newVal, oldVal) {
+        // need to listen for this too as the type of the call won't be know when it's created
+        if ($rootScope.currentCall && $rootScope.currentCall.type == 'video' && $rootScope.currentCall.state != 'connected') {
+            $scope.videoMode = 'mini';
         }
     });
 
@@ -150,6 +177,8 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even
         }
         call.onError = $scope.onCallError;
         call.onHangup = $scope.onCallHangup;
+        call.localVideoElement = angular.element('#localVideo')[0];
+        call.remoteVideoElement = angular.element('#remoteVideo')[0];
         $rootScope.currentCall = call;
     });
 
@@ -170,7 +199,7 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even
     
     $rootScope.onCallError = function(errStr) {
         $scope.feedback = errStr;
-    }
+    };
 
     $rootScope.onCallHangup = function(call) {
         if (call == $rootScope.currentCall) {
@@ -178,5 +207,5 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even
                 if (call == $rootScope.currentCall) $rootScope.currentCall = undefined;
             }, 4070);
         }
-    }
+    };
 }]);

webclient/app.css

@@ -94,6 +94,79 @@ a:active  { color: #000; }
     font-size: 80%;
 }
 
+#videoBackground {
+    position: absolute;
+    height: 100%;
+    width: 100%;
+    top: 32px;
+    left: 0px;
+    z-index: 1;
+    background-color: rgba(0,0,0,0.0);
+    pointer-events: none;
+    transition: background-color linear 500ms;
+}
+
+#videoBackground.large {
+    background-color: rgba(0,0,0,0.85);
+    pointer-events: auto;
+}
+
+#videoContainer {
+    position: relative;
+    max-width: 1280px;
+    margin: auto;
+}
+
+#videoContainerPadding {
+    width: 1280px;
+}
+
+#localVideo {
+    position: absolute;
+    width: 128px;
+    height: 72px;
+    z-index: 1;
+    transition: left linear 500ms, top linear 500ms, width linear 500ms, height linear 500ms;
+}
+
+#localVideo.mini {
+    top: 0px;
+    left: 130px;
+}
+
+#localVideo.large {
+    top: 70px;
+    left: 20px;
+}
+
+#localVideo.ended {
+    -webkit-filter: grayscale(1);
+    filter: grayscale(1);
+}
+
+#remoteVideo {
+    position: relative;
+    height: auto;
+    transition: left linear 500ms, top linear 500ms, width linear 500ms, height linear 500ms;
+}
+
+#remoteVideo.mini {
+    left: 260px;
+    top: 0px;
+    width: 128px;
+}
+
+#remoteVideo.large {
+    left: 0px;
+    top: 50px;
+    width: 100%;
+}
+
+#remoteVideo.ended {
+    -webkit-filter: grayscale(1);
+    filter: grayscale(1);
+}
+
 #headerContent {
     color: #ccc;
     max-width: 1280px;
@@ -101,6 +174,7 @@ a:active  { color: #000; }
     text-align: right;
     height: 32px;
     line-height: 32px;
+    position: relative;
 }
 
 #headerContent a:link,

webclient/components/matrix/matrix-call.js

@@ -56,6 +56,12 @@ angular.module('MatrixCall', [])
         // a queue for candidates waiting to go out. We try to amalgamate candidates into a single candidate message where possible
         this.candidateSendQueue = [];
         this.candidateSendTries = 0;
+
+        var self = this;
+        $rootScope.$watch(this.remoteVideoElement, function (oldValue, newValue) {
+            self.tryPlayRemoteStream();
+        });
+
     }
 
     MatrixCall.CALL_TIMEOUT = 60000;
@@ -76,13 +82,39 @@ angular.module('MatrixCall', [])
         return pc;
     }
 
-    MatrixCall.prototype.placeCall = function(config) {
+    MatrixCall.prototype.getUserMediaVideoContraints = function(callType) {
+        switch (callType) {
+            case 'voice':
+                return ({audio: true, video: false});
+            case 'video':
+                return ({audio: true, video: {
+                    mandatory: {
+                        minWidth: 640,
+                        maxWidth: 640,
+                        minHeight: 360,
+                        maxHeight: 360,
+                    }
+                }});
+        }
+    };
+
+    MatrixCall.prototype.placeVoiceCall = function() {
+        this.placeCallWithConstraints(this.getUserMediaVideoContraints('voice'));
+        this.type = 'voice';
+    };
+
+    MatrixCall.prototype.placeVideoCall = function(config) {
+        this.placeCallWithConstraints(this.getUserMediaVideoContraints('video'));
+        this.type = 'video';
+    };
+
+    MatrixCall.prototype.placeCallWithConstraints = function(constraints) {
         var self = this;
         matrixPhoneService.callPlaced(this);
-        navigator.getUserMedia({audio: config.audio, video: config.video}, function(s) { self.gotUserMediaForInvite(s); }, function(e) { self.getUserMediaFailed(e); });
+        navigator.getUserMedia(constraints, function(s) { self.gotUserMediaForInvite(s); }, function(e) { self.getUserMediaFailed(e); });
         this.state = 'wait_local_media';
         this.direction = 'outbound';
-        this.config = config;
+        this.config = constraints;
     };
 
     MatrixCall.prototype.initWithInvite = function(event) {
@@ -91,6 +123,17 @@ angular.module('MatrixCall', [])
         this.peerConn.setRemoteDescription(new RTCSessionDescription(this.msg.offer), this.onSetRemoteDescriptionSuccess, this.onSetRemoteDescriptionError);
         this.state = 'ringing';
         this.direction = 'inbound';
+
+        if (window.mozRTCPeerConnection) {
+            // firefox's RTCPeerConnection doesn't add streams until it starts getting media on them
+            // so we need to figure out whether a video channel has been offered by ourselves.
+            if (this.msg.offer.sdp.indexOf('m=video') > -1) {
+                this.type = 'video';
+            } else {
+                this.type = 'voice';
+            }
+        }
+
         var self = this;
         $timeout(function() {
             if (self.state == 'ringing') {
@@ -115,7 +158,7 @@ angular.module('MatrixCall', [])
         console.log("Answering call "+this.call_id);
         var self = this;
         if (!this.localAVStream && !this.waitForLocalAVStream) {
-            navigator.getUserMedia({audio: true, video: false}, function(s) { self.gotUserMediaForAnswer(s); }, function(e) { self.getUserMediaFailed(e); });
+            navigator.getUserMedia(this.getUserMediaVideoContraints(this.type), function(s) { self.gotUserMediaForAnswer(s); }, function(e) { self.getUserMediaFailed(e); });
             this.state = 'wait_local_media';
         } else if (this.localAVStream) {
             this.gotUserMediaForAnswer(this.localAVStream);
@@ -140,6 +183,11 @@ angular.module('MatrixCall', [])
     MatrixCall.prototype.hangup = function(suppressEvent) {
         console.log("Ending call "+this.call_id);
 
+        // pausing now keeps the last frame (ish) of the video call in the video element
+        // rather than it just turning black straight away
+        if (this.remoteVideoElement) this.remoteVideoElement.pause();
+        if (this.localVideoElement) this.localVideoElement.pause();
+
         this.stopAllMedia();
         if (this.peerConn) this.peerConn.close();
 
@@ -161,6 +209,13 @@ angular.module('MatrixCall', [])
         }
         if (this.state == 'ended') return;
 
+        if (this.localVideoElement && this.type == 'video') {
+            var vidTrack = stream.getVideoTracks()[0];
+            this.localVideoElement.src = URL.createObjectURL(stream);
+            this.localVideoElement.muted = true;
+            this.localVideoElement.play();
+        }
+
         this.localAVStream = stream;
         var audioTracks = stream.getAudioTracks();
         for (var i = 0; i < audioTracks.length; i++) {
@@ -182,6 +237,13 @@ angular.module('MatrixCall', [])
     MatrixCall.prototype.gotUserMediaForAnswer = function(stream) {
         if (this.state == 'ended') return;
 
+        if (this.localVideoElement && this.type == 'video') {
+            var vidTrack = stream.getVideoTracks()[0];
+            this.localVideoElement.src = URL.createObjectURL(stream);
+            this.localVideoElement.muted = true;
+            this.localVideoElement.play();
+        }
+
         this.localAVStream = stream;
         var audioTracks = stream.getAudioTracks();
         for (var i = 0; i < audioTracks.length; i++) {
@@ -192,7 +254,7 @@ angular.module('MatrixCall', [])
         var constraints = {
             'mandatory': {
                 'OfferToReceiveAudio': true,
-                'OfferToReceiveVideo': false
+                'OfferToReceiveVideo': this.type == 'video'
             },
         };
         this.peerConn.createAnswer(function(d) { self.createdAnswer(d); }, function(e) {}, constraints);
@@ -223,6 +285,7 @@ angular.module('MatrixCall', [])
         this.state = 'connecting';
     };
 
+
     MatrixCall.prototype.gotLocalOffer = function(description) {
         console.log("Created offer: "+description);
 
@@ -274,7 +337,7 @@ angular.module('MatrixCall', [])
     };
 
     MatrixCall.prototype.getUserMediaFailed = function() {
-        this.onError("Couldn't start capturing audio! Is your microphone set up?");
+        this.onError("Couldn't start capturing! Is your microphone set up?");
         this.hangup();
     };
 
@@ -310,6 +373,14 @@ angular.module('MatrixCall', [])
 
         this.remoteAVStream = s;
 
+        if (this.direction == 'inbound') {
+            if (s.getVideoTracks().length > 0) {
+                this.type = 'video';
+            } else {
+                this.type = 'voice';
+            }
+        }
+
         var self = this;
         forAllTracksOnStream(s, function(t) {
             // not currently implemented in chrome
@@ -319,9 +390,16 @@ angular.module('MatrixCall', [])
         event.stream.onended = function(e) { self.onRemoteStreamEnded(e); }; 
         // not currently implemented in chrome
         event.stream.onstarted = function(e) { self.onRemoteStreamStarted(e); };
-        var player = new Audio();
+
-        player.src = URL.createObjectURL(s);
+        this.tryPlayRemoteStream();
-        player.play();
+    };
+
+    MatrixCall.prototype.tryPlayRemoteStream = function(event) {
+        if (this.remoteVideoElement && this.remoteAVStream) {
+            var player = this.remoteVideoElement;
+            player.src = URL.createObjectURL(this.remoteAVStream);
+            player.play();
+        }
     };
 
     MatrixCall.prototype.onRemoteStreamStarted = function(event) {
@@ -352,10 +430,12 @@ angular.module('MatrixCall', [])
 
     MatrixCall.prototype.onHangupReceived = function() {
         console.log("Hangup received");
+        if (this.remoteVideoElement) this.remoteVideoElement.pause();
+        if (this.localVideoElement) this.localVideoElement.pause();
         this.state = 'ended';
         this.hangupParty = 'remote';
         this.stopAllMedia();
-        if (this.peerConn.signalingState != 'closed') this.peerConn.close();
+        if (this.peerConn && this.peerConn.signalingState != 'closed') this.peerConn.close();
         if (this.onHangup) this.onHangup(this);
     };
 
@@ -366,13 +446,15 @@ angular.module('MatrixCall', [])
             newCall.waitForLocalAVStream = true;
         } else if (this.state == 'create_offer') {
             console.log("Handing local stream to new call");
-            newCall.localAVStream = this.localAVStream;
+            newCall.gotUserMediaForAnswer(this.localAVStream);
             delete(this.localAVStream);
         } else if (this.state == 'invite_sent') {
             console.log("Handing local stream to new call");
-            newCall.localAVStream = this.localAVStream;
+            newCall.gotUserMediaForAnswer(this.localAVStream);
             delete(this.localAVStream);
         }
+        newCall.localVideoElement = this.localVideoElement;
+        newCall.remoteVideoElement = this.remoteVideoElement;
         this.successor = newCall;
         this.hangup(true);
     };

webclient/index.html

@@ -45,6 +45,13 @@
 </head>
 
 <body>
+    <div id="videoBackground" ng-class="videoMode">
+        <div id="videoContainer" ng-class="videoMode">
+            <div id="videoContainerPadding"></div>
+            <video id="localVideo" ng-class="[videoMode, currentCall.state]" ng-show="currentCall && currentCall.type == 'video' && (currentCall.state == 'connected' || currentCall.state == 'connecting' || currentCall.state == 'invite_sent' || currentCall.state == 'ended')"></video>
+            <video id="remoteVideo" ng-class="[videoMode, currentCall.state]" ng-show="currentCall && currentCall.type == 'video' && (currentCall.state == 'connected' || (currentCall.state == 'ended' && currentCall.didConnect))"></video>
+        </div>
+    </div>
 
     <div id="header">
         <!-- Do not show buttons on the login page -->
@@ -58,7 +65,8 @@
                     <br />
                     <span id="callState">
                         <span ng-show="currentCall.state == 'invite_sent'">Calling...</span>
-                        <span ng-show="currentCall.state == 'ringing'">Incoming Call</span>
+                        <span ng-show="currentCall.state == 'ringing' && currentCall && currentCall.type == 'video'">Incoming Video Call</span>
+                        <span ng-show="currentCall.state == 'ringing' && currentCall && currentCall.type == 'voice'">Incoming Voice Call</span>
                         <span ng-show="currentCall.state == 'connecting'">Call Connecting...</span>
                         <span ng-show="currentCall.state == 'connected'">Call Connected</span>
                         <span ng-show="currentCall.state == 'ended' && !currentCall.didConnect && currentCall.direction == 'outbound' && currentCall.hangupParty == 'remote'">Call Rejected</span>
@@ -71,7 +79,7 @@
                     </span>
                 </div>
                 <span ng-show="currentCall.state == 'ringing'">
-                    <button ng-click="answerCall()">Answer</button>
+                    <button ng-click="answerCall()">Answer {{ currentCall.type }} call</button>
                     <button ng-click="hangupCall()">Reject</button>
                 </span>
                 <button ng-click="hangupCall()" ng-show="currentCall && currentCall.state != 'ringing' && currentCall.state != 'ended' && currentCall.state != 'fledgling'">Hang up</button>
@@ -92,6 +100,7 @@
                     <source src="media/busy.mp3" type="audio/mpeg" />
                 </audio>
             </div>
+
             <a href id="headerUserId" ng-click='goToUserPage(user_id)'>{{ user_id }}</a>
             &nbsp;
             <button ng-click='goToPage("/")'>Home</button>

webclient/room/room-controller.js

@@ -860,7 +860,9 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput'])
         var call = new MatrixCall($scope.room_id);
         call.onError = $rootScope.onCallError;
         call.onHangup = $rootScope.onCallHangup;
-        call.placeCall({audio: true, video: false});
+        // remote video element is used for playing audio in voice calls
+        call.remoteVideoElement = angular.element('#remoteVideo')[0];
+        call.placeVoiceCall();
         $rootScope.currentCall = call;
     };
 
@@ -868,7 +870,9 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput'])
         var call = new MatrixCall($scope.room_id);
         call.onError = $rootScope.onCallError;
         call.onHangup = $rootScope.onCallHangup;
-        call.placeCall({audio: true, video: true});
+        call.localVideoElement = angular.element('#localVideo')[0];
+        call.remoteVideoElement = angular.element('#remoteVideo')[0];
+        call.placeVideoCall();
         $rootScope.currentCall = call;
     };
 

webclient/room/room.html

@@ -183,6 +183,13 @@
                         >
                     Voice Call
                 </button>
+                <button ng-click="startVideoCall()"
+                        ng-show="(currentCall == undefined || currentCall.state == 'ended') && memberCount() == 2"
+                        ng-disabled="state.permission_denied || !state.webRTCSupported"
+                        title ="{{ state.webRTCNotSupported ? '' : 'VoIP requires webRTC but your browser does not support it.'}}"
+                        >
+                    Video Call
+                </button>
             </div>
         
             {{ feedback }}