From 1fb2c831e824d89aa849fc2aee45f5f1162842b2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 17 Sep 2014 16:26:35 +0100 Subject: [PATCH 1/5] Video calling (in a tiny box at the moment) --- webclient/app-controller.js | 2 + webclient/app.css | 16 ++++ webclient/components/matrix/matrix-call.js | 86 +++++++++++++++++++--- webclient/index.html | 8 +- webclient/room/room-controller.js | 8 +- webclient/room/room.html | 1 + 6 files changed, 106 insertions(+), 15 deletions(-) diff --git a/webclient/app-controller.js b/webclient/app-controller.js index 633862448..e9912f886 100644 --- a/webclient/app-controller.js +++ b/webclient/app-controller.js @@ -150,6 +150,8 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even } call.onError = $scope.onCallError; call.onHangup = $scope.onCallHangup; + call.localVideoElement = angular.element('#localVideo')[0]; + call.remoteVideoElement = angular.element('#remoteVideo')[0]; $rootScope.currentCall = call; }); diff --git a/webclient/app.css b/webclient/app.css index 4a4ba7b8f..1845b3491 100755 --- a/webclient/app.css +++ b/webclient/app.css @@ -89,6 +89,21 @@ a:active { color: #000; } font-size: 80%; } +#localVideo { + position: absolute; + top: 32px; + left: 160px; + width: 128px; + height: 72px; +} +#remoteVideo { + position: absolute; + top: 32px; + left: 300px; + width: 128px; + height: 72px; +} + #headerContent { color: #ccc; max-width: 1280px; @@ -96,6 +111,7 @@ a:active { color: #000; } text-align: right; height: 32px; line-height: 32px; + position: relative; } #headerContent a:link, diff --git a/webclient/components/matrix/matrix-call.js b/webclient/components/matrix/matrix-call.js index bf1e61ad7..2f0bfddaf 100644 --- a/webclient/components/matrix/matrix-call.js +++ b/webclient/components/matrix/matrix-call.js @@ -51,6 +51,12 @@ angular.module('MatrixCall', []) // a queue for candidates waiting to go out. We try to amalgamate candidates into a single candidate message where possible this.candidateSendQueue = []; this.candidateSendTries = 0; + + var self = this; + $rootScope.$watch(this.remoteVideoElement, function (oldValue, newValue) { + self.tryPlayRemoteStream(); + }); + } MatrixCall.CALL_TIMEOUT = 60000; @@ -71,13 +77,39 @@ angular.module('MatrixCall', []) return pc; } - MatrixCall.prototype.placeCall = function(config) { + MatrixCall.prototype.getUserMediaVideoContraints = function(callType) { + switch (callType) { + case 'voice': + return ({audio: true, video: false}); + case 'video': + return ({audio: true, video: { + mandatory: { + minWidth: 640, + maxWidth: 640, + minHeight: 360, + maxHeight: 360, + } + }}); + } + }; + + MatrixCall.prototype.placeVoiceCall = function() { + this.placeCallWithConstraints(this.getUserMediaVideoContraints('voice')); + this.type = 'voice'; + }; + + MatrixCall.prototype.placeVideoCall = function(config) { + this.placeCallWithConstraints(this.getUserMediaVideoContraints('video')); + this.type = 'video'; + }; + + MatrixCall.prototype.placeCallWithConstraints = function(constraints) { var self = this; matrixPhoneService.callPlaced(this); - navigator.getUserMedia({audio: config.audio, video: config.video}, function(s) { self.gotUserMediaForInvite(s); }, function(e) { self.getUserMediaFailed(e); }); + navigator.getUserMedia(constraints, function(s) { self.gotUserMediaForInvite(s); }, function(e) { self.getUserMediaFailed(e); }); this.state = 'wait_local_media'; this.direction = 'outbound'; - this.config = config; + this.config = constraints; }; MatrixCall.prototype.initWithInvite = function(event) { @@ -110,7 +142,7 @@ angular.module('MatrixCall', []) console.log("Answering call "+this.call_id); var self = this; if (!this.localAVStream && !this.waitForLocalAVStream) { - navigator.getUserMedia({audio: true, video: false}, function(s) { self.gotUserMediaForAnswer(s); }, function(e) { self.getUserMediaFailed(e); }); + navigator.getUserMedia(this.getUserMediaVideoContraints(this.type), function(s) { self.gotUserMediaForAnswer(s); }, function(e) { self.getUserMediaFailed(e); }); this.state = 'wait_local_media'; } else if (this.localAVStream) { this.gotUserMediaForAnswer(this.localAVStream); @@ -156,6 +188,13 @@ angular.module('MatrixCall', []) } if (this.state == 'ended') return; + if (this.localVideoElement && this.type == 'video') { + var vidTrack = stream.getVideoTracks()[0]; + this.localVideoElement.src = URL.createObjectURL(stream); + this.localVideoElement.muted = true; + this.localVideoElement.play(); + } + this.localAVStream = stream; var audioTracks = stream.getAudioTracks(); for (var i = 0; i < audioTracks.length; i++) { @@ -177,6 +216,13 @@ angular.module('MatrixCall', []) MatrixCall.prototype.gotUserMediaForAnswer = function(stream) { if (this.state == 'ended') return; + if (this.localVideoElement && this.type == 'video') { + var vidTrack = stream.getVideoTracks()[0]; + this.localVideoElement.src = URL.createObjectURL(stream); + this.localVideoElement.muted = true; + this.localVideoElement.play(); + } + this.localAVStream = stream; var audioTracks = stream.getAudioTracks(); for (var i = 0; i < audioTracks.length; i++) { @@ -187,7 +233,7 @@ angular.module('MatrixCall', []) var constraints = { 'mandatory': { 'OfferToReceiveAudio': true, - 'OfferToReceiveVideo': false + 'OfferToReceiveVideo': this.type == 'video' }, }; this.peerConn.createAnswer(function(d) { self.createdAnswer(d); }, function(e) {}, constraints); @@ -218,6 +264,7 @@ angular.module('MatrixCall', []) this.state = 'connecting'; }; + MatrixCall.prototype.gotLocalOffer = function(description) { console.log("Created offer: "+description); @@ -305,6 +352,14 @@ angular.module('MatrixCall', []) this.remoteAVStream = s; + if (this.direction == 'inbound') { + if (s.getVideoTracks().length > 0) { + this.type = 'video'; + } else { + this.type = 'voice'; + } + } + var self = this; forAllTracksOnStream(s, function(t) { // not currently implemented in chrome @@ -314,9 +369,16 @@ angular.module('MatrixCall', []) event.stream.onended = function(e) { self.onRemoteStreamEnded(e); }; // not currently implemented in chrome event.stream.onstarted = function(e) { self.onRemoteStreamStarted(e); }; - var player = new Audio(); - player.src = URL.createObjectURL(s); - player.play(); + + this.tryPlayRemoteStream(); + }; + + MatrixCall.prototype.tryPlayRemoteStream = function(event) { + if (this.remoteVideoElement && this.remoteAVStream) { + var player = this.remoteVideoElement; + player.src = URL.createObjectURL(this.remoteAVStream); + player.play(); + } }; MatrixCall.prototype.onRemoteStreamStarted = function(event) { @@ -350,7 +412,7 @@ angular.module('MatrixCall', []) this.state = 'ended'; this.hangupParty = 'remote'; this.stopAllMedia(); - if (this.peerConn.signalingState != 'closed') this.peerConn.close(); + if (this.peerConn && this.peerConn.signalingState != 'closed') this.peerConn.close(); if (this.onHangup) this.onHangup(this); }; @@ -361,13 +423,15 @@ angular.module('MatrixCall', []) newCall.waitForLocalAVStream = true; } else if (this.state == 'create_offer') { console.log("Handing local stream to new call"); - newCall.localAVStream = this.localAVStream; + newCall.gotUserMediaForAnswer(this.localAVStream); delete(this.localAVStream); } else if (this.state == 'invite_sent') { console.log("Handing local stream to new call"); - newCall.localAVStream = this.localAVStream; + newCall.gotUserMediaForAnswer(this.localAVStream); delete(this.localAVStream); } + newCall.localVideoElement = this.localVideoElement; + newCall.remoteVideoElement = this.remoteVideoElement; this.successor = newCall; this.hangup(true); }; diff --git a/webclient/index.html b/webclient/index.html index 7e4dcb834..19b1a3b28 100644 --- a/webclient/index.html +++ b/webclient/index.html @@ -58,7 +58,8 @@
Calling... - Incoming Call + Incoming Video Call + Incoming Voice Call Call Connecting... Call Connected Call Rejected @@ -71,7 +72,7 @@ - + @@ -92,6 +93,9 @@ + + + {{ user_id }}   diff --git a/webclient/room/room-controller.js b/webclient/room/room-controller.js index 2c9a3836e..c8bcf88ff 100644 --- a/webclient/room/room-controller.js +++ b/webclient/room/room-controller.js @@ -837,7 +837,9 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) var call = new MatrixCall($scope.room_id); call.onError = $rootScope.onCallError; call.onHangup = $rootScope.onCallHangup; - call.placeCall({audio: true, video: false}); + // remote video element is used for playing audio in voice calls + call.remoteVideoElement = angular.element('#remoteVideo')[0]; + call.placeVoiceCall(); $rootScope.currentCall = call; }; @@ -845,7 +847,9 @@ angular.module('RoomController', ['ngSanitize', 'matrixFilter', 'mFileInput']) var call = new MatrixCall($scope.room_id); call.onError = $rootScope.onCallError; call.onHangup = $rootScope.onCallHangup; - call.placeCall({audio: true, video: true}); + call.localVideoElement = angular.element('#localVideo')[0]; + call.remoteVideoElement = angular.element('#remoteVideo')[0]; + call.placeVideoCall(); $rootScope.currentCall = call; }; diff --git a/webclient/room/room.html b/webclient/room/room.html index 9d617eadd..94077576a 100644 --- a/webclient/room/room.html +++ b/webclient/room/room.html @@ -173,6 +173,7 @@ + {{ feedback }} From e932e5237eaea4c08e6f7bcd849e4be6bd2e3f98 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Sep 2014 11:04:45 +0100 Subject: [PATCH 2/5] WIP video chat layout --- webclient/app-controller.js | 7 ++++-- webclient/app.css | 28 ++++++++++++++++++++++ webclient/components/matrix/matrix-call.js | 2 ++ webclient/index.html | 5 ++-- 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/webclient/app-controller.js b/webclient/app-controller.js index e9912f886..f63bb32f4 100644 --- a/webclient/app-controller.js +++ b/webclient/app-controller.js @@ -126,6 +126,7 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even angular.element('#ringAudio')[0].pause(); angular.element('#ringbackAudio')[0].pause(); angular.element('#callendAudio')[0].play(); + $scope.videoMode = undefined; } else if (newVal == 'ended' && oldVal == 'invite_sent' && $rootScope.currentCall.hangupParty == 'remote') { angular.element('#ringAudio')[0].pause(); angular.element('#ringbackAudio')[0].pause(); @@ -138,6 +139,8 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even angular.element('#ringbackAudio')[0].pause(); } else if (oldVal == 'ringing') { angular.element('#ringAudio')[0].pause(); + } else if (newVal == 'connected') { + $scope.videoMode = 'large'; } }); @@ -172,7 +175,7 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even $rootScope.onCallError = function(errStr) { $scope.feedback = errStr; - } + }; $rootScope.onCallHangup = function(call) { if (call == $rootScope.currentCall) { @@ -180,5 +183,5 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even if (call == $rootScope.currentCall) $rootScope.currentCall = undefined; }, 4070); } - } + }; }]); diff --git a/webclient/app.css b/webclient/app.css index 1845b3491..69e608287 100755 --- a/webclient/app.css +++ b/webclient/app.css @@ -89,19 +89,47 @@ a:active { color: #000; } font-size: 80%; } +#videoBackground { + position: absolute; + height: 100%; + width: 100%; + top: 32px; + left: 0px; + z-index: 1; + background-color: rgba(0,0,0,0.0); + transition: background-color linear 300ms; +} + +#videoBackground.large { + background-color: rgba(0,0,0,0.85); +} + #localVideo { position: absolute; top: 32px; left: 160px; width: 128px; height: 72px; + z-index: 2; } + #remoteVideo { position: absolute; top: 32px; left: 300px; width: 128px; height: 72px; + z-index: 2; + transition: all linear 300ms; +} + +#remoteVideo.large { + width: 100%; + height: auto; +} + +#remoteVideo.ended { + -webkit-filter: grayscale(1); } #headerContent { diff --git a/webclient/components/matrix/matrix-call.js b/webclient/components/matrix/matrix-call.js index 2f0bfddaf..636259297 100644 --- a/webclient/components/matrix/matrix-call.js +++ b/webclient/components/matrix/matrix-call.js @@ -167,6 +167,8 @@ angular.module('MatrixCall', []) MatrixCall.prototype.hangup = function(suppressEvent) { console.log("Ending call "+this.call_id); + this.remoteVideoElement.pause(); + this.stopAllMedia(); if (this.peerConn) this.peerConn.close(); diff --git a/webclient/index.html b/webclient/index.html index 19b1a3b28..05801a93b 100644 --- a/webclient/index.html +++ b/webclient/index.html @@ -45,6 +45,7 @@ +
- - + + {{ user_id }}   From 3bd8cbc62fd8ac47acd56ec50360259f6098c66b Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Sep 2014 15:51:30 +0100 Subject: [PATCH 3/5] Prettier and stabler video with basic support for viewing mode. For now, transition into 'large' mode is disabled. --- webclient/app-controller.js | 22 +++++++++++++-- webclient/app.css | 32 ++++++++++++++++------ webclient/components/matrix/matrix-call.js | 22 +++++++++++++-- webclient/index.html | 13 +++++---- 4 files changed, 70 insertions(+), 19 deletions(-) diff --git a/webclient/app-controller.js b/webclient/app-controller.js index f63bb32f4..7f48148aa 100644 --- a/webclient/app-controller.js +++ b/webclient/app-controller.js @@ -93,7 +93,13 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even }; $rootScope.$watch('currentCall', function(newVal, oldVal) { - if (!$rootScope.currentCall) return; + if (!$rootScope.currentCall) { + // This causes the still frame to be flushed out of the video elements, + // avoiding a flash of the last frame of the previous call when starting the next + angular.element('#localVideo')[0].load(); + angular.element('#remoteVideo')[0].load(); + return; + } var roomMembers = angular.copy($rootScope.events.rooms[$rootScope.currentCall.room_id].members); delete roomMembers[matrixService.config().user_id]; @@ -140,7 +146,19 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even } else if (oldVal == 'ringing') { angular.element('#ringAudio')[0].pause(); } else if (newVal == 'connected') { - $scope.videoMode = 'large'; + $timeout(function() { + //if ($scope.currentCall.type == 'video') $scope.videoMode = 'large'; + }, 5000); + } + + if ($rootScope.currentCall && $rootScope.currentCall.type == 'video' && $rootScope.currentCall.state != 'connected') { + $scope.videoMode = 'mini'; + } + }); + $rootScope.$watch('currentCall.type', function(newVal, oldVal) { + // need to listen for this too as the type of the call won't be know when it's created + if ($rootScope.currentCall && $rootScope.currentCall.type == 'video' && $rootScope.currentCall.state != 'connected') { + $scope.videoMode = 'mini'; } }); diff --git a/webclient/app.css b/webclient/app.css index 69e608287..7b6051e0b 100755 --- a/webclient/app.css +++ b/webclient/app.css @@ -97,30 +97,44 @@ a:active { color: #000; } left: 0px; z-index: 1; background-color: rgba(0,0,0,0.0); + pointer-events: none; transition: background-color linear 300ms; } #videoBackground.large { background-color: rgba(0,0,0,0.85); + pointer-events: auto; } -#localVideo { - position: absolute; +#videoContainer { + max-width: 1280px; + margin: auto; top: 32px; - left: 160px; +} + +#videoContainer.large { +} + +#localVideo.mini { + position: relative; + left: 120px; width: 128px; height: 72px; - z-index: 2; +} + +#localVideo.ended { + -webkit-filter: grayscale(1); } #remoteVideo { - position: absolute; - top: 32px; - left: 300px; + transition: all linear 300ms; +} + +#remoteVideo.mini { + position: relative; + left: 120px; width: 128px; height: 72px; - z-index: 2; - transition: all linear 300ms; } #remoteVideo.large { diff --git a/webclient/components/matrix/matrix-call.js b/webclient/components/matrix/matrix-call.js index 636259297..5ba782bac 100644 --- a/webclient/components/matrix/matrix-call.js +++ b/webclient/components/matrix/matrix-call.js @@ -65,7 +65,7 @@ angular.module('MatrixCall', []) var stunServer = 'stun:stun.l.google.com:19302'; var pc; if (window.mozRTCPeerConnection) { - pc = window.mozRTCPeerConnection({'url': stunServer}); + pc = new window.mozRTCPeerConnection({'url': stunServer}); } else { pc = new window.RTCPeerConnection({"iceServers":[{"urls":"stun:stun.l.google.com:19302"}]}); } @@ -118,6 +118,17 @@ angular.module('MatrixCall', []) this.peerConn.setRemoteDescription(new RTCSessionDescription(this.msg.offer), this.onSetRemoteDescriptionSuccess, this.onSetRemoteDescriptionError); this.state = 'ringing'; this.direction = 'inbound'; + + if (window.mozRTCPeerConnection) { + // firefox's RTCPeerConnection doesn't add streams until it starts getting media on them + // so we need to figure out whether a video channel has been offered by ourselves. + if (this.msg.offer.sdp.indexOf('m=video') > -1) { + this.type = 'video'; + } else { + this.type = 'voice'; + } + } + var self = this; $timeout(function() { if (self.state == 'ringing') { @@ -167,7 +178,10 @@ angular.module('MatrixCall', []) MatrixCall.prototype.hangup = function(suppressEvent) { console.log("Ending call "+this.call_id); - this.remoteVideoElement.pause(); + // pausing now keeps the last frame (ish) of the video call in the video element + // rather than it just turning black straight away + if (this.remoteVideoElement) this.remoteVideoElement.pause(); + if (this.localVideoElement) this.localVideoElement.pause(); this.stopAllMedia(); if (this.peerConn) this.peerConn.close(); @@ -318,7 +332,7 @@ angular.module('MatrixCall', []) }; MatrixCall.prototype.getUserMediaFailed = function() { - this.onError("Couldn't start capturing audio! Is your microphone set up?"); + this.onError("Couldn't start capturing! Is your microphone set up?"); this.hangup(); }; @@ -411,6 +425,8 @@ angular.module('MatrixCall', []) MatrixCall.prototype.onHangupReceived = function() { console.log("Hangup received"); + if (this.remoteVideoElement) this.remoteVideoElement.pause(); + if (this.localVideoElement) this.localVideoElement.pause(); this.state = 'ended'; this.hangupParty = 'remote'; this.stopAllMedia(); diff --git a/webclient/index.html b/webclient/index.html index 05801a93b..78a68753d 100644 --- a/webclient/index.html +++ b/webclient/index.html @@ -45,7 +45,12 @@ -
+
+
+ + +
+
- - {{ user_id }}   From 05050141520cd4055d23739946b18335eacc2bf6 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 18 Sep 2014 16:15:48 +0100 Subject: [PATCH 4/5] add unprefixed filter css as well --- webclient/app.css | 2 ++ 1 file changed, 2 insertions(+) diff --git a/webclient/app.css b/webclient/app.css index 7b6051e0b..fb92a0f43 100755 --- a/webclient/app.css +++ b/webclient/app.css @@ -124,6 +124,7 @@ a:active { color: #000; } #localVideo.ended { -webkit-filter: grayscale(1); + filter: grayscale(1); } #remoteVideo { @@ -144,6 +145,7 @@ a:active { color: #000; } #remoteVideo.ended { -webkit-filter: grayscale(1); + filter: grayscale(1); } #headerContent { From da8b5a53671911bd158865f7af4b04b3b0168dfa Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 19 Sep 2014 16:18:15 +0100 Subject: [PATCH 5/5] First working version of UI chrome for video calls. --- webclient/app-controller.js | 10 ++++++++-- webclient/app.css | 38 +++++++++++++++++++++++++------------ webclient/index.html | 3 ++- 3 files changed, 36 insertions(+), 15 deletions(-) diff --git a/webclient/app-controller.js b/webclient/app-controller.js index 7f48148aa..0e823b43e 100644 --- a/webclient/app-controller.js +++ b/webclient/app-controller.js @@ -26,6 +26,12 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even // Check current URL to avoid to display the logout button on the login page $scope.location = $location.path(); + + // disable nganimate for the local and remote video elements because ngAnimate appears + // to be buggy and leaves animation classes on the video elements causing them to show + // when they should not (their animations are pure CSS3) + $animate.enabled(false, angular.element('#localVideo')); + $animate.enabled(false, angular.element('#remoteVideo')); // Update the location state when the ng location changed $rootScope.$on('$routeChangeSuccess', function (event, current, previous) { @@ -147,8 +153,8 @@ angular.module('MatrixWebClientController', ['matrixService', 'mPresence', 'even angular.element('#ringAudio')[0].pause(); } else if (newVal == 'connected') { $timeout(function() { - //if ($scope.currentCall.type == 'video') $scope.videoMode = 'large'; - }, 5000); + if ($scope.currentCall.type == 'video') $scope.videoMode = 'large'; + }, 500); } if ($rootScope.currentCall && $rootScope.currentCall.type == 'video' && $rootScope.currentCall.state != 'connected') { diff --git a/webclient/app.css b/webclient/app.css index fb92a0f43..03dd5ec8b 100755 --- a/webclient/app.css +++ b/webclient/app.css @@ -98,7 +98,7 @@ a:active { color: #000; } z-index: 1; background-color: rgba(0,0,0,0.0); pointer-events: none; - transition: background-color linear 300ms; + transition: background-color linear 500ms; } #videoBackground.large { @@ -107,19 +107,31 @@ a:active { color: #000; } } #videoContainer { + position: relative; max-width: 1280px; margin: auto; - top: 32px; } -#videoContainer.large { +#videoContainerPadding { + width: 1280px; +} + +#localVideo { + position: absolute; + width: 128px; + height: 72px; + z-index: 1; + transition: left linear 500ms, top linear 500ms, width linear 500ms, height linear 500ms; } #localVideo.mini { - position: relative; - left: 120px; - width: 128px; - height: 72px; + top: 0px; + left: 130px; +} + +#localVideo.large { + top: 70px; + left: 20px; } #localVideo.ended { @@ -128,19 +140,21 @@ a:active { color: #000; } } #remoteVideo { - transition: all linear 300ms; + position: relative; + height: auto; + transition: left linear 500ms, top linear 500ms, width linear 500ms, height linear 500ms; } #remoteVideo.mini { - position: relative; - left: 120px; + left: 260px; + top: 0px; width: 128px; - height: 72px; } #remoteVideo.large { + left: 0px; + top: 50px; width: 100%; - height: auto; } #remoteVideo.ended { diff --git a/webclient/index.html b/webclient/index.html index 78a68753d..77686abcc 100644 --- a/webclient/index.html +++ b/webclient/index.html @@ -47,8 +47,9 @@
+
- +