From 64c0b7e18d7485cf58c672f5d08055498d7f1f0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Wed, 19 Aug 2020 18:09:10 +0200 Subject: [PATCH] Handle changes in the selected audio or video input devices during calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When an input device is changed during a call the old audio or video track is stopped, a new one is requested, and then the old track is replaced by the new one in the RTCPeerConnection. When a track is replaced in an RTCPeerConnection sometimes a renegotiation is needed. As neither the web UI nor the mobile apps support a proper renegotiation for now a forced reconnection is used. Signed-off-by: Daniel Calviño Sánchez --- src/utils/webrtc/models/LocalMediaModel.js | 40 ++++ src/utils/webrtc/simplewebrtc/localmedia.js | 212 ++++++++++++++++++++ src/utils/webrtc/simplewebrtc/peer.js | 58 ++++++ src/utils/webrtc/webrtc.js | 13 ++ 4 files changed, 323 insertions(+) diff --git a/src/utils/webrtc/models/LocalMediaModel.js b/src/utils/webrtc/models/LocalMediaModel.js index 21f5e6cfc7..abec519382 100644 --- a/src/utils/webrtc/models/LocalMediaModel.js +++ b/src/utils/webrtc/models/LocalMediaModel.js @@ -44,6 +44,7 @@ export default function LocalMediaModel() { this._handleLocalStreamBound = this._handleLocalStream.bind(this) this._handleLocalStreamRequestFailedRetryNoVideoBound = this._handleLocalStreamRequestFailedRetryNoVideo.bind(this) this._handleLocalStreamRequestFailedBound = this._handleLocalStreamRequestFailed.bind(this) + this._handleLocalStreamChangedBound = this._handleLocalStreamChanged.bind(this) this._handleLocalStreamStoppedBound = this._handleLocalStreamStopped.bind(this) this._handleAudioOnBound = this._handleAudioOn.bind(this) this._handleAudioOffBound = this._handleAudioOff.bind(this) @@ -116,6 +117,7 @@ LocalMediaModel.prototype = { this._webRtc.webrtc.off('localStream', this._handleLocalStreamBound) this._webRtc.webrtc.off('localStreamRequestFailedRetryNoVideo', this._handleLocalStreamRequestFailedBound) this._webRtc.webrtc.off('localStreamRequestFailed', this._handleLocalStreamRequestFailedBound) + this._webRtc.webrtc.off('localStreamChanged', this._handleLocalStreamChangedBound) this._webRtc.webrtc.off('localStreamStopped', this._handleLocalStreamStoppedBound) this._webRtc.webrtc.off('audioOn', this._handleAudioOnBound) this._webRtc.webrtc.off('audioOff', this._handleAudioOffBound) @@ -147,6 +149,7 @@ LocalMediaModel.prototype = { this._webRtc.webrtc.on('localStream', this._handleLocalStreamBound) this._webRtc.webrtc.on('localStreamRequestFailedRetryNoVideo', this._handleLocalStreamRequestFailedRetryNoVideoBound) this._webRtc.webrtc.on('localStreamRequestFailed', this._handleLocalStreamRequestFailedBound) + this._webRtc.webrtc.on('localStreamChanged', this._handleLocalStreamChangedBound) this._webRtc.webrtc.on('localStreamStopped', this._handleLocalStreamStoppedBound) this._webRtc.webrtc.on('audioOn', this._handleAudioOnBound) this._webRtc.webrtc.on('audioOff', this._handleAudioOffBound) @@ -225,6 +228,43 @@ LocalMediaModel.prototype = { } }, + _handleLocalStreamChanged: function(localStream) { + // Only a single local stream is assumed to be active at the same time. + this.set('localStream', localStream) + + this._updateMediaAvailability(localStream) + }, + + _updateMediaAvailability: function(localStream) { + if (localStream && localStream.getAudioTracks().length > 0) { + this.set('audioAvailable', true) + + if (!this.get('audioEnabled')) { + // Explicitly disable the audio to ensure that it will also be + // disabled in the other end. Otherwise the WebRTC media could + // be enabled. + this.disableAudio() + } + } else { + this.disableAudio() + this.set('audioAvailable', false) + } + + if (localStream && localStream.getVideoTracks().length > 0) { + this.set('videoAvailable', true) + + if (!this.get('videoEnabled')) { + // Explicitly disable the video to ensure that it will also be + // disabled in the other end. Otherwise the WebRTC media could + // be enabled. + this.disableVideo() + } + } else { + this.disableVideo() + this.set('videoAvailable', false) + } + }, + _handleLocalStreamStopped: function(localStream) { if (this.get('localStream') !== localStream) { return diff --git a/src/utils/webrtc/simplewebrtc/localmedia.js b/src/utils/webrtc/simplewebrtc/localmedia.js index 3f15a98a0a..8997b5ed07 100644 --- a/src/utils/webrtc/simplewebrtc/localmedia.js +++ b/src/utils/webrtc/simplewebrtc/localmedia.js @@ -60,6 +60,9 @@ function LocalMedia(opts) { this._audioMonitors = [] this.on('localScreenStopped', this._stopAudioMonitor.bind(this)) + + this._handleAudioInputIdChangedBound = this._handleAudioInputIdChanged.bind(this) + this._handleVideoInputIdChangedBound = this._handleVideoInputIdChanged.bind(this) } util.inherits(LocalMedia, WildEmitter) @@ -162,6 +165,9 @@ LocalMedia.prototype.start = function(mediaConstraints, cb, context) { self.emit('localStream', constraints, stream) + webrtcIndex.mediaDevicesManager.on('change:audioInputId', self._handleAudioInputIdChangedBound) + webrtcIndex.mediaDevicesManager.on('change:videoInputId', self._handleVideoInputIdChangedBound) + if (cb) { return cb(null, stream) } @@ -183,9 +189,215 @@ LocalMedia.prototype.start = function(mediaConstraints, cb, context) { }) } +LocalMedia.prototype._handleAudioInputIdChanged = function(mediaDevicesManager, audioInputId) { + const localStreamsChanged = [] + const localTracksReplaced = [] + + if (this.localStreams.length === 0 && audioInputId) { + // Force the creation of a new stream to add a new audio track to it. + localTracksReplaced.push({ track: null, stream: null }) + } + + this.localStreams.forEach(stream => { + if (stream.getAudioTracks().length === 0) { + localStreamsChanged.push(stream) + + localTracksReplaced.push({ track: null, stream }) + } + + stream.getAudioTracks().forEach(track => { + const settings = track.getSettings() + if (track.kind === 'audio' && settings && settings.deviceId !== audioInputId) { + track.stop() + + stream.removeTrack(track) + + if (!localStreamsChanged.includes(stream)) { + localStreamsChanged.push(stream) + } + + localTracksReplaced.push({ track, stream }) + } + }) + }) + + if (audioInputId === null) { + localStreamsChanged.forEach(stream => { + this.emit('localStreamChanged', stream) + }) + + localTracksReplaced.forEach(trackStreamPair => { + this.emit('localTrackReplaced', null, trackStreamPair.track, trackStreamPair.stream) + }) + + return + } + + if (localTracksReplaced.length === 0) { + return + } + + webrtcIndex.mediaDevicesManager.getUserMedia({ audio: true }).then(stream => { + // According to the specification "getUserMedia({ audio: true })" will + // return a single audio track. + const track = stream.getTracks()[0] + if (stream.getTracks().length > 1) { + console.error('More than a single audio track returned by getUserMedia, only the first one will be used') + } + + localTracksReplaced.forEach(trackStreamPair => { + const clonedTrack = track.clone() + + let stream = trackStreamPair.stream + let streamIndex = this.localStreams.indexOf(stream) + if (streamIndex < 0) { + stream = new MediaStream() + this.localStreams.push(stream) + streamIndex = this.localStreams.length - 1 + } + + stream.addTrack(clonedTrack) + + // The audio monitor stream is never disabled to be able to analyze + // it even when the stream sent is muted. + let audioMonitorStream + if (streamIndex > this._audioMonitorStreams.length - 1) { + audioMonitorStream = cloneLinkedStream(stream) + this._audioMonitorStreams.push(audioMonitorStream) + } else { + audioMonitorStream = this._audioMonitorStreams[streamIndex] + } + + if (this.config.detectSpeakingEvents) { + this._setupAudioMonitor(audioMonitorStream, this.config.harkOptions) + } + + clonedTrack.addEventListener('ended', () => { + if (isAllTracksEnded(stream)) { + this._removeStream(stream) + } + }) + + this.emit('localStreamChanged', stream) + this.emit('localTrackReplaced', clonedTrack, trackStreamPair.track, trackStreamPair.stream) + }) + + // After the clones were added to the local streams the original track + // is no longer needed. + track.stop() + }).catch(() => { + localStreamsChanged.forEach(stream => { + this.emit('localStreamChanged', stream) + }) + + localTracksReplaced.forEach(trackStreamPair => { + this.emit('localTrackReplaced', null, trackStreamPair.track, trackStreamPair.stream) + }) + }) +} + +LocalMedia.prototype._handleVideoInputIdChanged = function(mediaDevicesManager, videoInputId) { + const localStreamsChanged = [] + const localTracksReplaced = [] + + if (this.localStreams.length === 0 && videoInputId) { + // Force the creation of a new stream to add a new video track to it. + localTracksReplaced.push({ track: null, stream: null }) + } + + this.localStreams.forEach(stream => { + if (stream.getVideoTracks().length === 0) { + localStreamsChanged.push(stream) + + localTracksReplaced.push({ track: null, stream }) + } + + stream.getVideoTracks().forEach(track => { + const settings = track.getSettings() + if (track.kind === 'video' && settings && settings.deviceId !== videoInputId) { + track.stop() + + stream.removeTrack(track) + + if (!localStreamsChanged.includes(stream)) { + localStreamsChanged.push(stream) + } + + localTracksReplaced.push({ track, stream }) + } + }) + }) + + if (videoInputId === null) { + localStreamsChanged.forEach(stream => { + this.emit('localStreamChanged', stream) + }) + + localTracksReplaced.forEach(trackStreamPair => { + this.emit('localTrackReplaced', null, trackStreamPair.track, trackStreamPair.stream) + }) + + return + } + + if (localTracksReplaced.length === 0) { + return + } + + webrtcIndex.mediaDevicesManager.getUserMedia({ video: true }).then(stream => { + // According to the specification "getUserMedia({ video: true })" will + // return a single video track. + const track = stream.getTracks()[0] + if (stream.getTracks().length > 1) { + console.error('More than a single video track returned by getUserMedia, only the first one will be used') + } + + localTracksReplaced.forEach(trackStreamPair => { + const clonedTrack = track.clone() + + let stream = trackStreamPair.stream + if (!this.localStreams.includes(stream)) { + stream = new MediaStream() + this.localStreams.push(stream) + + const audioMonitorStream = cloneLinkedStream(stream) + this._audioMonitorStreams.push(audioMonitorStream) + } + + stream.addTrack(clonedTrack) + + clonedTrack.addEventListener('ended', () => { + if (isAllTracksEnded(stream)) { + this._removeStream(stream) + } + }) + + this.emit('localStreamChanged', stream) + this.emit('localTrackReplaced', clonedTrack, trackStreamPair.track, trackStreamPair.stream) + }) + + // After the clones were added to the local streams the original track + // is no longer needed. + track.stop() + }).catch(() => { + localStreamsChanged.forEach(stream => { + this.emit('localStreamChanged', stream) + }) + + localTracksReplaced.forEach(trackStreamPair => { + this.emit('localTrackReplaced', null, trackStreamPair.track, trackStreamPair.stream) + }) + }) +} + LocalMedia.prototype.stop = function(stream) { this.stopStream(stream) this.stopScreenShare(stream) + + if (!this.localStreams.length) { + webrtcIndex.mediaDevicesManager.off('change:audioInputId', this._handleAudioInputIdChangedBound) + webrtcIndex.mediaDevicesManager.off('change:videoInputId', this._handleVideoInputIdChangedBound) + } } LocalMedia.prototype.stopStream = function(stream) { diff --git a/src/utils/webrtc/simplewebrtc/peer.js b/src/utils/webrtc/simplewebrtc/peer.js index 7b49f05138..ef3704e9cc 100644 --- a/src/utils/webrtc/simplewebrtc/peer.js +++ b/src/utils/webrtc/simplewebrtc/peer.js @@ -77,6 +77,11 @@ function Peer(options) { } }) }) + + this.handleLocalTrackReplacedBound = this.handleLocalTrackReplaced.bind(this) + // TODO What would happen if the track is replaced while the peer is + // still negotiating the offer and answer? + this.parent.on('localTrackReplaced', this.handleLocalTrackReplacedBound) } // proxy events to parent @@ -377,6 +382,59 @@ Peer.prototype.end = function() { } this.pc.close() this.handleStreamRemoved() + this.parent.off('localTrackReplaced', this.handleLocalTrackReplacedBound) +} + +Peer.prototype.handleLocalTrackReplaced = function(newTrack, oldTrack, stream) { + let senderFound = false + + this.pc.getSenders().forEach(sender => { + if (sender.track !== oldTrack) { + return + } + + if (!sender.track && !newTrack) { + return + } + + if (!sender.kind && sender.track) { + sender.kind = sender.track.kind + } else if (!sender.kind) { + this.pc.getTransceivers().forEach(transceiver => { + if (transceiver.sender === sender) { + sender.kind = transceiver.mid + } + }) + } + + // A null track can match on audio and video senders, so it needs to be + // ensured that the sender kind and the new track kind are compatible. + // However, in some cases it may not be possible to know the sender + // kind. In those cases just go ahead and try to replace the track; if + // the kind does not match then replacing the track will fail, but this + // should not prevent replacing the track with a proper one later, nor + // affect any other sender. + if (!sender.track && sender.kind && sender.kind !== newTrack.kind) { + return + } + + senderFound = true + + sender.replaceTrack(newTrack).catch(error => { + if (error.name === 'InvalidModificationError') { + console.debug('Track could not be replaced, negotiation needed') + } else { + console.error('Track could not be replaced: ', error, oldTrack, newTrack) + } + }) + }) + + // If the call started when the audio or video device was not active there + // will be no sender for that type. In that case the track needs to be added + // instead of replaced. + if (!senderFound && newTrack) { + this.pc.addTrack(newTrack, stream) + } } Peer.prototype.handleRemoteStreamAdded = function(event) { diff --git a/src/utils/webrtc/webrtc.js b/src/utils/webrtc/webrtc.js index 9088bdece9..dcd9faf33e 100644 --- a/src/utils/webrtc/webrtc.js +++ b/src/utils/webrtc/webrtc.js @@ -630,6 +630,17 @@ export default function initWebRTC(signaling, _callParticipantCollection, _local signaling.forceReconnect(true, flags) } + function setHandlerForNegotiationNeeded(peer) { + peer.pc.addEventListener('negotiationneeded', function() { + // Negotiation needed will be first triggered before the connection + // is established, but forcing a reconnection should be done only + // once the connection was established. + if (peer.pc.iceConnectionState !== 'new' && peer.pc.iceConnectionState !== 'checking') { + forceReconnect(signaling) + } + }) + } + webrtc.on('createdPeer', function(peer) { console.debug('Peer created', peer) @@ -660,6 +671,8 @@ export default function initWebRTC(signaling, _callParticipantCollection, _local setHandlerForIceConnectionStateChange(peer) } + setHandlerForNegotiationNeeded(peer) + // Make sure required data channels exist for all peers. This // is required for peers that get created by SimpleWebRTC from // received "Offer" messages. Otherwise the "channelMessage"