Handle changes in the selected audio or video input devices during calls

When an input device is changed during a call the old audio or video
track is stopped, a new one is requested, and then the old track is
replaced by the new one in the RTCPeerConnection.

When a track is replaced in an RTCPeerConnection sometimes a
renegotiation is needed. As neither the web UI nor the mobile apps
support a proper renegotiation for now a forced reconnection is used.

Signed-off-by: Daniel Calviño Sánchez <danxuliu@gmail.com>

src/utils/webrtc/models/LocalMediaModel.js

@@ -44,6 +44,7 @@ export default function LocalMediaModel() {
 	this._handleLocalStreamBound = this._handleLocalStream.bind(this)
 	this._handleLocalStreamRequestFailedRetryNoVideoBound = this._handleLocalStreamRequestFailedRetryNoVideo.bind(this)
 	this._handleLocalStreamRequestFailedBound = this._handleLocalStreamRequestFailed.bind(this)
+	this._handleLocalStreamChangedBound = this._handleLocalStreamChanged.bind(this)
 	this._handleLocalStreamStoppedBound = this._handleLocalStreamStopped.bind(this)
 	this._handleAudioOnBound = this._handleAudioOn.bind(this)
 	this._handleAudioOffBound = this._handleAudioOff.bind(this)
@@ -116,6 +117,7 @@ LocalMediaModel.prototype = {
 			this._webRtc.webrtc.off('localStream', this._handleLocalStreamBound)
 			this._webRtc.webrtc.off('localStreamRequestFailedRetryNoVideo', this._handleLocalStreamRequestFailedBound)
 			this._webRtc.webrtc.off('localStreamRequestFailed', this._handleLocalStreamRequestFailedBound)
+			this._webRtc.webrtc.off('localStreamChanged', this._handleLocalStreamChangedBound)
 			this._webRtc.webrtc.off('localStreamStopped', this._handleLocalStreamStoppedBound)
 			this._webRtc.webrtc.off('audioOn', this._handleAudioOnBound)
 			this._webRtc.webrtc.off('audioOff', this._handleAudioOffBound)
@@ -147,6 +149,7 @@ LocalMediaModel.prototype = {
 		this._webRtc.webrtc.on('localStream', this._handleLocalStreamBound)
 		this._webRtc.webrtc.on('localStreamRequestFailedRetryNoVideo', this._handleLocalStreamRequestFailedRetryNoVideoBound)
 		this._webRtc.webrtc.on('localStreamRequestFailed', this._handleLocalStreamRequestFailedBound)
+		this._webRtc.webrtc.on('localStreamChanged', this._handleLocalStreamChangedBound)
 		this._webRtc.webrtc.on('localStreamStopped', this._handleLocalStreamStoppedBound)
 		this._webRtc.webrtc.on('audioOn', this._handleAudioOnBound)
 		this._webRtc.webrtc.on('audioOff', this._handleAudioOffBound)
@@ -225,6 +228,43 @@ LocalMediaModel.prototype = {
 		}
 	},
 
+	_handleLocalStreamChanged: function(localStream) {
+		// Only a single local stream is assumed to be active at the same time.
+		this.set('localStream', localStream)
+
+		this._updateMediaAvailability(localStream)
+	},
+
+	_updateMediaAvailability: function(localStream) {
+		if (localStream && localStream.getAudioTracks().length > 0) {
+			this.set('audioAvailable', true)
+
+			if (!this.get('audioEnabled')) {
+				// Explicitly disable the audio to ensure that it will also be
+				// disabled in the other end. Otherwise the WebRTC media could
+				// be enabled.
+				this.disableAudio()
+			}
+		} else {
+			this.disableAudio()
+			this.set('audioAvailable', false)
+		}
+
+		if (localStream && localStream.getVideoTracks().length > 0) {
+			this.set('videoAvailable', true)
+
+			if (!this.get('videoEnabled')) {
+				// Explicitly disable the video to ensure that it will also be
+				// disabled in the other end. Otherwise the WebRTC media could
+				// be enabled.
+				this.disableVideo()
+			}
+		} else {
+			this.disableVideo()
+			this.set('videoAvailable', false)
+		}
+	},
+
 	_handleLocalStreamStopped: function(localStream) {
 		if (this.get('localStream') !== localStream) {
 			return

src/utils/webrtc/simplewebrtc/localmedia.js

@@ -60,6 +60,9 @@ function LocalMedia(opts) {
 
 	this._audioMonitors = []
 	this.on('localScreenStopped', this._stopAudioMonitor.bind(this))
+
+	this._handleAudioInputIdChangedBound = this._handleAudioInputIdChanged.bind(this)
+	this._handleVideoInputIdChangedBound = this._handleVideoInputIdChanged.bind(this)
 }
 
 util.inherits(LocalMedia, WildEmitter)
@@ -162,6 +165,9 @@ LocalMedia.prototype.start = function(mediaConstraints, cb, context) {
 
 		self.emit('localStream', constraints, stream)
 
+		webrtcIndex.mediaDevicesManager.on('change:audioInputId', self._handleAudioInputIdChangedBound)
+		webrtcIndex.mediaDevicesManager.on('change:videoInputId', self._handleVideoInputIdChangedBound)
+
 		if (cb) {
 			return cb(null, stream)
 		}
@@ -183,9 +189,215 @@ LocalMedia.prototype.start = function(mediaConstraints, cb, context) {
 	})
 }
 
+LocalMedia.prototype._handleAudioInputIdChanged = function(mediaDevicesManager, audioInputId) {
+	const localStreamsChanged = []
+	const localTracksReplaced = []
+
+	if (this.localStreams.length === 0 && audioInputId) {
+		// Force the creation of a new stream to add a new audio track to it.
+		localTracksReplaced.push({ track: null, stream: null })
+	}
+
+	this.localStreams.forEach(stream => {
+		if (stream.getAudioTracks().length === 0) {
+			localStreamsChanged.push(stream)
+
+			localTracksReplaced.push({ track: null, stream })
+		}
+
+		stream.getAudioTracks().forEach(track => {
+			const settings = track.getSettings()
+			if (track.kind === 'audio' && settings && settings.deviceId !== audioInputId) {
+				track.stop()
+
+				stream.removeTrack(track)
+
+				if (!localStreamsChanged.includes(stream)) {
+					localStreamsChanged.push(stream)
+				}
+
+				localTracksReplaced.push({ track, stream })
+			}
+		})
+	})
+
+	if (audioInputId === null) {
+		localStreamsChanged.forEach(stream => {
+			this.emit('localStreamChanged', stream)
+		})
+
+		localTracksReplaced.forEach(trackStreamPair => {
+			this.emit('localTrackReplaced', null, trackStreamPair.track, trackStreamPair.stream)
+		})
+
+		return
+	}
+
+	if (localTracksReplaced.length === 0) {
+		return
+	}
+
+	webrtcIndex.mediaDevicesManager.getUserMedia({ audio: true }).then(stream => {
+		// According to the specification "getUserMedia({ audio: true })" will
+		// return a single audio track.
+		const track = stream.getTracks()[0]
+		if (stream.getTracks().length > 1) {
+			console.error('More than a single audio track returned by getUserMedia, only the first one will be used')
+		}
+
+		localTracksReplaced.forEach(trackStreamPair => {
+			const clonedTrack = track.clone()
+
+			let stream = trackStreamPair.stream
+			let streamIndex = this.localStreams.indexOf(stream)
+			if (streamIndex < 0) {
+				stream = new MediaStream()
+				this.localStreams.push(stream)
+				streamIndex = this.localStreams.length - 1
+			}
+
+			stream.addTrack(clonedTrack)
+
+			// The audio monitor stream is never disabled to be able to analyze
+			// it even when the stream sent is muted.
+			let audioMonitorStream
+			if (streamIndex > this._audioMonitorStreams.length - 1) {
+				audioMonitorStream = cloneLinkedStream(stream)
+				this._audioMonitorStreams.push(audioMonitorStream)
+			} else {
+				audioMonitorStream = this._audioMonitorStreams[streamIndex]
+			}
+
+			if (this.config.detectSpeakingEvents) {
+				this._setupAudioMonitor(audioMonitorStream, this.config.harkOptions)
+			}
+
+			clonedTrack.addEventListener('ended', () => {
+				if (isAllTracksEnded(stream)) {
+					this._removeStream(stream)
+				}
+			})
+
+			this.emit('localStreamChanged', stream)
+			this.emit('localTrackReplaced', clonedTrack, trackStreamPair.track, trackStreamPair.stream)
+		})
+
+		// After the clones were added to the local streams the original track
+		// is no longer needed.
+		track.stop()
+	}).catch(() => {
+		localStreamsChanged.forEach(stream => {
+			this.emit('localStreamChanged', stream)
+		})
+
+		localTracksReplaced.forEach(trackStreamPair => {
+			this.emit('localTrackReplaced', null, trackStreamPair.track, trackStreamPair.stream)
+		})
+	})
+}
+
+LocalMedia.prototype._handleVideoInputIdChanged = function(mediaDevicesManager, videoInputId) {
+	const localStreamsChanged = []
+	const localTracksReplaced = []
+
+	if (this.localStreams.length === 0 && videoInputId) {
+		// Force the creation of a new stream to add a new video track to it.
+		localTracksReplaced.push({ track: null, stream: null })
+	}
+
+	this.localStreams.forEach(stream => {
+		if (stream.getVideoTracks().length === 0) {
+			localStreamsChanged.push(stream)
+
+			localTracksReplaced.push({ track: null, stream })
+		}
+
+		stream.getVideoTracks().forEach(track => {
+			const settings = track.getSettings()
+			if (track.kind === 'video' && settings && settings.deviceId !== videoInputId) {
+				track.stop()
+
+				stream.removeTrack(track)
+
+				if (!localStreamsChanged.includes(stream)) {
+					localStreamsChanged.push(stream)
+				}
+
+				localTracksReplaced.push({ track, stream })
+			}
+		})
+	})
+
+	if (videoInputId === null) {
+		localStreamsChanged.forEach(stream => {
+			this.emit('localStreamChanged', stream)
+		})
+
+		localTracksReplaced.forEach(trackStreamPair => {
+			this.emit('localTrackReplaced', null, trackStreamPair.track, trackStreamPair.stream)
+		})
+
+		return
+	}
+
+	if (localTracksReplaced.length === 0) {
+		return
+	}
+
+	webrtcIndex.mediaDevicesManager.getUserMedia({ video: true }).then(stream => {
+		// According to the specification "getUserMedia({ video: true })" will
+		// return a single video track.
+		const track = stream.getTracks()[0]
+		if (stream.getTracks().length > 1) {
+			console.error('More than a single video track returned by getUserMedia, only the first one will be used')
+		}
+
+		localTracksReplaced.forEach(trackStreamPair => {
+			const clonedTrack = track.clone()
+
+			let stream = trackStreamPair.stream
+			if (!this.localStreams.includes(stream)) {
+				stream = new MediaStream()
+				this.localStreams.push(stream)
+
+				const audioMonitorStream = cloneLinkedStream(stream)
+				this._audioMonitorStreams.push(audioMonitorStream)
+			}
+
+			stream.addTrack(clonedTrack)
+
+			clonedTrack.addEventListener('ended', () => {
+				if (isAllTracksEnded(stream)) {
+					this._removeStream(stream)
+				}
+			})
+
+			this.emit('localStreamChanged', stream)
+			this.emit('localTrackReplaced', clonedTrack, trackStreamPair.track, trackStreamPair.stream)
+		})
+
+		// After the clones were added to the local streams the original track
+		// is no longer needed.
+		track.stop()
+	}).catch(() => {
+		localStreamsChanged.forEach(stream => {
+			this.emit('localStreamChanged', stream)
+		})
+
+		localTracksReplaced.forEach(trackStreamPair => {
+			this.emit('localTrackReplaced', null, trackStreamPair.track, trackStreamPair.stream)
+		})
+	})
+}
+
 LocalMedia.prototype.stop = function(stream) {
 	this.stopStream(stream)
 	this.stopScreenShare(stream)
+
+	if (!this.localStreams.length) {
+		webrtcIndex.mediaDevicesManager.off('change:audioInputId', this._handleAudioInputIdChangedBound)
+		webrtcIndex.mediaDevicesManager.off('change:videoInputId', this._handleVideoInputIdChangedBound)
+	}
 }
 
 LocalMedia.prototype.stopStream = function(stream) {

src/utils/webrtc/simplewebrtc/peer.js

@@ -77,6 +77,11 @@ function Peer(options) {
 				}
 			})
 		})
+
+		this.handleLocalTrackReplacedBound = this.handleLocalTrackReplaced.bind(this)
+		// TODO What would happen if the track is replaced while the peer is
+		// still negotiating the offer and answer?
+		this.parent.on('localTrackReplaced', this.handleLocalTrackReplacedBound)
 	}
 
 	// proxy events to parent
@@ -377,6 +382,59 @@ Peer.prototype.end = function() {
 	}
 	this.pc.close()
 	this.handleStreamRemoved()
+	this.parent.off('localTrackReplaced', this.handleLocalTrackReplacedBound)
+}
+
+Peer.prototype.handleLocalTrackReplaced = function(newTrack, oldTrack, stream) {
+	let senderFound = false
+
+	this.pc.getSenders().forEach(sender => {
+		if (sender.track !== oldTrack) {
+			return
+		}
+
+		if (!sender.track && !newTrack) {
+			return
+		}
+
+		if (!sender.kind && sender.track) {
+			sender.kind = sender.track.kind
+		} else if (!sender.kind) {
+			this.pc.getTransceivers().forEach(transceiver => {
+				if (transceiver.sender === sender) {
+					sender.kind = transceiver.mid
+				}
+			})
+		}
+
+		// A null track can match on audio and video senders, so it needs to be
+		// ensured that the sender kind and the new track kind are compatible.
+		// However, in some cases it may not be possible to know the sender
+		// kind. In those cases just go ahead and try to replace the track; if
+		// the kind does not match then replacing the track will fail, but this
+		// should not prevent replacing the track with a proper one later, nor
+		// affect any other sender.
+		if (!sender.track && sender.kind && sender.kind !== newTrack.kind) {
+			return
+		}
+
+		senderFound = true
+
+		sender.replaceTrack(newTrack).catch(error => {
+			if (error.name === 'InvalidModificationError') {
+				console.debug('Track could not be replaced, negotiation needed')
+			} else {
+				console.error('Track could not be replaced: ', error, oldTrack, newTrack)
+			}
+		})
+	})
+
+	// If the call started when the audio or video device was not active there
+	// will be no sender for that type. In that case the track needs to be added
+	// instead of replaced.
+	if (!senderFound && newTrack) {
+		this.pc.addTrack(newTrack, stream)
+	}
 }
 
 Peer.prototype.handleRemoteStreamAdded = function(event) {

src/utils/webrtc/webrtc.js

@@ -630,6 +630,17 @@ export default function initWebRTC(signaling, _callParticipantCollection, _local
 		signaling.forceReconnect(true, flags)
 	}
 
+	function setHandlerForNegotiationNeeded(peer) {
+		peer.pc.addEventListener('negotiationneeded', function() {
+			// Negotiation needed will be first triggered before the connection
+			// is established, but forcing a reconnection should be done only
+			// once the connection was established.
+			if (peer.pc.iceConnectionState !== 'new' && peer.pc.iceConnectionState !== 'checking') {
+				forceReconnect(signaling)
+			}
+		})
+	}
+
 	webrtc.on('createdPeer', function(peer) {
 		console.debug('Peer created', peer)
 
@@ -660,6 +671,8 @@ export default function initWebRTC(signaling, _callParticipantCollection, _local
 				setHandlerForIceConnectionStateChange(peer)
 			}
 
+			setHandlerForNegotiationNeeded(peer)
+
 			// Make sure required data channels exist for all peers. This
 			// is required for peers that get created by SimpleWebRTC from
 			// received "Offer" messages. Otherwise the "channelMessage"