Merge pull request #5779 from nextcloud/add-ui-feedback-when-local-participant-is-not-connected

Add UI feedback when local participant is not connected

src/components/CallView/shared/LocalVideo.vue

@@ -25,12 +25,16 @@
 		@mouseover="showShadow"
 		@mouseleave="hideShadow"
 		@click="handleClickVideo">
-		<video v-show="localMediaModel.attributes.videoEnabled"
-			id="localVideo"
-			ref="video"
-			disablePictureInPicture="true"
-			:class="videoClass"
-			class="video" />
+		<div v-show="localMediaModel.attributes.videoEnabled"
+			:class="videoWrapperClass"
+			class="videoWrapper">
+			<video
+				id="localVideo"
+				ref="video"
+				disablePictureInPicture="true"
+				:class="videoClass"
+				class="video" />
+		</div>
 		<div v-if="!localMediaModel.attributes.videoEnabled && !isSidebar" class="avatar-container">
 			<VideoBackground
 				v-if="isGrid || isStripe"
@@ -42,9 +46,10 @@
 				:disable-tooltip="true"
 				:show-user-status="false"
 				:user="userId"
-				:display-name="displayName" />
+				:display-name="displayName"
+				:class="avatarClass" />
 			<div v-if="!userId"
-				:class="avatarSizeClass"
+				:class="guestAvatarClass"
 				class="avatar guest">
 				{{ firstLetterOfGuestName }}
 			</div>
@@ -84,6 +89,7 @@ import {
 } from '@nextcloud/dialogs'
 import video from '../../../mixins/video.js'
 import VideoBackground from './VideoBackground'
+import { ConnectionState } from '../../../utils/webrtc/models/CallParticipantModel'
 
 export default {
 
@@ -139,8 +145,17 @@ export default {
 			return t('spreed', 'Back')
 		},
 
+		isNotConnected() {
+			// When there is no sender participant (when the MCU is not used, or
+			// if it is used but no peer object has been set yet) the local
+			// video is shown as connected.
+			return this.localCallParticipantModel.attributes.connectionState !== null
+				&& this.localCallParticipantModel.attributes.connectionState !== ConnectionState.CONNECTED && this.localCallParticipantModel.attributes.connectionState !== ConnectionState.COMPLETED
+		},
+
 		videoContainerClass() {
 			return {
+				'not-connected': this.isNotConnected,
 				speaking: this.localMediaModel.attributes.speaking,
 				'video-container-grid': this.isGrid,
 				'video-container-stripe': this.isStripe,
@@ -172,12 +187,26 @@ export default {
 			)
 		},
 
+		videoWrapperClass() {
+			return {
+				'icon-loading': this.isNotConnected,
+			}
+		},
+
 		avatarSize() {
 			return this.useConstrainedLayout ? 64 : 128
 		},
 
-		avatarSizeClass() {
-			return 'avatar-' + this.avatarSize + 'px'
+		avatarClass() {
+			return {
+				'icon-loading': this.isNotConnected,
+			}
+		},
+
+		guestAvatarClass() {
+			return Object.assign(this.avatarClass, {
+				['avatar-' + this.avatarSize + 'px']: true,
+			})
 		},
 
 		localStreamVideoError() {
@@ -296,6 +325,13 @@ export default {
 @include avatar-mixin(64px);
 @include avatar-mixin(128px);
 
+.not-connected {
+	video,
+	.avatar-container {
+		opacity: 0.5;
+	}
+}
+
 .video-container-grid {
 	position:relative;
 	height: 100%;
@@ -314,11 +350,18 @@ export default {
 	flex-direction: column;
 }
 
+.videoWrapper,
 .video {
 	height: 100%;
 	width: 100%;
 }
 
+.videoWrapper.icon-loading:after {
+	height: 60px;
+	width: 60px;
+	margin: -32px 0 0 -32px;
+}
+
 .video--fit {
 	/* Fit the frame */
 	object-fit: contain;

src/utils/webrtc/models/LocalCallParticipantModel.js

@@ -21,6 +21,8 @@
 
 import store from '../../../store/index.js'
 
+import { ConnectionState } from './CallParticipantModel'
+
 export default function LocalCallParticipantModel() {
 
 	this.attributes = {
@@ -28,11 +30,13 @@ export default function LocalCallParticipantModel() {
 		peer: null,
 		screenPeer: null,
 		guestName: null,
+		connectionState: null,
 	}
 
 	this._handlers = []
 
 	this._handleForcedMuteBound = this._handleForcedMute.bind(this)
+	this._handleExtendedIceConnectionStateChangeBound = this._handleExtendedIceConnectionStateChange.bind(this)
 
 }
 
@@ -107,7 +111,22 @@ LocalCallParticipantModel.prototype = {
 			console.warn('Mismatch between stored peer ID and ID of given peer: ', this.get('peerId'), peer.id)
 		}
 
+		if (this.get('peer')) {
+			this.get('peer').off('extendedIceConnectionStateChange', this._handleExtendedIceConnectionStateChangeBound)
+		}
+
 		this.set('peer', peer)
+
+		if (!this.get('peer')) {
+			this.set('connectionState', null)
+
+			return
+		}
+
+		// Reset state that depends on the Peer object.
+		this._handleExtendedIceConnectionStateChange(this.get('peer').pc.iceConnectionState)
+
+		this.get('peer').on('extendedIceConnectionStateChange', this._handleExtendedIceConnectionStateChangeBound)
 	},
 
 	setScreenPeer(screenPeer) {
@@ -132,4 +151,36 @@ LocalCallParticipantModel.prototype = {
 		this._trigger('forcedMute')
 	},
 
+	_handleExtendedIceConnectionStateChange(extendedIceConnectionState) {
+		switch (extendedIceConnectionState) {
+		case 'new':
+			this.set('connectionState', ConnectionState.NEW)
+			break
+		case 'checking':
+			this.set('connectionState', ConnectionState.CHECKING)
+			break
+		case 'connected':
+			this.set('connectionState', ConnectionState.CONNECTED)
+			break
+		case 'completed':
+			this.set('connectionState', ConnectionState.COMPLETED)
+			break
+		case 'disconnected':
+			this.set('connectionState', ConnectionState.DISCONNECTED)
+			break
+		case 'disconnected-long':
+			this.set('connectionState', ConnectionState.DISCONNECTED_LONG)
+			break
+		case 'failed':
+			this.set('connectionState', ConnectionState.FAILED)
+			break
+		// 'failed-no-restart' is not emitted by own peer
+		case 'closed':
+			this.set('connectionState', ConnectionState.CLOSED)
+			break
+		default:
+			console.error('Unexpected (extended) ICE connection state: ', extendedIceConnectionState)
+		}
+	},
+
 }

src/utils/webrtc/webrtc.js

@@ -691,6 +691,43 @@ export default function initWebRtc(signaling, _callParticipantCollection, _local
 		})
 	}
 
+	function setHandlerForOwnIceConnectionStateChange(peer) {
+		peer.pc.addEventListener('iceconnectionstatechange', function() {
+			peer.emit('extendedIceConnectionStateChange', peer.pc.iceConnectionState)
+
+			switch (peer.pc.iceConnectionState) {
+			case 'checking':
+				console.debug('Connecting own peer...', peer)
+
+				break
+			case 'connected':
+			case 'completed':
+				console.debug('Connection established (own peer).', peer)
+
+				break
+			case 'disconnected':
+				console.debug('Disconnected (own peer).', peer)
+
+				setTimeout(function() {
+					if (peer.pc.iceConnectionState !== 'disconnected') {
+						return
+					}
+
+					peer.emit('extendedIceConnectionStateChange', 'disconnected-long')
+				}, 5000)
+				break
+			case 'failed':
+				console.debug('Connection failed (own peer).', peer)
+
+				break
+			case 'closed':
+				console.debug('Connection closed (own peer).', peer)
+
+				break
+			}
+		})
+	}
+
 	const forceReconnect = function(signaling, flags) {
 		if (ownPeer) {
 			webrtc.removePeers(ownPeer.id)
@@ -856,7 +893,7 @@ export default function initWebRtc(signaling, _callParticipantCollection, _local
 
 		if (peer.type === 'video') {
 			if (peer.id === signaling.getSessionId()) {
-				console.debug('Not adding ICE connection state handler for own peer', peer)
+				setHandlerForOwnIceConnectionStateChange(peer)
 			} else {
 				setHandlerForIceConnectionStateChange(peer)
 			}