Add screen sharing feature during video conferencing

Fix External Services discovery, refresh the STUN/TURN at each sessions
Fix jingleSid handing and return proper errors for unknown Jingle session IQs

app/Info.php

@@ -216,7 +216,8 @@ class Info extends Model
 
     public function isJingle()
     {
-        return $this->hasFeature('urn:xmpp:jingle:apps:rtp:audio');
+        return $this->hasFeature('urn:xmpp:jingle:apps:rtp:audio')
+            && $this->hasFeature('urn:xmpp:jingle-message:0');
     }
 
     public function isMAM()
@@ -229,6 +230,11 @@ class Info extends Model
         return $this->hasFeature('urn:xmpp:mam:2');
     }
 
+    public function hasExternalServices()
+    {
+        return $this->hasFeature('urn:xmpp:extdisco:2');
+    }
+
     public function set($query, $node = false)
     {
         $from = (string)$query->attributes()->from;

app/Session.php

@@ -101,20 +101,6 @@ class Session extends Model
         $s->set('password', $password);
     }
 
-    public function setExternalservicesAttribute(array $externalServices)
-    {
-        $this->attributes['externalservices'] = serialize($externalServices);
-    }
-
-    public function getExternalservicesAttribute()
-    {
-        if (isset($this->attributes['externalservices'])) {
-            return unserialize($this->attributes['externalservices']);
-        }
-
-        return null;
-    }
-
     public function getUploadService()
     {
         return Info::where('server', 'like', '%' . $this->host)

app/widgets/CommunityAffiliations/communityaffiliations.js

@@ -2,7 +2,6 @@ var CommunityAffiliations = {
     update : function(jid) {
         var parts = MovimUtils.urlParts();
         if (parts.params.length > 0) {
-            console.log(MovimUtils.formToJson(jid));
             CommunityAffiliations_ajaxChangeAffiliation(
                 parts.params[0],
                 parts.params[1],

app/widgets/Presence/Presence.php

@@ -21,7 +21,6 @@ class Presence extends Base
         $this->registerEvent('mypresence', 'onMyPresence');
         $this->registerEvent('session_up', 'onSessionUp');
         $this->registerEvent('session_down', 'onSessionDown');
-        $this->registerEvent('externalservices_get_handle', 'onExternalServices');
     }
 
     public function onSessionUp()
@@ -36,15 +35,6 @@ class Presence extends Base
         $p->request();
     }
 
-    public function onExternalServices($packet)
-    {
-        $session = $this->user->session;
-        if ($session) {
-            $session->externalservices = $packet->content;
-            $session->save();
-        };
-    }
-
     public function onMyPresence($packet)
     {
         $this->rpc('MovimTpl.fill', '#presence_widget', $this->preparePresence());
@@ -72,7 +62,6 @@ class Presence extends Base
         $this->ajaxFeedRefresh();
         $this->ajaxServerDisco();
         $this->ajaxProfileRefresh();
-        $this->ajaxExternalServicesGet();
     }
 
     public function ajaxAskLogout()
@@ -144,14 +133,6 @@ class Presence extends Base
           ->request();
     }
 
-    // We discover the server services
-    public function ajaxExternalServicesGet()
-    {
-        $c = new \Moxl\Xec\Action\ExternalServices\Get;
-        $c->setTo($this->user->session->host)
-          ->request();
-    }
-
     // We refresh the profile
     public function ajaxProfileRefresh()
     {

app/widgets/Upload/upload.js

@@ -152,7 +152,6 @@ var Upload = {
 
         // If the preview system is there
         if (preview) {
-            console.log(file);
             var fileInfo = document.querySelector('#upload li.file');
             fileInfo.querySelector('p.name').innerText = Upload.name;
             var type = file.type ? file.type + ' · ' : '';

app/widgets/Visio/Visio.php

@@ -26,6 +26,48 @@ class Visio extends Base
         $this->registerEvent('jingle_sessionaccept', 'onAcceptSDP');
         $this->registerEvent('jingle_transportinfo', 'onCandidate');
         $this->registerEvent('jingle_sessionterminate', 'onTerminate');
+        $this->registerEvent('externalservices_get_handle', 'onExternalServices');
+        $this->registerEvent('externalservices_get_error', 'onExternalServicesError');
+    }
+
+    public function onExternalServices($packet)
+    {
+        $externalServices = [];
+        if ($packet->content) {
+            $turn = $stun = false;
+            foreach ($packet->content as $service) {
+                // One STUN/TURN server max
+                if ($service['type'] == 'stun' && $stun) continue;
+                if ($service['type'] == 'stun') $stun = true;
+                if ($service['type'] == 'turn' && $turn) continue;
+                if ($service['type'] == 'turn') $turn = true;
+
+                $url = $service['type'].':'.$service['host'];
+                $url .= !empty($service['port']) ? ':'.$service['port'] : '';
+                $item = ['urls' => $url];
+
+                if (isset($service['username']) && isset($service['password'])) {
+                    $item['username'] = $service['username'];
+                    $item['credential'] = $service['password'];
+                }
+
+                array_push($externalServices, $item);
+            }
+        }
+
+        if (!empty($externalServices)) {
+            $this->rpc('Visio.setServices', $externalServices);
+        } else {
+            $this->setDefaultServices();
+        }
+
+        $this->rpc('Visio.init');
+    }
+
+    public function onExternalServicesError($packet)
+    {
+        $this->setDefaultServices();
+        $this->rpc('Visio.init');
     }
 
     public function onPropose($packet)
@@ -130,6 +172,8 @@ class Visio extends Base
 
     public function ajaxAccept($to, $id)
     {
+        Session::start()->set('jingleSid', $id);
+
         $p = new SessionAccept;
         $p->setTo($to)
           ->setId($id)
@@ -152,6 +196,49 @@ class Visio extends Base
            ->request();
     }
 
+    public function ajaxResolveServices()
+    {
+        $info = \App\Info::where('server', $this->user->session->host)
+                    ->where('node', '')
+                    ->first();
+        if ($info && $info->hasExternalServices()) {
+            $c = new \Moxl\Xec\Action\ExternalServices\Get;
+            $c->setTo($this->user->session->host)
+              ->request();
+        } else {
+            $this->setDefaultServices();
+            $this->rpc('Visio.init');
+        }
+    }
+
+    public function setDefaultServices()
+    {
+        $servers = [
+            'stun:stun01.sipphone.com',
+            'stun:stun.ekiga.net',
+            'stun:stun.fwdnet.net',
+            'stun:stun.ideasip.com',
+            'stun:stun.iptel.org',
+            'stun:stun.rixtelecom.se',
+            'stun:stun.schlund.de',
+            'stun:stun.l.google.com:19305',
+            'stun:stun1.l.google.com:19305',
+            'stun:stun2.l.google.com:19305',
+            'stun:stun3.l.google.com:19305',
+            'stun:stun4.l.google.com:19305',
+            'stun:stunserver.org',
+            'stun:stun.softjoys.com',
+            'stun:stun.voiparound.com',
+            'stun:stun.voipbuster.com',
+            'stun:stun.voipstunt.com',
+            'stun:stun.voxgratia.org',
+            'stun:stun.xten.com'
+        ];
+
+        shuffle($servers);
+        $this->rpc('Visio.setServices', [['urls' => array_slice($servers, 0, 2)]]);
+    }
+
     public function ajaxSessionAccept($sdp, $to, $id)
     {
         $stj = new SDPtoJingle(
@@ -188,7 +275,7 @@ class Visio extends Base
 
     public function ajaxTerminate($to, $reason = 'success', $sid)
     {
-        $s = Session::start();
+        Session::start()->remove('jingleSid');
 
         $st = new SessionTerminate;
         $st->setTo($to)
@@ -199,30 +286,7 @@ class Visio extends Base
 
     public function display()
     {
-        $externalServices = [];
-        if ($this->user->session->externalservices) {
-            $turn = $stun = false;
-            foreach ($this->user->session->externalservices as $service) {
-                // One STUN/TURN server max
-                if ($service['type'] == 'stun' && $stun) continue;
-                if ($service['type'] == 'stun') $stun = true;
-                if ($service['type'] == 'turn' && $turn) continue;
-                if ($service['type'] == 'turn') $turn = true;
-
-                $url = $service['type'].':'.$service['host'];
-                $url .= !empty($service['port']) ? ':'.$service['port'] : '';
-                $item = ['urls' => $url];
-
-                if (isset($service['username']) && isset($service['password'])) {
-                    $item['username'] = $service['username'];
-                    $item['credential'] = $service['password'];
-                }
-
-                array_push($externalServices, $item);
-            }
-        }
         $this->view->assign('withvideo', $this->getView() == 'visio');
-        $this->view->assign('externalservices', json_encode($externalServices));
         $this->view->assign('contact', \App\Contact::firstOrNew(['id' => $this->get('f')]));
     }
 }

app/widgets/Visio/_visio_dialog.tpl

@@ -8,7 +8,14 @@
                         <img src="{$url}">
                     </p>
                 {/if}
-                <p class="normal center">{$contact->truename}</p>
+                <p class="normal center">
+                    {if="$withvideo"}
+                        {$c->__('visio.video_call')}
+                    {else}
+                        {$c->__('visio.audio_call')}
+                    {/if}
+                </p>
+                <p class="center">{$contact->truename}</p>
                 <p class="center">{$c->__('visio.calling')}</p>
             </div>
         </li>

app/widgets/Visio/locales.ini

@@ -1,5 +1,7 @@
 [visio]
 calling     = …is calling you
+video_call  = Incoming video call
+audio_call  = Incoming audio call
 ringing     = …ringing
 declined    = declined
 in_call     = in call

app/widgets/Visio/visio.css

@@ -11,7 +11,8 @@ body {
     display: none;
 }
 
-#visio #switch_camera:not(.enabled) {
+#visio #switch_camera:not(.enabled),
+#visio #screen_sharing:not(.enabled) {
     display: none;
 }
 
@@ -37,7 +38,8 @@ body {
     font-size: 1.8rem;
 }
 
-#visio video#video {
+#visio video#video,
+#visio video#screen_sharing_video.sharing {
     position: fixed;
     right: 1rem;
     bottom: 1rem;
@@ -52,8 +54,14 @@ body {
     background-size: 25%;
 }
 
+#visio video#screen_sharing_video:not(.sharing),
+#visio video#screen_sharing_video.sharing + video#video {
+    display: none;
+}
+
 @media screen and (max-width: 800px) {
-    #visio video#video {
+    #visio video#video,
+    #visio video#screen_sharing_video.sharing {
         width: 15rem;
     }
 }

app/widgets/Visio/visio.js

@@ -12,12 +12,15 @@ var Visio = {
     remoteVideo: null,
     localAudio: null,
     remoteAudio: null,
+    screenSharing: null,
 
     calling: false,
 
     videoSelect: undefined,
     switchCamera: undefined,
 
+    services: [],
+
     init: function(id) {
         Visio.from = MovimUtils.urlParts().params[0];
 
@@ -28,43 +31,14 @@ var Visio = {
         if (Visio.withVideo) {
             Visio.localVideo = document.getElementById('video');
             Visio.remoteVideo = document.getElementById('remote_video');
+            Visio.screenSharing = document.getElementById('screen_sharing_video');
         }
 
         Visio.localAudio = document.getElementById('audio');
         Visio.remoteAudio = document.getElementById('remote_audio');
 
-        var iceServers = [];
-        if (Visio.externalServices.length > 0) {
-            iceServers = Visio.externalServices
-        } else {
-            const servers = ['stun:stun01.sipphone.com',
-                'stun:stun.ekiga.net',
-                'stun:stun.fwdnet.net',
-                'stun:stun.ideasip.com',
-                'stun:stun.iptel.org',
-                'stun:stun.rixtelecom.se',
-                'stun:stun.schlund.de',
-                'stun:stun.l.google.com:19302',
-                'stun:stun1.l.google.com:19302',
-                'stun:stun2.l.google.com:19302',
-                'stun:stun3.l.google.com:19302',
-                'stun:stun4.l.google.com:19302',
-                'stun:stunserver.org',
-                'stun:stun.softjoys.com',
-                'stun:stun.voiparound.com',
-                'stun:stun.voipbuster.com',
-                'stun:stun.voipstunt.com',
-                'stun:stun.voxgratia.org',
-                'stun:stun.xten.com'
-            ];
-
-            const shuffled = servers.sort(() => 0.5 - Math.random());
-            iceServers = [{urls: shuffled.slice(0, 2)}];
-        }
-
-
         var configuration = {
-            'iceServers': iceServers
+            'iceServers': Visio.services
         };
 
         Visio.pc = new RTCPeerConnection(configuration);
@@ -104,6 +78,10 @@ var Visio = {
         Visio.gotStream();
     },
 
+    setServices: function(services) {
+        Visio.services = services;
+    },
+
     gotStream: function() {
         if (Visio.withVideo) {
             // On Android where you can't have both camera enabled at the same time
@@ -135,12 +113,9 @@ var Visio = {
                 Visio.localVideo.srcObject = stream;
 
                 // Switch camera
-                let videoTrack = stream.getVideoTracks()[0];
-                var sender = Visio.pc.getSenders().find(s => s.track && s.track.kind == videoTrack.kind);
+                VisioUtils.pcReplaceTrack(stream);
 
-                if (sender) {
-                    sender.replaceTrack(videoTrack);
-                }
+                VisioUtils.enableScreenSharingButton();
             }
 
             VisioUtils.handleAudio();
@@ -163,6 +138,14 @@ var Visio = {
         }, logError);
     },
 
+    gotQuickStream: function() {
+        VisioUtils.pcReplaceTrack(Visio.localVideo.srcObject);
+    },
+
+    gotScreen: function() {
+        VisioUtils.pcReplaceTrack(Visio.screenSharing.srcObject);
+    },
+
     onCandidate: function(candidate, mid, mlineindex) {
         Visio.pc.addIceCandidate(new RTCIceCandidate({
             'candidate': candidate.substr(2),
@@ -274,7 +257,7 @@ MovimWebsocket.attach(() => {
         Visio.withVideo = true;
     }
 
-    Visio.init();
+    Visio_ajaxResolveServices();
 });
 
 window.onbeforeunload = () => {

app/widgets/Visio/visio.tpl

@@ -8,7 +8,7 @@
                 <span id="toggle_fullscreen" class="control icon color transparent active" onclick="VisioUtils.toggleFullScreen()">
                     <i class="material-icons">fullscreen</i>
                 </span>
-                <span id="toggle_audio" class="control icon color transparent active" onclick="VisioUtils.toggleAudio()">
+                <span id="toggle_audio" class="divided control icon color transparent active" onclick="VisioUtils.toggleAudio()">
                     <i class="material-icons">mic</i>
                 </span>
                 {if="$withvideo"}
@@ -18,6 +18,9 @@
                     <span id="switch_camera" class="control icon color transparent active">
                         <i class="material-icons">switch_camera</i>
                     </span>
+                    <span id="screen_sharing" class="control icon color transparent active" onclick="VisioUtils.toggleScreenSharing()">
+                        <i class="material-icons">screen_share</i>
+                    </span>
                 {/if}
                 <div><p></p></div>
             </li>
@@ -45,6 +48,7 @@
 
     {if="$withvideo"}
         <video id="remote_video" autoplay poster="/theme/img/empty.png"></video>
+        <video id="screen_sharing_video" autoplay muted poster="/theme/img/empty.png"></video>
         <video id="video" autoplay muted poster="/theme/img/empty.png"></video>
     {/if}
 
@@ -64,6 +68,4 @@ Visio.states = {
     ended: '{$c->__('visio.ended')}',
     declined: '{$c->__('visio.declined')}'
 };
-
-Visio.externalServices = {autoescape="off"}{$externalservices}{/autoescape};
 </script>

app/widgets/Visio/visio_utils.js

@@ -34,7 +34,7 @@ var VisioUtils = {
 
             instant_L = Math.sqrt(sum_L / inpt_L.length);
             VisioUtils.max_level_L = Math.max(VisioUtils.max_level_L, instant_L);
-            instant_L = Math.max(instant_L, VisioUtils.old_level_L -0.0005 );
+            instant_L = Math.max(instant_L, VisioUtils.old_level_L -0.001 );
             VisioUtils.old_level_L = instant_L;
 
             var level = (instant_L/VisioUtils.max_level_L);
@@ -197,6 +197,41 @@ var VisioUtils = {
         }
     },
 
+    enableScreenSharingButton: function() {
+        document.querySelector('#screen_sharing').classList.add('enabled');
+    },
+
+    toggleScreenSharing: async function() {
+        var button = document.querySelector('#screen_sharing i');
+        if (Visio.screenSharing.srcObject == null) {
+            try {
+                Visio.screenSharing.srcObject = await navigator.mediaDevices.getDisplayMedia({
+                    video: {
+                        cursor: "always"
+                    },
+                    audio: false
+                });
+
+                Visio.screenSharing.classList.add('sharing');
+                Visio.switchCamera.classList.add('disabled');
+                button.innerText = 'stop_screen_share';
+
+                Visio.gotScreen();
+            } catch(err) {
+                console.error("Error: " + err);
+            }
+        } else {
+            Visio.screenSharing.srcObject.getTracks().forEach(track => track.stop());
+            Visio.screenSharing.srcObject = null;
+            Visio.screenSharing.classList.remove('sharing');
+            Visio.switchCamera.classList.remove('disabled');
+
+            button.innerText = 'screen_share';
+
+            Visio.gotQuickStream();
+        }
+    },
+
     switchCameraSetup: function() {
         Visio.videoSelect = document.querySelector('#visio select#visio_source');
         navigator.mediaDevices.enumerateDevices().then(devices => VisioUtils.gotDevices(devices));
@@ -214,6 +249,15 @@ var VisioUtils = {
         };
     },
 
+    pcReplaceTrack: function(stream) {
+        let videoTrack = stream.getVideoTracks()[0];
+        var sender = Visio.pc.getSenders().find(s => s.track && s.track.kind == videoTrack.kind);
+
+        if (sender) {
+            sender.replaceTrack(videoTrack);
+        }
+    },
+
     gotDevices: function(deviceInfos) {
         Visio.videoSelect.innerText = '';
 

database/migrations/20200422151151_remove_external_services_to_sessions_table.php

@@ -0,0 +1,21 @@
+<?php
+
+use Movim\Migration;
+use Illuminate\Database\Schema\Blueprint;
+
+class RemoveExternalServicesToSessionsTable extends Migration
+{
+    public function up()
+    {
+        $this->schema->table('sessions', function (Blueprint $table) {
+            $table->dropColumn('externalservices');
+        });
+    }
+
+    public function down()
+    {
+        $this->schema->table('sessions', function (Blueprint $table) {
+            $table->text('externalservices')->nullable();
+        });
+    }
+}

lib/moxl/src/Moxl/Stanza/Jingle.php

@@ -93,4 +93,17 @@ class Jingle
         $xml = \Moxl\API::iqWrapper($jingle, $to, 'set');
         \Moxl\API::request($xml);
     }
+
+    public static function unknownSession($to, $id)
+    {
+        $dom = new \DOMDocument('1.0', 'UTF-8');
+        $error = $dom->createElement('error');
+        $error->setAttribute('type', 'cancel');
+
+        $fni = $dom->createElementNS('urn:xmpp:jingle:errors:1', 'unknown-session');
+        $error->appendChild($fni);
+
+        $xml = \Moxl\API::iqWrapper($error, $to, 'error', $id);
+        \Moxl\API::request($xml);
+    }
 }

lib/moxl/src/Moxl/Utils.php

@@ -51,6 +51,7 @@ class Utils
             'urn:xmpp:jingle:transports:ice-udp:0',
             'urn:xmpp:jingle:transports:ice-udp:1',
             'urn:xmpp:jingle:apps:rtp:rtcp-fb:0',
+            'urn:xmpp:jingle-message:0',
 
             'http://jabber.org/protocol/muc',
             'http://jabber.org/protocol/nick',

lib/moxl/src/Moxl/Xec/Action/ExternalServices/Get.php

@@ -39,4 +39,9 @@ class Get extends Action
             $this->deliver();
         }
     }
+
+    public function error($stanza, $parent = false)
+    {
+        $this->deliver();
+    }
 }

lib/moxl/src/Moxl/Xec/Payload/Jingle.php

@@ -27,6 +27,8 @@
 namespace Moxl\Xec\Payload;
 
 use Moxl\Stanza\Ack;
+use Moxl\Stanza\Jingle as JingleStanza;
+use Movim\Session;
 
 class Jingle extends Payload
 {
@@ -37,8 +39,6 @@ class Jingle extends Payload
 
         $action = (string)$stanza->attributes()->action;
 
-        Ack::send($from, $id);
-
         $userid = \App\User::me()->id;
         $message = new \App\Message;
         $message->user_id = $userid;
@@ -48,25 +48,33 @@ class Jingle extends Payload
         $message->published = gmdate('Y-m-d H:i:s');
         $message->thread = (string)$stanza->attributes()->sid;
 
-        switch ($action) {
-            case 'session-initiate':
-                $message->type = 'jingle_start';
-                $message->save();
-                $this->event('jingle_sessioninitiate', [$stanza, $from]);
-                break;
-            case 'transport-info':
-                $this->event('jingle_transportinfo', $stanza);
-                break;
-            case 'session-terminate':
-                $message->type = 'jingle_end';
-                $message->save();
-                $this->event('jingle_sessionterminate', (string)$stanza->reason->children()[0]->getName());
-                break;
-            case 'session-accept':
-                $message->type = 'jingle_start';
-                $message->save();
-                $this->event('jingle_sessionaccept', $stanza);
-                break;
+        $sid = Session::start()->get('jingleSid');
+
+        if ($sid == $message->thread) {
+            Ack::send($from, $id);
+
+            switch ($action) {
+                case 'session-initiate':
+                    $message->type = 'jingle_start';
+                    $message->save();
+                    $this->event('jingle_sessioninitiate', [$stanza, $from]);
+                    break;
+                case 'transport-info':
+                    $this->event('jingle_transportinfo', $stanza);
+                    break;
+                case 'session-terminate':
+                    $message->type = 'jingle_end';
+                    $message->save();
+                    $this->event('jingle_sessionterminate', (string)$stanza->reason->children()[0]->getName());
+                    break;
+                case 'session-accept':
+                    $message->type = 'jingle_start';
+                    $message->save();
+                    $this->event('jingle_sessionaccept', $stanza);
+                    break;
+            }
+        } else {
+            JingleStanza::unknownSession($from, $id);
         }
     }
 }