From f4af954834d4923c14aa58cc6495b7a908278b14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Jaussoin?= Date: Sun, 1 Dec 2019 23:35:04 +0100 Subject: [PATCH] Refactor visio --- app/widgets/Visio/Visio.php | 30 ++-- app/widgets/Visio/visio.js | 30 ++-- app/widgets/Visio/visio2.js | 189 +++++++++++++++++++++++ app/widgets/VisioLink/VisioLink.php | 19 +++ app/widgets/VisioLink/visiolink.js | 10 +- lib/JingletoSDP.php | 21 ++- lib/SDPtoJingle.php | 71 ++++++--- lib/moxl/src/Moxl/Xec/Payload/Jingle.php | 2 +- 8 files changed, 309 insertions(+), 63 deletions(-) create mode 100644 app/widgets/Visio/visio2.js diff --git a/app/widgets/Visio/Visio.php b/app/widgets/Visio/Visio.php index da0459fc1..ef55d9831 100644 --- a/app/widgets/Visio/Visio.php +++ b/app/widgets/Visio/Visio.php @@ -12,6 +12,7 @@ class Visio extends Base { $this->addcss('visio.css'); $this->addjs('visio.js'); + $this->addjs('visio2.js'); $this->registerEvent('jingle_sessioninitiate', 'onSDP'); $this->registerEvent('jingle_transportinfo', 'onCandidate'); @@ -22,10 +23,6 @@ class Visio extends Base public function onSDP($data) { list($stanza, $from) = $data; - $jts = new JingletoSDP($stanza); - - $s = Session::start(); - $s->set('sdp', $jts->generate()); $contact = \App\Contact::firstOrNew(['id' => cleanJid($from)]); @@ -47,7 +44,7 @@ class Visio extends Base ); } - public function ajaxAskInit() + /*public function ajaxAskInit() { $s = Session::start(); if ($s->get('sdp')) { @@ -56,12 +53,12 @@ class Visio extends Base } else { $this->rpc('Visio.init'); } - } + }*/ public function onAccept($stanza) { $jts = new JingletoSDP($stanza); - $this->rpc('Visio.onSDP', $jts->generate(), 'answer'); + $this->rpc('Visio.onAcceptSDP', $jts->generate()); } public function onCandidate($stanza) @@ -69,7 +66,7 @@ class Visio extends Base $jts = new JingletoSDP($stanza); $sdp = $jts->generate(); - $s = Session::start(); + /*$s = Session::start(); $candidates = $s->get('candidates'); if (!$candidates) { @@ -78,12 +75,13 @@ class Visio extends Base array_push($candidates, [$sdp, $jts->name, substr($jts->name, -1, 1)]); - $s->set('candidates', $candidates); + $s->set('candidates', $candidates);*/ $this->rpc('Visio.onCandidate', $sdp, $jts->name, substr($jts->name, -1, 1)); + //Visio.consume } - public function ajaxGetCandidates() + /*public function ajaxGetCandidates() { $s = Session::start(); $candidates = $s->get('candidates'); @@ -93,17 +91,17 @@ class Visio extends Base } $s->remove('candidates'); - } + }*/ public function onTerminate($stanza) { - $this->clearCandidates(); + //$this->clearCandidates(); $this->rpc('Visio.onTerminate'); } public function ajaxInitiate($sdp, $to) { - $this->clearCandidates(); + //$this->clearCandidates(); $stj = new SDPtoJingle( $sdp->sdp, @@ -152,7 +150,7 @@ class Visio extends Base public function ajaxTerminate($to, $reason = 'success') { - $this->clearCandidates(); + //$this->clearCandidates(); $s = Session::start(); @@ -163,11 +161,11 @@ class Visio extends Base ->request(); } - private function clearCandidates() + /*private function clearCandidates() { $s = Session::start(); $s->remove('candidates'); - } + }*/ public function display() { diff --git a/app/widgets/Visio/visio.js b/app/widgets/Visio/visio.js index 4cc2775b1..0f9ac188b 100644 --- a/app/widgets/Visio/visio.js +++ b/app/widgets/Visio/visio.js @@ -1,4 +1,4 @@ -function logError(error) { +/*function logError(error) { console.log(error.name + ': ' + error.message); console.log(error); } @@ -15,7 +15,6 @@ var Visio = { videoSelect: undefined, switchCamera: undefined, - localCreated: false, setFrom: from => Visio.from = from, @@ -59,6 +58,11 @@ var Visio = { } Visio.toggleMainButton(); + + // If we received an offer, we need to answer + if (Visio.pc.remoteDescription.type == 'offer') { + Visio.pc.createAnswer(Visio.localDescCreated, logError); + } }); } }, @@ -128,11 +132,6 @@ var Visio = { new RTCSessionDescription({'sdp': sdp + "\n", 'type': type}), () => { Visio_ajaxGetCandidates(); - - // If we received an offer, we need to answer - if (Visio.pc.remoteDescription.type == 'offer') { - Visio.pc.createAnswer(Visio.localDescCreated, logError); - } }, (error) => { Visio.goodbye('incompatible-parameters'); @@ -142,8 +141,10 @@ var Visio = { }, localDescCreated: function(desc) { - Visio.localCreated = true; - Visio.pc.setLocalDescription(desc, Visio.toggleMainButton, logError); + Visio.pc.setLocalDescription(desc, () => { + Visio_ajaxAccept(Visio.pc.localDescription, Visio.from); + Visio.toggleMainButton(); + }, logError); }, onCandidates: function(candidates) { @@ -285,11 +286,6 @@ var Visio = { }; }, - answer: () => { - Visio.localCreated = false; - Visio_ajaxAccept(Visio.pc.localDescription, Visio.from); - }, - hello: function() { Visio.pc.createOffer((desc) => { Visio.pc.setLocalDescription( @@ -305,9 +301,6 @@ var Visio = { Visio_ajaxTerminate(Visio.from, reason); }, - /* - * UI Status - */ toggleMainButton: function() { button = document.getElementById('main'); state = document.querySelector('p.state'); @@ -318,8 +311,6 @@ var Visio = { button.classList.add('disabled'); if (Visio.pc) { - if (Visio.localCreated) Visio.answer(); - let length = Visio.pc.getSenders().length; if (Visio.pc.iceConnectionState != 'closed' @@ -432,3 +423,4 @@ MovimWebsocket.attach(() => { window.onbeforeunload = () => { Visio.goodbye(); } +*/ \ No newline at end of file diff --git a/app/widgets/Visio/visio2.js b/app/widgets/Visio/visio2.js new file mode 100644 index 000000000..6396981dd --- /dev/null +++ b/app/widgets/Visio/visio2.js @@ -0,0 +1,189 @@ +function logError(error) { + console.log(error.name + ': ' + error.message); + console.log(error); +} + +var Visio = { + from: null, + localVideo: null, + remoteVideo: null, + + init: function() { + Visio.from = MovimUtils.urlParts().params.join('/'); + Visio.localVideo = document.getElementById('video'); + Visio.remoteVideo = document.getElementById('remote_video'); + + /*const servers = ['stun:stun01.sipphone.com', + 'stun:stun.ekiga.net', + 'stun:stun.fwdnet.net', + 'stun:stun.ideasip.com', + 'stun:stun.iptel.org', + 'stun:stun.rixtelecom.se', + 'stun:stun.schlund.de', + 'stun:stun.l.google.com:19302', + 'stun:stun1.l.google.com:19302', + 'stun:stun2.l.google.com:19302', + 'stun:stun3.l.google.com:19302', + 'stun:stun4.l.google.com:19302', + 'stun:stunserver.org', + 'stun:stun.softjoys.com', + 'stun:stun.voiparound.com', + 'stun:stun.voipbuster.com', + 'stun:stun.voipstunt.com', + 'stun:stun.voxgratia.org', + 'stun:stun.xten.com' + ]; + + const shuffled = servers.sort(() => 0.5 - Math.random()); + + var configuration = { + 'iceServers': [ + {urls: shuffled.slice(0, 2)} + ] + };*/ + const configuration = { + iceServers: [{ + urls: 'stun:stun.l.google.com:19302' + }] + }; + Visio.pc = new RTCPeerConnection(configuration); + + Visio.pc.onicecandidate = evt => { + console.log('SEND CANDIDATE'); + if (evt.candidate) { + console.log(evt.candidate.candidate); + Visio_ajaxCandidate(evt.candidate, Visio.from); + } + }; + + /*Visio.pc.ontrack = event => { + console.log('TRACK'); + console.log(event); + const stream = event.streams[0]; + if (!Visio.remoteVideo.srcObject || Visio.remoteVideo.srcObject.id !== stream.id) { + console.log(event.track.kind); + if (event.track.kind === 'video') Visio.remoteVideo.srcObject = stream; + } + };*/ + + Visio.pc.onaddstream = function(event) { + Visio.remoteVideo.srcObject = event.stream; + }; + + const remoteSDP = localStorage.getItem('sdp'); + + // If we are calling + if (remoteSDP === null) { + console.log('CALLING'); + Visio.pc.onnegotiationneeded = function() { + Visio.pc.createOffer().then(function(offer) { + return Visio.pc.setLocalDescription(offer); + }) + .then(function() { + Visio_ajaxInitiate(Visio.pc.localDescription, Visio.from); + }); + } + } else { + console.log('CALLED'); + // If we are called + localStorage.removeItem('sdp'); + Visio.pc.setRemoteDescription(new RTCSessionDescription({'sdp': remoteSDP + "\n", 'type': 'offer'}), () => { + if (Visio.pc.remoteDescription.type === 'offer') { + Visio.pc.createAnswer().then(function(answer) { + return Visio.pc.setLocalDescription(answer); + }).then(function() { + Visio.consumeCandidates(); + Visio_ajaxAccept(Visio.pc.localDescription, Visio.from); + }).catch(logError); + } + }, logError); + } + + navigator.mediaDevices.getUserMedia({ + audio: true, + video: true, + }).then(stream => { + Visio.localVideo.srcObject = stream; + stream.getTracks().forEach(track => Visio.pc.addTrack(track, stream)); + }, logError); + }, + + consumeCandidates: function() { + const candidates = localStorage.getObject('candidates'); + + if (candidates) { + candidates.forEach(candidate => Visio.onCandidate(candidate[0], candidate[1], candidate[2])); + //localStorage.removeItem('candidates'); + } + }, + + onCandidate: function(candidate, mid, mlineindex) { + if (mid == '') mlineindex = 1; + + console.log('RECEIVED CANDIDATE'); + if (Visio.pc.remoteDescription == null) return; + console.log(candidate); + + candidate = new RTCIceCandidate({ + 'candidate': candidate, + 'sdpMid': mid, + 'sdpMLineIndex' : mlineindex + }); + + Visio.pc.addIceCandidate(candidate, e => {}); + }, + + onAcceptSDP: function(sdp) { + Visio.pc.setRemoteDescription( + new RTCSessionDescription({'sdp': sdp + "\n", 'type': 'answer'}), () => { + Visio.consumeCandidates(); + }, + (error) => { + Visio.goodbye('incompatible-parameters'); + logError(error) + } + ); + }, + + onTerminate: () => { + let localStream = Visio.localVideo.srcObject; + + if (localStream) { + localStream.getTracks().forEach(track => track.stop()); + } + + let remoteStream = Visio.remoteVideo.srcObject; + + if (remoteStream) { + remoteStream.getTracks().forEach(track => track.stop()); + } + + Visio.localVideo.srcObject = null; + Visio.remoteVideo.srcObject = null; + Visio.localVideo.classList.add('hide'); + + if (Visio.pc) Visio.pc.close(); + + document.querySelector('p.state').innerText = Visio.states.ended; + button = document.querySelector('#main'); + + button.className = 'button action color red'; + button.querySelector('i').className = 'material-icons'; + button.querySelector('i').innerText = 'close'; + + button.onclick = () => window.close(); + }, + + goodbye: (reason) => { + Visio.onTerminate(); + Visio_ajaxTerminate(Visio.from, reason); + }, +} + +MovimWebsocket.attach(() => { + Visio.init(); +}); + +window.onbeforeunload = () => { + Visio.goodbye(); +} \ No newline at end of file diff --git a/app/widgets/VisioLink/VisioLink.php b/app/widgets/VisioLink/VisioLink.php index 73a4a333d..57d1bd6ad 100644 --- a/app/widgets/VisioLink/VisioLink.php +++ b/app/widgets/VisioLink/VisioLink.php @@ -6,10 +6,29 @@ class VisioLink extends Base { public function load() { + $this->registerEvent('jingle_transportinfo', 'onCandidate'); + $this->registerEvent('jingle_sessioninitiate', 'onSDP'); + $this->addjs('visiolink.js'); $this->addcss('visiolink.css'); } + public function onSDP($data) + { + list($stanza, $from) = $data; + $jts = new JingletoSDP($stanza); + + $this->rpc('VisioLink.setSDP', $jts->generate()); + } + + public function onCandidate($stanza) + { + $jts = new JingletoSDP($stanza); + $sdp = $jts->generate(); + + $this->rpc('VisioLink.setCandidate', [$sdp, $jts->name, substr($jts->name, -1, 1)]); + } + public function ajaxDecline($to) { $visio = new Visio; diff --git a/app/widgets/VisioLink/visiolink.js b/app/widgets/VisioLink/visiolink.js index b6503ed61..8062ea928 100644 --- a/app/widgets/VisioLink/visiolink.js +++ b/app/widgets/VisioLink/visiolink.js @@ -1,5 +1,13 @@ var VisioLink = { - candidates: [], + setSDP: function(remoteSDP) { + localStorage.setItem('sdp', remoteSDP); + }, + + setCandidate: function(remoteCandidate) { + var candidates = localStorage.getObject('candidates'); + if (!candidates) candidates = []; + candidates.push(remoteCandidate); + }, reset: function() { VisioLink.window = null; diff --git a/lib/JingletoSDP.php b/lib/JingletoSDP.php index 240b6646a..a7668d231 100644 --- a/lib/JingletoSDP.php +++ b/lib/JingletoSDP.php @@ -100,7 +100,9 @@ class JingletoSDP $sdp_media .= "\r\na=ice-pwd:".$content->transport->attributes()->pwd; } - if (isset($content->transport->attributes()->ufrag)) { + // ufrag can be alone without a password for candidates + if (isset($content->transport->attributes()->ufrag) + && isset($content->transport->attributes()->pwd)) { $sdp_media .= "\r\na=ice-ufrag:".$content->transport->attributes()->ufrag; } @@ -306,6 +308,17 @@ class JingletoSDP ' network-id '.$payload->attributes()->{'network-id'}; } + if (isset($payload->attributes()->{'network-cost'})) { + $sdp_media .= + ' network-id '.$payload->attributes()->{'network-cost'}; + } + + // ufrag in candidate transport + if (isset($content->transport->attributes()->ufrag)) { + $sdp_media .= + ' ufrag '.$content->transport->attributes()->ufrag; + } + $media_header_last_ip = $payload->attributes()->ip; break; @@ -341,9 +354,13 @@ class JingletoSDP $sdp_media_header = $sdp_media_header.' '.implode(' ', $media_header_ids); + $ipVersion = filter_var($media_header_last_ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) + ? 'IP6' + : 'IP4'; + $sdp_medias .= $sdp_media_header. - "\r\nc=IN IP4 ".$media_header_last_ip. + "\r\nc=IN ".$ipVersion." ".$media_header_last_ip. $sdp_media; //"\r\na=sendrecv"; diff --git a/lib/SDPtoJingle.php b/lib/SDPtoJingle.php index aaa924ec6..867592767 100644 --- a/lib/SDPtoJingle.php +++ b/lib/SDPtoJingle.php @@ -22,7 +22,7 @@ class SDPtoJingle private $rtcp_fb_cache = []; private $regex = [ - 'candidate' => "/^a=candidate:(\w{1,32}) (\d{1,5}) (udp|tcp) (\d{1,10}) ([a-zA-Z0-9:\.]{1,45}) (\d{1,5}) (typ) (host|srflx|prflx|relay)( (raddr) ([a-zA-Z0-9:\.]{1,45}) (rport) (\d{1,5}))?( (generation) (\d) (network) (\d) (id) ([a-zA-Z0-9]{1,45}))?/i", //à partir de generation les attr sont spécifiques à XMPP..autant l'enlever de la REGEX et les traiter à part? En théorie ils peuvent être dans n'importe quel ordre. + 'candidate' => "/^a=candidate:(\w{1,32}) (\d{1,5}) (udp|tcp) (\d{1,10}) ([a-zA-Z0-9:\.]{1,45}) (\d{1,5}) (typ) (host|srflx|prflx|relay) (.+)?/i", 'sess_id' => "/^o=(\S+) (\d+)/i", 'group' => "/^a=group:(\S+) (.+)/i", 'rtpmap' => "/^a=rtpmap:(\d+) (([^\s\/]+)(\/(\d+)(\/([^\s\/]+))?)?)?/i", @@ -352,9 +352,10 @@ class SDPtoJingle $this->initContent(); $this->addName(); - $generation = $network = $id = $networkid = false; + //$generation = $network = $id = $networkid = false; +\Utils::debug($this->sdp); - if ($key = array_search("generation", $matches)) { + /*if ($key = array_search("generation", $matches)) { $generation = $matches[($key+1)]; } if ($key = array_search("network", $matches)) { @@ -365,43 +366,65 @@ class SDPtoJingle } if ($key = array_search("network-id", $matches)) { $networkid = $matches[($key+1)]; - } + }*/ - if (isset($matches[11]) && isset($matches[13])) { + /*if (isset($matches[11]) && isset($matches[13])) { $reladdr = $matches[11]; $relport = $matches[13]; } else { $reladdr = $relport = null; - } + }*/ $candidate = $this->transport->addChild('candidate'); - $candidate->addAttribute('component', $matches[2]); $candidate->addAttribute('foundation', $matches[1]); + $candidate->addAttribute('component', $matches[2]); + $candidate->addAttribute('protocol', $matches[3]); + $candidate->addAttribute('priority', $matches[4]); + $candidate->addAttribute('ip', $matches[5]); + $candidate->addAttribute('port', $matches[6]); + $candidate->addAttribute('type', $matches[8]); - if ($generation) { - $candidate->addAttribute('generation', $generation); + // We have other arguments + $args = []; + if (isset($matches[9])) { + $keyValues = explode(' ', $matches[9]); + foreach ($keyValues as $key) + + foreach (array_chunk($keyValues, 2) as $pair) { + list($key, $value) = $pair; + $args[$key] = $value; + } + \Utils::debug(serialize($args)); } - if ($id) { - $candidate->addAttribute('id', $id); + + if (isset($args['generation'])) { + $candidate->addAttribute('generation', $args['generation']); } - if ($network) { - $candidate->addAttribute('network', $network); + if (isset($args['id'])) { + $candidate->addAttribute('id', $args['id']); } - if ($networkid) { - $candidate->addAttribute('network-id', $networkid); + if (isset($args['network'])) { + $candidate->addAttribute('network', $args['network']); + } + if (isset($args['network-id'])) { + $candidate->addAttribute('network-id', $args['network-id']); + } + if (isset($args['network-cost'])) { + $candidate->addAttribute('network-cost', $args['network-cost']); + } + if (isset($args['raddr'])) { + $candidate->addAttribute('rel-addr', $args['raddr']); + } + if (isset($args['rport'])) { + $candidate->addAttribute('rel-port', $args['rport']); } - $candidate->addAttribute('ip', $matches[5]); - $candidate->addAttribute('port', $matches[6]); - $candidate->addAttribute('priority', $matches[4]); - $candidate->addAttribute('protocol', $matches[3]); - $candidate->addAttribute('type', $matches[8]); - - if ($reladdr) { - $candidate->addAttribute('rel-addr', $reladdr); - $candidate->addAttribute('rel-port', $relport); + // ufrag to the transport + if (isset($args['ufrag'])) { + $this->transport->addAttribute('ufrag', $args['ufrag']); } + break; } } diff --git a/lib/moxl/src/Moxl/Xec/Payload/Jingle.php b/lib/moxl/src/Moxl/Xec/Payload/Jingle.php index ad72e36fb..c13e5a197 100644 --- a/lib/moxl/src/Moxl/Xec/Payload/Jingle.php +++ b/lib/moxl/src/Moxl/Xec/Payload/Jingle.php @@ -38,7 +38,7 @@ class Jingle extends Payload $action = (string)$stanza->attributes()->action; - Ack::send($from, $id); + //Ack::send($from, $id); $userid = \App\User::me()->id; $message = new \App\Message;