Merge pull request #294 from open-easyrtc/multi-stream

Multiple video streams for the easyrtc adapter #269
networked-aframe · Oct 31, 2021 · 00501a7 · 00501a7
2 parents bb75bf8 + 6a88d96
commit 00501a7
Show file tree

Hide file tree

Showing 9 changed files with 245 additions and 50 deletions.
diff --git a/dist/networked-aframe.js b/dist/networked-aframe.js
diff --git a/dist/networked-aframe.min.js b/dist/networked-aframe.min.js
diff --git a/examples/basic-multi-streams.html b/examples/basic-multi-streams.html
@@ -0,0 +1,121 @@
+<html>
+  <head>
+    <meta charset="utf-8">
+    <title>Multi Streams Example — Networked-Aframe</title>
+    <meta name="description" content="Dev Example — Networked-Aframe">
+
+    <script src="https://aframe.io/releases/1.2.0/aframe.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/2.3.0/socket.io.slim.js"></script>
+    <script src="/easyrtc/easyrtc.js"></script>
+    <script src="/dist/networked-aframe.js"></script>
+
+    <script src="https://unpkg.com/aframe-randomizer-components@^3.0.1/dist/aframe-randomizer-components.min.js"></script>
+    <!--<script src="https://unpkg.com/aframe-particle-system-component@1.0.5/dist/aframe-particle-system-component.min.js"></script>-->
+    <script src="https://cdn.jsdelivr.net/gh/oneWaveAdrian/aframe-particle-system-component@aframe-1.2.0-upgrade/dist/aframe-particle-system-component.min.js"></script>
+    <script src="/js/spawn-in-circle.component.js"></script>
+  </head>
+  <body>
+    <a-scene networked-scene="
+      room: basic-multi-stream;
+      debug: true;
+      adapter: easyrtc;
+      audio: false;
+      video: true;
+    ">
+      <a-assets>
+
+        <img id="grid" src="https://img.gs/bbdkhfbzkk/stretch/https://i.imgur.com/25P1geh.png" crossorigin="anonymous">
+        <img id="sky" src="https://i.imgur.com/WqlqEkq.jpg" crossorigin="anonymous" />
+
+        <!-- Templates -->
+
+        <!-- Avatar -->
+        <template id="avatar-template">
+          <a-entity class="avatar">
+             <a-plane  color="#FFF" width="4" height="3" position="0 .6 0"  material="side: front" networked-video-source></a-plane>
+             <a-plane  color="#FFF" width="2" height="1" position="0 .7 0"  material="side: front" networked-video-source="streamName: screen"></a-plane>
+          </a-entity>
+        </template>
+
+        <!-- /Templates -->
+      </a-assets>
+
+      <a-entity id="player"
+        networked="template:#avatar-template;attachTemplateToLocal:false;"
+        camera
+        position="0 1.6 0"
+        spawn-in-circle="radius:3"
+        wasd-controls look-controls
+        >
+        <a-sphere class="head"
+          visible="false"
+          random-color
+        ></a-sphere>
+      </a-entity>
+
+      <a-entity position="0 0 0"
+        geometry="primitive: plane; width: 10000; height: 10000;" rotation="-90 0 0"
+        material="src: #grid; repeat: 10000 10000; transparent: true; metalness:0.6; roughness: 0.4; sphericalEnvMap: #sky;"></a-entity>
+
+      <a-entity light="color: #ccccff; intensity: 1; type: ambient;" visible=""></a-entity>
+      <a-entity light="color: #ffaaff; intensity: 1.5" position="5 5 5"></a-entity>
+
+      <a-sky src="#sky" rotation="0 -90 0"></a-sky>
+      <a-entity id="particles" particle-system="preset: snow"></a-entity>
+    </a-scene>
+
+    <!-- GitHub Corner. -->
+    <a href="https://github.com/networked-aframe/networked-aframe" class="github-corner">
+      <svg width="80" height="80" viewBox="0 0 250 250" style="fill:#222; color:#fff; position: absolute; top: 0; border: 0; right: 0;">
+        <path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path>
+      </svg>
+    </a>
+    <button id="screen-btn" type="button" class="camera-btn">Share screen</button>
+    <style>.camera-btn{position:absolute;cursor:pointer;bottom:3%;left:3%;background:#fff;height:40px;width:130px;border-radius:30px}.github-corner:hover .octo-arm{animation:octocat-wave 560ms ease-in-out}@keyframes octocat-wave{0%,100%{transform:rotate(0)}20%,60%{transform:rotate(-25deg)}40%,80%{transform:rotate(10deg)}}@media (max-width:500px){.github-corner:hover .octo-arm{animation:none}.github-corner .octo-arm{animation:octocat-wave 560ms ease-in-out}}
+    </style>
+
+    <script>
+      // Camera status
+      let screenEnabled = false;
+      // Camera button ele
+      const screenBtnEle = document.getElementById('screen-btn');
+
+      // On mobile remove elements that are resource heavy
+      const isMobile = AFRAME.utils.device.isMobile();
+
+      if (isMobile) {
+        const particles = document.getElementById('particles');
+        particles.parentNode.removeChild(particles);
+      }
+
+      // Define custom schema for syncing avatar color, set by random-color
+      NAF.schemas.add({
+        template: '#avatar-template',
+        components: [
+          'position',
+          'rotation'
+        ]
+      });
+
+      // Called by Networked-Aframe when connected to server
+      function onConnect () {
+        console.log("onConnect", new Date());
+
+        // Handle screen button click (Off and On)
+        screenBtnEle.addEventListener('click', function() {
+          if (screenEnabled) {
+            NAF.connection.adapter.removeLocalMediaStream("screen");
+            screenEnabled = false;
+            screenBtnEle.textContent = 'Share screen';
+          } else {
+            navigator.mediaDevices.getDisplayMedia().then((stream) => {
+              NAF.connection.adapter.addLocalMediaStream(stream, "screen");
+              screenEnabled = true;
+              screenBtnEle.textContent = 'Stop Screen';
+            });
+          }
+        });
+      }
+    </script>
+  </body>
+</html>
diff --git a/examples/index.html b/examples/index.html
@@ -45,6 +45,10 @@ <h1>Networked-Aframe Examples</h1>
       <a href="basic-video.html">Video Streaming</a>
       <span>Example of using texture video with WebRTC</span>
     </div>
+    <div>
+      <a href="basic-multi-streams.html">Multi Streams</a>
+      <span>Example of using texture video with WebRTC from multiple local streams</span>
+    </div>
     <div>
       <a href="shooter-2.html">Shooter</a>
       <span>Press spacebar to shoot. Example for spawning networked entities at runtime.</span>

diff --git a/package-lock.json b/package-lock.json
diff --git a/src/NetworkConnection.js b/src/NetworkConnection.js
@@ -90,7 +90,7 @@ class NetworkConnection {
     for (var id in oldOccupantList) {
       var clientFound = newOccupantList[id];
       if (!clientFound) {
-        NAF.log.write('Closing stream to ', id);
+        NAF.log.write('Closing stream to', id);
         this.adapter.closeStreamConnection(id);
       }
     }
@@ -101,7 +101,7 @@ class NetworkConnection {
     for (var id in occupantList) {
       var startConnection = this.isNewClient(id) && this.adapter.shouldStartConnectionTo(occupantList[id]);
       if (startConnection) {
-        NAF.log.write('Opening datachannel to ', id);
+        NAF.log.write('Opening datachannel to', id);
         this.adapter.startStreamConnection(id);
       }
     }

diff --git a/src/adapters/EasyRtcAdapter.js b/src/adapters/EasyRtcAdapter.js
@@ -11,11 +11,21 @@ class EasyRtcAdapter extends NoOpAdapter {
     this.room = "default";
 
     this.mediaStreams = {};
+    this.remoteClients = {};
     this.pendingMediaRequests = new Map();
 
     this.serverTimeRequests = 0;
     this.timeOffsets = [];
     this.avgTimeOffset = 0;
+
+    this.easyrtc.setPeerOpenListener((clientId) => {
+      const clientConnection = this.easyrtc.getPeerConnectionByUserId(clientId);
+      this.remoteClients[clientId] = clientConnection;
+    });
+
+    this.easyrtc.setPeerClosedListener((clientId) => {
+      delete this.remoteClients[clientId];
+    });
   }
 
   setServerUrl(url) {
@@ -128,7 +138,7 @@ class EasyRtcAdapter extends NoOpAdapter {
   }
 
   closeStreamConnection(clientId) {
-    // Handled by easyrtc
+    this.easyrtc.hangup(clientId);
   }
 
   sendData(clientId, dataType, data) {
@@ -173,58 +183,113 @@ class EasyRtcAdapter extends NoOpAdapter {
     }
   }
 
-  getMediaStream(clientId, type = "audio") {
-    if (this.mediaStreams[clientId]) {
-      NAF.log.write(`Already had ${type} for ${clientId}`);
-      return Promise.resolve(this.mediaStreams[clientId][type]);
+  getMediaStream(clientId, streamName = "audio") {
+
+    if (this.mediaStreams[clientId] && this.mediaStreams[clientId][streamName]) {
+      NAF.log.write(`Already had ${streamName} for ${clientId}`);
+      return Promise.resolve(this.mediaStreams[clientId][streamName]);
     } else {
-      NAF.log.write(`Waiting on ${type} for ${clientId}`);
+      NAF.log.write(`Waiting on ${streamName} for ${clientId}`);
+
+      // Create initial pendingMediaRequests with audio|video alias
       if (!this.pendingMediaRequests.has(clientId)) {
-        this.pendingMediaRequests.set(clientId, {});
+        const pendingMediaRequests = {};
 
         const audioPromise = new Promise((resolve, reject) => {
-          this.pendingMediaRequests.get(clientId).audio = { resolve, reject };
-        });
+          pendingMediaRequests.audio = { resolve, reject };
+        }).catch(e => NAF.log.warn(`${clientId} getMediaStream Audio Error`, e));
+        pendingMediaRequests.audio.promise = audioPromise;
+
         const videoPromise = new Promise((resolve, reject) => {
-          this.pendingMediaRequests.get(clientId).video = { resolve, reject };
-        });
+          pendingMediaRequests.video = { resolve, reject };
+        }).catch(e => NAF.log.warn(`${clientId} getMediaStream Video Error`, e));
+        pendingMediaRequests.video.promise = videoPromise;
 
-        this.pendingMediaRequests.get(clientId).audio.promise = audioPromise;
-        this.pendingMediaRequests.get(clientId).video.promise = videoPromise;
+        this.pendingMediaRequests.set(clientId, pendingMediaRequests);
+      }
 
-        audioPromise.catch(e => NAF.log.warn(`${clientId} getMediaStream Audio Error`, e));
-        videoPromise.catch(e => NAF.log.warn(`${clientId} getMediaStream Video Error`, e));
+      const pendingMediaRequests = this.pendingMediaRequests.get(clientId);
+
+      // Create initial pendingMediaRequests with streamName
+      if (!pendingMediaRequests[streamName]) {
+        const streamPromise = new Promise((resolve, reject) => {
+          pendingMediaRequests[streamName] = { resolve, reject };
+        }).catch(e => NAF.log.warn(`${clientId} getMediaStream "${streamName}"" Error`, e))
+        pendingMediaRequests[streamName].promise = streamPromise;
       }
-      return this.pendingMediaRequests.get(clientId)[type].promise;
+
+      return this.pendingMediaRequests.get(clientId)[streamName].promise;
     }
   }
 
-  setMediaStream(clientId, stream) {
-    // Safari doesn't like it when you use single a mixed media stream where one of the tracks is inactive, so we
-    // split the tracks into two streams.
-    const audioStream = new MediaStream();
-    try {
-      stream.getAudioTracks().forEach(track => audioStream.addTrack(track));
-    } catch(e) {
-      NAF.log.warn(`${clientId} setMediaStream Audio Error`, e);
-    }
+  setMediaStream(clientId, stream, streamName) {
 
-    const videoStream = new MediaStream();
-    try {
-      stream.getVideoTracks().forEach(track => videoStream.addTrack(track));
-    } catch (e) {
-      NAF.log.warn(`${clientId} setMediaStream Video Error`, e);
-    }
+    const clientMediaStreams = this.mediaStreams[clientId] = this.mediaStreams[clientId] || new Map();
+    clientMediaStreams.set(streamName, stream);
 
-    this.mediaStreams[clientId] = { audio: audioStream, video: videoStream };
-
-    // Resolve the promise for the user's media stream if it exists.
     if (this.pendingMediaRequests.has(clientId)) {
-      this.pendingMediaRequests.get(clientId).audio.resolve(audioStream);
-      this.pendingMediaRequests.get(clientId).video.resolve(videoStream);
+      const pendingMediaRequests = this.pendingMediaRequests.get(clientId);
+
+      // Add mediaStreams audio streamName alias if does not exist yet
+      const audioTracks = stream.getAudioTracks();
+      if (!clientMediaStreams.has('audio') && audioTracks.length > 0) {
+        // Safari doesn't like it when you use single a mixed media stream where one of the tracks is inactive, so we
+        // split the tracks into two streams.
+        const audioStream = new MediaStream();
+        try {
+          audioTracks.forEach(track => audioStream.addTrack(track));
+          clientMediaStreams.set('audio', audioStream)
+        } catch(e) {
+          NAF.log.warn(`${clientId} setMediaStream "audio" alias Error`, e);
+        }
+
+        // Resolve the promise for the user's media stream audio alias if it exists.
+        pendingMediaRequests.audio.resolve(audioStream);
+      }
+
+      // Add mediaStreams video streamName alias if does not exist yet
+      const videoTracks = stream.getVideoTracks();
+      if (!clientMediaStreams.has('video') && videoTracks.length > 0) {
+        // Safari doesn't like it when you use single a mixed media stream where one of the tracks is inactive, so we
+        // split the tracks into two streams.
+        const videoStream = new MediaStream();
+        try {
+          videoTracks.forEach(track => videoStream.addTrack(track));
+          clientMediaStreams.set('video', videoStream)
+        } catch(e) {
+          NAF.log.warn(`${clientId} setMediaStream "video" alias Error`, e);
+        }
+
+        // Resolve the promise for the user's media stream video alias if it exists.
+        pendingMediaRequests.video.resolve(videoStream);
+      }
+
+      // Resolve the promise for the user's media stream by StreamName if it exists.
+      if (pendingMediaRequests[streamName]) {
+        pendingMediaRequests[streamName].resolve(stream);
+      }
     }
   }
 
+  addLocalMediaStream(stream, streamName) {
+    const easyrtc = this.easyrtc;
+    streamName = streamName || stream.id;
+    this.setMediaStream("local", stream, streamName);
+    easyrtc.register3rdPartyLocalMediaStream(stream, streamName);
+
+    // Add local stream to existing connections
+    Object.keys(this.remoteClients).forEach((clientId) => {
+      if (easyrtc.getConnectStatus(clientId) !== easyrtc.NOT_CONNECTED) {
+        easyrtc.addStreamToCall(clientId, streamName);
+      }
+    });
+  }
+
+  removeLocalMediaStream(streamName) {
+    this.easyrtc.closeLocalMediaStream(streamName);
+    delete this.mediaStreams["local"][streamName];
+  }
+
   enableMicrophone(enabled) {
     this.easyrtc.enableMicrophone(enabled);
   }
@@ -246,14 +311,17 @@ class EasyRtcAdapter extends NoOpAdapter {
 
     this.easyrtc.setStreamAcceptor(this.setMediaStream.bind(this));
 
-    this.easyrtc.setOnStreamClosed(function(easyrtcid) {
-      delete that.mediaStreams[easyrtcid];
+    this.easyrtc.setOnStreamClosed(function(clientId, stream, streamName) {
+      delete this.mediaStreams[clientId][streamName];
     });
 
     if (that.easyrtc.audioEnabled || that.easyrtc.videoEnabled) {
-      this.easyrtc.initMediaSource(
+      navigator.mediaDevices.getUserMedia({
+        video: that.easyrtc.videoEnabled,
+        audio: that.easyrtc.audioEnabled
+      }).then(
         function(stream) {
-          that.setMediaStream(that.easyrtc.myEasyrtcid, stream);
+          that.addLocalMediaStream(stream, "default");
           that.easyrtc.connect(that.app, connectSuccess, connectFailure);
         },
         function(errorCode, errmesg) {

diff --git a/src/components/networked-audio-source.js b/src/components/networked-audio-source.js
@@ -3,6 +3,7 @@ var naf = require('../NafIndex');
 
 AFRAME.registerComponent('networked-audio-source', {
   schema: {
+    streamName: { default: 'audio' },
     positional: { default: true },
     distanceModel: {
       default: "inverse",
@@ -23,7 +24,7 @@ AFRAME.registerComponent('networked-audio-source', {
       const ownerId = networkedEl.components.networked.data.owner;
 
       if (ownerId) {
-        NAF.connection.adapter.getMediaStream(ownerId)
+        NAF.connection.adapter.getMediaStream(ownerId, this.data.streamName)
           .then(this._setMediaStream)
           .catch((e) => naf.log.error(`Error getting media stream for ${ownerId}`, e));
       } else {

diff --git a/src/components/networked-video-source.js b/src/components/networked-video-source.js
@@ -4,6 +4,7 @@ var naf = require('../NafIndex');
 AFRAME.registerComponent('networked-video-source', {
 
   schema: {
+    streamName: { default: 'video' },
   },
 
   dependencies: ['material'],
@@ -19,7 +20,7 @@ AFRAME.registerComponent('networked-video-source', {
       const ownerId = networkedEl.components.networked.data.owner;
 
       if (ownerId) {
-        NAF.connection.adapter.getMediaStream(ownerId, "video")
+        NAF.connection.adapter.getMediaStream(ownerId, this.data.streamName)
           .then(this._setMediaStream)
           .catch((e) => naf.log.error(`Error getting media stream for ${ownerId}`, e));
       } else {