[feature request] Camera feed on an HTMLVideoElement

[marcusx2]

Hello, not sure if I can request this here, but the threejs examples have everything on a canvas element. I was wondering if you could leave the webcam feed on an HTMLVideoElement instead, and let the canvas be responsible only for drawing the face effects. Essentially the canvas element would overlay the video element. Is there any particular reason everything goes to the canvas element? If the webcam feed was on a HTMLVideoElement, it'd allow the users to have control over the webcam. ARJS for example has a parameter called videoTexture, where if true it injects the webcam feed in the canvas, but if false the webcam feed stays in a video element. See this for example. If you see in the inspector, the webcam feed is just a video element. Having the option would be nice.

If this can't be done, is there an easy way to modify the code to do this? Thank you for your time.

EDIT

Upon further inspection, I only found getUserMedia in the jeelizFaceFilter.js file. Nothing in the threejs, which seems to suggest that it's jeeliz itself that is responsible for getting access to the webcam and using a video element or not.

[marcusx2]

I solved the issue after further investigation. It wasn't that hard. Just had to make some changes to the JeelizThreeHelper.

/*
  Helper for Three.js
*/

const JeelizThreeHelper = (function(){
  // internal settings:
  const _settings = {
    rotationOffsetX: 0.0, // negative -> look upper. in radians
    pivotOffsetYZ: [0.2, 0.6],// YZ of the distance between the center of the cube and the pivot
    
    detectionThreshold: 0.8, // sensibility, between 0 and 1. Less -> more sensitive
    detectionHysteresis: 0.02,

    //tweakMoveYRotateX: 0,//0.5, // tweak value: move detection window along Y axis when rotate the face around X (look up <-> down)
    
    cameraMinVideoDimFov: 35 // Field of View for the smallest dimension of the video in degrees
  };

  // private vars:
  let _threeRenderer = null,
      _threeScene = null,
      _threeTranslation = null;

  let _maxFaces = -1,
      _isMultiFaces = false,
      _detectCallback = null,
      _isSeparateThreeCanvas = false,
      _faceFilterCv = null,
      _videoElement = null,
      _isDetected = false,
      _scaleW = 1,
      _canvasAspectRatio = -1;

  const _threeCompositeObjects = [];
    
  let _gl = null;


  // private funcs:
  function destroy(){
    _threeCompositeObjects.splice(0);
  }


  function create_threeCompositeObjects(){
    for (let i=0; i<_maxFaces; ++i){
      // COMPOSITE OBJECT WHICH WILL TRACK A DETECTED FACE
      const threeCompositeObject = new THREE.Object3D();
      threeCompositeObject.frustumCulled = false;
      threeCompositeObject.visible = false;

      _threeCompositeObjects.push(threeCompositeObject);
      _threeScene.add(threeCompositeObject);
    }
  }


  function detect(detectState){
    _threeCompositeObjects.forEach(function(threeCompositeObject, i){
      _isDetected = threeCompositeObject.visible;
      const ds = detectState[i];
      if (_isDetected && ds.detected < _settings.detectionThreshold-_settings.detectionHysteresis){
        
        // DETECTION LOST
        if (_detectCallback) _detectCallback(i, false);
        threeCompositeObject.visible = false;
      } else if (!_isDetected && ds.detected > _settings.detectionThreshold+_settings.detectionHysteresis){
        
        // FACE DETECTED
        if (_detectCallback) _detectCallback(i, true);
        threeCompositeObject.visible = true;
      }
    }); //end loop on all detection slots
  }


  function update_poses(ds, threeCamera){
    // tan( <horizontal FoV> / 2 ):
    const halfTanFOVX = Math.tan(threeCamera.aspect * threeCamera.fov * Math.PI/360); //tan(<horizontal FoV>/2), in radians (threeCamera.fov is vertical FoV)

    _threeCompositeObjects.forEach(function(threeCompositeObject, i){
      if (!threeCompositeObject.visible) return;
      const detectState = ds[i];

      // tweak Y position depending on rx:
      //const tweak = _settings.tweakMoveYRotateX * Math.tan(detectState.rx);
      const cz = Math.cos(detectState.rz), sz = Math.sin(detectState.rz);
      
      // relative width of the detection window (1-> whole width of the detection window):
      const W = detectState.s * _scaleW;

      // distance between the front face of the cube and the camera:
      const DFront = 1 / ( 2 * W * halfTanFOVX );
      
      // D is the distance between the center of the unit cube and the camera:
      const D = DFront + 0.5;

      // coords in 2D of the center of the detection window in the viewport:
      const xv = detectState.x * _scaleW;
      const yv = detectState.y * _scaleW;

      // coords in 3D of the center of the cube (in the view coordinates system):
      const z = -D;   // minus because view coordinate system Z goes backward
      const x = xv * D * halfTanFOVX;
      const y = yv * D * halfTanFOVX / _canvasAspectRatio;

      // set position before pivot:
      threeCompositeObject.position.set(-sz*_settings.pivotOffsetYZ[0], -cz*_settings.pivotOffsetYZ[0], -_settings.pivotOffsetYZ[1]);

      // set rotation and apply it to position:
      threeCompositeObject.rotation.set(detectState.rx+_settings.rotationOffsetX, detectState.ry, detectState.rz, "ZYX");
      threeCompositeObject.position.applyEuler(threeCompositeObject.rotation);

      // add translation part:
      _threeTranslation.set(x, y+_settings.pivotOffsetYZ[0], z+_settings.pivotOffsetYZ[1]);
      threeCompositeObject.position.add(_threeTranslation);
    }); //end loop on composite objects
  }


  //public methods:
  const that = {
    // launched with the same spec object than callbackReady. set spec.threeCanvasId to the ID of the threeCanvas to be in 2 canvas mode:
    init: function(spec, detectCallback){
      destroy();

      _maxFaces = spec.maxFacesDetected;
      _gl = spec.GL;
      _faceFilterCv = spec.canvasElement;
      _isMultiFaces = (_maxFaces>1);
      _videoElement = spec.videoElement;
	  _videoElement.id = 'jeeFaceFilterVideo';
	  _videoElement.setAttribute('style', 'width: 100vw;height: 100vh;object-fit: cover;position: absolute;top: 0;left: 0; transform: scale(-1, 1);');
	  document.body.appendChild(_videoElement);

      // enable 2 canvas mode if necessary:
      let threeCanvas = null;
      if (spec.threeCanvasId){
        _isSeparateThreeCanvas = true;
        // adjust the threejs canvas size to the threejs canvas:
        threeCanvas = document.getElementById(spec.threeCanvasId);
        threeCanvas.setAttribute('width', _faceFilterCv.width);
        threeCanvas.setAttribute('height', _faceFilterCv.height);
      } else {
        threeCanvas = _faceFilterCv;
      }

      if (typeof(detectCallback) !== 'undefined'){
        _detectCallback = detectCallback;
      }

       // init THREE.JS context:
      _threeRenderer = new THREE.WebGLRenderer({
        context: (_isSeparateThreeCanvas) ? null : _gl,
        canvas: threeCanvas,
        alpha: (_isSeparateThreeCanvas || spec.alpha) ? true : false,
        preserveDrawingBuffer: true // to make image capture possible
      });

      _threeScene = new THREE.Scene();
      _threeTranslation = new THREE.Vector3();

      create_threeCompositeObjects();

      // handle device orientation change:
      window.addEventListener('orientationchange', function(){
        setTimeout(JEELIZFACEFILTER.resize, 1000);
      }, false);
      
      const returnedDict = {
        renderer: _threeRenderer,
        scene: _threeScene
      };
      if (_isMultiFaces){
        returnedDict.faceObjects = _threeCompositeObjects
      } else {
        returnedDict.faceObject = _threeCompositeObjects[0];
      }
      return returnedDict;
    }, //end that.init()


    detect: function(detectState){
      const ds = (_isMultiFaces) ? detectState : [detectState];

      // update detection states:
      detect(ds);
    },


    get_isDetected: function() {
      return _isDetected;
    },


    render: function(detectState, threeCamera){
      const ds = (_isMultiFaces) ? detectState : [detectState];

      // update detection states then poses:
      detect(ds);
      update_poses(ds, threeCamera);

      if (_isSeparateThreeCanvas) {
      } else {
        // reinitialize the state of THREE.JS because JEEFACEFILTER have changed stuffs:
        // -> can be VERY costly !
        _threeRenderer.state.reset();        
      }

      // trigger the render of the THREE.JS SCENE:
      _threeRenderer.render(_threeScene, threeCamera);
    },


    sortFaces: function(bufferGeometry, axis, isInv){ // sort faces long an axis
      // Useful when a bufferGeometry has alpha: we should render the last faces first
      const axisOffset = {X:0, Y:1, Z:2}[axis.toUpperCase()];
      const sortWay = (isInv) ? -1 : 1;

      // fill the faces array:
      const nFaces = bufferGeometry.index.count/3;
      const faces = new Array(nFaces);
      for (let i=0; i<nFaces; ++i){
        faces[i] = [bufferGeometry.index.array[3*i], bufferGeometry.index.array[3*i+1], bufferGeometry.index.array[3*i+2]];
      }

      // compute centroids:
      const aPos = bufferGeometry.attributes.position.array;
      const centroids = faces.map(function(face, faceIndex){
        return [
          (aPos[3*face[0]]+aPos[3*face[1]]+aPos[3*face[2]])/3,       // X
          (aPos[3*face[0]+1]+aPos[3*face[1]+1]+aPos[3*face[2]+1])/3, // Y
          (aPos[3*face[0]+2]+aPos[3*face[1]+2]+aPos[3*face[2]+2])/3, // Z
          face
        ];
      });

      // sort centroids:
      centroids.sort(function(ca, cb){
        return (ca[axisOffset]-cb[axisOffset]) * sortWay;
      });

      // reorder bufferGeometry faces:
      centroids.forEach(function(centroid, centroidIndex){
        const face = centroid[3];
        bufferGeometry.index.array[3*centroidIndex] = face[0];
        bufferGeometry.index.array[3*centroidIndex+1] = face[1];
        bufferGeometry.index.array[3*centroidIndex+2] = face[2];
      });
    }, //end sortFaces


    // create an occluder, IE a transparent object which writes on the depth buffer:
    create_threejsOccluder: function(occluderURL, callback){
      const occluderMesh = new THREE.Mesh();
      new THREE.BufferGeometryLoader().load(occluderURL, function(occluderGeometry){
        const mat = new THREE.ShaderMaterial({
          vertexShader: THREE.ShaderLib.basic.vertexShader,
          fragmentShader: "precision lowp float;\n void main(void){\n gl_FragColor=vec4(1.,0.,0.,1.);\n }",
          uniforms: THREE.ShaderLib.basic.uniforms,
          colorWrite: false
        });
        
        occluderMesh.renderOrder = -1; //render first
        occluderMesh.material = mat;
        occluderMesh.geometry = occluderGeometry;
        if (typeof(callback)!=='undefined' && callback) callback(occluderMesh);
      });
      return occluderMesh;
    },
    

    set_pivotOffsetYZ: function(pivotOffset) {
      _settings.pivotOffsetYZ = pivotOffset;
    },


    create_camera: function(zNear, zFar){
      const threeCamera = new THREE.PerspectiveCamera(1, 1, (zNear) ? zNear : 0.1, (zFar) ? zFar : 100);
      that.update_camera(threeCamera);

      return threeCamera;
    },


    update_camera: function(threeCamera){
      // compute aspectRatio:
      const canvasElement = _threeRenderer.domElement;
      const cvw = canvasElement.width;
      const cvh = canvasElement.height;
      _canvasAspectRatio = cvw / cvh;

      // compute vertical field of view:
      const vw = _videoElement.videoWidth;
      const vh = _videoElement.videoHeight;
      const videoAspectRatio = vw / vh;
      const fovFactor = (vh > vw) ? (1.0 / videoAspectRatio) : 1.0;
      const fov = _settings.cameraMinVideoDimFov * fovFactor;
      console.log('INFO in JeelizThreeHelper - update_camera(): Estimated vertical video FoV is', fov);
      
      // compute X and Y offsets in pixels:
      let scale = 1.0;
      if (_canvasAspectRatio > videoAspectRatio) {
        // the canvas is more in landscape format than the video, so we crop top and bottom margins:
        scale = cvw / vw;
      } else {
        // the canvas is more in portrait format than the video, so we crop right and left margins:
        scale = cvh / vh;
      }
      const cvws = vw * scale, cvhs = vh * scale;
      const offsetX = (cvws - cvw) / 2.0;
      const offsetY = (cvhs - cvh) / 2.0;
      _scaleW = cvw / cvws;

      // apply parameters:
      threeCamera.aspect = _canvasAspectRatio;
      threeCamera.fov = fov;
      console.log('INFO in JeelizThreeHelper.update_camera(): camera vertical estimated FoV is', fov, 'deg');
      threeCamera.setViewOffset(cvws, cvhs, offsetX, offsetY, cvw, cvh);
      threeCamera.updateProjectionMatrix();

      // update drawing area:
      _threeRenderer.setSize(cvw, cvh, false);
      _threeRenderer.setViewport(0, 0, cvw, cvh);
    }, //end update_camera()


    resize: function(w, h, threeCamera){
      _threeRenderer.domElement.width = w;
      _threeRenderer.domElement.height = h;
      JEELIZFACEFILTER.resize();
      if (threeCamera){
        that.update_camera(threeCamera);
      }
    }
  }
  return that;
})();


// Export ES6 module:
try {
  module.exports = JeelizThreeHelper;
} catch(e){
  console.log('JeelizThreeHelper ES6 Module not exported');  
  window.JeelizThreeHelper = JeelizThreeHelper;
}

[marcusx2]

I celebrated too soon. It works on desktop chrome but it breaks on Android Chrome...trying the modified file with this experience for fullscreen.

[marcusx2]

Closing because there's an issue with the fullscreen threejs example, which I reported on the threejs github. The issue it's not working on Android Chrome seems to be because the face stretches, and the helmet fits on the stretched face, not a non-stretched face...

EDIT

Created issue here.