diff --git a/src/plugin.js b/src/plugin.js index bdd26dff..f492846d 100644 --- a/src/plugin.js +++ b/src/plugin.js @@ -1,31 +1,18 @@ import {version as VERSION} from '../package.json'; import window from 'global/window'; +import document from 'global/document'; import WebVRPolyfill from 'webvr-polyfill'; import videojs from 'video.js'; import * as THREE from 'three'; import VRControls from 'three/examples/js/controls/VRControls.js'; import VREffect from 'three/examples/js/effects/VREffect.js'; import OrbitControls from 'three/examples/js/controls/OrbitControls.js'; -import rgbFragmentShader from './rgb-fragment-shader'; -import rgbaFragmentShader from './rgba-fragment-shader'; -import vertexShader from './vertex-shader'; +import * as utils from './utils'; // import controls so they get regisetered with videojs import './cardboard-button'; import './big-vr-play-button'; -const validProjections = [ - '360', - '360_LR', - '360_TB', - '360_CUBE', - 'NONE', - 'AUTO', - 'Sphere', - 'Cube', - 'equirectangular' -]; - // Default options for the plugin. const defaults = { projection: 'AUTO', @@ -43,54 +30,14 @@ const errors = { headline: '360 not supported on this device', type: '360_NOT_SUPPORTED', message: "Your browser does not support 360. See webvr.info for assistance." + }, + 'web-vr-hls-cors-not-supported': { + headline: '360 HLS video not supported on this device', + type: '360_NOT_SUPPORTED', + message: "Your browser/device does not support HLS 360 video. See webvr.info for assistance." } }; -const getInternalProjectionName = function(projection) { - if (!projection) { - return; - } - - projection = projection.toString().trim(); - - if ((/sphere/i).test(projection)) { - return '360'; - } - - if ((/cube/i).test(projection)) { - return '360_CUBE'; - } - - if ((/equirectangular/i).test(projection)) { - return '360'; - } - - for (let i = 0; i < validProjections.length; i++) { - if (new RegExp('^' + validProjections[i] + '$', 'i').test(projection)) { - return validProjections[i]; - } - } - -}; - -const isHLS = function(currentType) { - // hls video types - const hlsTypes = [ - // Apple santioned - 'application/vnd.apple.mpegurl', - // Very common - 'application/x-mpegurl', - // Included for completeness - 'video/x-mpegurl', - 'video/mpegurl', - 'application/mpegurl' - ]; - - // if the current type has a case insensitivie match from the list above - // this is hls - return hlsTypes.some((type) => (new RegExp(`^${type}$`, 'i')).test(currentType)); -}; - const Plugin = videojs.getPlugin('plugin'); const Component = videojs.getComponent('Component'); @@ -115,8 +62,9 @@ class VR extends Plugin { player.errors({errors}); } - // we need this as IE 11 reports that it has a VR display, but isnt compatible with Video as a Texture. for example - if (videojs.browser.IE_VERSION) { + // IE 11 does not support enough webgl to be supported + // older safari does not support cors, so it wont work + if (videojs.browser.IE_VERSION || !utils.corsSupport) { this.triggerError_({code: 'web-vr-not-supported', dismiss: false}); return; } @@ -132,7 +80,7 @@ class VR extends Plugin { } changeProjection_(projection) { - projection = getInternalProjectionName(projection); + projection = utils.getInternalProjectionName(projection); // don't change to an invalid projection if (!projection) { projection = 'NONE'; @@ -147,7 +95,7 @@ class VR extends Plugin { // mediainfo cannot be set to auto or we would infinite loop here // each source should know wether they are 360 or not, if using AUTO if (this.player_.mediainfo && this.player_.mediainfo.projection && this.player_.mediainfo.projection !== 'AUTO') { - const autoProjection = getInternalProjectionName(this.player_.mediainfo.projection); + const autoProjection = utils.getInternalProjectionName(this.player_.mediainfo.projection); return this.changeProjection_(autoProjection); } @@ -285,7 +233,15 @@ class VR extends Plugin { return this.vrDisplay.requestAnimationFrame(fn); } - return super.requestAnimationFrame(fn); + return Component.prototype.requestAnimationFrame.call(this, fn); + } + + cancelAnimationFrame(id) { + if (this.vrDisplay) { + return this.vrDisplay.cancelAnimationFrame(id); + } + + return Component.prototype.cancelAnimationFrame.call(this, id); } togglePlay_() { @@ -297,17 +253,26 @@ class VR extends Plugin { } animate_() { + if (!this.initialized_) { + return; + } if (this.getVideoEl_().readyState === this.getVideoEl_().HAVE_ENOUGH_DATA) { if (this.videoTexture) { this.videoTexture.needsUpdate = true; } } + // This draws the current video data as an image to a canvas every render. That canvas is used + // as a texture by webgl. Normally the video is used directly and we don't have to do this, but + // HLS video textures on iOS >= 11 is currently broken, so we have to support those browser + // in a roundabout way. + if (this.videoImageContext_) { + this.videoImageContext_.drawImage(this.getVideoEl_(), 0, 0, this.videoImage_.width, this.videoImage_.height); + } + this.controls3d.update(); this.effect.render(this.scene, this.camera); - this.animationFrameId_ = this.requestAnimationFrame(this.animate_); - if (window.navigator.getGamepads) { // Grab all gamepads const gamepads = window.navigator.getGamepads(); @@ -330,12 +295,20 @@ class VR extends Plugin { } } this.camera.getWorldDirection(this.cameraVector); + + this.animationFrameId_ = this.requestAnimationFrame(this.animate_); } handleResize_() { const width = this.player_.currentWidth(); const height = this.player_.currentHeight(); + // when dealing with a non video + if (this.videoImage_) { + this.videoImage_.width = width; + this.videoImage_.height = height; + } + this.effect.setSize(width, height, false); this.camera.aspect = width / height; this.camera.updateProjectionMatrix(); @@ -343,8 +316,8 @@ class VR extends Plugin { setProjection(projection) { - if (!getInternalProjectionName(projection)) { - videojs.log.error('videojs-vr: please pass a valid projection: ' + validProjections.join(', ')); + if (!utils.getInternalProjectionName(projection)) { + videojs.log.error('videojs-vr: please pass a valid projection ' + utils.validProjections.join(', ')); return; } @@ -366,46 +339,41 @@ class VR extends Plugin { this.scene = new THREE.Scene(); - this.videoTexture = new THREE.VideoTexture(this.getVideoEl_()); - - this.videoTexture.generateMipmaps = false; - this.videoTexture.minFilter = THREE.LinearFilter; - this.videoTexture.magFilter = THREE.LinearFilter; - - // iOS and macOS HLS fix/hacks + // We opted to stop using a video texture on safari due to + // various bugs that exist when using it. This gives us worse performance + // but it will actually work on all recent version of safari. See + // the following issues for more info on this: + // // https://bugs.webkit.org/show_bug.cgi?id=163866#c3 - // https://github.com/mrdoob/three.js/issues/9754 - // On iOS with HLS, color space is wrong and texture is flipped on Y axis - // On macOS, just need to flip texture Y axis - - if (isHLS() && videojs.browser.IS_ANY_SAFARI) { - this.log('Safari + iOS + HLS = flipY and colorspace hack'); - this.videoTexture.format = THREE.RGBAFormat; - this.videoTexture.flipY = false; - } else if (isHLS() && videojs.browser.IS_SAFARI) { - this.log('Safari + HLS = flipY hack'); - this.videoTexture.format = THREE.RGBFormat; - this.videoTexture.flipY = false; - } else { - this.videoTexture.format = THREE.RGBFormat; - } + // https://bugs.webkit.org/show_bug.cgi?id=179417 + if (videojs.browser.IS_ANY_SAFARI && utils.isHLS(this.player_.currentSource().type)) { + this.log('Video texture is not supported using image canvas hack'); + this.videoImage_ = document.createElement('canvas'); + this.videoImage_.width = this.player_.currentWidth(); + this.videoImage_.height = this.player_.currentHeight(); - if ((this.videoTexture.format === THREE.RGBAFormat || this.videoTexture.format === THREE.RGBFormat) && this.videoTexture.flipY === false) { - let fragmentShader = rgbFragmentShader; + this.videoImageContext_ = this.videoImage_.getContext('2d'); + this.videoImageContext_.fillStyle = '#000000'; - if (this.videoTexture.format === THREE.RGBAFormat) { - fragmentShader = rgbaFragmentShader; - } + this.videoTexture = new THREE.Texture(this.videoImage_); - this.movieMaterial = new THREE.ShaderMaterial({ - uniforms: {texture: {value: this.videoTexture}}, - vertexShader, - fragmentShader - }); + this.videoTexture.wrapS = THREE.ClampToEdgeWrapping; + this.videoTexture.wrapT = THREE.ClampToEdgeWrapping; + this.videoTexture.flipY = true; } else { - this.movieMaterial = new THREE.MeshBasicMaterial({ map: this.videoTexture, overdraw: true, side: THREE.DoubleSide }); + this.log('Video texture is supported using that'); + this.videoTexture = new THREE.VideoTexture(this.getVideoEl_()); } + // shared regardless of wether VideoTexture is used or + // an image canvas is used + this.videoTexture.generateMipmaps = false; + this.videoTexture.minFilter = THREE.LinearFilter; + this.videoTexture.magFilter = THREE.LinearFilter; + this.videoTexture.format = THREE.RGBFormat; + + this.movieMaterial = new THREE.MeshBasicMaterial({ map: this.videoTexture, overdraw: true, side: THREE.DoubleSide }); + this.changeProjection_(this.currentProjection_); if (this.currentProjection_ === 'NONE') { @@ -438,6 +406,21 @@ class VR extends Plugin { antialias: true }); + const webglContext = this.renderer.getContext('webgl'); + const oldTexImage2D = webglContext.texImage2D; + + /* this is a workaround since threejs uses try catch */ + webglContext.texImage2D = (...args) => { + try { + return oldTexImage2D.apply(webglContext, args); + } catch (e) { + this.reset(); + this.player_.pause(); + this.triggerError_({code: 'web-vr-hls-cors-not-supported', dismiss: false}); + throw new Error(e); + } + }; + this.renderer.setSize(this.player_.currentWidth(), this.player_.currentHeight(), false); this.effect = new VREffect(this.renderer); @@ -501,7 +484,7 @@ class VR extends Plugin { this.controls3d = new OrbitControls(this.camera, this.renderedCanvas); this.controls3d.target.set(0, 0, -1); } - this.requestAnimationFrame(this.animate_); + this.animationFrameId_ = this.requestAnimationFrame(this.animate_); }); } else if (window.navigator.getVRDevices) { this.triggerError_({code: 'web-vr-out-of-date', dismiss: false}); @@ -583,6 +566,14 @@ class VR extends Plugin { this.cancelAnimationFrame(this.animationFrameId_); } + if (this.videoImage_) { + this.videoImage_ = null; + } + + if (this.videoImageContext_) { + this.videoImageContext_ = null; + } + this.initialized_ = false; } @@ -592,8 +583,6 @@ class VR extends Plugin { } } -VR.prototype.requestAnimationFrame = Component.prototype.requestAnimationFrame; -VR.prototype.cancelAnimationFrame = Component.prototype.cancelAnimationFrame; VR.prototype.setTimeout = Component.prototype.setTimeout; VR.prototype.clearTimeout = Component.prototype.clearTimeout; diff --git a/src/rgb-fragment-shader.js b/src/rgb-fragment-shader.js deleted file mode 100644 index 708384a1..00000000 --- a/src/rgb-fragment-shader.js +++ /dev/null @@ -1,5 +0,0 @@ -module.exports = `uniform sampler2D texture; -varying vec2 vUV; -void main() { - gl_FragColor = texture2D(texture, vUV); -}`; diff --git a/src/rgba-fragment-shader.js b/src/rgba-fragment-shader.js deleted file mode 100644 index 05de9b0d..00000000 --- a/src/rgba-fragment-shader.js +++ /dev/null @@ -1,5 +0,0 @@ -module.exports = `uniform sampler2D texture; -varying vec2 vUV; -void main() { - gl_FragColor = texture2D(texture, vUV).bgra; -}`; diff --git a/src/utils.js b/src/utils.js new file mode 100644 index 00000000..1e833909 --- /dev/null +++ b/src/utils.js @@ -0,0 +1,67 @@ +import document from 'global/document'; + +// check if the browser supports cors +export const corsSupport = (function() { + const video = document.createElement('video'); + + video.crossOrigin = 'anonymous'; + + return video.hasAttribute('crossorigin'); +})(); + +export const isHLS = function(currentType) { + // hls video types + const hlsTypes = [ + // Apple santioned + 'application/vnd.apple.mpegurl', + // Very common + 'application/x-mpegurl', + // Included for completeness + 'video/x-mpegurl', + 'video/mpegurl', + 'application/mpegurl' + ]; + + // if the current type has a case insensitivie match from the list above + // this is hls + return hlsTypes.some((type) => (RegExp(`^${type}$`, 'i')).test(currentType)); +}; + +export const validProjections = [ + '360', + '360_LR', + '360_TB', + '360_CUBE', + 'NONE', + 'AUTO', + 'Sphere', + 'Cube', + 'equirectangular' +]; + +export const getInternalProjectionName = function(projection) { + if (!projection) { + return; + } + + projection = projection.toString().trim(); + + if ((/sphere/i).test(projection)) { + return '360'; + } + + if ((/cube/i).test(projection)) { + return '360_CUBE'; + } + + if ((/equirectangular/i).test(projection)) { + return '360'; + } + + for (let i = 0; i < validProjections.length; i++) { + if (new RegExp('^' + validProjections[i] + '$', 'i').test(projection)) { + return validProjections[i]; + } + } + +}; diff --git a/src/vertex-shader.js b/src/vertex-shader.js deleted file mode 100644 index 9aa41971..00000000 --- a/src/vertex-shader.js +++ /dev/null @@ -1,5 +0,0 @@ -module.exports = `varying vec2 vUV; -void main() { - vUV = vec2( uv.x, 1.0 - uv.y ); - gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 ); -}`;