From b73deddd0651743e4326913c1ef76830660b0d2f Mon Sep 17 00:00:00 2001 From: brandonocasey Date: Mon, 22 Jan 2018 16:43:00 -0500 Subject: [PATCH] fix: safari hls --- src/plugin.js | 158 ++++++++++++++---------------------- src/rgb-fragment-shader.js | 5 -- src/rgba-fragment-shader.js | 5 -- src/utils.js | 99 ++++++++++++++++++++++ src/vertex-shader.js | 5 -- 5 files changed, 158 insertions(+), 114 deletions(-) delete mode 100644 src/rgb-fragment-shader.js delete mode 100644 src/rgba-fragment-shader.js create mode 100644 src/utils.js delete mode 100644 src/vertex-shader.js diff --git a/src/plugin.js b/src/plugin.js index bdd26dff..48b19f99 100644 --- a/src/plugin.js +++ b/src/plugin.js @@ -1,31 +1,18 @@ import {version as VERSION} from '../package.json'; import window from 'global/window'; +import document from 'global/document'; import WebVRPolyfill from 'webvr-polyfill'; import videojs from 'video.js'; import * as THREE from 'three'; import VRControls from 'three/examples/js/controls/VRControls.js'; import VREffect from 'three/examples/js/effects/VREffect.js'; import OrbitControls from 'three/examples/js/controls/OrbitControls.js'; -import rgbFragmentShader from './rgb-fragment-shader'; -import rgbaFragmentShader from './rgba-fragment-shader'; -import vertexShader from './vertex-shader'; +import * as utils from './utils'; // import controls so they get regisetered with videojs import './cardboard-button'; import './big-vr-play-button'; -const validProjections = [ - '360', - '360_LR', - '360_TB', - '360_CUBE', - 'NONE', - 'AUTO', - 'Sphere', - 'Cube', - 'equirectangular' -]; - // Default options for the plugin. const defaults = { projection: 'AUTO', @@ -46,51 +33,6 @@ const errors = { } }; -const getInternalProjectionName = function(projection) { - if (!projection) { - return; - } - - projection = projection.toString().trim(); - - if ((/sphere/i).test(projection)) { - return '360'; - } - - if ((/cube/i).test(projection)) { - return '360_CUBE'; - } - - if ((/equirectangular/i).test(projection)) { - return '360'; - } - - for (let i = 0; i < validProjections.length; i++) { - if (new RegExp('^' + validProjections[i] + '$', 'i').test(projection)) { - return validProjections[i]; - } - } - -}; - -const isHLS = function(currentType) { - // hls video types - const hlsTypes = [ - // Apple santioned - 'application/vnd.apple.mpegurl', - // Very common - 'application/x-mpegurl', - // Included for completeness - 'video/x-mpegurl', - 'video/mpegurl', - 'application/mpegurl' - ]; - - // if the current type has a case insensitivie match from the list above - // this is hls - return hlsTypes.some((type) => (new RegExp(`^${type}$`, 'i')).test(currentType)); -}; - const Plugin = videojs.getPlugin('plugin'); const Component = videojs.getComponent('Component'); @@ -115,8 +57,9 @@ class VR extends Plugin { player.errors({errors}); } - // we need this as IE 11 reports that it has a VR display, but isnt compatible with Video as a Texture. for example - if (videojs.browser.IE_VERSION) { + // IE 11 does not support enough webgl to be supported + // older safari does not support cors, so it wont work + if (videojs.browser.IE_VERSION || !utils.corsSupport) { this.triggerError_({code: 'web-vr-not-supported', dismiss: false}); return; } @@ -132,7 +75,7 @@ class VR extends Plugin { } changeProjection_(projection) { - projection = getInternalProjectionName(projection); + projection = utils.getInternalProjectionName(projection); // don't change to an invalid projection if (!projection) { projection = 'NONE'; @@ -147,7 +90,7 @@ class VR extends Plugin { // mediainfo cannot be set to auto or we would infinite loop here // each source should know wether they are 360 or not, if using AUTO if (this.player_.mediainfo && this.player_.mediainfo.projection && this.player_.mediainfo.projection !== 'AUTO') { - const autoProjection = getInternalProjectionName(this.player_.mediainfo.projection); + const autoProjection = utils.getInternalProjectionName(this.player_.mediainfo.projection); return this.changeProjection_(autoProjection); } @@ -303,6 +246,14 @@ class VR extends Plugin { } } + // This draws the current video data as an image to a canvas every render. That canvas is used + // as a texture by webgl. Normally the video is used directly and we don't have to do this, but + // HLS video textures on iOS >= 11 is currently broken, so we have to support those browser + // in a roundabout way. + if (this.videoImageContext_) { + this.videoImageContext_.drawImage(this.getVideoEl_(), 0, 0, this.videoImage_.width, this.videoImage_.height); + } + this.controls3d.update(); this.effect.render(this.scene, this.camera); @@ -336,6 +287,12 @@ class VR extends Plugin { const width = this.player_.currentWidth(); const height = this.player_.currentHeight(); + // when dealing with a non video + if (this.videoImage_) { + this.videoImage_.width = width; + this.videoImage_.height = height; + } + this.effect.setSize(width, height, false); this.camera.aspect = width / height; this.camera.updateProjectionMatrix(); @@ -343,8 +300,8 @@ class VR extends Plugin { setProjection(projection) { - if (!getInternalProjectionName(projection)) { - videojs.log.error('videojs-vr: please pass a valid projection: ' + validProjections.join(', ')); + if (!utils.getInternalProjectionName(projection)) { + videojs.log.error('videojs-vr: please pass a valid projection ' + utils.validProjections.join(', ')); return; } @@ -366,46 +323,41 @@ class VR extends Plugin { this.scene = new THREE.Scene(); - this.videoTexture = new THREE.VideoTexture(this.getVideoEl_()); - - this.videoTexture.generateMipmaps = false; - this.videoTexture.minFilter = THREE.LinearFilter; - this.videoTexture.magFilter = THREE.LinearFilter; - - // iOS and macOS HLS fix/hacks + // We opted to stop using a video texture on safari due to + // various bugs that exist when using it. This gives us worse performance + // but it will actually work on all recent version of safari. See + // the following issues for more info on this: + // // https://bugs.webkit.org/show_bug.cgi?id=163866#c3 - // https://github.com/mrdoob/three.js/issues/9754 - // On iOS with HLS, color space is wrong and texture is flipped on Y axis - // On macOS, just need to flip texture Y axis - - if (isHLS() && videojs.browser.IS_ANY_SAFARI) { - this.log('Safari + iOS + HLS = flipY and colorspace hack'); - this.videoTexture.format = THREE.RGBAFormat; - this.videoTexture.flipY = false; - } else if (isHLS() && videojs.browser.IS_SAFARI) { - this.log('Safari + HLS = flipY hack'); - this.videoTexture.format = THREE.RGBFormat; - this.videoTexture.flipY = false; - } else { - this.videoTexture.format = THREE.RGBFormat; - } + // https://bugs.webkit.org/show_bug.cgi?id=179417 + if (videojs.browser.IS_ANY_SAFARI && utils.isHLS(this.player_.currentSource().type)) { + this.log('Video texture is not supported using image canvas hack'); + this.videoImage_ = document.createElement('canvas'); + this.videoImage_.width = this.player_.currentWidth(); + this.videoImage_.height = this.player_.currentHeight(); - if ((this.videoTexture.format === THREE.RGBAFormat || this.videoTexture.format === THREE.RGBFormat) && this.videoTexture.flipY === false) { - let fragmentShader = rgbFragmentShader; + this.videoImageContext_ = this.videoImage_.getContext('2d'); + this.videoImageContext_.fillStyle = '#000000'; - if (this.videoTexture.format === THREE.RGBAFormat) { - fragmentShader = rgbaFragmentShader; - } + this.videoTexture = new THREE.Texture(this.videoImage_); - this.movieMaterial = new THREE.ShaderMaterial({ - uniforms: {texture: {value: this.videoTexture}}, - vertexShader, - fragmentShader - }); + this.videoTexture.wrapS = THREE.ClampToEdgeWrapping; + this.videoTexture.wrapT = THREE.ClampToEdgeWrapping; + this.videoTexture.flipY = true; } else { - this.movieMaterial = new THREE.MeshBasicMaterial({ map: this.videoTexture, overdraw: true, side: THREE.DoubleSide }); + this.log('Video texture is supported using that'); + this.videoTexture = new THREE.VideoTexture(this.getVideoEl_()); } + // shared regardless of wether VideoTexture is used or + // an image canvas is used + this.videoTexture.generateMipmaps = false; + this.videoTexture.minFilter = THREE.LinearFilter; + this.videoTexture.magFilter = THREE.LinearFilter; + this.videoTexture.format = THREE.RGBFormat; + + this.movieMaterial = new THREE.MeshBasicMaterial({ map: this.videoTexture, overdraw: true, side: THREE.DoubleSide }); + this.changeProjection_(this.currentProjection_); if (this.currentProjection_ === 'NONE') { @@ -583,6 +535,14 @@ class VR extends Plugin { this.cancelAnimationFrame(this.animationFrameId_); } + if (this.videoImage_) { + this.videoImage_ = null; + } + + if (this.videoImageContext_) { + this.videoImageContext_ = null; + } + this.initialized_ = false; } diff --git a/src/rgb-fragment-shader.js b/src/rgb-fragment-shader.js deleted file mode 100644 index 708384a1..00000000 --- a/src/rgb-fragment-shader.js +++ /dev/null @@ -1,5 +0,0 @@ -module.exports = `uniform sampler2D texture; -varying vec2 vUV; -void main() { - gl_FragColor = texture2D(texture, vUV); -}`; diff --git a/src/rgba-fragment-shader.js b/src/rgba-fragment-shader.js deleted file mode 100644 index 05de9b0d..00000000 --- a/src/rgba-fragment-shader.js +++ /dev/null @@ -1,5 +0,0 @@ -module.exports = `uniform sampler2D texture; -varying vec2 vUV; -void main() { - gl_FragColor = texture2D(texture, vUV).bgra; -}`; diff --git a/src/utils.js b/src/utils.js new file mode 100644 index 00000000..6124e52c --- /dev/null +++ b/src/utils.js @@ -0,0 +1,99 @@ +import window from 'global/window'; +import document from 'global/document'; +import videojs from 'video.js'; + +const USER_AGENT = window && window.navigator && window.navigator.userAgent || ''; + +// get the major/minor/patch for safari desktop an ios +// should be used in conjunction with videojs.browser +export const safariVersion = (function() { + const version = {major: 0, minor: 0, patch: 0}; + + if (!videojs.browser.IS_ANY_SAFARI) { + return version; + } + + const matches = (/Version\/(\d+)(\.\d+)?(\.\d+)? /i).exec(USER_AGENT); + + if (!matches || matches.length === 0) { + return version; + } + + version.major = parseInt(matches[1], 10); + + if (matches[2]) { + version.minor = parseInt(matches[2].replace('.', ''), 10); + } + + if (matches[3]) { + version.patch = parseInt(matches[3].replace('.', ''), 10); + } + + return version; +})(); + +// check if the browser supports cors +export const corsSupport = (function() { + const video = document.createElement('video'); + + video.crossOrigin = 'anonymous'; + + return video.hasAttribute('crossorigin'); +})(); + +export const isHLS = function(currentType) { + // hls video types + const hlsTypes = [ + // Apple santioned + 'application/vnd.apple.mpegurl', + // Very common + 'application/x-mpegurl', + // Included for completeness + 'video/x-mpegurl', + 'video/mpegurl', + 'application/mpegurl' + ]; + + // if the current type has a case insensitivie match from the list above + // this is hls + return hlsTypes.some((type) => (RegExp(`^${type}$`, 'i')).test(currentType)); +}; + +export const validProjections = [ + '360', + '360_LR', + '360_TB', + '360_CUBE', + 'NONE', + 'AUTO', + 'Sphere', + 'Cube', + 'equirectangular' +]; + +export const getInternalProjectionName = function(projection) { + if (!projection) { + return; + } + + projection = projection.toString().trim(); + + if ((/sphere/i).test(projection)) { + return '360'; + } + + if ((/cube/i).test(projection)) { + return '360_CUBE'; + } + + if ((/equirectangular/i).test(projection)) { + return '360'; + } + + for (let i = 0; i < validProjections.length; i++) { + if (new RegExp('^' + validProjections[i] + '$', 'i').test(projection)) { + return validProjections[i]; + } + } + +}; diff --git a/src/vertex-shader.js b/src/vertex-shader.js deleted file mode 100644 index 9aa41971..00000000 --- a/src/vertex-shader.js +++ /dev/null @@ -1,5 +0,0 @@ -module.exports = `varying vec2 vUV; -void main() { - vUV = vec2( uv.x, 1.0 - uv.y ); - gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 ); -}`;