Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: safari hls #48

Merged
merged 3 commits into from
Jan 31, 2018
Merged

fix: safari hls #48

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
70fbcbc
fix: safari hls
brandonocasey Jan 22, 2018
1995f20
add an error for cors issues
brandonocasey Jan 31, 2018
e5d761b
fix animation frame looping
brandonocasey Jan 31, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
198 changes: 93 additions & 105 deletions src/plugin.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,31 +1,18 @@
import {version as VERSION} from '../package.json';
import window from 'global/window';
import document from 'global/document';
import WebVRPolyfill from 'webvr-polyfill';
import videojs from 'video.js';
import * as THREE from 'three';
import VRControls from 'three/examples/js/controls/VRControls.js';
import VREffect from 'three/examples/js/effects/VREffect.js';
import OrbitControls from 'three/examples/js/controls/OrbitControls.js';
import rgbFragmentShader from './rgb-fragment-shader';
import rgbaFragmentShader from './rgba-fragment-shader';
import vertexShader from './vertex-shader';
import * as utils from './utils';

// import controls so they get regisetered with videojs
import './cardboard-button';
import './big-vr-play-button';

const validProjections = [
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

moved to utils

'360',
'360_LR',
'360_TB',
'360_CUBE',
'NONE',
'AUTO',
'Sphere',
'Cube',
'equirectangular'
];

// Default options for the plugin.
const defaults = {
projection: 'AUTO',
Expand All @@ -43,54 +30,14 @@ const errors = {
headline: '360 not supported on this device',
type: '360_NOT_SUPPORTED',
message: "Your browser does not support 360. See <a href='http://webvr.info'>webvr.info</a> for assistance."
},
'web-vr-hls-cors-not-supported': {
headline: '360 HLS video not supported on this device',
type: '360_NOT_SUPPORTED',
message: "Your browser/device does not support HLS 360 video. See <a href='http://webvr.info'>webvr.info</a> for assistance."
}
};

const getInternalProjectionName = function(projection) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

moved to utils

if (!projection) {
return;
}

projection = projection.toString().trim();

if ((/sphere/i).test(projection)) {
return '360';
}

if ((/cube/i).test(projection)) {
return '360_CUBE';
}

if ((/equirectangular/i).test(projection)) {
return '360';
}

for (let i = 0; i < validProjections.length; i++) {
if (new RegExp('^' + validProjections[i] + '$', 'i').test(projection)) {
return validProjections[i];
}
}

};

const isHLS = function(currentType) {
// hls video types
const hlsTypes = [
// Apple santioned
'application/vnd.apple.mpegurl',
// Very common
'application/x-mpegurl',
// Included for completeness
'video/x-mpegurl',
'video/mpegurl',
'application/mpegurl'
];

// if the current type has a case insensitivie match from the list above
// this is hls
return hlsTypes.some((type) => (new RegExp(`^${type}$`, 'i')).test(currentType));
};

const Plugin = videojs.getPlugin('plugin');
const Component = videojs.getComponent('Component');

Expand All @@ -115,8 +62,9 @@ class VR extends Plugin {
player.errors({errors});
}

// we need this as IE 11 reports that it has a VR display, but isnt compatible with Video as a Texture. for example
if (videojs.browser.IE_VERSION) {
// IE 11 does not support enough webgl to be supported
// older safari does not support cors, so it wont work
if (videojs.browser.IE_VERSION || !utils.corsSupport) {
Copy link
Contributor Author

@brandonocasey brandonocasey Jan 22, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

older version of safari that don't support cors at all (<9 wont work), they dont work right now.

this.triggerError_({code: 'web-vr-not-supported', dismiss: false});
return;
}
Expand All @@ -132,7 +80,7 @@ class VR extends Plugin {
}

changeProjection_(projection) {
projection = getInternalProjectionName(projection);
projection = utils.getInternalProjectionName(projection);
// don't change to an invalid projection
if (!projection) {
projection = 'NONE';
Expand All @@ -147,7 +95,7 @@ class VR extends Plugin {
// mediainfo cannot be set to auto or we would infinite loop here
// each source should know wether they are 360 or not, if using AUTO
if (this.player_.mediainfo && this.player_.mediainfo.projection && this.player_.mediainfo.projection !== 'AUTO') {
const autoProjection = getInternalProjectionName(this.player_.mediainfo.projection);
const autoProjection = utils.getInternalProjectionName(this.player_.mediainfo.projection);

return this.changeProjection_(autoProjection);
}
Expand Down Expand Up @@ -285,7 +233,15 @@ class VR extends Plugin {
return this.vrDisplay.requestAnimationFrame(fn);
}

return super.requestAnimationFrame(fn);
return Component.prototype.requestAnimationFrame.call(this, fn);
}

cancelAnimationFrame(id) {
if (this.vrDisplay) {
return this.vrDisplay.cancelAnimationFrame(id);
}

return Component.prototype.cancelAnimationFrame.call(this, id);
}

togglePlay_() {
Expand All @@ -297,17 +253,26 @@ class VR extends Plugin {
}

animate_() {
if (!this.initialized_) {
return;
}
if (this.getVideoEl_().readyState === this.getVideoEl_().HAVE_ENOUGH_DATA) {
if (this.videoTexture) {
this.videoTexture.needsUpdate = true;
}
}

// This draws the current video data as an image to a canvas every render. That canvas is used
// as a texture by webgl. Normally the video is used directly and we don't have to do this, but
// HLS video textures on iOS >= 11 is currently broken, so we have to support those browser
// in a roundabout way.
if (this.videoImageContext_) {
this.videoImageContext_.drawImage(this.getVideoEl_(), 0, 0, this.videoImage_.width, this.videoImage_.height);
}

this.controls3d.update();
this.effect.render(this.scene, this.camera);

this.animationFrameId_ = this.requestAnimationFrame(this.animate_);

if (window.navigator.getGamepads) {
// Grab all gamepads
const gamepads = window.navigator.getGamepads();
Expand All @@ -330,21 +295,29 @@ class VR extends Plugin {
}
}
this.camera.getWorldDirection(this.cameraVector);

this.animationFrameId_ = this.requestAnimationFrame(this.animate_);
}

handleResize_() {
const width = this.player_.currentWidth();
const height = this.player_.currentHeight();

// when dealing with a non video
if (this.videoImage_) {
this.videoImage_.width = width;
this.videoImage_.height = height;
}

this.effect.setSize(width, height, false);
this.camera.aspect = width / height;
this.camera.updateProjectionMatrix();
}

setProjection(projection) {

if (!getInternalProjectionName(projection)) {
videojs.log.error('videojs-vr: please pass a valid projection: ' + validProjections.join(', '));
if (!utils.getInternalProjectionName(projection)) {
videojs.log.error('videojs-vr: please pass a valid projection ' + utils.validProjections.join(', '));
return;
}

Expand All @@ -366,46 +339,41 @@ class VR extends Plugin {

this.scene = new THREE.Scene();

this.videoTexture = new THREE.VideoTexture(this.getVideoEl_());

this.videoTexture.generateMipmaps = false;
this.videoTexture.minFilter = THREE.LinearFilter;
this.videoTexture.magFilter = THREE.LinearFilter;

// iOS and macOS HLS fix/hacks
// We opted to stop using a video texture on safari due to
// various bugs that exist when using it. This gives us worse performance
// but it will actually work on all recent version of safari. See
// the following issues for more info on this:
//
// https://bugs.webkit.org/show_bug.cgi?id=163866#c3
// https://github.com/mrdoob/three.js/issues/9754
// On iOS with HLS, color space is wrong and texture is flipped on Y axis
// On macOS, just need to flip texture Y axis

if (isHLS() && videojs.browser.IS_ANY_SAFARI) {
this.log('Safari + iOS + HLS = flipY and colorspace hack');
this.videoTexture.format = THREE.RGBAFormat;
this.videoTexture.flipY = false;
} else if (isHLS() && videojs.browser.IS_SAFARI) {
this.log('Safari + HLS = flipY hack');
this.videoTexture.format = THREE.RGBFormat;
this.videoTexture.flipY = false;
} else {
this.videoTexture.format = THREE.RGBFormat;
}
// https://bugs.webkit.org/show_bug.cgi?id=179417
if (videojs.browser.IS_ANY_SAFARI && utils.isHLS(this.player_.currentSource().type)) {
Copy link
Contributor Author

@brandonocasey brandonocasey Jan 22, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

previously we had a hack to get video textures working here. This uses a canvas image texture that is updated to the current video frame instead. This makes safari 10 and 11 both work for all versions but it can be a bit slower on worse hardware, and it probably doesn't look as good.

Copy link
Contributor Author

@brandonocasey brandonocasey Jan 22, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From my testing on an iPAD It preforms well enough that I think its a solution.

this.log('Video texture is not supported using image canvas hack');
this.videoImage_ = document.createElement('canvas');
this.videoImage_.width = this.player_.currentWidth();
this.videoImage_.height = this.player_.currentHeight();

if ((this.videoTexture.format === THREE.RGBAFormat || this.videoTexture.format === THREE.RGBFormat) && this.videoTexture.flipY === false) {
let fragmentShader = rgbFragmentShader;
this.videoImageContext_ = this.videoImage_.getContext('2d');
this.videoImageContext_.fillStyle = '#000000';

if (this.videoTexture.format === THREE.RGBAFormat) {
fragmentShader = rgbaFragmentShader;
}
this.videoTexture = new THREE.Texture(this.videoImage_);

this.movieMaterial = new THREE.ShaderMaterial({
uniforms: {texture: {value: this.videoTexture}},
vertexShader,
fragmentShader
});
this.videoTexture.wrapS = THREE.ClampToEdgeWrapping;
this.videoTexture.wrapT = THREE.ClampToEdgeWrapping;
this.videoTexture.flipY = true;
} else {
this.movieMaterial = new THREE.MeshBasicMaterial({ map: this.videoTexture, overdraw: true, side: THREE.DoubleSide });
this.log('Video texture is supported using that');
this.videoTexture = new THREE.VideoTexture(this.getVideoEl_());
}

// shared regardless of wether VideoTexture is used or
// an image canvas is used
this.videoTexture.generateMipmaps = false;
this.videoTexture.minFilter = THREE.LinearFilter;
this.videoTexture.magFilter = THREE.LinearFilter;
this.videoTexture.format = THREE.RGBFormat;

this.movieMaterial = new THREE.MeshBasicMaterial({ map: this.videoTexture, overdraw: true, side: THREE.DoubleSide });

this.changeProjection_(this.currentProjection_);

if (this.currentProjection_ === 'NONE') {
Expand Down Expand Up @@ -438,6 +406,20 @@ class VR extends Plugin {
antialias: true
});

const webglContext = this.renderer.getContext('webgl');
const oldTexImage2D = webglContext.texImage2D;

webglContext.texImage2D = (...args) => {
try {
return oldTexImage2D.apply(webglContext, args);
} catch (e) {
this.reset();
this.player_.pause();
this.triggerError_({code: 'web-vr-hls-cors-not-supported', dismiss: false});
throw new Error(e);
}
};

this.renderer.setSize(this.player_.currentWidth(), this.player_.currentHeight(), false);
this.effect = new VREffect(this.renderer);

Expand Down Expand Up @@ -501,7 +483,7 @@ class VR extends Plugin {
this.controls3d = new OrbitControls(this.camera, this.renderedCanvas);
this.controls3d.target.set(0, 0, -1);
}
this.requestAnimationFrame(this.animate_);
this.animationFrameId_ = this.requestAnimationFrame(this.animate_);
});
} else if (window.navigator.getVRDevices) {
this.triggerError_({code: 'web-vr-out-of-date', dismiss: false});
Expand Down Expand Up @@ -583,6 +565,14 @@ class VR extends Plugin {
this.cancelAnimationFrame(this.animationFrameId_);
}

if (this.videoImage_) {
this.videoImage_ = null;
}

if (this.videoImageContext_) {
this.videoImageContext_ = null;
}

this.initialized_ = false;
}

Expand All @@ -592,8 +582,6 @@ class VR extends Plugin {
}
}

VR.prototype.requestAnimationFrame = Component.prototype.requestAnimationFrame;
VR.prototype.cancelAnimationFrame = Component.prototype.cancelAnimationFrame;
VR.prototype.setTimeout = Component.prototype.setTimeout;
VR.prototype.clearTimeout = Component.prototype.clearTimeout;

Expand Down
5 changes: 0 additions & 5 deletions src/rgb-fragment-shader.js
View file
Open in desktop

This file was deleted.

5 changes: 0 additions & 5 deletions src/rgba-fragment-shader.js
View file
Open in desktop

This file was deleted.

Loading