✨ [amp-story-360] Add scene orientation support (fixes ampproject#29514…

…) (ampproject#30138)

* Base element with a working example

* Add amp-img to test file

* Fix base test, add play/pause/rewind methods

* revert example to proper state

* revert example to master state (for now)

* actually revert example to master state (with new line)

* Add comment in zuho's README.amp

* Implement play/pause support for amp-story-360 in amp-story-page

* Fix test

* Update/add validator tests

* restore missing newlines (wat)

* more newline nonsense

* Add tilt attributes to validator

* Update example

* Add orientation correction support to zuho library

* Replacing tilt API with Euler angles for scene orientation correction

* Remove unused Quaternion class

* Whitespace & validator text fixes

* Lint fix

* Lint fix 2

* Lint fix 3 the fixening

examples/amp-story-360.amp.html

@@ -19,7 +19,8 @@
         poster-portrait-src="example.com/poster.jpg">
       <amp-story-page id="page1">
         <amp-story-grid-layer template="fill">
-          <amp-story-360 layout="fill" heading-start="45" pitch-start="-20" heading-end="115" pitch-end="10" zoom-end="1.5" duration="3s" controls="gyroscope">
+          <amp-story-360 layout="fill" heading-start="45" pitch-start="-20" heading-end="115" pitch-end="10" zoom-end="1.5" duration="3s" controls="gyroscope"
+          scene-pitch="-5">
             <!-- Curiosity self-portrat, Sol 1462 (September 2016) Credit: NASA / JPL / MSSS / Seán Doran -->
             <!-- https://informal.jpl.nasa.gov/museum/360-video -->
             <amp-img src="img/SeanDoran-Quela-sol1462-edited_ver2-sm.png" width="1998" height="999"></amp-img>

extensions/amp-story-360/0.1/amp-story-360.js

@@ -240,19 +240,31 @@ export class AmpStory360 extends AMP.BaseElement {
 
     /** @private {boolean} */
     this.isOnActivePage_ = false;
+
+    /** @private {number} */
+    this.sceneHeading_ = 0;
+
+    /** @private {number} */
+    this.scenePitch_ = 0;
+
+    /** @private {number} */
+    this.sceneRoll_ = 0;
   }
 
   /** @override */
   buildCallback() {
     const attr = (name) => this.element.getAttribute(name);
+    const attrAsFloat = (name, fallbackValue = 0) => {
+      return parseFloat(attr(name) || fallbackValue);
+    };
 
     if (attr('duration')) {
       this.duration_ = timeStrToMillis(attr('duration')) || 0;
     }
 
-    const startHeading = parseFloat(attr('heading-start') || 0);
-    const startPitch = parseFloat(attr('pitch-start') || 0);
-    const startZoom = parseFloat(attr('zoom-start') || 1);
+    const startHeading = attrAsFloat('heading-start');
+    const startPitch = attrAsFloat('pitch-start');
+    const startZoom = attrAsFloat('zoom-start', 1);
     this.orientations_.push(
       CameraOrientation.fromDegrees(startHeading, startPitch, startZoom)
     );
@@ -262,14 +274,24 @@ export class AmpStory360 extends AMP.BaseElement {
       attr('pitch-end') !== undefined ||
       attr('zoom-end') !== undefined
     ) {
-      const endHeading = parseFloat(attr('heading-end') || startHeading);
-      const endPitch = parseFloat(attr('pitch-end') || startPitch);
-      const endZoom = parseFloat(attr('zoom-end') || startZoom);
+      const endHeading = attrAsFloat('heading-end', startHeading);
+      const endPitch = attrAsFloat('pitch-end', startPitch);
+      const endZoom = attrAsFloat('zoom-end', startZoom);
       this.orientations_.push(
         CameraOrientation.fromDegrees(endHeading, endPitch, endZoom)
       );
     }
 
+    if (
+      attr('scene-heading') !== undefined ||
+      attr('scene-pitch') !== undefined ||
+      attr('scene-roll') !== undefined
+    ) {
+      this.sceneHeading_ = attrAsFloat('scene-heading');
+      this.scenePitch_ = attrAsFloat('scene-pitch');
+      this.sceneRoll_ = attrAsFloat('scene-roll');
+    }
+
     const container = this.element.ownerDocument.createElement('div');
     this.canvas_ = this.element.ownerDocument.createElement('canvas');
     this.element.appendChild(container);
@@ -528,6 +550,11 @@ export class AmpStory360 extends AMP.BaseElement {
           const img = this.checkImageReSize_(
             dev().assertElement(this.element.querySelector('img'))
           );
+          this.renderer_.setImageOrientation(
+            this.sceneHeading_,
+            this.scenePitch_,
+            this.sceneRoll_
+          );
           this.renderer_.setImage(img);
           this.renderer_.resize();
           if (this.orientations_.length < 1) {

extensions/amp-story-360/0.1/test/validator-amp-story-360.html

@@ -39,7 +39,7 @@
         poster-portrait-src="example.com/poster.jpg">
     <amp-story-page id="cover">
       <amp-story-grid-layer template="fill">
-        <amp-story-360 layout="fill" heading-start="95" pitch-start="-10" zoom-start="4" heading-end="-45" pitch-end="-20" zoom-end="1" duration="30s">
+        <amp-story-360 layout="fill" heading-start="95" pitch-start="-10" zoom-start="4" heading-end="-45" pitch-end="-20" zoom-end="1" duration="30s" scene-heading="8.5" scene-pitch="5" scene-roll="-1">
           <amp-img src="img/panorama1.jpg" width="7168" height="3584"></amp-img>
         </amp-story-360>
      </amp-story-grid-layer>

extensions/amp-story-360/0.1/test/validator-amp-story-360.out

@@ -40,7 +40,7 @@ PASS
 |          poster-portrait-src="example.com/poster.jpg">
 |      <amp-story-page id="cover">
 |        <amp-story-grid-layer template="fill">
-|          <amp-story-360 layout="fill" heading-start="95" pitch-start="-10" zoom-start="4" heading-end="-45" pitch-end="-20" zoom-end="1" duration="30s">
+|          <amp-story-360 layout="fill" heading-start="95" pitch-start="-10" zoom-start="4" heading-end="-45" pitch-end="-20" zoom-end="1" duration="30s" scene-heading="8.5" scene-pitch="5" scene-roll="-1">
 |            <amp-img src="img/panorama1.jpg" width="7168" height="3584"></amp-img>
 |          </amp-story-360>
 |       </amp-story-grid-layer>

extensions/amp-story-360/validator-amp-story-360.protoascii

@@ -58,6 +58,18 @@ tags: {  # <amp-story-360>
     name: "zoom-end"
     value_regex: "\\d+\\.?\\d*"
   }
+  attrs: {
+    name: "scene-heading"
+    value_regex: "-?\\d+\\.?\\d*"
+  }
+  attrs: {
+    name: "scene-pitch"
+    value_regex: "-?\\d+\\.?\\d*"
+  }
+  attrs: {
+    name: "scene-roll"
+    value_regex: "-?\\d+\\.?\\d*"
+  }
   child_tags: {
     mandatory_num_child_tags: 1
     child_tag_name_oneof: "AMP-IMG"

third_party/zuho/README.amp

@@ -15,3 +15,4 @@ Local Modifications:
 - Refactored into module, exporting Renderer and Matrix classes
 - Removed String.raw calls (breaks the build)
 - Swapped X-axis projection (original code was horizontally mirroring the source image)
+- Added setImageOrientation(heading, pitch, roll) method to correct the orientation of the texture (credits: Nicolas Barradeau - https://github.com/nicoptere).

third_party/zuho/zuho.js

@@ -15,20 +15,12 @@ const SHADERS = {
     uniform sampler2D uTex;
     varying vec2      vPos;
 
-    bool unproject(vec2, out vec3);
-
     vec4 sample(float dx, float dy) {
-      vec2 p = vPos + uPxSize * vec2(dx, dy);
-      vec3 q;
-      if(unproject(p, q)) {
-        vec3 dir = normalize(uRot * q);
-        float u = (-0.5 / pi) * atan(dir[1], dir[0]) + 0.5;
-        float v = (1.0 / pi) * acos(dir[2]);
-        return texture2D(uTex, vec2(u, v));
-      }
-      else {
-        return vec4(0.0);
-      }
+      vec3 q = vec3(vPos + uPxSize * vec2(dx, dy), -1.0);
+      vec3 dir = normalize(uRot * q);
+      float u = (-0.5 / pi) * atan(dir[1], dir[0]) + 0.5;
+      float v = (1.0 / pi) * acos(dir[2]);
+      return texture2D(uTex, vec2(u, v));
     }
   `,
   fragSourceFast: `
@@ -45,22 +37,22 @@ const SHADERS = {
     void main() {
       // (2, 3) halton vector sequences.
       vec4 acc = (1.0 / 16.0) * (
-        (((sampleSq( 1.0 /  2.0 - 0.5,  1.0 /  3.0 - 0.5) +
-           sampleSq( 1.0 /  4.0 - 0.5,  2.0 /  3.0 - 0.5)) +
-          (sampleSq( 3.0 /  4.0 - 0.5,  1.0 /  9.0 - 0.5) +
-           sampleSq( 1.0 /  8.0 - 0.5,  4.0 /  9.0 - 0.5))) +
-         ((sampleSq( 5.0 /  8.0 - 0.5,  7.0 /  9.0 - 0.5) +
-           sampleSq( 3.0 /  8.0 - 0.5,  2.0 /  9.0 - 0.5)) +
-          (sampleSq( 7.0 /  8.0 - 0.5,  5.0 /  9.0 - 0.5) +
-           sampleSq( 1.0 / 16.0 - 0.5,  8.0 /  9.0 - 0.5)))) +
-        (((sampleSq( 9.0 / 16.0 - 0.5,  1.0 / 27.0 - 0.5) +
-           sampleSq( 5.0 / 16.0 - 0.5, 10.0 / 27.0 - 0.5)) +
+        (((sampleSq(1.0 / 2.0 - 0.5,  1.0 / 3.0 - 0.5) +
+           sampleSq(1.0 / 4.0 - 0.5,  2.0 / 3.0 - 0.5)) +
+          (sampleSq(3.0 / 4.0 - 0.5,  1.0 / 9.0 - 0.5) +
+           sampleSq(1.0 / 8.0 - 0.5,  4.0 / 9.0 - 0.5))) +
+         ((sampleSq(5.0 / 8.0 - 0.5,  7.0 / 9.0 - 0.5) +
+           sampleSq(3.0 / 8.0 - 0.5,  2.0 / 9.0 - 0.5)) +
+          (sampleSq(7.0 / 8.0 - 0.5,  5.0 / 9.0 - 0.5) +
+           sampleSq(1.0 / 16.0 - 0.5,  8.0 / 9.0 - 0.5)))) +
+        (((sampleSq(9.0 / 16.0 - 0.5,  1.0 / 27.0 - 0.5) +
+           sampleSq(5.0 / 16.0 - 0.5, 10.0 / 27.0 - 0.5)) +
           (sampleSq(13.0 / 16.0 - 0.5, 19.0 / 27.0 - 0.5) +
-           sampleSq( 3.0 / 16.0 - 0.5,  4.0 / 27.0 - 0.5))) +
+           sampleSq(3.0 / 16.0 - 0.5,  4.0 / 27.0 - 0.5))) +
          ((sampleSq(11.0 / 16.0 - 0.5, 13.0 / 27.0 - 0.5) +
-           sampleSq( 7.0 / 16.0 - 0.5, 22.0 / 27.0 - 0.5)) +
+           sampleSq(7.0 / 16.0 - 0.5, 22.0 / 27.0 - 0.5)) +
           (sampleSq(15.0 / 16.0 - 0.5,  7.0 / 27.0 - 0.5) +
-           sampleSq( 1.0 / 32.0 - 0.5, 16.0 / 27.0 - 0.5))))
+           sampleSq(1.0 / 32.0 - 0.5, 16.0 / 27.0 - 0.5))))
      );
       gl_FragColor = vec4(sqrt(acc.xyz), acc.w);
     }
@@ -77,23 +69,14 @@ const SHADERS = {
   `,
 };
 
-const MAPPING = {
-  azPerspective: `
-    bool unproject(vec2 p, out vec3 q) {
-      q = vec3(p, -1.0);
-      return true;
-    }
-  `,
-};
-
 export class Matrix {
   static mul(n, x, y) {
     console.assert(x.length == n * n && y.length == n * n);
     const z = new Float32Array(n * n);
-    for(let i = 0; i < n; ++i) {
-      for(let j = 0; j < n; ++j) {
+    for (let i = 0; i < n; ++i) {
+      for (let j = 0; j < n; ++j) {
         let sum = 0.0;
-        for(let k = 0; k < n; ++k) {
+        for (let k = 0; k < n; ++k) {
           sum += x[i * n + k] * y[k * n + j];
         }
         z[i * n + j] = sum;
@@ -105,7 +88,7 @@ export class Matrix {
   static identity(n) {
     const z = new Float32Array(n * n);
     z.fill(0.0);
-    for(let i = 0; i < n; ++i) {
+    for (let i = 0; i < n; ++i) {
       z[i * n + i] = 1.0;
     }
     return z;
@@ -127,21 +110,21 @@ export class Matrix {
 export class Renderer {
   constructor(canvas) {
     const params = {
-        alpha: true,
-        depth: false,
-        stencil: false,
-        antialias: false,
-        premultipliedAlpha: true,
+      alpha: true,
+      depth: false,
+      stencil: false,
+      antialias: false,
+      premultipliedAlpha: true,
     };
-    const gl = this.gl =
-      	canvas.getContext("webgl", params) ||
-      	canvas.getContext("experimental-webgl", params);
+    const gl = this.gl = canvas.getContext('webgl', params) ||
+        canvas.getContext('experimental-webgl', params);
 
     this.canvas = canvas;
     this.resize();
 
     this.rotation = null;
     this.scale = 1;
+    this.orientation = null;
 
     this.vertShader = gl.createShader(gl.VERTEX_SHADER);
     this.fragShaderFast = gl.createShader(gl.FRAGMENT_SHADER);
@@ -153,13 +136,13 @@ export class Renderer {
     gl.attachShader(this.progFast, this.fragShaderFast);
     gl.attachShader(this.progSlow, this.vertShader);
     gl.attachShader(this.progSlow, this.fragShaderSlow);
-    this.setMapping(MAPPING.azPerspective);
+    this.setMapping();
     this.setCamera(Matrix.identity(3), 1.0);
 
     this.vbo = gl.createBuffer();
     gl.bindBuffer(gl.ARRAY_BUFFER, this.vbo);
-    const vertices = new Float32Array(
-    		[-1.0, -1.0, +1.0, -1.0, -1.0, +1.0, +1.0, +1.0]);
+    const vertices =
+        new Float32Array([-1.0, -1.0, +1.0, -1.0, -1.0, +1.0, +1.0, +1.0]);
     gl.bufferData(gl.ARRAY_BUFFER, vertices, gl.STATIC_DRAW);
 
     this.tex = gl.createTexture();
@@ -180,11 +163,19 @@ export class Renderer {
     gl.bindTexture(gl.TEXTURE_2D, null);
   }
 
-  setMapping(code) {
-    this.compile_(this.fragShaderFast, SHADERS.fragSourceCommon 
-    		+ SHADERS.fragSourceFast + code);
-    this.compile_(this.fragShaderSlow, SHADERS.fragSourceCommon 
-    		+ SHADERS.fragSourceSlow + code);
+  setImageOrientation(heading = 0, pitch = 0, roll = 0) {
+    const RAD = Math.PI / 180;
+    this.orientation =
+        this.euler_(RAD * heading, RAD * pitch, RAD * roll);
+  }
+
+  setMapping(code = '') {
+    this.compile_(
+        this.fragShaderFast,
+        SHADERS.fragSourceCommon + SHADERS.fragSourceFast + code);
+    this.compile_(
+        this.fragShaderSlow,
+        SHADERS.fragSourceCommon + SHADERS.fragSourceSlow + code);
     this.link_(this.progFast);
     this.link_(this.progSlow);
   }
@@ -196,7 +187,7 @@ export class Renderer {
 
   resize() {
     const rect = this.canvas.getBoundingClientRect();
-    this.canvas.width  = rect.width  * devicePixelRatio;
+    this.canvas.width = rect.width * devicePixelRatio;
     this.canvas.height = rect.height * devicePixelRatio;
     this.gl.viewport(0.0, 0.0, this.canvas.width, this.canvas.height);
   }
@@ -208,22 +199,29 @@ export class Renderer {
 
     const prog = fast ? this.progFast : this.progSlow;
     gl.useProgram(prog);
-    const f = this.scale / Math.sqrt(gl.drawingBufferWidth * 
-        gl.drawingBufferHeight);
+    const f =
+        this.scale / Math.sqrt(gl.drawingBufferWidth * gl.drawingBufferHeight);
     const sx = f * gl.drawingBufferWidth;
     const sy = f * gl.drawingBufferHeight;
-    gl.uniformMatrix3fv(gl.getUniformLocation(prog, "uRot"), false, 
-    		this.rotation);
-    gl.uniform2f(gl.getUniformLocation(prog, "uScale"), sx, sy);
-    gl.uniform1f(gl.getUniformLocation(prog, "uPxSize"), 2.0 * f);
+    gl.uniformMatrix3fv(
+        gl.getUniformLocation(prog, 'uRot'), false, this.rotation);
+    gl.uniform2f(gl.getUniformLocation(prog, 'uScale'), sx, sy);
+    gl.uniform1f(gl.getUniformLocation(prog, 'uPxSize'), 2.0 * f);
+
+    if (!this.orientation) {
+      this.orientation = Matrix.identity(3);
+    }
+    gl.uniformMatrix3fv(
+        gl.getUniformLocation(prog, 'uRot'), false,
+        Matrix.mul(3, this.rotation, this.orientation));
 
     gl.enableVertexAttribArray(0);
     gl.bindBuffer(gl.ARRAY_BUFFER, this.vbo);
     gl.vertexAttribPointer(0, 2, gl.FLOAT, false, 0, 0);
 
     gl.activeTexture(gl.TEXTURE0);
     gl.bindTexture(gl.TEXTURE_2D, this.tex);
-    gl.uniform1i(gl.getUniformLocation(prog, "uTex"), 0);
+    gl.uniform1i(gl.getUniformLocation(prog, 'uTex'), 0);
 
     gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
 
@@ -238,7 +236,7 @@ export class Renderer {
     gl.shaderSource(shader, src);
     gl.compileShader(shader);
     const log = gl.getShaderInfoLog(shader);
-    if(log.length > 0) {
+    if (log.length > 0) {
       console.log(log);
     }
   }
@@ -248,8 +246,32 @@ export class Renderer {
     const gl = this.gl;
     gl.linkProgram(prog);
     const log = gl.getProgramInfoLog(prog);
-    if(log.length > 0) {
+    if (log.length > 0) {
       console.log(log);
     }
   }
-}
+
+  /** @private */
+  euler_(heading, pitch, roll) {
+    const te = Matrix.identity(3);
+    const x = -roll, y = -pitch, z = heading;
+
+    const a = Math.cos(x), b = Math.sin(x);
+    const c = Math.cos(y), d = Math.sin(y);
+    const e = Math.cos(z), f = Math.sin(z);
+
+    const ae = a * e, af = a * f, be = b * e, bf = b * f;
+
+    te[0] = c * e;
+    te[3] = -c * f;
+    te[6] = d;
+    te[1] = af + be * d;
+    te[4] = ae - bf * d;
+    te[7] = -b * c;
+    te[2] = bf - ae * d;
+    te[5] = be + af * d;
+    te[8] = a * c;
+
+    return te;
+  }
+}