Merge 877697f into c8e6bf3

visgl · Sep 9, 2019 · 336645e · 336645e
2 parents c8e6bf3 + 877697f
commit 336645e
Show file tree

Hide file tree

Showing 15 changed files with 201 additions and 0 deletions.
diff --git a/docs/api-reference/core/fetch-progress.md b/docs/api-reference/core/fetch-progress.md
@@ -0,0 +1,27 @@
+# fetchProgress
+
+> This function is still experimental
+
+A function that tracks a fetch response object and calls `onProgress` callbacks.
+
+## Usage
+
+```js
+import {_fetchProgress} from '@loaders.gl/core';
+
+function onProgress(percent, {loadedBytes, totalBytes}) {
+  console.log(`${percent}% ${Math.round(loadedBytes/1000)} of ${Math.round(totalBytes/1000)} Kbytes`);
+}
+
+async function main() {
+  const response = await _fetchProgress(fetch(PROGRESS_IMAGE_URL, onProgress),
+  const data = await response.arrayBuffer();
+  // At this point, onProgress will have been called one or more times.
+  ...
+}
+```
+
+## _fetchProgress(response : Response | Promise, onProgress : function, onDone : function, onError : function) : Response
+
+`onProgress: (percent: number, {loadedBytes : number, totalBytes : number}) => void`
+
diff --git a/examples/progress/app.js b/examples/progress/app.js
@@ -0,0 +1,39 @@
+// Forked from github AnthumChris/fetch-progress-indicators under MIT license
+/* global fetch, document, URL */
+import {_fetchProgress} from '@loaders.gl/core';
+
+const PROGRESS_IMAGE_URL = 'https://fetch-progress.anthum.com/30kbps/images/sunrise-baseline.jpg';
+// 'https://raw.githubusercontent.com/uber-web/loaders.gl/master/modules/images/test/progress/sunrise-baseline.jpg';
+// 'https://raw.githubusercontent.com/uber-web/loaders.gl/master/modules/images/test/data/img1-preview.jpeg';
+
+const elStatus = document.getElementById('status');
+const elProgress = document.getElementById('progress');
+const elImage = document.getElementById('img');
+
+async function main() {
+  elStatus.innerHTML = 'Downloading with fetch()...';
+
+  try {
+    const response = await _fetchProgress(
+      fetch(PROGRESS_IMAGE_URL),
+      (percent, {loadedBytes, totalBytes}) => {
+        elProgress.innerHTML = `${percent}% ${Math.round(loadedBytes / 1000)} of ${Math.round(
+          totalBytes / 1000
+        )} Kbytes`;
+      }
+    );
+    const data = await response.blob();
+    elStatus.innerHTML = 'download completed';
+    elImage.src = URL.createObjectURL(data);
+  } catch (error) {
+    console.error(error); // eslint-disable-line
+    elStatus.innerHTML = error;
+  }
+}
+
+main();
+
+/*
+unction progress(percent {loadedBytes, totalBytes}) {
+elProgress.innerHTML = Math.round(loadedBytes/totalBytes*100)+'%';
+*/
diff --git a/examples/progress/index.html b/examples/progress/index.html
@@ -0,0 +1,13 @@
+<!doctype html>
+<script type="module">
+  import './app.js';
+</script>
+<body>
+  <main>
+    <!-- Forked from github AnthumChris/fetch-progress-indicators under MIT license -->
+    <div id="status">&nbsp;</div>
+    <h1 id="progress">&nbsp;</h1>
+    <div><button onclick="window.location.reload()">Reload</button></div>
+    <img id="img" />
+  </main>
+</body>
diff --git a/examples/progress/package.json b/examples/progress/package.json
@@ -0,0 +1,17 @@
+{
+  "license": "MIT",
+  "description": "Minimal example of using loaders.gl with webpack.",
+  "scripts": {
+    "start": "webpack-dev-server --progress --hot --open -d",
+    "start-local": "webpack-dev-server --env.local --progress --hot --open"
+  },
+  "dependencies": {
+    "@loaders.gl/core": "^1.2.2",
+    "@loaders.gl/obj": "^1.2.2"
+  },
+  "devDependencies": {
+    "html-webpack-plugin": "^3.2.0",
+    "webpack": "^4.3.0",
+    "webpack-dev-server": "^3.1.1"
+  }
+}
diff --git a/examples/progress/webpack.config.js b/examples/progress/webpack.config.js
@@ -0,0 +1,16 @@
+const {resolve} = require('path');
+
+const CONFIG = {
+  mode: 'development',
+
+  entry: {
+    app: resolve('./app.js')
+  },
+
+  node: {
+    fs: 'empty'
+  }
+};
+
+// This line enables bundling against src in this repo rather than installed module
+module.exports = env => (env ? require('../webpack.config.local')(CONFIG)(env) : CONFIG);
diff --git a/modules/core/src/index.js b/modules/core/src/index.js
@@ -65,6 +65,8 @@ export {default as _WorkerThread} from './worker-utils/worker-thread';
 export {default as _WorkerFarm} from './worker-utils/worker-farm';
 export {default as _WorkerPool} from './worker-utils/worker-pool';
 
+export {default as _fetchProgress} from './lib/progress/fetch-progress';
+
 // FOR TESTING
 export {_unregisterLoaders} from './lib/register-loaders';
 

diff --git a/modules/core/src/lib/progress/fetch-progress.js b/modules/core/src/lib/progress/fetch-progress.js
@@ -0,0 +1,61 @@
+// Forked from github AnthumChris/fetch-progress-indicators under MIT license
+/* global Response, ReadableStream */
+
+// Intercepts the Response stream and creates a new Response
+export default async function fetchProgress(
+  response,
+  onProgress,
+  onDone = () => {},
+  onError = () => {}
+) {
+  response = await response;
+  if (!response.ok) {
+    // ERROR checking needs to be done separately
+    return response;
+  }
+  if (!response.body) {
+    // 'ReadableStream not yet supported in this browser.
+    return response;
+  }
+  const contentLength = response.headers.get('content-length');
+  const totalBytes = contentLength && parseInt(contentLength, 10);
+  if (!(contentLength > 0)) {
+    return response;
+  }
+  // Currently override only implemented in browser
+  if (typeof ReadableStream === 'undefined') {
+    return response;
+  }
+
+  // Create a new stream that invisbly wraps original stream
+  const progressStream = new ReadableStream({
+    start(controller) {
+      const reader = response.body.getReader();
+      read(controller, reader, 0, totalBytes, onProgress, onDone, onError);
+    }
+  });
+
+  return new Response(progressStream);
+}
+
+// Forward to original streams controller
+// TODO - this causes a crazy deep "async stack"... rewrite as async iterator?
+// eslint-disable-next-line max-params
+async function read(controller, reader, loadedBytes, totalBytes, onProgress, onDone, onError) {
+  try {
+    const {done, value} = await reader.read();
+    if (done) {
+      onDone();
+      controller.close();
+      return;
+    }
+    loadedBytes += value.byteLength;
+    const percent = Math.round((loadedBytes / totalBytes) * 100);
+    onProgress(percent, {loadedBytes, totalBytes});
+    controller.enqueue(value);
+    await read(controller, reader, loadedBytes, totalBytes, onProgress, onDone, onError);
+  } catch (error) {
+    controller.error(error);
+    onError(error);
+  }
+}
diff --git a/modules/core/test/index.js b/modules/core/test/index.js
@@ -4,6 +4,7 @@ import './worker-utils';
 
 import './lib/fetch';
 import './lib/loader-utils';
+import './lib/progress/fetch-progress.spec';
 
 import './lib/parse.spec';
 import './lib/load.spec';

diff --git a/modules/core/test/lib/progress/fetch-progress.spec.js b/modules/core/test/lib/progress/fetch-progress.spec.js
@@ -0,0 +1,18 @@
+import test from 'tape-promise/tape';
+import {fetchFile, _fetchProgress} from '@loaders.gl/core';
+
+const PROGRESS_IMAGE_URL = '@loaders.gl/images/test/data/img1-preview.jpeg';
+
+test('progress#fetchProgress', async t => {
+  t.ok(_fetchProgress, '_fetchProgress defined');
+  const response = await _fetchProgress(
+    fetchFile(PROGRESS_IMAGE_URL),
+    (percent, {loadedBytes, totalBytes}) => {
+      t.ok(Number.isFinite(percent));
+      t.ok(Number.isFinite(loadedBytes));
+      t.ok(Number.isFinite(totalBytes));
+      t.end();
+    }
+  );
+  await response.arrayBuffer();
+});
diff --git a/modules/images/test/data/README.md b/modules/images/test/data/README.md
@@ -0,0 +1,3 @@
+# Data Licenses
+
+- progress folder images, copied under MIT license from AnthumChris/fetch-progress-indicators.
diff --git a/modules/images/test/data/progress/logo-streams-300.png b/modules/images/test/data/progress/logo-streams-300.png
diff --git a/modules/images/test/data/progress/sunrise-baseline.jpg b/modules/images/test/data/progress/sunrise-baseline.jpg
diff --git a/modules/images/test/data/progress/sunrise-progressive.jpg b/modules/images/test/data/progress/sunrise-progressive.jpg
diff --git a/modules/polyfills/src/fetch-node/response.node.js b/modules/polyfills/src/fetch-node/response.node.js
@@ -40,6 +40,7 @@ export default class NodeFetchResponse {
 
   get headers() {
     this._headers = this._headers || this._getHeaders();
+    return this._headers;
   }
 
   // Returns a readable stream to the "body" of the response (or file)

diff --git a/modules/polyfills/test/fetch-node/fetch.node.spec.js b/modules/polyfills/test/fetch-node/fetch.node.spec.js
@@ -9,6 +9,7 @@ const PLY_CUBE_ATT_URL = resolvePath('@loaders.gl/ply/test/data/cube_att.ply');
 test('fetch polyfill (Node.js)#fetch()', async t => {
   if (!isBrowser) {
     const response = await fetch(PLY_CUBE_ATT_URL);
+    t.ok(response.headers, 'fetch polyfill successfully returned headers under Node.js');
     const data = await response.text();
     t.ok(data, 'fetch polyfill successfully loaded data under Node.js');
   }
@@ -19,6 +20,8 @@ test('fetch polyfill (Node.js)#fetch() ignores url query params when loading fil
   if (!isBrowser) {
     const response = await fetch(`${PLY_CUBE_ATT_URL}?v=1.2.3`);
     const data = await response.text();
+    // TODO - strip query params when doing stat in headers...
+    // t.ok(response.headers, 'fetch polyfill successfully returned headers under Node.js');
     t.ok(data, 'fetch polyfill successfully loaded data under Node.js');
   }
   t.end();