feat: add upload progress events (#1044)

Makefile

@@ -49,7 +49,7 @@ node_modules/.link:
 
 test: coverage check
 
-test-samples: node_modules/.link
+test-samples: node_modules/.link build
 	mocha build/test/samples
 
 watch:

samples/youtube/upload.js

@@ -21,17 +21,16 @@ const {google} = require('googleapis');
 const sampleClient = require('../sampleclient');
 const fs = require('fs');
 
-const FILENAME = process.argv[2];
-
 // initialize the Youtube API library
 const youtube = google.youtube({
   version: 'v3',
   auth: sampleClient.oAuth2Client
 });
 
 // very basic example of uploading a video to youtube
-function uploadVideo () {
-  const req = youtube.videos.insert({
+function runSample (fileName, callback) {
+  const fileSize = fs.statSync(fileName).size;
+  youtube.videos.insert({
     part: 'id,snippet,status',
     notifySubscribers: false,
     resource: {
@@ -44,43 +43,43 @@ function uploadVideo () {
       }
     },
     media: {
-      body: fs.createReadStream(FILENAME)
+      body: fs.createReadStream(fileName)
+    }
+  }, {
+    // Use the `onUploadProgress` event from Axios to track the
+    // number of bytes uploaded to this point.
+    onUploadProgress: evt => {
+      const progress = (evt.bytesRead / fileSize) * 100;
+      process.stdout.clearLine();
+      process.stdout.cursorTo(0);
+      process.stdout.write(`${Math.round(progress)}% complete`);
     }
-  }, (err, data) => {
+  }, (err, res) => {
     if (err) {
       throw err;
     }
-    console.log(data);
-    process.exit();
+    console.log('\n\n');
+    console.log(res.data);
+    callback(res.data);
   });
-
-  const fileSize = fs.statSync(FILENAME).size;
-
-  // show some progress
-  const id = setInterval(() => {
-    const uploadedBytes = req.req.connection._bytesDispatched;
-    const uploadedMBytes = uploadedBytes / 1000000;
-    const progress = uploadedBytes > fileSize
-      ? 100 : (uploadedBytes / fileSize) * 100;
-    process.stdout.clearLine();
-    process.stdout.cursorTo(0);
-    process.stdout.write(uploadedMBytes.toFixed(2) + ' MBs uploaded. ' +
-     progress.toFixed(2) + '% completed.');
-    if (progress === 100) {
-      process.stdout.write('\nDone uploading, waiting for response...\n');
-      clearInterval(id);
-    }
-  }, 250);
 }
 
 const scopes = [
   'https://www.googleapis.com/auth/youtube.upload',
   'https://www.googleapis.com/auth/youtube'
 ];
 
-sampleClient.authenticate(scopes, err => {
-  if (err) {
-    throw err;
-  }
-  uploadVideo();
-});
+if (module === require.main) {
+  const fileName = process.argv[2];
+  sampleClient.authenticate(scopes, err => {
+    if (err) {
+      throw err;
+    }
+    runSample(fileName, () => { /* sample complete */ });
+  });
+}
+
+module.exports = {
+  runSample,
+  client: sampleClient.oAuth2Client
+};

src/lib/apirequest.ts

@@ -158,6 +158,7 @@ export function createAPIRequest<T>(
       const boundary = uuid.v4();
       const finale = `--${boundary}--`;
       const rStream = new stream.PassThrough();
+      const pStream = new ProgressStream();
       const isStream = isReadableStream(multipart[1].body);
       headers['Content-Type'] = `multipart/related; boundary=${boundary}`;
       for (const part of multipart) {
@@ -168,7 +169,15 @@ export function createAPIRequest<T>(
           rStream.push(part.body);
           rStream.push('\r\n');
         } else {
-          part.body.pipe(rStream, {end: false});
+          // Axios does not natively support onUploadProgress in node.js.
+          // Pipe through the pStream first to read the number of bytes read
+          // for the purpose of tracking progress.
+          pStream.on('progress', bytesRead => {
+            if (options.onUploadProgress) {
+              options.onUploadProgress({bytesRead});
+            }
+          });
+          part.body.pipe(pStream).pipe(rStream, {end: false});
           part.body.on('end', () => {
             rStream.push('\r\n');
             rStream.push(finale);
@@ -225,9 +234,17 @@ export function createAPIRequest<T>(
   }
 }
 
-export class BaseAPI {
-  protected _options: GlobalOptions;
-  constructor(options: GlobalOptions) {
-    this._options = options || {};
+/**
+ * Basic Passthrough Stream that records the number of bytes read
+ * every time the cursor is moved.
+ */
+class ProgressStream extends stream.Transform {
+  bytesRead = 0;
+  // tslint:disable-next-line: no-any
+  _transform(chunk: any, encoding: string, callback: Function) {
+    this.bytesRead += chunk.length;
+    this.emit('progress', this.bytesRead);
+    this.push(chunk);
+    callback();
   }
 }

test/samples/test.samples.youtube.ts

@@ -0,0 +1,54 @@
+// Copyright 2018, Google, LLC.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import * as assert from 'assert';
+import * as fs from 'fs';
+import * as nock from 'nock';
+import * as os from 'os';
+import * as path from 'path';
+
+import {Utils} from './../utils';
+
+nock.disableNetConnect();
+
+const samples = {
+  upload: require('../../../samples/youtube/upload')
+};
+
+for (const p in samples) {
+  if (samples[p]) {
+    samples[p].client.credentials = {access_token: 'not-a-token'};
+  }
+}
+
+const someFile = path.resolve('test/fixtures/public.pem');
+
+describe('YouTube samples', () => {
+  afterEach(() => {
+    nock.cleanAll();
+  });
+
+  it('should upload a video', done => {
+    const scope =
+        nock(Utils.baseUrl)
+            .post(
+                `/upload/youtube/v3/videos?part=id%2Csnippet%2Cstatus&notifySubscribers=false&uploadType=multipart`)
+            .reply(200, {kind: 'youtube#video'});
+    samples.upload.runSample(someFile, data => {
+      assert(data);
+      assert.equal(data.kind, 'youtube#video');
+      scope.done();
+      done();
+    });
+  });
+});

test/test.media.ts

@@ -91,6 +91,36 @@ describe('Media', () => {
     localGmail = google.gmail('v1');
   });
 
+  it('should post progress for uploads', async () => {
+    const scope =
+        nock(Utils.baseUrl)
+            .post(
+                '/upload/youtube/v3/videos?part=id%2Csnippet&notifySubscribers=false&uploadType=multipart')
+            .reply(200);
+    const fileName = path.join(__dirname, '../../test/fixtures/mediabody.txt');
+    const fileSize = (await pify(fs.stat)(fileName)).size;
+    const google = new GoogleApis();
+    const youtube = google.youtube('v3');
+    const progressEvents = new Array<number>();
+    const res = await pify(youtube.videos.insert)(
+        {
+          part: 'id,snippet',
+          notifySubscribers: false,
+          resource: {
+            snippet: {
+              title: 'Node.js YouTube Upload Test',
+              description:
+                  'Testing YouTube upload via Google APIs Node.js Client'
+            }
+          },
+          media: {body: fs.createReadStream(fileName)}
+        },
+        {onUploadProgress: evt => progressEvents.push(evt.bytesRead)});
+    assert(progressEvents.length > 0);
+    assert.equal(progressEvents[0], fileSize);
+    scope.done();
+  });
+
   it('should post with uploadType=multipart if resource and media set',
      async () => {
        nock(Utils.baseUrl)