Unify payload to include readable streams

Add simple progress indication when debugging HTTP streaming
architect · Sep 29, 2023 · ebe5a0a · ebe5a0a
1 parent 1e6a8e8
commit ebe5a0a
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 8 deletions.
diff --git a/plugins/s3/src/put-object.mjs b/plugins/s3/src/put-object.mjs
@@ -210,7 +210,7 @@ const PutObject = {
           stream.push(null)
         }
       })
-      canonicalReq.stream = stream
+      canonicalReq.payload = stream
       return canonicalReq
     }
   },

diff --git a/readme.md b/readme.md
@@ -171,9 +171,10 @@ The following parameters may be passed with individual client requests; only `se
 - **`headers` (object)**
   - Header names + values to be added to your request
   - By default, all headers are included in [authentication via AWS signature v4](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html)
-- **`payload` (object or string)**
+- **`payload` (object, buffer, readable stream, string)**
   - Aliases: `body`, `data`, `json`
-  - As a convenience, any passed objects are automatically JSON-encoded (with the appropriate `content-type` header set, if not already present); strings pass through
+  - As a convenience, any passed objects are automatically JSON-encoded (with the appropriate `content-type` header set, if not already present); buffers and strings simply pass through as is
+  - Passing a Node.js readable stream is currently experimental; this initiates an HTTP data stream to the API endpoint instead of writing a normal HTTP body payload
 - **`query` (object)**
   - Serialize the passed object and append it to your `endpoint` as a query string in your request
 - **`service` (string) [required]**

diff --git a/src/request.js b/src/request.js
@@ -1,4 +1,5 @@
 let qs = require('querystring')
+let { Readable } = require('stream')
 let aws4 = require('aws4')
 let { globalServices, semiGlobalServices } = require('./services')
 let { is } = require('./validate')
@@ -40,8 +41,11 @@ module.exports = function request (params, creds, region, config, metadata) {
 
     // Body - JSON-ify payload where convenient!
     let body = params.payload || params.body || params.data || params.json
+    let isBuffer = body instanceof Buffer
+    let isStream = body instanceof Readable
+
     // Lots of potentially weird valid json (like just a null), deal with it if / when we need to I guess
-    if (typeof body === 'object' && !(body instanceof Buffer)) {
+    if (typeof body === 'object' && !isBuffer && !isStream) {
       // Backfill content-type if it's just an object
       if (!contentType) contentType = 'application/json'
 
@@ -59,8 +63,11 @@ module.exports = function request (params, creds, region, config, metadata) {
       // Final JSON encoding
       params.body = JSON.stringify(body)
     }
-    // Everything else just passes through
-    else params.body = body
+    // Everything besides streams pass through for signing
+    else {
+      /* istanbul ignore next */
+      params.body = isStream ? undefined : body
+    }
 
     // Finalize headers, content-type
     if (contentType) headers['content-type'] = contentType
@@ -104,7 +111,8 @@ module.exports = function request (params, creds, region, config, metadata) {
     if (config.debug) {
       let { method = 'GET', service, host, path, port = '', headers, protocol, body } = options
       let truncatedBody
-      if (body instanceof Buffer) truncatedBody = `<body buffer of ${body.length}b>`
+      /**/ if (isBuffer) truncatedBody = `<body buffer of ${body.length}b>`
+      else if (isStream) truncatedBody = `<readable stream>`
       else truncatedBody = body?.length > 1000 ? body?.substring(0, 1000) + '...' : body
       console.error('[aws-lite] Requesting:', {
         service,
@@ -150,8 +158,18 @@ module.exports = function request (params, creds, region, config, metadata) {
         port: options.port,
       }
     }))
+
     /* istanbul ignore next */ // TODO remove and test
-    if (options.stream) options.stream.pipe(req)
+    if (isStream) {
+      body.pipe(req)
+      if (config.debug) {
+        let bytes = 0
+        body.on('data', chunk => {
+          bytes += chunk.length
+          console.error(`Bytes streamed: ${bytes}`)
+        })
+      }
+    }
     else req.end(options.body || '')
   })
 }