Merge branch 'main' into async-configstores

* main: Bump typescript from 4.9.3 to 5.0.4 (tus#425) Bump sinon from 15.0.0 to 15.0.4 (tus#426) @tus/gcs-store: allow user to pass bucket instance to GCSStore (tus#388) Fix e2e test typecheking (tus#429) Update incorrect comment (tus#430) @tus/s3-store: add missing error code in termination extension (tus#413) @tus/s3-store@1.0.0-beta.5 @tus/s3-store: fix uncaught exception when cancelling an upload (tus#412) @tus/s3-store@1.0.0-beta.4 @tus/s3-store: fix uncaught exception when removing files (tus#410) @tus/s3-store: add termination extension (tus#401) @tus/server@1.0.0-beta.5 Bump @google-cloud/storage from 6.8.0 to 6.9.3 (tus#404) Bump http-cache-semantics from 4.1.0 to 4.1.1 (tus#393) Bump eslint-config-prettier from 8.5.0 to 8.6.0 (tus#392) Bump cookiejar from 2.1.3 to 2.1.4 (tus#383) Bump aws-sdk from 2.1269.0 to 2.1325.0 (tus#403) @tus/server: refactor and improve request validator (tus#402) Fix demo (tus#386)
mitjap · May 16, 2023 · 5a36abd · 5a36abd
2 parents 1f0a3dc + 8b8be66
commit 5a36abd
Show file tree

Hide file tree

Showing 27 changed files with 821 additions and 820 deletions.
diff --git a/README.md b/README.md
@@ -110,28 +110,28 @@ The tus protocol supports optional [extensions][]. Below is a table of the suppo
 | [Creation With Upload][] | ✅                                | ✅                            | ✅                              |
 | [Expiration][]           | ✅                                | ❌                            | ❌                              |
 | [Checksum][]             | ❌                                | ❌                            | ❌                              |
-| [Termination][]          | ✅                                | ❌                            | ❌                              |
+| [Termination][]          | ✅                                | ✅                            | ❌                              |
 | [Concatenation][]        | ❌                                | ❌                            | ❌                              |
 
 ## Demos
 
 Start the demo server using Local File Storage
 
 ```bash
-yarn workspace demo start
+yarn build && yarn demo
 ```
 
 Start up the demo server using AWS S3. The environment variables `AWS_BUCKET`,
 `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_REGION` need to be present.
 
 ```bash
-yarn workspace demo start:s3
+yarn build && yarn demo:s3
 ```
 
 Start up the demo server using Google Cloud Storage. A `keyfile.json` needs to be present in the root of the repository.
 
 ```bash
-yarn workspace demo start:gcs
+yarn build && yarn demo:gcs
 ```
 
 Then navigate to the demo ([localhost:1080](http://localhost:1080)) which uses [`tus-js-client`](https://github.com/tus/tus-js-client).

diff --git a/demo/server.js b/demo/server.js
@@ -59,7 +59,7 @@ const writeFile = (req, res) => {
   if (!filename.startsWith('/dist/')) {
     filename = '/demos/browser' + filename
   }
-  filename = path.join(process.cwd(), '/node_modules/tus-js-client', filename)
+  filename = path.join(process.cwd(), '../node_modules/tus-js-client', filename)
   fs.readFile(filename, 'binary', (err, file) => {
     if (err) {
       res.writeHead(500, {'Content-Type': 'text/plain'})

diff --git a/packages/eslint-config-custom/package.json b/packages/eslint-config-custom/package.json
@@ -12,7 +12,7 @@
     "@typescript-eslint/eslint-plugin": "^5.45.1",
     "@typescript-eslint/parser": "^5.47.1",
     "eslint": "^8.29.0",
-    "eslint-config-prettier": "^8.5.0",
+    "eslint-config-prettier": "^8.6.0",
     "eslint-config-turbo": "latest",
     "eslint-plugin-prettier": "^4.2.1",
     "prettier": "^2.8.1"

diff --git a/packages/gcs-store/README.md b/packages/gcs-store/README.md
@@ -29,12 +29,14 @@ npm install @tus/gcs-store
 const {Server} = require('@tus/server')
 const {GCSStore} = require('@tus/gcs-store')
 
+const {Storage} = require('@google-cloud/storage');
+
+const storage = new Storage({keyFilename: 'key.json'})
+
 const server = new Server({
   path: '/files',
   datastore: new GCSStore({
-    projectId: 'id',
-    keyFilename: path.resolve('./some-path', 'keyfile.json'),
-    bucket: 'tus-node-server-ci',
+    bucket: storage.bucket('tus-node-server-ci'),
   }),
 })
 // ...
@@ -46,19 +48,11 @@ This package exports `GCSStore`. There is no default export.
 
 ### `new GCSStore(options)`
 
-Creates a new Google Cloud Storage store with options.
-
-#### `options.projectId`
-
-The GCS project ID (`string`).
-
-#### `options.keyFilename`
-
-Path to the keyfile with credentials (`string`).
+Creates a new Google Cloud Storage store by passing a GCS bucket instance.
 
 #### `options.bucket`
 
-The bucket name.
+The bucket instance
 
 ## Extensions
 

diff --git a/packages/gcs-store/index.ts b/packages/gcs-store/index.ts
@@ -1,23 +1,17 @@
-import {Storage, Bucket} from '@google-cloud/storage'
+import {Bucket} from '@google-cloud/storage'
 import stream from 'node:stream'
 import http from 'node:http'
 import debug from 'debug'
 
 import {ERRORS, TUS_RESUMABLE} from '@tus/server'
 import {Upload, DataStore} from '@tus/server'
 
-type Options = {
-  bucket: string
-  projectId: string
-  keyFilename: string
-}
-
 const log = debug('tus-node-server:stores:gcsstore')
 
+type Options = {bucket: Bucket}
+
 export class GCSStore extends DataStore {
   bucket: Bucket
-  bucket_name: string
-  gcs: Storage
 
   constructor(options: Options) {
     super()
@@ -26,13 +20,9 @@ export class GCSStore extends DataStore {
       throw new Error('GCSDataStore must have a bucket')
     }
 
+    this.bucket = options.bucket
+
     this.extensions = ['creation', 'creation-with-upload', 'creation-defer-length']
-    this.bucket_name = options.bucket
-    this.gcs = new Storage({
-      projectId: options.projectId,
-      keyFilename: options.keyFilename,
-    })
-    this.bucket = this.gcs.bucket(this.bucket_name)
   }
 
   create(file: Upload): Promise<Upload> {

diff --git a/packages/gcs-store/package.json b/packages/gcs-store/package.json
@@ -21,10 +21,10 @@
     "test": "mocha test.ts --timeout 30000 --exit --extension ts --require ts-node/register"
   },
   "dependencies": {
-    "@google-cloud/storage": "^6.2.2",
     "debug": "^4.3.3"
   },
   "devDependencies": {
+    "@google-cloud/storage": "^6.9.3",
     "@tus/server": "workspace:^",
     "@types/debug": "^4.1.7",
     "@types/mocha": "^10.0.1",
@@ -35,7 +35,8 @@
     "should": "^13.2.3",
     "typescript": "latest"
   },
-  "peerdependencies": {
+  "peerDependencies": {
+    "@google-cloud/storage": "*",
     "@tus/server": "workspace:^"
   },
   "engines": {

diff --git a/packages/gcs-store/test.ts b/packages/gcs-store/test.ts
@@ -4,6 +4,8 @@ import {GCSStore} from './'
 
 import * as shared from '../../test/stores.test'
 
+import {Storage} from '@google-cloud/storage'
+
 const fixturesPath = path.resolve('../', '../', 'test', 'fixtures')
 const storePath = path.resolve('../', '../', 'test', 'output')
 
@@ -16,10 +18,13 @@ describe('GCSStore', () => {
   })
 
   beforeEach(function () {
-    this.datastore = new GCSStore({
+    const storage = new Storage({
       projectId: 'tus-node-server',
       keyFilename: path.resolve('../', '../', 'keyfile.json'),
-      bucket: 'tus-node-server-ci',
+    })
+
+    this.datastore = new GCSStore({
+      bucket: storage.bucket('tus-node-server-ci'),
     })
   })
 

diff --git a/packages/s3-store/README.md b/packages/s3-store/README.md
@@ -2,7 +2,7 @@
 
 > 👉 **Note**: since 1.0.0 packages are split and published under the `@tus` scope.
 > The old package, `tus-node-server`, is considered unstable and will only receive security fixes.
-> Make sure to use the new packages, currently in beta at `1.0.0-beta.1`.
+> Make sure to use the new package, currently in beta at `1.0.0-beta.5`.
 
 ## Contents
 
@@ -67,7 +67,7 @@ but may increase it to not exceed the S3 10K parts limit.
 
 Options to pass to the AWS S3 SDK.
 Checkout the [`S3ClientConfig`](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/s3clientconfig.html)
-docs for the supported options. You need to at least set the `region`, `bucket` name, and your preferred method of authentication. 
+docs for the supported options. You need to at least set the `region`, `bucket` name, and your preferred method of authentication.
 
 ## Extensions
 
@@ -79,9 +79,13 @@ The tus protocol supports optional [extensions][]. Below is a table of the suppo
 | [Creation With Upload][] | ✅              |
 | [Expiration][]           | ❌              |
 | [Checksum][]             | ❌              |
-| [Termination][]          | ❌              |
+| [Termination][]          | ✅              |
 | [Concatenation][]        | ❌              |
 
+### Termination
+
+After a multipart upload is aborted, no additional parts can be uploaded using that upload ID. The storage consumed by any previously uploaded parts will be freed. However, if any part uploads are currently in progress, those part uploads might or might not succeed. As a result, it might be necessary to set an [S3 Lifecycle configuration](https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpu-abort-incomplete-mpu-lifecycle-config.html) to abort incomplete multipart uploads.
+
 ## Examples
 
 ### Example: using `credentials` to fetch credentials inside a AWS container

diff --git a/packages/s3-store/index.ts b/packages/s3-store/index.ts
@@ -72,7 +72,12 @@ export class S3Store extends DataStore {
     super()
     const {partSize, s3ClientConfig} = options
     const {bucket, ...restS3ClientConfig} = s3ClientConfig
-    this.extensions = ['creation', 'creation-with-upload', 'creation-defer-length']
+    this.extensions = [
+      'creation',
+      'creation-with-upload',
+      'creation-defer-length',
+      'termination',
+    ]
     this.bucket = bucket
     this.preferredPartSize = partSize || 8 * 1024 * 1024
     this.client = new aws.S3(restS3ClientConfig)
@@ -257,19 +262,23 @@ export class S3Store extends DataStore {
               ])
             }
 
+            const readable = fs.createReadStream(path)
+            readable.on('error', reject)
             if (partSize > this.minPartSize || isFinalChunk) {
-              await this.uploadPart(metadata, fs.createReadStream(path), partNumber)
+              await this.uploadPart(metadata, readable, partNumber)
               offset += partSize
             } else {
-              await this.uploadIncompletePart(incompletePartId, fs.createReadStream(path))
+              await this.uploadIncompletePart(incompletePartId, readable)
             }
 
             bytesUploaded += partSize
             resolve()
           } catch (error) {
             reject(error)
           } finally {
-            fsProm.rm(path).catch(/* ignore */)
+            fsProm.rm(path).catch(() => {
+              /* ignore */
+            })
           }
         })
 
@@ -517,4 +526,36 @@ export class S3Store extends DataStore {
 
     this.saveMetadata(file, upload_id)
   }
+
+  public async remove(id: string): Promise<void> {
+    try {
+      const {upload_id} = await this.getMetadata(id)
+      if (upload_id) {
+        await this.client
+          .abortMultipartUpload({
+            Bucket: this.bucket,
+            Key: id,
+            UploadId: upload_id,
+          })
+          .promise()
+      }
+    } catch (error) {
+      if (error?.code && ['NotFound', 'NoSuchKey', 'NoSuchUpload'].includes(error.code)) {
+        log('remove: No file found.', error)
+        throw ERRORS.FILE_NOT_FOUND
+      }
+      throw error
+    }
+
+    await this.client
+      .deleteObjects({
+        Bucket: this.bucket,
+        Delete: {
+          Objects: [{Key: id}, {Key: `${id}.info`}],
+        },
+      })
+      .promise()
+
+    this.clearCache(id)
+  }
 }
diff --git a/packages/s3-store/package.json b/packages/s3-store/package.json
@@ -1,7 +1,7 @@
 {
   "$schema": "https://json.schemastore.org/package.json",
   "name": "@tus/s3-store",
-  "version": "1.0.0-beta.3",
+  "version": "1.0.0-beta.5",
   "description": "AWS S3 store for @tus/server",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -21,7 +21,7 @@
     "test": "mocha test.ts --timeout 40000 --exit --extension ts --require ts-node/register"
   },
   "dependencies": {
-    "aws-sdk": "^2.1064.0",
+    "aws-sdk": "^2.1325.0",
     "debug": "^4.3.3"
   },
   "devDependencies": {

diff --git a/packages/s3-store/test.ts b/packages/s3-store/test.ts
@@ -73,8 +73,7 @@ describe('S3DataStore', function () {
 
   shared.shouldHaveStoreMethods()
   shared.shouldCreateUploads()
-  // Termination extension not implemented yet
-  // shared.shouldRemoveUploads()
+  shared.shouldRemoveUploads() // Termination extension
   shared.shouldWriteUploads()
   shared.shouldHandleOffset()
   shared.shouldDeclareUploadLength() // Creation-defer-length extension

diff --git a/packages/server/README.md b/packages/server/README.md
@@ -2,7 +2,7 @@
 
 > 👉 **Note**: since 1.0.0 packages are split and published under the `@tus` scope.
 > The old package, `tus-node-server`, is considered unstable and will only receive security fixes.
-> Make sure to use the new package, currently in beta at `1.0.0-beta.2`.
+> Make sure to use the new package, currently in beta at `1.0.0-beta.5`.
 
 ## Contents
 

diff --git a/packages/server/package.json b/packages/server/package.json
@@ -1,7 +1,7 @@
 {
   "$schema": "https://json.schemastore.org/package.json",
   "name": "@tus/server",
-  "version": "1.0.0-beta.4",
+  "version": "1.0.0-beta.5",
   "description": "Tus resumable upload protocol in Node.js",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -35,7 +35,7 @@
     "mocha": "^10.1.0",
     "node-mocks-http": "^1.12.1",
     "should": "^13.2.3",
-    "sinon": "^15.0.0",
+    "sinon": "^15.0.4",
     "supertest": "^6.3.2",
     "ts-node": "^10.9.1",
     "tsconfig": "*",

diff --git a/packages/server/src/handlers/HeadHandler.ts b/packages/server/src/handlers/HeadHandler.ts
@@ -47,8 +47,9 @@ export class HeadHandler extends BaseHandler {
     }
 
     if (file.metadata !== undefined) {
-      // If the size of the upload is known, the Server MUST include
-      // the Upload-Length header in the response.
+      // If an upload contains additional metadata, responses to HEAD
+      // requests MUST include the Upload-Metadata header and its value
+      // as specified by the Client during the creation.
       res.setHeader('Upload-Metadata', Metadata.stringify(file.metadata) as string)
     }
 

diff --git a/packages/server/src/handlers/PostHandler.ts b/packages/server/src/handlers/PostHandler.ts
@@ -2,7 +2,7 @@ import debug from 'debug'
 
 import {BaseHandler} from './BaseHandler'
 import {Upload, Uid, Metadata} from '../models'
-import {RequestValidator} from '../validators/RequestValidator'
+import {validateHeader} from '../validators/HeaderValidator'
 import {EVENTS, ERRORS} from '../constants'
 
 import type http from 'node:http'
@@ -60,10 +60,12 @@ export class PostHandler extends BaseHandler {
     }
 
     let metadata
-    try {
-      metadata = Metadata.parse(upload_metadata)
-    } catch (error) {
-      throw ERRORS.INVALID_METADATA
+    if ('upload-metadata' in req.headers) {
+      try {
+        metadata = Metadata.parse(upload_metadata)
+      } catch {
+        throw ERRORS.INVALID_METADATA
+      }
     }
 
     const upload = new Upload({
@@ -95,7 +97,7 @@ export class PostHandler extends BaseHandler {
     } = {}
 
     // The request MIGHT include a Content-Type header when using creation-with-upload extension
-    if (!RequestValidator.isInvalidHeader('content-type', req.headers['content-type'])) {
+    if (validateHeader('content-type', req.headers['content-type'])) {
       newOffset = await this.store.write(req, upload.id, 0)
       headers['Upload-Offset'] = newOffset.toString()
       isFinal = newOffset === Number.parseInt(upload_length as string, 10)