diff --git a/package.json b/package.json index 21c138793..9fc6bc605 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,6 @@ "arrify": "^2.0.0", "async-retry": "^1.3.3", "compressible": "^2.0.12", - "configstore": "^5.0.0", "date-and-time": "^2.0.0", "duplexify": "^4.0.0", "ent": "^2.2.0", @@ -82,7 +81,6 @@ "@grpc/proto-loader": "^0.6.0", "@types/async-retry": "^1.4.3", "@types/compressible": "^2.0.0", - "@types/configstore": "^5.0.0", "@types/date-and-time": "^0.13.0", "@types/ent": "^2.2.1", "@types/extend": "^3.0.0", diff --git a/samples/package.json b/samples/package.json index 269aa3bfc..05efabc26 100644 --- a/samples/package.json +++ b/samples/package.json @@ -4,7 +4,7 @@ "license": "Apache-2.0", "author": "Google Inc.", "engines": { - "node": ">=8" + "node": ">=10" }, "repository": "googleapis/nodejs-storage", "private": true, diff --git a/samples/uploadDirectory.js b/samples/uploadDirectory.js index a9efc8d79..62ff62279 100644 --- a/samples/uploadDirectory.js +++ b/samples/uploadDirectory.js @@ -19,7 +19,10 @@ // description: Uploads full hierarchy of a local directory to a bucket. // usage: node files.js upload-directory -async function main(bucketName, directoryPath) { +function main( + bucketName = 'your-unique-bucket-name', + directoryPath = './local/path/to/directory' +) { // [START upload_directory] /** * TODO(developer): Uncomment the following lines before running the sample. @@ -36,68 +39,53 @@ async function main(bucketName, directoryPath) { // Creates a client const storage = new Storage(); + const {promisify} = require('util'); const fs = require('fs'); const path = require('path'); - const fileList = []; - async function uploadDirectory() { - // Get a list of files from the specified directory - let dirCtr = 1; - let itemCtr = 0; - const pathDirName = path.dirname(directoryPath); - - getFiles(directoryPath); - - function getFiles(directory) { - fs.readdir(directory, (err, items) => { - dirCtr--; - itemCtr += items.length; - items.forEach(item => { - const fullPath = path.join(directory, item); - fs.stat(fullPath, (err, stat) => { - itemCtr--; - if (stat.isFile()) { - fileList.push(fullPath); - } else if (stat.isDirectory()) { - dirCtr++; - getFiles(fullPath); - } - if (dirCtr === 0 && itemCtr === 0) { - onComplete(); - } - }); - }); - }); + const readdir = promisify(fs.readdir); + const stat = promisify(fs.stat); + + async function* getFiles(directory = '.') { + for (const file of await readdir(directory)) { + const fullPath = path.join(directory, file); + const stats = await stat(fullPath); + + if (stats.isDirectory()) { + yield* getFiles(fullPath); + } + + if (stats.isFile()) { + yield fullPath; + } } + } - async function onComplete() { - const resp = await Promise.all( - fileList.map(filePath => { - let destination = path.relative(pathDirName, filePath); - // If running on Windows - if (process.platform === 'win32') { - destination = destination.replace(/\\/g, '/'); - } - return storage - .bucket(bucketName) - .upload(filePath, {destination}) - .then( - uploadResp => ({fileName: destination, status: uploadResp[0]}), - err => ({fileName: destination, response: err}) - ); - }) - ); - - const successfulUploads = - fileList.length - resp.filter(r => r.status instanceof Error).length; - console.log( - `${successfulUploads} files uploaded to ${bucketName} successfully.` - ); + async function uploadDirectory() { + const bucket = storage.bucket(bucketName); + let successfulUploads = 0; + + for await (const filePath of getFiles(directoryPath)) { + try { + const dirname = path.dirname(directoryPath); + const destination = path.relative(dirname, filePath); + + await bucket.upload(filePath, {destination}); + + console.log(`Successfully uploaded: ${filePath}`); + successfulUploads++; + } catch (e) { + console.error(`Error uploading ${filePath}:`, e); + } } + + console.log( + `${successfulUploads} files uploaded to ${bucketName} successfully.` + ); } uploadDirectory().catch(console.error); // [END upload_directory] } -main(...process.argv.slice(2)).catch(console.error); +main(...process.argv.slice(2)); diff --git a/src/file.ts b/src/file.ts index 9d6cb0b8d..760aa8993 100644 --- a/src/file.ts +++ b/src/file.ts @@ -196,7 +196,6 @@ export type PredefinedAcl = export interface CreateResumableUploadOptions { chunkSize?: number; - configPath?: string; metadata?: Metadata; origin?: string; offset?: number; @@ -1545,8 +1544,6 @@ class File extends ServiceObject { */ /** * @typedef {object} CreateResumableUploadOptions - * @property {string} [configPath] A full JSON file path to use with - * `gcs-resumable-upload`. This maps to the {@link https://github.com/yeoman/configstore/tree/0df1ec950d952b1f0dfb39ce22af8e505dffc71a#configpath| configstore option by the same name}. * @property {object} [metadata] Metadata to set on the file. * @property {number} [offset] The starting byte of the upload stream for resuming an interrupted upload. * @property {string} [origin] Origin header to set for the upload. @@ -1648,7 +1645,6 @@ class File extends ServiceObject { authClient: this.storage.authClient, apiEndpoint: this.storage.apiEndpoint, bucket: this.bucket.name, - configPath: options.configPath, customRequestOptions: this.getRequestInterceptors().reduce( (reqOpts, interceptorFn) => interceptorFn(reqOpts), {} @@ -1674,9 +1670,6 @@ class File extends ServiceObject { /** * @typedef {object} CreateWriteStreamOptions Configuration options for File#createWriteStream(). - * @property {string} [configPath] **This only applies to resumable - * uploads.** A full JSON file path to use with `gcs-resumable-upload`. - * This maps to the {@link https://github.com/yeoman/configstore/tree/0df1ec950d952b1f0dfb39ce22af8e505dffc71a#configpath| configstore option by the same name}. * @property {string} [contentType] Alias for * `options.metadata.contentType`. If set to `auto`, the file name is used * to determine the contentType. @@ -1744,12 +1737,6 @@ class File extends ServiceObject { * Resumable uploads are automatically enabled and must be shut off explicitly * by setting `options.resumable` to `false`. * - * Resumable uploads require write access to the $HOME directory. Through - * {@link https://www.npmjs.com/package/configstore| `config-store`}, some metadata - * is stored. By default, if the directory is not writable, we will fall back - * to a simple upload. However, if you explicitly request a resumable upload, - * and we cannot write to the config directory, we will return a - * `ResumableUploadError`. * *

* There is some overhead when using a resumable upload that can cause @@ -1899,71 +1886,7 @@ class File extends ServiceObject { this.startSimpleUpload_(fileWriteStream, options); return; } - - if (options.configPath) { - this.startResumableUpload_(fileWriteStream, options); - return; - } - - // The logic below attempts to mimic the resumable upload library, - // gcs-resumable-upload. That library requires a writable configuration - // directory in order to work. If we wait for that library to discover any - // issues, we've already started a resumable upload which is difficult to back - // out of. We want to catch any errors first, so we can choose a simple, non- - // resumable upload instead. - - // Same as configstore (used by gcs-resumable-upload): - // https://github.com/yeoman/configstore/blob/f09f067e50e6a636cfc648a6fc36a522062bd49d/index.js#L11 - const configDir = xdgBasedir.config || os.tmpdir(); - - fs.access(configDir, fs.constants.W_OK, accessErr => { - if (!accessErr) { - // A configuration directory exists, and it's writable. gcs-resumable-upload - // should have everything it needs to work. - this.startResumableUpload_(fileWriteStream, options); - return; - } - - // The configuration directory is either not writable, or it doesn't exist. - // gcs-resumable-upload will attempt to create it for the user, but we'll try - // it now to confirm that it won't have any issues. That way, if we catch the - // issue before we start the resumable upload, we can instead start a simple - // upload. - fs.mkdir(configDir, {mode: 0o0700}, err => { - if (!err) { - // We successfully created a configuration directory that - // gcs-resumable-upload will use. - this.startResumableUpload_(fileWriteStream, options); - return; - } - - if (options.resumable) { - // The user wanted a resumable upload, but we couldn't create a - // configuration directory, which means gcs-resumable-upload will fail. - - // Determine if the issue is that the directory does not exist or - // if the directory exists, but is not writable. - const error = new ResumableUploadError( - [ - 'A resumable upload could not be performed. The directory,', - `${configDir}, is not writable. You may try another upload,`, - 'this time setting `options.resumable` to `false`.', - ].join(' ') - ); - fs.access(configDir, fs.constants.R_OK, noReadErr => { - if (noReadErr) { - error.additionalInfo = 'The directory does not exist.'; - } else { - error.additionalInfo = 'The directory is read-only.'; - } - stream.destroy(error); - }); - } else { - // The user didn't care, resumable or not. Fall back to simple upload. - this.startSimpleUpload_(fileWriteStream, options); - } - }); - }); + this.startResumableUpload_(fileWriteStream, options); }); fileWriteStream.on('response', stream.emit.bind(stream, 'response')); @@ -2030,50 +1953,6 @@ class File extends ServiceObject { return stream as Writable; } - /** - * Delete failed resumable upload file cache. - * - * Resumable file upload cache the config file to restart upload in case of - * failure. In certain scenarios, the resumable upload will not works and - * upload file cache needs to be deleted to upload the same file. - * - * Following are some of the scenarios. - * - * Resumable file upload failed even though the file is successfully saved - * on the google storage and need to clean up a resumable file cache to - * update the same file. - * - * Resumable file upload failed due to pre-condition - * (i.e generation number is not matched) and want to upload a same - * file with the new generation number. - * - * @example - * ``` - * const {Storage} = require('@google-cloud/storage'); - * const storage = new Storage(); - * const myBucket = storage.bucket('my-bucket'); - * - * const file = myBucket.file('my-file', { generation: 0 }); - * const contents = 'This is the contents of the file.'; - * - * file.save(contents, function(err) { - * if (err) { - * file.deleteResumableCache(); - * } - * }); - * - * ``` - */ - deleteResumableCache() { - const uploadStream = resumableUpload.upload({ - bucket: this.bucket.name, - file: this.name, - generation: this.generation, - retryOptions: this.storage.retryOptions, - }); - uploadStream.deleteConfig(); - } - download(options?: DownloadOptions): Promise; download(options: DownloadOptions, callback: DownloadCallback): void; download(callback: DownloadCallback): void; @@ -3971,7 +3850,6 @@ class File extends ServiceObject { authClient: this.storage.authClient, apiEndpoint: this.storage.apiEndpoint, bucket: this.bucket.name, - configPath: options.configPath, customRequestOptions: this.getRequestInterceptors().reduce( (reqOpts, interceptorFn) => interceptorFn(reqOpts), {} diff --git a/src/gcs-resumable-upload.ts b/src/gcs-resumable-upload.ts index d7b7bef30..c769fab00 100644 --- a/src/gcs-resumable-upload.ts +++ b/src/gcs-resumable-upload.ts @@ -13,7 +13,6 @@ // limitations under the License. import AbortController from 'abort-controller'; -import * as ConfigStore from 'configstore'; import {createHash} from 'crypto'; import * as extend from 'extend'; import { @@ -101,12 +100,6 @@ export interface UploadConfig { ) => Promise> | GaxiosPromise; }; - /** - * Where the gcs-resumable-upload configuration file should be stored on your - * system. This maps to the configstore option by the same name. - */ - configPath?: string; - /** * Create a separate request per chunk. * @@ -257,7 +250,6 @@ export class Upload extends Pumpify { uri?: string; userProject?: string; encryption?: Encryption; - configStore: ConfigStore; uriProvidedManually: boolean; numBytesWritten = 0; numRetries = 0; @@ -338,14 +330,9 @@ export class Upload extends Pumpify { if (cfg.private) this.predefinedAcl = 'private'; if (cfg.public) this.predefinedAcl = 'publicRead'; - const configPath = cfg.configPath; - this.configStore = new ConfigStore('gcs-resumable-upload', null, { - configPath, - }); - const autoRetry = cfg.retryOptions.autoRetry; this.uriProvidedManually = !!cfg.uri; - this.uri = cfg.uri || this.get('uri'); + this.uri = cfg.uri; this.numBytesWritten = 0; this.numRetries = 0; //counter for number of retries currently executed if (!autoRetry) { @@ -379,11 +366,10 @@ export class Upload extends Pumpify { if (this.uri) { this.continueUploading(); } else { - this.createURI((err, uri) => { + this.createURI(err => { if (err) { return this.destroy(err); } - this.set({uri}); this.startUploading(); return; }); @@ -650,15 +636,6 @@ export class Upload extends Pumpify { this.offset = 0; } - // Check if we're uploading the expected object - if (this.numBytesWritten === 0) { - const isSameObject = await this.ensureUploadingSameObject(); - if (!isSameObject) { - // `ensureUploadingSameObject` will restart the upload. - return; - } - } - // Check if the offset (server) is too far behind the current stream if (this.offset < this.numBytesWritten) { const delta = this.numBytesWritten - this.offset; @@ -818,7 +795,6 @@ export class Upload extends Pumpify { resp.data.size = Number(resp.data.size); } this.emit('metadata', resp.data); - this.deleteConfig(); // Allow the object (Upload) to continue naturally so the user's // "finish" event fires. @@ -826,49 +802,6 @@ export class Upload extends Pumpify { } } - /** - * Check if this is the same content uploaded previously. This caches a - * slice of the first chunk, then compares it with the first byte of - * incoming data. - * - * @returns if the request is ok to continue as-is - */ - private async ensureUploadingSameObject() { - // A queue for the upstream data - const upstreamQueue = this.upstreamIterator( - 16, - true // we just want one chunk for this validation - ); - - const upstreamChunk = await upstreamQueue.next(); - const chunk = upstreamChunk.value - ? upstreamChunk.value.chunk - : Buffer.alloc(0); - - // Put the original chunk back into the buffer as we just wanted to 'peek' - // at the stream for validation. - this.unshiftChunkBuffer(chunk); - - let cachedFirstChunk = this.get('firstChunk'); - const firstChunk = chunk.valueOf(); - - if (!cachedFirstChunk) { - // This is a new upload. Cache the first chunk. - this.set({uri: this.uri, firstChunk}); - } else { - // this continues an upload in progress. check if the bytes are the same - cachedFirstChunk = Buffer.from(cachedFirstChunk); - const nextChunk = Buffer.from(firstChunk); - if (Buffer.compare(cachedFirstChunk, nextChunk) !== 0) { - // this data is not the same. start a new upload - this.restart(); - return false; - } - } - - return true; - } - private async getAndSetOffset() { const opts: GaxiosOptions = { method: 'PUT', @@ -887,30 +820,6 @@ export class Upload extends Pumpify { this.offset = 0; } catch (e) { const err = e as GaxiosError; - const resp = err.response; - // we don't return a 404 to the user if they provided the resumable - // URI. if we're just using the configstore file to tell us that this - // file exists, and it turns out that it doesn't (the 404), that's - // probably stale config data. - if ( - resp && - resp.status === NOT_FOUND_STATUS_CODE && - !this.uriProvidedManually - ) { - this.restart(); - return; - } - - // this resumable upload is unrecoverable (bad data or service error). - // - - // https://github.com/googleapis/gcs-resumable-upload/issues/15 - // - - // https://github.com/googleapis/gcs-resumable-upload/pull/16#discussion_r80363774 - if (resp && resp.status === TERMINATED_UPLOAD_STATUS_CODE) { - this.restart(); - return; - } - this.destroy(err); } } @@ -986,31 +895,15 @@ export class Upload extends Pumpify { } this.lastChunkSent = Buffer.alloc(0); - this.deleteConfig(); - this.createURI((err, uri) => { + this.createURI(err => { if (err) { return this.destroy(err); } - this.set({uri}); this.startUploading(); return; }); } - private get(prop: string) { - const store = this.configStore.get(this.cacheKey); - return store && store[prop]; - } - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private set(props: any) { - this.configStore.set(this.cacheKey, props); - } - - deleteConfig() { - this.configStore.delete(this.cacheKey); - } - /** * @return {bool} is the request good? */ diff --git a/system-test/kitchen.ts b/system-test/kitchen.ts index 3a0182c0d..2e782013c 100644 --- a/system-test/kitchen.ts +++ b/system-test/kitchen.ts @@ -60,7 +60,7 @@ describe('gcs-resumable-upload', () => { bucket: bucketName, file: filePath, retryOptions: retryOptions, - }).deleteConfig(); + }); }); it('should work', done => { @@ -178,39 +178,4 @@ describe('gcs-resumable-upload', () => { done(); }); }); - - it('should set custom config file', done => { - const uploadOptions = { - bucket: bucketName, - file: filePath, - metadata: {contentType: 'image/jpg'}, - retryOptions: retryOptions, - configPath: path.join( - os.tmpdir(), - `test-gcs-resumable-${Date.now()}.json` - ), - }; - let uploadSucceeded = false; - - fs.createReadStream(filePath) - .on('error', done) - .pipe(upload(uploadOptions)) - .on('error', done) - .on('response', resp => { - uploadSucceeded = resp.status === 200; - }) - .on('finish', () => { - assert.strictEqual(uploadSucceeded, true); - - const configData = JSON.parse( - fs.readFileSync(uploadOptions.configPath, 'utf8') - ); - const keyName = `${uploadOptions.bucket}/${uploadOptions.file}`.replace( - path.extname(filePath), - '' - ); - assert.ok(Object.keys(configData).includes(keyName)); - done(); - }); - }); }); diff --git a/system-test/storage.ts b/system-test/storage.ts index e4c51d7a4..f5cdade45 100644 --- a/system-test/storage.ts +++ b/system-test/storage.ts @@ -2474,7 +2474,6 @@ describe('storage', () => { fs.stat(FILES.big.path, (err, metadata) => { assert.ifError(err); - // Use a random name to force an empty ConfigStore cache. const file = bucket.file(generateName()); const fileSize = metadata.size; upload({interrupt: true}, err => { diff --git a/test/file.ts b/test/file.ts index 9ad63e692..29b7aa96d 100644 --- a/test/file.ts +++ b/test/file.ts @@ -1771,7 +1771,6 @@ describe('File', () => { it('should create a resumable upload URI', done => { const options = { - configPath: '/Users/user/.config/here', metadata: { contentType: 'application/json', }, @@ -1806,7 +1805,6 @@ describe('File', () => { assert.strictEqual(opts.authClient, storage.authClient); assert.strictEqual(opts.apiEndpoint, storage.apiEndpoint); assert.strictEqual(opts.bucket, bucket.name); - assert.strictEqual(opts.configPath, options.configPath); assert.strictEqual(opts.file, file.name); assert.strictEqual(opts.generation, file.generation); assert.strictEqual(opts.key, file.encryptionKey); @@ -1856,7 +1854,6 @@ describe('File', () => { }, }); const options = { - configPath: '/Users/user/.config/here', metadata: { contentType: 'application/json', }, @@ -1887,7 +1884,6 @@ describe('File', () => { assert.strictEqual(opts.authClient, storage.authClient); assert.strictEqual(opts.apiEndpoint, storage.apiEndpoint); assert.strictEqual(opts.bucket, bucket.name); - assert.strictEqual(opts.configPath, options.configPath); assert.strictEqual(opts.file, file.name); assert.strictEqual(opts.generation, file.generation); assert.strictEqual(opts.key, file.encryptionKey); @@ -2026,21 +2022,6 @@ describe('File', () => { writable.write('data'); }); - it('should start a resumable upload if configPath is provided', done => { - const options = { - metadata: METADATA, - configPath: '/config/path.json', - }; - const writable = file.createWriteStream(options); - - file.startResumableUpload_ = (stream: {}, options_: {}) => { - assert.deepStrictEqual(options_, options); - done(); - }; - - writable.write('data'); - }); - it('should start a resumable upload if specified', done => { const options = { metadata: METADATA, @@ -2057,127 +2038,6 @@ describe('File', () => { writable.write('data'); }); - it('should check if xdg-basedir is writable', done => { - const fakeDir = 'fake-xdg-dir'; - - xdgConfigOverride = fakeDir; - - Object.assign(fakeFs, { - access(dir: {}) { - assert.strictEqual(dir, fakeDir); - done(); - }, - }); - - file.createWriteStream({resumable: true}).write('data'); - }); - - it('should fall back to checking tmpdir', done => { - const fakeDir = 'fake-tmp-dir'; - - xdgConfigOverride = false; - - fakeOs.tmpdir = () => { - return fakeDir; - }; - - Object.assign(fakeFs, { - access(dir: {}) { - assert.strictEqual(dir, fakeDir); - done(); - }, - }); - - file.createWriteStream({resumable: true}).write('data'); - }); - - describe('config directory does not exist', () => { - const CONFIG_DIR = path.join(os.tmpdir(), `/fake-xdg-dir/${Date.now()}`); - - beforeEach(() => { - xdgConfigOverride = CONFIG_DIR; - fakeFs.access = fsCached.access; - }); - - it('should attempt to create the config directory', done => { - Object.assign(fakeFs, { - mkdir(dir: string, options: {}) { - assert.strictEqual(dir, CONFIG_DIR); - assert.deepStrictEqual(options, {mode: 0o0700}); - done(); - }, - }); - - const writable = file.createWriteStream({resumable: true}); - writable.write('data'); - }); - - it('should start a resumable upload if config directory created successfully', done => { - Object.assign(fakeFs, { - mkdir(dir: string, options: {}, callback: Function) { - callback(); - }, - }); - - file.startResumableUpload_ = () => { - // If no error is thrown here, we know the request completed successfully. - done(); - }; - - file.createWriteStream().write('data'); - }); - - it('should return error if resumable was requested, but a config directory could not be created', done => { - Object.assign(fakeFs, { - mkdir(dir: string, options: {}, callback: Function) { - callback(new Error()); - }, - }); - - const writable = file.createWriteStream({resumable: true}); - - writable.on('error', (err: ResumableUploadError) => { - assert.strictEqual(err.name, 'ResumableUploadError'); - assert.strictEqual( - err.message, - [ - 'A resumable upload could not be performed. The directory,', - `${CONFIG_DIR}, is not writable. You may try another upload,`, - 'this time setting `options.resumable` to `false`.', - ].join(' ') - ); - assert.strictEqual( - err.additionalInfo, - 'The directory does not exist.' - ); - - done(); - }); - - writable.write('data'); - }); - - it('should fallback to a simple upload if the config directory could not be created', done => { - const options = { - metadata: METADATA, - customValue: true, - }; - - Object.assign(fakeFs, { - mkdir(dir: string, options: {}, callback: Function) { - callback(new Error()); - }, - }); - - file.startSimpleUpload_ = (stream: Stream, _options: {}) => { - assert.deepStrictEqual(_options, options); - done(); - }; - - file.createWriteStream(options).write('data'); - }); - }); - it('should default to a resumable upload', done => { const writable = file.createWriteStream({ metadata: METADATA, @@ -2572,30 +2432,6 @@ describe('File', () => { }); }); - describe('deleteResumableCache', () => { - it('should delete resumable file upload cache', done => { - file.generation = 123; - - resumableUploadOverride = { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - upload(opts: any) { - assert.strictEqual(opts.bucket, file.bucket.name); - assert.strictEqual(opts.file, file.name); - assert.strictEqual(opts.generation, file.generation); - assert.strictEqual(opts.retryOptions, file.storage.retryOptions); - assert.strictEqual(opts.params, file.preconditionOpts); - - return { - deleteConfig: () => { - done(); - }, - }; - }, - }; - file.deleteResumableCache(); - }); - }); - describe('download', () => { let fileReadStream: Readable; @@ -4776,7 +4612,6 @@ describe('File', () => { describe('starting', () => { it('should start a resumable upload', done => { const options = { - configPath: '/Users/user/.config/here', metadata: {}, offset: 1234, public: true, @@ -4819,7 +4654,6 @@ describe('File', () => { assert.strictEqual(opts.authClient, authClient); assert.strictEqual(opts.apiEndpoint, storage.apiEndpoint); assert.strictEqual(opts.bucket, bucket.name); - assert.strictEqual(opts.configPath, options.configPath); assert.deepStrictEqual(opts.customRequestOptions, { headers: { a: 'b', diff --git a/test/gcs-resumable-upload.ts b/test/gcs-resumable-upload.ts index 7ab580756..abfdad91e 100644 --- a/test/gcs-resumable-upload.ts +++ b/test/gcs-resumable-upload.ts @@ -46,23 +46,6 @@ class AbortController { } } -let configData = {} as {[index: string]: {}}; -class ConfigStore { - constructor(packageName: string, defaults: object, config: object) { - this.set('packageName', packageName); - this.set('config', config); - } - delete(key: string) { - delete configData[key]; - } - get(key: string) { - return configData[key]; - } - set(key: string, value: {}) { - configData[key] = value; - } -} - const RESUMABLE_INCOMPLETE_STATUS_CODE = 308; /** 256 KiB */ const CHUNK_SIZE_MULTIPLE = 2 ** 18; @@ -109,13 +92,11 @@ describe('gcs-resumable-upload', () => { before(() => { mockery.registerMock('abort-controller', {default: AbortController}); - mockery.registerMock('configstore', ConfigStore); mockery.enable({useCleanCache: true, warnOnUnregistered: false}); upload = require('../src/gcs-resumable-upload').upload; }); beforeEach(() => { - configData = {}; REQ_OPTS = {url: 'http://fake.local'}; up = upload({ bucket: BUCKET, @@ -312,22 +293,6 @@ describe('gcs-resumable-upload', () => { assert.strictEqual(up.predefinedAcl, 'private'); }); - it('should create a ConfigStore instance', () => { - assert.strictEqual(configData.packageName, 'gcs-resumable-upload'); - }); - - it('should set the configPath', () => { - const configPath = '/custom/config/path'; - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const up = upload({ - bucket: BUCKET, - file: FILE, - configPath, - retryOptions: RETRY_OPTIONS, - }); - assert.deepStrictEqual(configData.config, {configPath}); - }); - it('should set numBytesWritten to 0', () => { assert.strictEqual(up.numBytesWritten, 0); }); @@ -355,7 +320,7 @@ describe('gcs-resumable-upload', () => { assert.strictEqual(up.contentLength, '*'); }); - it('should localize the uri or get one from config', () => { + it('should localize the uri', () => { const uri = 'http://www.blah.com/'; const upWithUri = upload({ bucket: BUCKET, @@ -365,15 +330,6 @@ describe('gcs-resumable-upload', () => { }); assert.strictEqual(upWithUri.uriProvidedManually, true); assert.strictEqual(upWithUri.uri, uri); - - configData[`${BUCKET}/${FILE}`] = {uri: 'fake-uri'}; - const up = upload({ - bucket: BUCKET, - file: FILE, - retryOptions: RETRY_OPTIONS, - }); - assert.strictEqual(up.uriProvidedManually, false); - assert.strictEqual(up.uri, 'fake-uri'); }); it('should not have `chunkSize` by default', () => { @@ -488,18 +444,6 @@ describe('gcs-resumable-upload', () => { }; up.emit('writing'); }); - - it('should save the uri to config on first write event', done => { - const uri = 'http://newly-created-uri'; - up.createURI = (callback: CreateUriCallback) => { - callback(null, uri); - }; - up.set = (props: {}) => { - assert.deepStrictEqual(props, {uri}); - done(); - }; - up.emit('writing'); - }); }); }); @@ -1273,12 +1217,6 @@ describe('gcs-resumable-upload', () => { up.responseHandler(RESP); }); - it('should delete the config', done => { - const RESP = {data: '', status: 200}; - up.deleteConfig = done; - up.responseHandler(RESP); - }); - it('should emit `prepareFinish` when request succeeds', done => { const RESP = {data: '', status: 200}; up.once('prepareFinish', done); @@ -1355,91 +1293,6 @@ describe('gcs-resumable-upload', () => { }); }); - describe('#ensureUploadingSameObject', () => { - let chunk = Buffer.alloc(0); - - beforeEach(() => { - chunk = crypto.randomBytes(512); - up.upstreamChunkBuffer = chunk; - }); - - it('should not alter the chunk buffer', async () => { - await up.ensureUploadingSameObject(); - - assert.equal(Buffer.compare(up.upstreamChunkBuffer, chunk), 0); - }); - - describe('first write', () => { - it('should get the first chunk', async () => { - let calledGet = false; - up.get = (prop: string) => { - assert.strictEqual(prop, 'firstChunk'); - calledGet = true; - }; - - const result = await up.ensureUploadingSameObject(); - - assert(result); - assert(calledGet); - }); - - describe('new upload', () => { - it('should save the uri and first chunk (16 bytes) if its not cached', done => { - const URI = 'uri'; - up.uri = URI; - up.get = () => {}; - up.set = (props: {uri?: string; firstChunk: Buffer}) => { - const firstChunk = chunk.slice(0, 16); - assert.deepStrictEqual(props.uri, URI); - assert.strictEqual(Buffer.compare(props.firstChunk, firstChunk), 0); - done(); - }; - up.ensureUploadingSameObject(); - }); - }); - - describe('continued upload', () => { - beforeEach(() => { - up.restart = () => {}; - }); - - it('should not `#restart` and return `true` if cache is the same', async () => { - up.upstreamChunkBuffer = Buffer.alloc(512, 'a'); - up.get = (param: string) => { - return param === 'firstChunk' ? Buffer.alloc(16, 'a') : undefined; - }; - - let calledRestart = false; - up.restart = () => { - calledRestart = true; - }; - - const result = await up.ensureUploadingSameObject(); - - assert(result); - assert.equal(calledRestart, false); - }); - - it('should `#restart` and return `false` if different', async () => { - up.upstreamChunkBuffer = Buffer.alloc(512, 'a'); - up.get = (param: string) => { - return param === 'firstChunk' ? Buffer.alloc(16, 'b') : undefined; - }; - - let calledRestart = false; - up.restart = () => { - calledRestart = true; - }; - - const result = await up.ensureUploadingSameObject(); - - assert(calledRestart); - assert.equal(result, false); - }); - }); - }); - }); - describe('#getAndSetOffset', () => { const RANGE = 123456; const RESP = {status: 308, headers: {range: `range-${RANGE}`}}; @@ -1460,50 +1313,6 @@ describe('gcs-resumable-upload', () => { up.getAndSetOffset(); }); - describe('restart on 404', () => { - const RESP = {status: 404} as GaxiosResponse; - const ERROR = new Error(':(') as GaxiosError; - ERROR.response = RESP; - - beforeEach(() => { - up.makeRequest = async () => { - throw ERROR; - }; - }); - - it('should restart the upload', done => { - up.restart = done; - up.getAndSetOffset(); - }); - - it('should not restart if URI provided manually', done => { - up.uriProvidedManually = true; - up.restart = done; // will cause test to fail - up.on('error', (err: Error) => { - assert.strictEqual(err, ERROR); - done(); - }); - up.getAndSetOffset(); - }); - }); - - describe('restart on 410', () => { - const ERROR = new Error(':(') as GaxiosError; - const RESP = {status: 410} as GaxiosResponse; - ERROR.response = RESP; - - beforeEach(() => { - up.makeRequest = async () => { - throw ERROR; - }; - }); - - it('should restart the upload', done => { - up.restart = done; - up.getAndSetOffset(); - }); - }); - it('should set the offset from the range', async () => { up.makeRequest = async () => RESP; await up.getAndSetOffset(); @@ -1833,11 +1642,6 @@ describe('gcs-resumable-upload', () => { up.restart(); }); - it('should delete the config', done => { - up.deleteConfig = done; - up.restart(); - }); - describe('starting a new upload', () => { it('should create a new URI', done => { up.createURI = () => { @@ -1862,21 +1666,6 @@ describe('gcs-resumable-upload', () => { up.restart(); }); - it('should save the uri to config when restarting', done => { - const uri = 'http://newly-created-uri'; - - up.createURI = (callback: Function) => { - callback(null, uri); - }; - - up.set = (props: {}) => { - assert.deepStrictEqual(props, {uri}); - done(); - }; - - up.restart(); - }); - it('should start uploading', done => { up.createURI = (callback: Function) => { up.startUploading = done; @@ -1887,51 +1676,6 @@ describe('gcs-resumable-upload', () => { }); }); - describe('#get', () => { - it('should return the value from the config store', () => { - const prop = 'property'; - const value = 'abc'; - up.configStore = { - get(name: string) { - assert.strictEqual(name, up.cacheKey); - const obj: {[i: string]: string} = {}; - obj[prop] = value; - return obj; - }, - }; - assert.strictEqual(up.get(prop), value); - }); - }); - - describe('#set', () => { - it('should set the value to the config store', done => { - const props = {setting: true}; - up.configStore = { - set(name: string, prps: {}) { - assert.strictEqual(name, up.cacheKey); - assert.strictEqual(prps, props); - done(); - }, - }; - up.set(props); - }); - }); - - describe('#deleteConfig', () => { - it('should delete the entry from the config store', done => { - const props = {setting: true}; - - up.configStore = { - delete(name: string) { - assert.strictEqual(name, up.cacheKey); - done(); - }, - }; - - up.deleteConfig(props); - }); - }); - describe('#onResponse', () => { beforeEach(() => { up.numRetries = 0;