From 6efd387fe078870aff3a37b66e5ac628786b506e Mon Sep 17 00:00:00 2001 From: Sameena Shaffeeullah Date: Fri, 12 Aug 2022 18:08:13 +0000 Subject: [PATCH 01/10] started copy sample --- samples/copyFile.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/samples/copyFile.js b/samples/copyFile.js index d0118ee2d..94cda7b94 100644 --- a/samples/copyFile.js +++ b/samples/copyFile.js @@ -49,11 +49,22 @@ function main( const storage = new Storage(); async function copyFile() { + const copyDestination = storage.bucket(destBucketName).file(destFileName); + + // Optional: set a generation-match precondition to avoid potential race + // conditions and data corruptions. The request to upload is aborted if the + // object's generation number does not match your precondition. + // For a destination object that does not yet exist, set the ifGenerationMatch precondition + // to 0 + const copyOptions = { + ifGenerationMatch: copyDestination.metadata.generation + } + // Copies the file to the other bucket await storage .bucket(srcBucketName) .file(srcFilename) - .copy(storage.bucket(destBucketName).file(destFileName)); + .copy(copyDestination, copyOptions); console.log( `gs://${srcBucketName}/${srcFilename} copied to gs://${destBucketName}/${destFileName}` From 26f47dc23ecacc4b3c9f55a85460129be8dd49f4 Mon Sep 17 00:00:00 2001 From: Sameena Shaffeeullah Date: Thu, 18 Aug 2022 17:22:53 +0000 Subject: [PATCH 02/10] samples: add preconditions to storage.objects.rewrite & storage.objects.delete samples --- samples/changeFileCSEKToCMEK.js | 18 ++++++++++++++++-- samples/composeFile.js | 15 +++++++++++++-- samples/copyFile.js | 23 ++++++++++++++--------- samples/copyOldVersionOfFile.js | 19 +++++++++++++++++-- samples/deleteFile.js | 18 ++++++++++++++++-- samples/fileChangeStorageClass.js | 16 ++++++++++++++-- samples/moveFile.js | 21 +++++++++++++++++++-- samples/rotateEncryptionKey.js | 15 +++++++++++++-- samples/system-test/encryption.test.js | 10 +++++++--- samples/system-test/files.test.js | 10 ++++++---- 10 files changed, 135 insertions(+), 30 deletions(-) diff --git a/samples/changeFileCSEKToCMEK.js b/samples/changeFileCSEKToCMEK.js index 6e126a275..6cc70e5dd 100644 --- a/samples/changeFileCSEKToCMEK.js +++ b/samples/changeFileCSEKToCMEK.js @@ -23,7 +23,8 @@ function main( bucketName = 'my-bucket', fileName = 'test.txt', encryptionKey = 'my-encription-key', - kmsKeyName = 'my-kms-key' + kmsKeyName = 'my-kms-key', + generationMatchPrecondition = 0 ) { // [START storage_object_csek_to_cmek] /** @@ -49,13 +50,26 @@ function main( const storage = new Storage(); async function changeFileCSEKToCMEK() { + + const rotateEncryptionKeyOptions = { + kmsKeyName, + // Optional: set a generation-match precondition to avoid potential race + // conditions and data corruptions. The request to upload is aborted if the + // object's generation number does not match your precondition. + preconditionOpts: { + ifGenerationMatch: generationMatchPrecondition, + } + }; + + console.log(rotateEncryptionKeyOptions) + await storage .bucket(bucketName) .file(fileName, { encryptionKey: Buffer.from(encryptionKey, 'base64'), }) .rotateEncryptionKey({ - kmsKeyName, + rotateEncryptionKeyOptions, }); console.log( diff --git a/samples/composeFile.js b/samples/composeFile.js index e5ecdd572..3f1bbfb80 100644 --- a/samples/composeFile.js +++ b/samples/composeFile.js @@ -23,7 +23,8 @@ function main( bucketName = 'my-bucket', firstFileName = 'file-one.txt', secondFileName = 'file-two.txt', - destinationFileName = 'file-one-two.txt' + destinationFileName = 'file-one-two.txt', + generationMatchPrecondition = 0 ) { // [START storage_compose_file] /** @@ -51,7 +52,17 @@ function main( const bucket = storage.bucket(bucketName); const sources = [firstFileName, secondFileName]; - await bucket.combine(sources, destinationFileName); + // Optional: + // Set a generation-match precondition to avoid potential race conditions + // and data corruptions. The request to upload is aborted if the object's + // generation number does not match your precondition. For a destination + // object that does not yet exist, set the ifGenerationMatch precondition to 0 + // If the destination object already exists in your bucket, set instead a + // generation-match precondition using its generation number. + const combineOptions = { + ifGenerationMatch: generationMatchPrecondition, + }; + await bucket.combine(sources, destinationFileName, combineOptions); console.log( `New composite file ${destinationFileName} was created by combining ${firstFileName} and ${secondFileName}` diff --git a/samples/copyFile.js b/samples/copyFile.js index 94cda7b94..8bddb7921 100644 --- a/samples/copyFile.js +++ b/samples/copyFile.js @@ -24,7 +24,8 @@ function main( srcBucketName = 'my-bucket', srcFilename = 'test2.txt', destBucketName = 'my-bucket', - destFileName = 'test3.txt' + destFileName = 'test3.txt', + generationMatchPrecondition = 0 ) { // [START storage_copy_file] /** @@ -50,15 +51,19 @@ function main( async function copyFile() { const copyDestination = storage.bucket(destBucketName).file(destFileName); - - // Optional: set a generation-match precondition to avoid potential race - // conditions and data corruptions. The request to upload is aborted if the - // object's generation number does not match your precondition. - // For a destination object that does not yet exist, set the ifGenerationMatch precondition - // to 0 + + // Optional: + // Set a generation-match precondition to avoid potential race conditions + // and data corruptions. The request to upload is aborted if the object's + // generation number does not match your precondition. For a destination + // object that does not yet exist, set the ifGenerationMatch precondition to 0 + // If the destination object already exists in your bucket, set instead a + // generation-match precondition using its generation number. const copyOptions = { - ifGenerationMatch: copyDestination.metadata.generation - } + preconditionOpts: { + ifGenerationMatch: generationMatchPrecondition, + }, + }; // Copies the file to the other bucket await storage diff --git a/samples/copyOldVersionOfFile.js b/samples/copyOldVersionOfFile.js index d60dcb430..84f34b180 100644 --- a/samples/copyOldVersionOfFile.js +++ b/samples/copyOldVersionOfFile.js @@ -24,7 +24,8 @@ function main( srcFilename = 'test2.txt', destBucketName = 'my-bucket', destFileName = 'test3.txt', - generation = 1 + generation = 1, + generationMatchPrecondition = 0 ) { // [START storage_copy_file_archived_generation] /** @@ -53,12 +54,26 @@ function main( async function copyOldVersionOfFile() { // Copies the file to the other bucket + + // Optional: + // Set a generation-match precondition to avoid potential race conditions + // and data corruptions. The request to upload is aborted if the object's + // generation number does not match your precondition. For a destination + // object that does not yet exist, set the ifGenerationMatch precondition to 0 + // If the destination object already exists in your bucket, set instead a + // generation-match precondition using its generation number. + const copyOptions = { + preconditionOpts: { + ifGenerationMatch: generationMatchPrecondition, + }, + }; + await storage .bucket(srcBucketName) .file(srcFilename, { generation, }) - .copy(storage.bucket(destBucketName).file(destFileName)); + .copy(storage.bucket(destBucketName).file(destFileName), copyOptions); console.log( `Generation ${generation} of file ${srcFilename} in bucket ${srcBucketName} was copied to ${destFileName} in bucket ${destBucketName}` diff --git a/samples/deleteFile.js b/samples/deleteFile.js index f5ec0c526..37391dabf 100644 --- a/samples/deleteFile.js +++ b/samples/deleteFile.js @@ -20,7 +20,11 @@ * at https://cloud.google.com/storage/docs. */ -function main(bucketName = 'my-bucket', fileName = 'test.txt') { +function main( + bucketName = 'my-bucket', + fileName = 'test.txt', + generationMatchPrecondition = 0 +) { // [START storage_delete_file] /** * TODO(developer): Uncomment the following lines before running the sample. @@ -37,8 +41,18 @@ function main(bucketName = 'my-bucket', fileName = 'test.txt') { // Creates a client const storage = new Storage(); + // Optional: + // Set a generation-match precondition to avoid potential race conditions + // and data corruptions. The request to upload is aborted if the object's + // generation number does not match your precondition. For a destination + // object that does not yet exist, set the ifGenerationMatch precondition to 0 + // If the destination object already exists in your bucket, set instead a + // generation-match precondition using its generation number. + const deleteOptions = { + ifGenerationMatch: generationMatchPrecondition, + }; async function deleteFile() { - await storage.bucket(bucketName).file(fileName).delete(); + await storage.bucket(bucketName).file(fileName).delete(deleteOptions); console.log(`gs://${bucketName}/${fileName} deleted`); } diff --git a/samples/fileChangeStorageClass.js b/samples/fileChangeStorageClass.js index 3fe8c3dd2..c44a3d184 100644 --- a/samples/fileChangeStorageClass.js +++ b/samples/fileChangeStorageClass.js @@ -22,7 +22,8 @@ function main( bucketName = 'my-bucket', fileName = 'file.txt', - storageClass = 'standard' + storageClass = 'standard', + generationMatchPrecondition = 0 ) { // [START storage_change_file_storage_class] // Imports the Google Cloud client library @@ -46,10 +47,21 @@ function main( // const storageClass = 'coldline'; async function fileChangeStorageClass() { + // Optional: + // Set a generation-match precondition to avoid potential race conditions + // and data corruptions. The request to upload is aborted if the object's + // generation number does not match your precondition. For a destination + // object that does not yet exist, set the ifGenerationMatch precondition to 0 + // If the destination object already exists in your bucket, set instead a + // generation-match precondition using its generation number. + const setStorageClassOptions = { + ifGenerationMatch: generationMatchPrecondition, + }; + await storage .bucket(bucketName) .file(fileName) - .setStorageClass(storageClass); + .setStorageClass(storageClass, setStorageClassOptions); console.log(`${fileName} has been set to ${storageClass}`); } diff --git a/samples/moveFile.js b/samples/moveFile.js index 723cc043a..dcd9a7425 100644 --- a/samples/moveFile.js +++ b/samples/moveFile.js @@ -23,7 +23,8 @@ function main( bucketName = 'my-bucket', srcFileName = 'test.txt', - destFileName = 'test2.txt' + destFileName = 'test2.txt', + generationMatchPrecondition = 0 ) { // [START storage_move_file] /** @@ -45,8 +46,24 @@ function main( const storage = new Storage(); async function moveFile() { + // Optional: + // Set a generation-match precondition to avoid potential race conditions + // and data corruptions. The request to upload is aborted if the object's + // generation number does not match your precondition. For a destination + // object that does not yet exist, set the ifGenerationMatch precondition to 0 + // If the destination object already exists in your bucket, set instead a + // generation-match precondition using its generation number. + const moveOptions = { + preconditionOpts: { + ifGenerationMatch: generationMatchPrecondition, + }, + }; + // Moves the file within the bucket - await storage.bucket(bucketName).file(srcFileName).move(destFileName); + await storage + .bucket(bucketName) + .file(srcFileName) + .move(destFileName, moveOptions); console.log( `gs://${bucketName}/${srcFileName} moved to gs://${bucketName}/${destFileName}` diff --git a/samples/rotateEncryptionKey.js b/samples/rotateEncryptionKey.js index 1b81f4709..3399d9388 100644 --- a/samples/rotateEncryptionKey.js +++ b/samples/rotateEncryptionKey.js @@ -24,7 +24,8 @@ function main( bucketName = 'my-bucket', fileName = 'test.txt', oldKey = process.env.GOOGLE_CLOUD_KMS_KEY_US, - newKey = process.env.GOOGLE_CLOUD_KMS_KEY_ASIA + newKey = process.env.GOOGLE_CLOUD_KMS_KEY_ASIA, + generationMatchPrecondition = 0 ) { // [START storage_rotate_encryption_key] /** @@ -53,13 +54,23 @@ function main( const storage = new Storage(); async function rotateEncryptionKey() { + const rotateEncryptionKeyOptions = { + encryptionKey: Buffer.from(newKey, 'base64'), + + // Optional: set a generation-match precondition to avoid potential race + // conditions and data corruptions. The request to upload is aborted if the + // object's generation number does not match your precondition. + preconditionOpts: { + ifGenerationMatch: generationMatchPrecondition, + } + }; await storage .bucket(bucketName) .file(fileName, { encryptionKey: Buffer.from(oldKey, 'base64'), }) .rotateEncryptionKey({ - encryptionKey: Buffer.from(newKey, 'base64'), + rotateEncryptionKeyOptions }); console.log('Encryption key rotated successfully'); diff --git a/samples/system-test/encryption.test.js b/samples/system-test/encryption.test.js index 4a2f3a319..92d0c713e 100644 --- a/samples/system-test/encryption.test.js +++ b/samples/system-test/encryption.test.js @@ -77,11 +77,14 @@ it('should download a file', () => { fs.statSync(downloadFilePath); }); -it('should rotate keys', () => { +it('should rotate keys', async () => { const newKey = crypto.randomBytes(32).toString('base64'); + const [metadata] = await storage.bucket(bucketName).file(fileName).getMetadata(); const output = execSync( - `node rotateEncryptionKey.js ${bucketName} ${fileName} ${key} ${newKey}` + `node rotateEncryptionKey.js ${bucketName} ${fileName} ${key} ${newKey} ${metadata.generation}` ); + console.log(output); + console.log("sameena above") assert.include(output, 'Encryption key rotated successfully'); }); @@ -90,9 +93,10 @@ it('should convert CSEK to KMS key', async () => { const file = bucket.file(encryptedFileName, { encryptionKey: Buffer.from(key, 'base64'), }); + const [metadata] = await storage.bucket(bucketName).file(fileName).getMetadata(); await file.save('secret data', {resumable: false}); const output = execSync( - `node changeFileCSEKToCMEK.js ${bucketName} ${encryptedFileName} ${key} ${kmsKeyName}` + `node changeFileCSEKToCMEK.js ${bucketName} ${encryptedFileName} ${key} ${kmsKeyName} ${metadata.generation}` ); assert.include( output, diff --git a/samples/system-test/files.test.js b/samples/system-test/files.test.js index a509ef256..a616be50f 100644 --- a/samples/system-test/files.test.js +++ b/samples/system-test/files.test.js @@ -43,6 +43,7 @@ const folderPath = path.join(cwd, 'resources'); const downloadFilePath = path.join(cwd, 'downloaded.txt'); const startByte = 0; const endByte = 20; +const doesNotExistPrecondition = 0; const fileContent = fs.readFileSync(filePath, 'utf-8'); @@ -219,7 +220,7 @@ describe('file', () => { it('should move a file', async () => { const output = execSync( - `node moveFile.js ${bucketName} ${fileName} ${movedFileName}` + `node moveFile.js ${bucketName} ${fileName} ${movedFileName} ${doesNotExistPrecondition}` ); assert.include( output, @@ -231,7 +232,7 @@ describe('file', () => { it('should copy a file', async () => { const output = execSync( - `node copyFile.js ${bucketName} ${movedFileName} ${bucketName} ${copiedFileName}` + `node copyFile.js ${bucketName} ${movedFileName} ${bucketName} ${copiedFileName} ${doesNotExistPrecondition}` ); assert.include( output, @@ -438,7 +439,7 @@ describe('file', () => { it('should set storage class for a file', async () => { const output = execSync( - `node fileChangeStorageClass.js ${bucketName} ${copiedFileName} standard` + `node fileChangeStorageClass.js ${bucketName} ${copiedFileName} standard ${doesNotExistPrecondition}` ); assert.include(output, `${copiedFileName} has been set to standard`); const [metadata] = await storage @@ -477,8 +478,9 @@ describe('file', () => { }); it('should delete a file', async () => { + const [metadata] = await bucket.file(copiedFileName).getMetadata(); const output = execSync( - `node deleteFile.js ${bucketName} ${copiedFileName}` + `node deleteFile.js ${bucketName} ${copiedFileName} ${metadata.generation}` ); assert.match( output, From 49fa4d582b6903e5d8c3438b1df3982e6a3e24af Mon Sep 17 00:00:00 2001 From: Sameena Shaffeeullah Date: Thu, 18 Aug 2022 17:28:23 +0000 Subject: [PATCH 03/10] linted files --- samples/changeFileCSEKToCMEK.js | 5 ++--- samples/rotateEncryptionKey.js | 4 ++-- samples/system-test/encryption.test.js | 12 ++++++++---- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/samples/changeFileCSEKToCMEK.js b/samples/changeFileCSEKToCMEK.js index 6cc70e5dd..eb6745abd 100644 --- a/samples/changeFileCSEKToCMEK.js +++ b/samples/changeFileCSEKToCMEK.js @@ -50,7 +50,6 @@ function main( const storage = new Storage(); async function changeFileCSEKToCMEK() { - const rotateEncryptionKeyOptions = { kmsKeyName, // Optional: set a generation-match precondition to avoid potential race @@ -58,10 +57,10 @@ function main( // object's generation number does not match your precondition. preconditionOpts: { ifGenerationMatch: generationMatchPrecondition, - } + }, }; - console.log(rotateEncryptionKeyOptions) + console.log(rotateEncryptionKeyOptions); await storage .bucket(bucketName) diff --git a/samples/rotateEncryptionKey.js b/samples/rotateEncryptionKey.js index 3399d9388..3f0f174da 100644 --- a/samples/rotateEncryptionKey.js +++ b/samples/rotateEncryptionKey.js @@ -62,7 +62,7 @@ function main( // object's generation number does not match your precondition. preconditionOpts: { ifGenerationMatch: generationMatchPrecondition, - } + }, }; await storage .bucket(bucketName) @@ -70,7 +70,7 @@ function main( encryptionKey: Buffer.from(oldKey, 'base64'), }) .rotateEncryptionKey({ - rotateEncryptionKeyOptions + rotateEncryptionKeyOptions, }); console.log('Encryption key rotated successfully'); diff --git a/samples/system-test/encryption.test.js b/samples/system-test/encryption.test.js index 92d0c713e..125b22059 100644 --- a/samples/system-test/encryption.test.js +++ b/samples/system-test/encryption.test.js @@ -79,12 +79,13 @@ it('should download a file', () => { it('should rotate keys', async () => { const newKey = crypto.randomBytes(32).toString('base64'); - const [metadata] = await storage.bucket(bucketName).file(fileName).getMetadata(); + const [metadata] = await storage + .bucket(bucketName) + .file(fileName) + .getMetadata(); const output = execSync( `node rotateEncryptionKey.js ${bucketName} ${fileName} ${key} ${newKey} ${metadata.generation}` ); - console.log(output); - console.log("sameena above") assert.include(output, 'Encryption key rotated successfully'); }); @@ -93,7 +94,10 @@ it('should convert CSEK to KMS key', async () => { const file = bucket.file(encryptedFileName, { encryptionKey: Buffer.from(key, 'base64'), }); - const [metadata] = await storage.bucket(bucketName).file(fileName).getMetadata(); + const [metadata] = await storage + .bucket(bucketName) + .file(fileName) + .getMetadata(); await file.save('secret data', {resumable: false}); const output = execSync( `node changeFileCSEKToCMEK.js ${bucketName} ${encryptedFileName} ${key} ${kmsKeyName} ${metadata.generation}` From 5f1ed990c3b8aba1afcd3b05c128f3d7ad1251ee Mon Sep 17 00:00:00 2001 From: Sameena Shaffeeullah Date: Thu, 18 Aug 2022 18:01:51 +0000 Subject: [PATCH 04/10] samples: storage.objects.insert precondition samples --- samples/system-test/encryption.test.js | 3 ++- samples/system-test/files.test.js | 7 ++++--- samples/uploadEncryptedFile.js | 12 +++++++++++- samples/uploadFile.js | 18 ++++++++++++++---- samples/uploadFileWithKmsKey.js | 17 ++++++++++++++--- samples/uploadWithoutAuthentication.js | 22 +++++++++++++++++----- 6 files changed, 62 insertions(+), 17 deletions(-) diff --git a/samples/system-test/encryption.test.js b/samples/system-test/encryption.test.js index 4a2f3a319..40ad523d0 100644 --- a/samples/system-test/encryption.test.js +++ b/samples/system-test/encryption.test.js @@ -34,6 +34,7 @@ const kmsKeyName = process.env.GOOGLE_CLOUD_KMS_KEY_US; const fileName = 'test.txt'; const filePath = path.join(__dirname, '../resources', fileName); const downloadFilePath = path.join(__dirname, '../resources/downloaded.txt'); +const doesNotExistPrecondition = 0; const key = crypto.randomBytes(32).toString('base64'); @@ -56,7 +57,7 @@ it('should generate a key', () => { it('should upload a file', async () => { const output = execSync( - `node uploadEncryptedFile.js ${bucketName} ${filePath} ${fileName} ${key}` + `node uploadEncryptedFile.js ${bucketName} ${filePath} ${fileName} ${key} ${doesNotExistPrecondition}` ); assert.match( output, diff --git a/samples/system-test/files.test.js b/samples/system-test/files.test.js index a509ef256..fe8a7b320 100644 --- a/samples/system-test/files.test.js +++ b/samples/system-test/files.test.js @@ -43,6 +43,7 @@ const folderPath = path.join(cwd, 'resources'); const downloadFilePath = path.join(cwd, 'downloaded.txt'); const startByte = 0; const endByte = 20; +const doesNotExistPrecondition = 0; const fileContent = fs.readFileSync(filePath, 'utf-8'); @@ -61,7 +62,7 @@ describe('file', () => { it('should upload a file', async () => { const output = execSync( - `node uploadFile.js ${bucketName} ${filePath} ${fileName}` + `node uploadFile.js ${bucketName} ${filePath} ${fileName} ${doesNotExistPrecondition}` ); assert.match(output, new RegExp(`${filePath} uploaded to ${bucketName}`)); const [exists] = await bucket.file(fileName).exists(); @@ -84,7 +85,7 @@ describe('file', () => { it('should upload a file without authentication', async () => { const output = execSync( - `node uploadWithoutAuthentication.js ${bucketName} ${fileContents} ${fileName}` + `node uploadWithoutAuthentication.js ${bucketName} ${fileContents} ${fileName} ${doesNotExistPrecondition}` ); assert.match(output, new RegExp(`${fileName} uploaded to ${bucketName}`)); const [exists] = await bucket.file(fileName).exists(); @@ -111,7 +112,7 @@ describe('file', () => { it('should upload a file with a kms key', async () => { const output = execSync( - `node uploadFileWithKmsKey.js ${bucketName} ${filePath} ${kmsKeyName}` + `node uploadFileWithKmsKey.js ${bucketName} ${filePath} ${kmsKeyName} ${doesNotExistPrecondition}` ); assert.include( output, diff --git a/samples/uploadEncryptedFile.js b/samples/uploadEncryptedFile.js index 10e25c3e8..8dd156cc0 100644 --- a/samples/uploadEncryptedFile.js +++ b/samples/uploadEncryptedFile.js @@ -17,7 +17,8 @@ function main( bucketName = 'my-bucket', filePath = path.join(__dirname, '../resources', 'test.txt'), destFileName = 'test.txt', - key = process.env.GOOGLE_CLOUD_KMS_KEY_US + key = process.env.GOOGLE_CLOUD_KMS_KEY_US, + generationMatchPrecondition = 0 ) { // [START storage_upload_encrypted_file] /** @@ -45,6 +46,15 @@ function main( const options = { destination: destFileName, encryptionKey: Buffer.from(key, 'base64'), + + // Optional: + // Set a generation-match precondition to avoid potential race conditions + // and data corruptions. The request to upload is aborted if the object's + // generation number does not match your precondition. For a destination + // object that does not yet exist, set the ifGenerationMatch precondition to 0 + // If the destination object already exists in your bucket, set instead a + // generation-match precondition using its generation number. + preconditionOpts: {ifGenerationMatch: generationMatchPrecondition}, }; await storage.bucket(bucketName).upload(filePath, options); diff --git a/samples/uploadFile.js b/samples/uploadFile.js index 5cb56e8e8..1538e4ff1 100644 --- a/samples/uploadFile.js +++ b/samples/uploadFile.js @@ -15,7 +15,8 @@ function main( bucketName = 'my-bucket', filePath = './local/path/to/file.txt', - destFileName = 'file.txt' + destFileName = 'file.txt', + generationMatchPrecondition = 0 ) { // [START storage_upload_file] /** @@ -37,10 +38,19 @@ function main( const storage = new Storage(); async function uploadFile() { - await storage.bucket(bucketName).upload(filePath, { + const options = { destination: destFileName, - }); - + // Optional: + // Set a generation-match precondition to avoid potential race conditions + // and data corruptions. The request to upload is aborted if the object's + // generation number does not match your precondition. For a destination + // object that does not yet exist, set the ifGenerationMatch precondition to 0 + // If the destination object already exists in your bucket, set instead a + // generation-match precondition using its generation number. + preconditionOpts: {ifGenerationMatch: generationMatchPrecondition}, + }; + + await storage.bucket(bucketName).upload(filePath, options); console.log(`${filePath} uploaded to ${bucketName}`); } diff --git a/samples/uploadFileWithKmsKey.js b/samples/uploadFileWithKmsKey.js index 8e2de5fd1..771638abc 100644 --- a/samples/uploadFileWithKmsKey.js +++ b/samples/uploadFileWithKmsKey.js @@ -23,7 +23,8 @@ function main( bucketName = 'my-bucket', filePath = 'test.txt', - kmsKeyName = process.env.GOOGLE_CLOUD_KMS_KEY_US + kmsKeyName = process.env.GOOGLE_CLOUD_KMS_KEY_US, + generationMatchPrecondition = 0 ) { // [START storage_upload_with_kms_key] /** @@ -45,9 +46,19 @@ function main( const storage = new Storage(); async function uploadFileWithKmsKey() { - await storage.bucket(bucketName).upload(filePath, { + const options = { kmsKeyName, - }); + // Optional: + // Set a generation-match precondition to avoid potential race conditions + // and data corruptions. The request to upload is aborted if the object's + // generation number does not match your precondition. For a destination + // object that does not yet exist, set the ifGenerationMatch precondition to 0 + // If the destination object already exists in your bucket, set instead a + // generation-match precondition using its generation number. + preconditionOpts: {ifGenerationMatch: generationMatchPrecondition}, + }; + + await storage.bucket(bucketName).upload(filePath, options); console.log(`${filePath} uploaded to ${bucketName} using ${kmsKeyName}.`); } diff --git a/samples/uploadWithoutAuthentication.js b/samples/uploadWithoutAuthentication.js index 12922c4a3..f51e56940 100644 --- a/samples/uploadWithoutAuthentication.js +++ b/samples/uploadWithoutAuthentication.js @@ -15,7 +15,8 @@ function main( bucketName = 'my-bucket', contents = 'these are my file contents', - destFileName = 'file.txt' + destFileName = 'file.txt', + generationMatchPrecondition = 0 ) { // [START storage_upload_without_authentication] /** @@ -43,13 +44,24 @@ function main( // you can make requests without credentials. const [location] = await file.createResumableUpload(); //auth required - // Passes the location to file.save so you don't need to - // authenticate this call - await file.save(contents, { + const options = { uri: location, resumable: true, validation: false, - }); + + // Optional: + // Set a generation-match precondition to avoid potential race conditions + // and data corruptions. The request to upload is aborted if the object's + // generation number does not match your precondition. For a destination + // object that does not yet exist, set the ifGenerationMatch precondition to 0 + // If the destination object already exists in your bucket, set instead a + // generation-match precondition using its generation number. + preconditionOpts: {ifGenerationMatch: generationMatchPrecondition}, + }; + + // Passes the location to file.save so you don't need to + // authenticate this call + await file.save(contents, options); console.log(`${destFileName} uploaded to ${bucketName}`); } From 7f3b330c29f6d54f101ecd86f69fa334681cd87e Mon Sep 17 00:00:00 2001 From: Sameena Shaffeeullah Date: Thu, 18 Aug 2022 18:14:10 +0000 Subject: [PATCH 05/10] updated test --- samples/system-test/files.test.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/system-test/files.test.js b/samples/system-test/files.test.js index fe8a7b320..56214dd55 100644 --- a/samples/system-test/files.test.js +++ b/samples/system-test/files.test.js @@ -111,8 +111,9 @@ describe('file', () => { }); it('should upload a file with a kms key', async () => { + const [metadata] = await bucket.file(fileName).getMetadata(); const output = execSync( - `node uploadFileWithKmsKey.js ${bucketName} ${filePath} ${kmsKeyName} ${doesNotExistPrecondition}` + `node uploadFileWithKmsKey.js ${bucketName} ${filePath} ${kmsKeyName} ${metadata.generation}` ); assert.include( output, From ca1b4b783a021163be698748c43b03b4fb9c2255 Mon Sep 17 00:00:00 2001 From: Sameena Shaffeeullah Date: Thu, 18 Aug 2022 11:25:45 -0700 Subject: [PATCH 06/10] samples: storage.objects.insert precondition samples (#2047) * test: add retries (#2039) * refactor: remove unused `restart` private method (#2038) Co-authored-by: Sameena Shaffeeullah * fix: Retry `EPIPE` Connection Errors + Attempt Retries in Resumable Upload Connection Errors (#2040) * feat: Add `epipe` as retryable error * fix: capture and retry potential connection errors * test: Add tests and remove logs * test: set `retryOptions` by copy rather than reference * fix: grammar * chore(main): release 6.4.1 (#2036) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> * test: add delay to bucket tests to reduce rate limiting errors (#2043) * samples: storage.objects.insert precondition samples * updated test Co-authored-by: Daniel Bankhead Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> Co-authored-by: Denis DelGrosso <85250797+ddelgrosso1@users.noreply.github.com> --- CHANGELOG.md | 8 +++ package.json | 2 +- samples/package.json | 2 +- samples/system-test/encryption.test.js | 3 +- samples/system-test/files.test.js | 7 +- samples/uploadEncryptedFile.js | 12 +++- samples/uploadFile.js | 18 ++++-- samples/uploadFileWithKmsKey.js | 17 ++++- samples/uploadWithoutAuthentication.js | 22 +++++-- src/resumable-upload.ts | 47 +++++++------- src/storage.ts | 9 +-- system-test/storage.ts | 15 ++++- test/index.ts | 14 ++++ test/resumable-upload.ts | 89 ++++++++++---------------- 14 files changed, 160 insertions(+), 105 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35b1f48e8..911c4bab4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ [1]: https://www.npmjs.com/package/@google-cloud/storage?activeTab=versions +## [6.4.1](https://github.com/googleapis/nodejs-storage/compare/v6.4.0...v6.4.1) (2022-08-12) + + +### Bug Fixes + +* Remove `pumpify` ([#2029](https://github.com/googleapis/nodejs-storage/issues/2029)) ([edc1d64](https://github.com/googleapis/nodejs-storage/commit/edc1d64069a6038c301c3b775f116fbf69b10b28)) +* Retry `EPIPE` Connection Errors + Attempt Retries in Resumable Upload Connection Errors ([#2040](https://github.com/googleapis/nodejs-storage/issues/2040)) ([ba35321](https://github.com/googleapis/nodejs-storage/commit/ba35321c3fef9a1874ff8fbea43885e2e7746b4f)) + ## [6.4.0](https://github.com/googleapis/nodejs-storage/compare/v6.3.0...v6.4.0) (2022-08-10) diff --git a/package.json b/package.json index 626823622..6619818c2 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@google-cloud/storage", "description": "Cloud Storage Client Library for Node.js", - "version": "6.4.0", + "version": "6.4.1", "license": "Apache-2.0", "author": "Google Inc.", "engines": { diff --git a/samples/package.json b/samples/package.json index bcb163130..70ebac861 100644 --- a/samples/package.json +++ b/samples/package.json @@ -17,7 +17,7 @@ }, "dependencies": { "@google-cloud/pubsub": "^3.0.0", - "@google-cloud/storage": "^6.4.0", + "@google-cloud/storage": "^6.4.1", "node-fetch": "^2.6.7", "uuid": "^8.0.0", "yargs": "^16.0.0" diff --git a/samples/system-test/encryption.test.js b/samples/system-test/encryption.test.js index 125b22059..e970df61e 100644 --- a/samples/system-test/encryption.test.js +++ b/samples/system-test/encryption.test.js @@ -34,6 +34,7 @@ const kmsKeyName = process.env.GOOGLE_CLOUD_KMS_KEY_US; const fileName = 'test.txt'; const filePath = path.join(__dirname, '../resources', fileName); const downloadFilePath = path.join(__dirname, '../resources/downloaded.txt'); +const doesNotExistPrecondition = 0; const key = crypto.randomBytes(32).toString('base64'); @@ -56,7 +57,7 @@ it('should generate a key', () => { it('should upload a file', async () => { const output = execSync( - `node uploadEncryptedFile.js ${bucketName} ${filePath} ${fileName} ${key}` + `node uploadEncryptedFile.js ${bucketName} ${filePath} ${fileName} ${key} ${doesNotExistPrecondition}` ); assert.match( output, diff --git a/samples/system-test/files.test.js b/samples/system-test/files.test.js index a616be50f..f783eeac2 100644 --- a/samples/system-test/files.test.js +++ b/samples/system-test/files.test.js @@ -62,7 +62,7 @@ describe('file', () => { it('should upload a file', async () => { const output = execSync( - `node uploadFile.js ${bucketName} ${filePath} ${fileName}` + `node uploadFile.js ${bucketName} ${filePath} ${fileName} ${doesNotExistPrecondition}` ); assert.match(output, new RegExp(`${filePath} uploaded to ${bucketName}`)); const [exists] = await bucket.file(fileName).exists(); @@ -85,7 +85,7 @@ describe('file', () => { it('should upload a file without authentication', async () => { const output = execSync( - `node uploadWithoutAuthentication.js ${bucketName} ${fileContents} ${fileName}` + `node uploadWithoutAuthentication.js ${bucketName} ${fileContents} ${fileName} ${doesNotExistPrecondition}` ); assert.match(output, new RegExp(`${fileName} uploaded to ${bucketName}`)); const [exists] = await bucket.file(fileName).exists(); @@ -111,8 +111,9 @@ describe('file', () => { }); it('should upload a file with a kms key', async () => { + const [metadata] = await bucket.file(fileName).getMetadata(); const output = execSync( - `node uploadFileWithKmsKey.js ${bucketName} ${filePath} ${kmsKeyName}` + `node uploadFileWithKmsKey.js ${bucketName} ${filePath} ${kmsKeyName} ${metadata.generation}` ); assert.include( output, diff --git a/samples/uploadEncryptedFile.js b/samples/uploadEncryptedFile.js index 10e25c3e8..8dd156cc0 100644 --- a/samples/uploadEncryptedFile.js +++ b/samples/uploadEncryptedFile.js @@ -17,7 +17,8 @@ function main( bucketName = 'my-bucket', filePath = path.join(__dirname, '../resources', 'test.txt'), destFileName = 'test.txt', - key = process.env.GOOGLE_CLOUD_KMS_KEY_US + key = process.env.GOOGLE_CLOUD_KMS_KEY_US, + generationMatchPrecondition = 0 ) { // [START storage_upload_encrypted_file] /** @@ -45,6 +46,15 @@ function main( const options = { destination: destFileName, encryptionKey: Buffer.from(key, 'base64'), + + // Optional: + // Set a generation-match precondition to avoid potential race conditions + // and data corruptions. The request to upload is aborted if the object's + // generation number does not match your precondition. For a destination + // object that does not yet exist, set the ifGenerationMatch precondition to 0 + // If the destination object already exists in your bucket, set instead a + // generation-match precondition using its generation number. + preconditionOpts: {ifGenerationMatch: generationMatchPrecondition}, }; await storage.bucket(bucketName).upload(filePath, options); diff --git a/samples/uploadFile.js b/samples/uploadFile.js index 5cb56e8e8..1538e4ff1 100644 --- a/samples/uploadFile.js +++ b/samples/uploadFile.js @@ -15,7 +15,8 @@ function main( bucketName = 'my-bucket', filePath = './local/path/to/file.txt', - destFileName = 'file.txt' + destFileName = 'file.txt', + generationMatchPrecondition = 0 ) { // [START storage_upload_file] /** @@ -37,10 +38,19 @@ function main( const storage = new Storage(); async function uploadFile() { - await storage.bucket(bucketName).upload(filePath, { + const options = { destination: destFileName, - }); - + // Optional: + // Set a generation-match precondition to avoid potential race conditions + // and data corruptions. The request to upload is aborted if the object's + // generation number does not match your precondition. For a destination + // object that does not yet exist, set the ifGenerationMatch precondition to 0 + // If the destination object already exists in your bucket, set instead a + // generation-match precondition using its generation number. + preconditionOpts: {ifGenerationMatch: generationMatchPrecondition}, + }; + + await storage.bucket(bucketName).upload(filePath, options); console.log(`${filePath} uploaded to ${bucketName}`); } diff --git a/samples/uploadFileWithKmsKey.js b/samples/uploadFileWithKmsKey.js index 8e2de5fd1..771638abc 100644 --- a/samples/uploadFileWithKmsKey.js +++ b/samples/uploadFileWithKmsKey.js @@ -23,7 +23,8 @@ function main( bucketName = 'my-bucket', filePath = 'test.txt', - kmsKeyName = process.env.GOOGLE_CLOUD_KMS_KEY_US + kmsKeyName = process.env.GOOGLE_CLOUD_KMS_KEY_US, + generationMatchPrecondition = 0 ) { // [START storage_upload_with_kms_key] /** @@ -45,9 +46,19 @@ function main( const storage = new Storage(); async function uploadFileWithKmsKey() { - await storage.bucket(bucketName).upload(filePath, { + const options = { kmsKeyName, - }); + // Optional: + // Set a generation-match precondition to avoid potential race conditions + // and data corruptions. The request to upload is aborted if the object's + // generation number does not match your precondition. For a destination + // object that does not yet exist, set the ifGenerationMatch precondition to 0 + // If the destination object already exists in your bucket, set instead a + // generation-match precondition using its generation number. + preconditionOpts: {ifGenerationMatch: generationMatchPrecondition}, + }; + + await storage.bucket(bucketName).upload(filePath, options); console.log(`${filePath} uploaded to ${bucketName} using ${kmsKeyName}.`); } diff --git a/samples/uploadWithoutAuthentication.js b/samples/uploadWithoutAuthentication.js index 12922c4a3..f51e56940 100644 --- a/samples/uploadWithoutAuthentication.js +++ b/samples/uploadWithoutAuthentication.js @@ -15,7 +15,8 @@ function main( bucketName = 'my-bucket', contents = 'these are my file contents', - destFileName = 'file.txt' + destFileName = 'file.txt', + generationMatchPrecondition = 0 ) { // [START storage_upload_without_authentication] /** @@ -43,13 +44,24 @@ function main( // you can make requests without credentials. const [location] = await file.createResumableUpload(); //auth required - // Passes the location to file.save so you don't need to - // authenticate this call - await file.save(contents, { + const options = { uri: location, resumable: true, validation: false, - }); + + // Optional: + // Set a generation-match precondition to avoid potential race conditions + // and data corruptions. The request to upload is aborted if the object's + // generation number does not match your precondition. For a destination + // object that does not yet exist, set the ifGenerationMatch precondition to 0 + // If the destination object already exists in your bucket, set instead a + // generation-match precondition using its generation number. + preconditionOpts: {ifGenerationMatch: generationMatchPrecondition}, + }; + + // Passes the location to file.save so you don't need to + // authenticate this call + await file.save(contents, options); console.log(`${destFileName} uploaded to ${bucketName}`); } diff --git a/src/resumable-upload.ts b/src/resumable-upload.ts index 0a3e6809b..ba2918632 100644 --- a/src/resumable-upload.ts +++ b/src/resumable-upload.ts @@ -742,9 +742,18 @@ export class Upload extends Writable { responseReceived = true; this.responseHandler(resp); } - } catch (err) { - const e = err as Error; - this.destroy(e); + } catch (e) { + const err = e as ApiError; + + if (this.retryOptions.retryableErrorFn!(err)) { + this.attemptDelayedRetry({ + status: NaN, + data: err, + }); + return; + } + + this.destroy(err); } } @@ -833,7 +842,16 @@ export class Upload extends Writable { } this.offset = 0; } catch (e) { - const err = e as GaxiosError; + const err = e as ApiError; + + if (this.retryOptions.retryableErrorFn!(err)) { + this.attemptDelayedRetry({ + status: NaN, + data: err, + }); + return; + } + this.destroy(err); } } @@ -899,25 +917,6 @@ export class Upload extends Writable { return successfulRequest ? res : null; } - private restart() { - if (this.numBytesWritten) { - const message = - 'Attempting to restart an upload after unrecoverable bytes have been written from upstream. Stopping as this could result in data loss. Initiate a new upload to continue.'; - - this.emit('error', new RangeError(message)); - return; - } - - this.lastChunkSent = Buffer.alloc(0); - this.createURI(err => { - if (err) { - return this.destroy(err); - } - this.startUploading(); - return; - }); - } - /** * @return {bool} is the request good? */ @@ -941,7 +940,7 @@ export class Upload extends Writable { /** * @param resp GaxiosResponse object from previous attempt */ - private attemptDelayedRetry(resp: GaxiosResponse) { + private attemptDelayedRetry(resp: Pick) { if (this.numRetries < this.retryOptions.maxRetries!) { if ( resp.status === NOT_FOUND_STATUS_CODE && diff --git a/src/storage.ts b/src/storage.ts index 5268cdd7b..a40d1b1bb 100644 --- a/src/storage.ts +++ b/src/storage.ts @@ -261,11 +261,12 @@ const IDEMPOTENCY_STRATEGY_DEFAULT = IdempotencyStrategy.RetryConditional; * @return {boolean} True if the API request should be retried, false otherwise. */ export const RETRYABLE_ERR_FN_DEFAULT = function (err?: ApiError) { - const isConnectionProblem = (reason: string | undefined) => { + const isConnectionProblem = (reason: string) => { return ( - (reason && reason.includes('eai_again')) || //DNS lookup error + reason.includes('eai_again') || // DNS lookup error reason === 'econnreset' || - reason === 'unexpected connection closure' + reason === 'unexpected connection closure' || + reason === 'epipe' ); }; @@ -284,7 +285,7 @@ export const RETRYABLE_ERR_FN_DEFAULT = function (err?: ApiError) { if (err.errors) { for (const e of err.errors) { const reason = e?.reason?.toString().toLowerCase(); - if (isConnectionProblem(reason)) { + if (reason && isConnectionProblem(reason)) { return true; } } diff --git a/system-test/storage.ts b/system-test/storage.ts index fb499aa88..c7614368e 100644 --- a/system-test/storage.ts +++ b/system-test/storage.ts @@ -71,7 +71,9 @@ describe('storage', () => { const RETENTION_DURATION_SECONDS = 10; const storage = new Storage({ - retryOptions: {idempotencyStrategy: IdempotencyStrategy.RetryAlways}, + retryOptions: { + idempotencyStrategy: IdempotencyStrategy.RetryAlways, + }, }); const bucket = storage.bucket(generateName()); @@ -152,7 +154,10 @@ describe('storage', () => { // eslint-disable-next-line @typescript-eslint/no-var-requires const {Storage} = require('../src'); storageWithoutAuth = new Storage({ - retryOptions: {idempotencyStrategy: IdempotencyStrategy.RetryAlways}, + retryOptions: { + idempotencyStrategy: IdempotencyStrategy.RetryAlways, + retryDelayMultiplier: 3, + }, }); }); @@ -219,6 +224,12 @@ describe('storage', () => { describe('acls', () => { describe('buckets', () => { + // Some bucket update operations have a rate limit. + // Introduce a delay between tests to avoid getting an error. + beforeEach(done => { + setTimeout(done, 1000); + }); + it('should get access controls', async () => { const accessControls = await bucket.acl.get(); assert(Array.isArray(accessControls)); diff --git a/test/index.ts b/test/index.ts index 8283b41e6..56d08e3db 100644 --- a/test/index.ts +++ b/test/index.ts @@ -311,6 +311,20 @@ describe('Storage', () => { assert.strictEqual(calledWith.retryOptions.retryableErrorFn(error), true); }); + it('should retry a broken pipe error', () => { + const storage = new Storage({ + projectId: PROJECT_ID, + }); + const calledWith = storage.calledWith_[0]; + const error = new ApiError('Broken pipe'); + error.errors = [ + { + reason: 'EPIPE', + }, + ]; + assert.strictEqual(calledWith.retryOptions.retryableErrorFn(error), true); + }); + it('should not retry a 999 error', () => { const storage = new Storage({ projectId: PROJECT_ID, diff --git a/test/resumable-upload.ts b/test/resumable-upload.ts index bfc2bd618..9ab05933e 100644 --- a/test/resumable-upload.ts +++ b/test/resumable-upload.ts @@ -112,7 +112,7 @@ describe('resumable-upload', () => { userProject: USER_PROJECT, authConfig: {keyFile}, apiEndpoint: API_ENDPOINT, - retryOptions: RETRY_OPTIONS, + retryOptions: {...RETRY_OPTIONS}, }); }); @@ -1028,6 +1028,22 @@ describe('resumable-upload', () => { up.startUploading(); }); + it('should retry retryable errors if the request failed', done => { + const error = new Error('Error.'); + + // mock as retryable + up.retryOptions.retryableErrorFn = () => true; + + up.on('error', done); + up.attemptDelayedRetry = () => done(); + + up.makeRequestStream = async () => { + throw error; + }; + + up.startUploading(); + }); + describe('request preparation', () => { // a convenient handle for getting the request options let reqOpts: GaxiosOptions; @@ -1384,6 +1400,22 @@ describe('resumable-upload', () => { await up.getAndSetOffset(); assert.strictEqual(up.offset, 0); }); + + it('should retry retryable errors if the request failed', done => { + const error = new Error('Error.'); + + // mock as retryable + up.retryOptions.retryableErrorFn = () => true; + + up.on('error', done); + up.attemptDelayedRetry = () => done(); + + up.makeRequest = async () => { + throw error; + }; + + up.getAndSetOffset(); + }); }); describe('#makeRequest', () => { @@ -1679,61 +1711,6 @@ describe('resumable-upload', () => { }); }); - describe('#restart', () => { - beforeEach(() => { - up.createURI = () => {}; - }); - - it('should throw if `numBytesWritten` is not 0', done => { - up.numBytesWritten = 8; - - up.on('error', (error: Error) => { - assert(error instanceof RangeError); - assert( - /Attempting to restart an upload after unrecoverable bytes have been written/.test( - error.message - ) - ); - done(); - }); - - up.restart(); - }); - - describe('starting a new upload', () => { - it('should create a new URI', done => { - up.createURI = () => { - done(); - }; - - up.restart(); - }); - - it('should destroy stream if it cannot create a URI', done => { - const error = new Error(':('); - - up.createURI = (callback: Function) => { - callback(error); - }; - - up.destroy = (err: Error) => { - assert.strictEqual(err, error); - done(); - }; - - up.restart(); - }); - - it('should start uploading', done => { - up.createURI = (callback: Function) => { - up.startUploading = done; - callback(); - }; - up.restart(); - }); - }); - }); - describe('#onResponse', () => { beforeEach(() => { up.numRetries = 0; From 490ba27f38f950f614a9d8cac1586f509dd1491d Mon Sep 17 00:00:00 2001 From: Sameena Shaffeeullah Date: Thu, 18 Aug 2022 11:27:16 -0700 Subject: [PATCH 07/10] Revert "samples: storage.objects.insert precondition samples (#2047)" (#2048) This reverts commit ca1b4b783a021163be698748c43b03b4fb9c2255. --- CHANGELOG.md | 8 --- package.json | 2 +- samples/package.json | 2 +- samples/system-test/encryption.test.js | 3 +- samples/system-test/files.test.js | 7 +- samples/uploadEncryptedFile.js | 12 +--- samples/uploadFile.js | 18 ++---- samples/uploadFileWithKmsKey.js | 17 +---- samples/uploadWithoutAuthentication.js | 22 ++----- src/resumable-upload.ts | 47 +++++++------- src/storage.ts | 9 ++- system-test/storage.ts | 15 +---- test/index.ts | 14 ---- test/resumable-upload.ts | 89 ++++++++++++++++---------- 14 files changed, 105 insertions(+), 160 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 911c4bab4..35b1f48e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,14 +4,6 @@ [1]: https://www.npmjs.com/package/@google-cloud/storage?activeTab=versions -## [6.4.1](https://github.com/googleapis/nodejs-storage/compare/v6.4.0...v6.4.1) (2022-08-12) - - -### Bug Fixes - -* Remove `pumpify` ([#2029](https://github.com/googleapis/nodejs-storage/issues/2029)) ([edc1d64](https://github.com/googleapis/nodejs-storage/commit/edc1d64069a6038c301c3b775f116fbf69b10b28)) -* Retry `EPIPE` Connection Errors + Attempt Retries in Resumable Upload Connection Errors ([#2040](https://github.com/googleapis/nodejs-storage/issues/2040)) ([ba35321](https://github.com/googleapis/nodejs-storage/commit/ba35321c3fef9a1874ff8fbea43885e2e7746b4f)) - ## [6.4.0](https://github.com/googleapis/nodejs-storage/compare/v6.3.0...v6.4.0) (2022-08-10) diff --git a/package.json b/package.json index 6619818c2..626823622 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@google-cloud/storage", "description": "Cloud Storage Client Library for Node.js", - "version": "6.4.1", + "version": "6.4.0", "license": "Apache-2.0", "author": "Google Inc.", "engines": { diff --git a/samples/package.json b/samples/package.json index 70ebac861..bcb163130 100644 --- a/samples/package.json +++ b/samples/package.json @@ -17,7 +17,7 @@ }, "dependencies": { "@google-cloud/pubsub": "^3.0.0", - "@google-cloud/storage": "^6.4.1", + "@google-cloud/storage": "^6.4.0", "node-fetch": "^2.6.7", "uuid": "^8.0.0", "yargs": "^16.0.0" diff --git a/samples/system-test/encryption.test.js b/samples/system-test/encryption.test.js index e970df61e..125b22059 100644 --- a/samples/system-test/encryption.test.js +++ b/samples/system-test/encryption.test.js @@ -34,7 +34,6 @@ const kmsKeyName = process.env.GOOGLE_CLOUD_KMS_KEY_US; const fileName = 'test.txt'; const filePath = path.join(__dirname, '../resources', fileName); const downloadFilePath = path.join(__dirname, '../resources/downloaded.txt'); -const doesNotExistPrecondition = 0; const key = crypto.randomBytes(32).toString('base64'); @@ -57,7 +56,7 @@ it('should generate a key', () => { it('should upload a file', async () => { const output = execSync( - `node uploadEncryptedFile.js ${bucketName} ${filePath} ${fileName} ${key} ${doesNotExistPrecondition}` + `node uploadEncryptedFile.js ${bucketName} ${filePath} ${fileName} ${key}` ); assert.match( output, diff --git a/samples/system-test/files.test.js b/samples/system-test/files.test.js index f783eeac2..a616be50f 100644 --- a/samples/system-test/files.test.js +++ b/samples/system-test/files.test.js @@ -62,7 +62,7 @@ describe('file', () => { it('should upload a file', async () => { const output = execSync( - `node uploadFile.js ${bucketName} ${filePath} ${fileName} ${doesNotExistPrecondition}` + `node uploadFile.js ${bucketName} ${filePath} ${fileName}` ); assert.match(output, new RegExp(`${filePath} uploaded to ${bucketName}`)); const [exists] = await bucket.file(fileName).exists(); @@ -85,7 +85,7 @@ describe('file', () => { it('should upload a file without authentication', async () => { const output = execSync( - `node uploadWithoutAuthentication.js ${bucketName} ${fileContents} ${fileName} ${doesNotExistPrecondition}` + `node uploadWithoutAuthentication.js ${bucketName} ${fileContents} ${fileName}` ); assert.match(output, new RegExp(`${fileName} uploaded to ${bucketName}`)); const [exists] = await bucket.file(fileName).exists(); @@ -111,9 +111,8 @@ describe('file', () => { }); it('should upload a file with a kms key', async () => { - const [metadata] = await bucket.file(fileName).getMetadata(); const output = execSync( - `node uploadFileWithKmsKey.js ${bucketName} ${filePath} ${kmsKeyName} ${metadata.generation}` + `node uploadFileWithKmsKey.js ${bucketName} ${filePath} ${kmsKeyName}` ); assert.include( output, diff --git a/samples/uploadEncryptedFile.js b/samples/uploadEncryptedFile.js index 8dd156cc0..10e25c3e8 100644 --- a/samples/uploadEncryptedFile.js +++ b/samples/uploadEncryptedFile.js @@ -17,8 +17,7 @@ function main( bucketName = 'my-bucket', filePath = path.join(__dirname, '../resources', 'test.txt'), destFileName = 'test.txt', - key = process.env.GOOGLE_CLOUD_KMS_KEY_US, - generationMatchPrecondition = 0 + key = process.env.GOOGLE_CLOUD_KMS_KEY_US ) { // [START storage_upload_encrypted_file] /** @@ -46,15 +45,6 @@ function main( const options = { destination: destFileName, encryptionKey: Buffer.from(key, 'base64'), - - // Optional: - // Set a generation-match precondition to avoid potential race conditions - // and data corruptions. The request to upload is aborted if the object's - // generation number does not match your precondition. For a destination - // object that does not yet exist, set the ifGenerationMatch precondition to 0 - // If the destination object already exists in your bucket, set instead a - // generation-match precondition using its generation number. - preconditionOpts: {ifGenerationMatch: generationMatchPrecondition}, }; await storage.bucket(bucketName).upload(filePath, options); diff --git a/samples/uploadFile.js b/samples/uploadFile.js index 1538e4ff1..5cb56e8e8 100644 --- a/samples/uploadFile.js +++ b/samples/uploadFile.js @@ -15,8 +15,7 @@ function main( bucketName = 'my-bucket', filePath = './local/path/to/file.txt', - destFileName = 'file.txt', - generationMatchPrecondition = 0 + destFileName = 'file.txt' ) { // [START storage_upload_file] /** @@ -38,19 +37,10 @@ function main( const storage = new Storage(); async function uploadFile() { - const options = { + await storage.bucket(bucketName).upload(filePath, { destination: destFileName, - // Optional: - // Set a generation-match precondition to avoid potential race conditions - // and data corruptions. The request to upload is aborted if the object's - // generation number does not match your precondition. For a destination - // object that does not yet exist, set the ifGenerationMatch precondition to 0 - // If the destination object already exists in your bucket, set instead a - // generation-match precondition using its generation number. - preconditionOpts: {ifGenerationMatch: generationMatchPrecondition}, - }; - - await storage.bucket(bucketName).upload(filePath, options); + }); + console.log(`${filePath} uploaded to ${bucketName}`); } diff --git a/samples/uploadFileWithKmsKey.js b/samples/uploadFileWithKmsKey.js index 771638abc..8e2de5fd1 100644 --- a/samples/uploadFileWithKmsKey.js +++ b/samples/uploadFileWithKmsKey.js @@ -23,8 +23,7 @@ function main( bucketName = 'my-bucket', filePath = 'test.txt', - kmsKeyName = process.env.GOOGLE_CLOUD_KMS_KEY_US, - generationMatchPrecondition = 0 + kmsKeyName = process.env.GOOGLE_CLOUD_KMS_KEY_US ) { // [START storage_upload_with_kms_key] /** @@ -46,19 +45,9 @@ function main( const storage = new Storage(); async function uploadFileWithKmsKey() { - const options = { + await storage.bucket(bucketName).upload(filePath, { kmsKeyName, - // Optional: - // Set a generation-match precondition to avoid potential race conditions - // and data corruptions. The request to upload is aborted if the object's - // generation number does not match your precondition. For a destination - // object that does not yet exist, set the ifGenerationMatch precondition to 0 - // If the destination object already exists in your bucket, set instead a - // generation-match precondition using its generation number. - preconditionOpts: {ifGenerationMatch: generationMatchPrecondition}, - }; - - await storage.bucket(bucketName).upload(filePath, options); + }); console.log(`${filePath} uploaded to ${bucketName} using ${kmsKeyName}.`); } diff --git a/samples/uploadWithoutAuthentication.js b/samples/uploadWithoutAuthentication.js index f51e56940..12922c4a3 100644 --- a/samples/uploadWithoutAuthentication.js +++ b/samples/uploadWithoutAuthentication.js @@ -15,8 +15,7 @@ function main( bucketName = 'my-bucket', contents = 'these are my file contents', - destFileName = 'file.txt', - generationMatchPrecondition = 0 + destFileName = 'file.txt' ) { // [START storage_upload_without_authentication] /** @@ -44,24 +43,13 @@ function main( // you can make requests without credentials. const [location] = await file.createResumableUpload(); //auth required - const options = { + // Passes the location to file.save so you don't need to + // authenticate this call + await file.save(contents, { uri: location, resumable: true, validation: false, - - // Optional: - // Set a generation-match precondition to avoid potential race conditions - // and data corruptions. The request to upload is aborted if the object's - // generation number does not match your precondition. For a destination - // object that does not yet exist, set the ifGenerationMatch precondition to 0 - // If the destination object already exists in your bucket, set instead a - // generation-match precondition using its generation number. - preconditionOpts: {ifGenerationMatch: generationMatchPrecondition}, - }; - - // Passes the location to file.save so you don't need to - // authenticate this call - await file.save(contents, options); + }); console.log(`${destFileName} uploaded to ${bucketName}`); } diff --git a/src/resumable-upload.ts b/src/resumable-upload.ts index ba2918632..0a3e6809b 100644 --- a/src/resumable-upload.ts +++ b/src/resumable-upload.ts @@ -742,18 +742,9 @@ export class Upload extends Writable { responseReceived = true; this.responseHandler(resp); } - } catch (e) { - const err = e as ApiError; - - if (this.retryOptions.retryableErrorFn!(err)) { - this.attemptDelayedRetry({ - status: NaN, - data: err, - }); - return; - } - - this.destroy(err); + } catch (err) { + const e = err as Error; + this.destroy(e); } } @@ -842,16 +833,7 @@ export class Upload extends Writable { } this.offset = 0; } catch (e) { - const err = e as ApiError; - - if (this.retryOptions.retryableErrorFn!(err)) { - this.attemptDelayedRetry({ - status: NaN, - data: err, - }); - return; - } - + const err = e as GaxiosError; this.destroy(err); } } @@ -917,6 +899,25 @@ export class Upload extends Writable { return successfulRequest ? res : null; } + private restart() { + if (this.numBytesWritten) { + const message = + 'Attempting to restart an upload after unrecoverable bytes have been written from upstream. Stopping as this could result in data loss. Initiate a new upload to continue.'; + + this.emit('error', new RangeError(message)); + return; + } + + this.lastChunkSent = Buffer.alloc(0); + this.createURI(err => { + if (err) { + return this.destroy(err); + } + this.startUploading(); + return; + }); + } + /** * @return {bool} is the request good? */ @@ -940,7 +941,7 @@ export class Upload extends Writable { /** * @param resp GaxiosResponse object from previous attempt */ - private attemptDelayedRetry(resp: Pick) { + private attemptDelayedRetry(resp: GaxiosResponse) { if (this.numRetries < this.retryOptions.maxRetries!) { if ( resp.status === NOT_FOUND_STATUS_CODE && diff --git a/src/storage.ts b/src/storage.ts index a40d1b1bb..5268cdd7b 100644 --- a/src/storage.ts +++ b/src/storage.ts @@ -261,12 +261,11 @@ const IDEMPOTENCY_STRATEGY_DEFAULT = IdempotencyStrategy.RetryConditional; * @return {boolean} True if the API request should be retried, false otherwise. */ export const RETRYABLE_ERR_FN_DEFAULT = function (err?: ApiError) { - const isConnectionProblem = (reason: string) => { + const isConnectionProblem = (reason: string | undefined) => { return ( - reason.includes('eai_again') || // DNS lookup error + (reason && reason.includes('eai_again')) || //DNS lookup error reason === 'econnreset' || - reason === 'unexpected connection closure' || - reason === 'epipe' + reason === 'unexpected connection closure' ); }; @@ -285,7 +284,7 @@ export const RETRYABLE_ERR_FN_DEFAULT = function (err?: ApiError) { if (err.errors) { for (const e of err.errors) { const reason = e?.reason?.toString().toLowerCase(); - if (reason && isConnectionProblem(reason)) { + if (isConnectionProblem(reason)) { return true; } } diff --git a/system-test/storage.ts b/system-test/storage.ts index c7614368e..fb499aa88 100644 --- a/system-test/storage.ts +++ b/system-test/storage.ts @@ -71,9 +71,7 @@ describe('storage', () => { const RETENTION_DURATION_SECONDS = 10; const storage = new Storage({ - retryOptions: { - idempotencyStrategy: IdempotencyStrategy.RetryAlways, - }, + retryOptions: {idempotencyStrategy: IdempotencyStrategy.RetryAlways}, }); const bucket = storage.bucket(generateName()); @@ -154,10 +152,7 @@ describe('storage', () => { // eslint-disable-next-line @typescript-eslint/no-var-requires const {Storage} = require('../src'); storageWithoutAuth = new Storage({ - retryOptions: { - idempotencyStrategy: IdempotencyStrategy.RetryAlways, - retryDelayMultiplier: 3, - }, + retryOptions: {idempotencyStrategy: IdempotencyStrategy.RetryAlways}, }); }); @@ -224,12 +219,6 @@ describe('storage', () => { describe('acls', () => { describe('buckets', () => { - // Some bucket update operations have a rate limit. - // Introduce a delay between tests to avoid getting an error. - beforeEach(done => { - setTimeout(done, 1000); - }); - it('should get access controls', async () => { const accessControls = await bucket.acl.get(); assert(Array.isArray(accessControls)); diff --git a/test/index.ts b/test/index.ts index 56d08e3db..8283b41e6 100644 --- a/test/index.ts +++ b/test/index.ts @@ -311,20 +311,6 @@ describe('Storage', () => { assert.strictEqual(calledWith.retryOptions.retryableErrorFn(error), true); }); - it('should retry a broken pipe error', () => { - const storage = new Storage({ - projectId: PROJECT_ID, - }); - const calledWith = storage.calledWith_[0]; - const error = new ApiError('Broken pipe'); - error.errors = [ - { - reason: 'EPIPE', - }, - ]; - assert.strictEqual(calledWith.retryOptions.retryableErrorFn(error), true); - }); - it('should not retry a 999 error', () => { const storage = new Storage({ projectId: PROJECT_ID, diff --git a/test/resumable-upload.ts b/test/resumable-upload.ts index 9ab05933e..bfc2bd618 100644 --- a/test/resumable-upload.ts +++ b/test/resumable-upload.ts @@ -112,7 +112,7 @@ describe('resumable-upload', () => { userProject: USER_PROJECT, authConfig: {keyFile}, apiEndpoint: API_ENDPOINT, - retryOptions: {...RETRY_OPTIONS}, + retryOptions: RETRY_OPTIONS, }); }); @@ -1028,22 +1028,6 @@ describe('resumable-upload', () => { up.startUploading(); }); - it('should retry retryable errors if the request failed', done => { - const error = new Error('Error.'); - - // mock as retryable - up.retryOptions.retryableErrorFn = () => true; - - up.on('error', done); - up.attemptDelayedRetry = () => done(); - - up.makeRequestStream = async () => { - throw error; - }; - - up.startUploading(); - }); - describe('request preparation', () => { // a convenient handle for getting the request options let reqOpts: GaxiosOptions; @@ -1400,22 +1384,6 @@ describe('resumable-upload', () => { await up.getAndSetOffset(); assert.strictEqual(up.offset, 0); }); - - it('should retry retryable errors if the request failed', done => { - const error = new Error('Error.'); - - // mock as retryable - up.retryOptions.retryableErrorFn = () => true; - - up.on('error', done); - up.attemptDelayedRetry = () => done(); - - up.makeRequest = async () => { - throw error; - }; - - up.getAndSetOffset(); - }); }); describe('#makeRequest', () => { @@ -1711,6 +1679,61 @@ describe('resumable-upload', () => { }); }); + describe('#restart', () => { + beforeEach(() => { + up.createURI = () => {}; + }); + + it('should throw if `numBytesWritten` is not 0', done => { + up.numBytesWritten = 8; + + up.on('error', (error: Error) => { + assert(error instanceof RangeError); + assert( + /Attempting to restart an upload after unrecoverable bytes have been written/.test( + error.message + ) + ); + done(); + }); + + up.restart(); + }); + + describe('starting a new upload', () => { + it('should create a new URI', done => { + up.createURI = () => { + done(); + }; + + up.restart(); + }); + + it('should destroy stream if it cannot create a URI', done => { + const error = new Error(':('); + + up.createURI = (callback: Function) => { + callback(error); + }; + + up.destroy = (err: Error) => { + assert.strictEqual(err, error); + done(); + }; + + up.restart(); + }); + + it('should start uploading', done => { + up.createURI = (callback: Function) => { + up.startUploading = done; + callback(); + }; + up.restart(); + }); + }); + }); + describe('#onResponse', () => { beforeEach(() => { up.numRetries = 0; From 8057ce0a10952d6fa153f02907bafc5c9def64ac Mon Sep 17 00:00:00 2001 From: Sameena Shaffeeullah Date: Thu, 18 Aug 2022 18:48:25 +0000 Subject: [PATCH 08/10] storage.objects.patch sample updates --- samples/fileSetMetadata.js | 39 ++++++++++++++++++-------- samples/releaseEventBasedHold.js | 22 ++++++++++++--- samples/releaseTemporaryHold.js | 22 ++++++++++++--- samples/setEventBasedHold.js | 22 ++++++++++++--- samples/setTemporaryHold.js | 22 ++++++++++++--- samples/system-test/bucketLock.test.js | 22 +++++++++------ samples/system-test/files.test.js | 6 ++-- 7 files changed, 117 insertions(+), 38 deletions(-) diff --git a/samples/fileSetMetadata.js b/samples/fileSetMetadata.js index 1fd178532..15a3fdb3c 100644 --- a/samples/fileSetMetadata.js +++ b/samples/fileSetMetadata.js @@ -19,7 +19,11 @@ // description: Set file metadata. // usage: node fileSetMetadata.js -function main(bucketName = 'my-bucket', fileName = 'file.txt') { +function main( + bucketName = 'my-bucket', + fileName = 'file.txt', + generationMatchPrecondition = 0 +) { // [START storage_set_metadata] // Imports the Google Cloud client library const {Storage} = require('@google-cloud/storage'); @@ -37,23 +41,34 @@ function main(bucketName = 'my-bucket', fileName = 'file.txt') { // const fileName = 'your-file-name'; async function setFileMetadata() { + // Optional: set a generation-match precondition to avoid potential race + // conditions and data corruptions. The request to upload is aborted if the + // object's generation number does not match your precondition. + const options = { + ifGenerationMatch: generationMatchPrecondition, + }; + // Set file metadata. const [metadata] = await storage .bucket(bucketName) .file(fileName) - .setMetadata({ - // Predefined metadata for server e.g. 'cacheControl', 'contentDisposition', - // 'contentEncoding', 'contentLanguage', 'contentType' - contentDisposition: 'attachment; filename*=utf-8\'\'"anotherImage.jpg"', - contentType: 'image/jpeg', + .setMetadata( + { + // Predefined metadata for server e.g. 'cacheControl', 'contentDisposition', + // 'contentEncoding', 'contentLanguage', 'contentType' + contentDisposition: + 'attachment; filename*=utf-8\'\'"anotherImage.jpg"', + contentType: 'image/jpeg', - // A note or actionable items for user e.g. uniqueId, object description, - // or other useful information. - metadata: { - description: 'file description...', - modified: '1900-01-01', + // A note or actionable items for user e.g. uniqueId, object description, + // or other useful information. + metadata: { + description: 'file description...', + modified: '1900-01-01', + }, }, - }); + options + ); console.log( 'Updated metadata for object', diff --git a/samples/releaseEventBasedHold.js b/samples/releaseEventBasedHold.js index e73dce03f..1360b8f84 100644 --- a/samples/releaseEventBasedHold.js +++ b/samples/releaseEventBasedHold.js @@ -20,7 +20,11 @@ * at https://cloud.google.com/storage/docs/bucket-lock */ -function main(bucketName = 'my-bucket', fileName = 'test.txt') { +function main( + bucketName = 'my-bucket', + fileName = 'test.txt', + generationMatchPrecondition = 0 +) { // [START storage_release_event_based_hold] /** * TODO(developer): Uncomment the following lines before running the sample. @@ -38,9 +42,19 @@ function main(bucketName = 'my-bucket', fileName = 'test.txt') { const storage = new Storage(); async function releaseEventBasedHold() { - await storage.bucket(bucketName).file(fileName).setMetadata({ - eventBasedHold: false, - }); + // Optional: set a generation-match precondition to avoid potential race + // conditions and data corruptions. The request to upload is aborted if the + // object's generation number does not match your precondition. + const options = { + ifGenerationMatch: generationMatchPrecondition, + }; + + await storage.bucket(bucketName).file(fileName).setMetadata( + { + eventBasedHold: false, + }, + options + ); console.log(`Event-based hold was released for ${fileName}.`); } diff --git a/samples/releaseTemporaryHold.js b/samples/releaseTemporaryHold.js index 970236a3f..ae9d013b9 100644 --- a/samples/releaseTemporaryHold.js +++ b/samples/releaseTemporaryHold.js @@ -20,7 +20,11 @@ * at https://cloud.google.com/storage/docs/bucket-lock */ -function main(bucketName = 'my-bucket', fileName = 'test.txt') { +function main( + bucketName = 'my-bucket', + fileName = 'test.txt', + generationMatchPrecondition = 0 +) { // [START storage_release_temporary_hold] /** * TODO(developer): Uncomment the following lines before running the sample. @@ -38,9 +42,19 @@ function main(bucketName = 'my-bucket', fileName = 'test.txt') { const storage = new Storage(); async function releaseTemporaryHold() { - await storage.bucket(bucketName).file(fileName).setMetadata({ - temporaryHold: false, - }); + // Optional: set a generation-match precondition to avoid potential race + // conditions and data corruptions. The request to upload is aborted if the + // object's generation number does not match your precondition. + const options = { + ifGenerationMatch: generationMatchPrecondition, + }; + + await storage.bucket(bucketName).file(fileName).setMetadata( + { + temporaryHold: false, + }, + options + ); console.log(`Temporary hold was released for ${fileName}.`); } diff --git a/samples/setEventBasedHold.js b/samples/setEventBasedHold.js index 06a09070b..0a8b46c54 100644 --- a/samples/setEventBasedHold.js +++ b/samples/setEventBasedHold.js @@ -20,7 +20,11 @@ * at https://cloud.google.com/storage/docs/bucket-lock */ -function main(bucketName = 'my-bucket', fileName = 'test.txt') { +function main( + bucketName = 'my-bucket', + fileName = 'test.txt', + generationMatchPrecondition = 0 +) { // [START storage_set_event_based_hold] /** * TODO(developer): Uncomment the following lines before running the sample. @@ -38,10 +42,20 @@ function main(bucketName = 'my-bucket', fileName = 'test.txt') { const storage = new Storage(); async function setEventBasedHold() { + // Optional: set a generation-match precondition to avoid potential race + // conditions and data corruptions. The request to upload is aborted if the + // object's generation number does not match your precondition. + const options = { + ifGenerationMatch: generationMatchPrecondition, + }; + // Set event-based hold - await storage.bucket(bucketName).file(fileName).setMetadata({ - eventBasedHold: true, - }); + await storage.bucket(bucketName).file(fileName).setMetadata( + { + eventBasedHold: true, + }, + options + ); console.log(`Event-based hold was set for ${fileName}.`); } diff --git a/samples/setTemporaryHold.js b/samples/setTemporaryHold.js index d83ac10d3..23a8ae016 100644 --- a/samples/setTemporaryHold.js +++ b/samples/setTemporaryHold.js @@ -20,7 +20,11 @@ * at https://cloud.google.com/storage/docs/bucket-lock */ -function main(bucketName = 'my-bucket', fileName = 'test.txt') { +function main( + bucketName = 'my-bucket', + fileName = 'test.txt', + generationMatchPrecondition = 0 +) { // [START storage_set_temporary_hold] /** * TODO(developer): Uncomment the following lines before running the sample. @@ -38,9 +42,19 @@ function main(bucketName = 'my-bucket', fileName = 'test.txt') { const storage = new Storage(); async function setTemporaryHold() { - await storage.bucket(bucketName).file(fileName).setMetadata({ - temporaryHold: true, - }); + // Optional: set a generation-match precondition to avoid potential race + // conditions and data corruptions. The request to upload is aborted if the + // object's generation number does not match your precondition. + const options = { + ifGenerationMatch: generationMatchPrecondition, + }; + + await storage.bucket(bucketName).file(fileName).setMetadata( + { + temporaryHold: true, + }, + options + ); console.log(`Temporary hold was set for ${fileName}.`); } diff --git a/samples/system-test/bucketLock.test.js b/samples/system-test/bucketLock.test.js index 2ea645471..105240d6f 100644 --- a/samples/system-test/bucketLock.test.js +++ b/samples/system-test/bucketLock.test.js @@ -88,16 +88,18 @@ it('should disable default event-based hold on a bucket', () => { ); }); -it('should set an event-based hold on a file', () => { +it('should set an event-based hold on a file', async () => { + const [metadata] = await bucket.file(fileName).getMetadata(); const output = execSync( - `node setEventBasedHold.js ${bucketName} ${fileName}` + `node setEventBasedHold.js ${bucketName} ${fileName} ${metadata.generation}` ); assert.match(output, new RegExp(`Event-based hold was set for ${fileName}`)); }); -it('should release an event-based hold on a file', () => { +it('should release an event-based hold on a file', async () => { + const [metadata] = await bucket.file(fileName).getMetadata(); const output = execSync( - `node releaseEventBasedHold.js ${bucketName} ${fileName}` + `node releaseEventBasedHold.js ${bucketName} ${fileName} ${metadata.generation}` ); assert.match( output, @@ -113,14 +115,18 @@ it('should remove a retention policy on a bucket', () => { ); }); -it('should set an temporary hold on a file', () => { - const output = execSync(`node setTemporaryHold.js ${bucketName} ${fileName}`); +it('should set an temporary hold on a file', async () => { + const [metadata] = await bucket.file(fileName).getMetadata(); + const output = execSync( + `node setTemporaryHold.js ${bucketName} ${fileName} ${metadata.generation}` + ); assert.match(output, new RegExp(`Temporary hold was set for ${fileName}.`)); }); -it('should release an temporary hold on a file', () => { +it('should release an temporary hold on a file', async () => { + const [metadata] = await bucket.file(fileName).getMetadata(); const output = execSync( - `node releaseTemporaryHold.js ${bucketName} ${fileName}` + `node releaseTemporaryHold.js ${bucketName} ${fileName} ${metadata.generation}` ); assert.match( output, diff --git a/samples/system-test/files.test.js b/samples/system-test/files.test.js index f783eeac2..ca4806bfc 100644 --- a/samples/system-test/files.test.js +++ b/samples/system-test/files.test.js @@ -421,14 +421,16 @@ describe('file', () => { assert.include(output, `Name: ${copiedFileName}`); }); - it('should set metadata for a file', () => { + it('should set metadata for a file', async () => { + const [metadata] = await bucket.file(copiedFileName).getMetadata(); + // used in sample const userMetadata = { description: 'file description...', modified: '1900-01-01', }; const output = execSync( - `node fileSetMetadata.js ${bucketName} ${copiedFileName}` + `node fileSetMetadata.js ${bucketName} ${copiedFileName} ${metadata.generation} ` ); assert.match( From bc5088661e6db3c7496dcd2a95118e10e8535d40 Mon Sep 17 00:00:00 2001 From: Sameena Shaffeeullah Date: Thu, 18 Aug 2022 20:16:43 +0000 Subject: [PATCH 09/10] destinationGenerationMatchPrecondition --- samples/composeFile.js | 4 ++-- samples/copyFile.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/composeFile.js b/samples/composeFile.js index 3f1bbfb80..332c3da9c 100644 --- a/samples/composeFile.js +++ b/samples/composeFile.js @@ -24,7 +24,7 @@ function main( firstFileName = 'file-one.txt', secondFileName = 'file-two.txt', destinationFileName = 'file-one-two.txt', - generationMatchPrecondition = 0 + destinationGenerationMatchPrecondition = 0 ) { // [START storage_compose_file] /** @@ -60,7 +60,7 @@ function main( // If the destination object already exists in your bucket, set instead a // generation-match precondition using its generation number. const combineOptions = { - ifGenerationMatch: generationMatchPrecondition, + ifGenerationMatch: destinationGenerationMatchPrecondition, }; await bucket.combine(sources, destinationFileName, combineOptions); diff --git a/samples/copyFile.js b/samples/copyFile.js index 8bddb7921..10f1eb4a6 100644 --- a/samples/copyFile.js +++ b/samples/copyFile.js @@ -25,7 +25,7 @@ function main( srcFilename = 'test2.txt', destBucketName = 'my-bucket', destFileName = 'test3.txt', - generationMatchPrecondition = 0 + destinationGenerationMatchPrecondition = 0 ) { // [START storage_copy_file] /** @@ -61,7 +61,7 @@ function main( // generation-match precondition using its generation number. const copyOptions = { preconditionOpts: { - ifGenerationMatch: generationMatchPrecondition, + ifGenerationMatch: destinationGenerationMatchPrecondition, }, }; From 42e535772ff2078a8573ea213b6ca5062a15773e Mon Sep 17 00:00:00 2001 From: Sameena Shaffeeullah Date: Thu, 18 Aug 2022 21:17:50 +0000 Subject: [PATCH 10/10] renamed variable --- samples/copyOldVersionOfFile.js | 4 ++-- samples/moveFile.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/copyOldVersionOfFile.js b/samples/copyOldVersionOfFile.js index 84f34b180..11c8973b4 100644 --- a/samples/copyOldVersionOfFile.js +++ b/samples/copyOldVersionOfFile.js @@ -25,7 +25,7 @@ function main( destBucketName = 'my-bucket', destFileName = 'test3.txt', generation = 1, - generationMatchPrecondition = 0 + destinationGenerationMatchPrecondition = 0 ) { // [START storage_copy_file_archived_generation] /** @@ -64,7 +64,7 @@ function main( // generation-match precondition using its generation number. const copyOptions = { preconditionOpts: { - ifGenerationMatch: generationMatchPrecondition, + ifGenerationMatch: destinationGenerationMatchPrecondition, }, }; diff --git a/samples/moveFile.js b/samples/moveFile.js index dcd9a7425..ce700648e 100644 --- a/samples/moveFile.js +++ b/samples/moveFile.js @@ -24,7 +24,7 @@ function main( bucketName = 'my-bucket', srcFileName = 'test.txt', destFileName = 'test2.txt', - generationMatchPrecondition = 0 + destinationGenerationMatchPrecondition = 0 ) { // [START storage_move_file] /** @@ -55,7 +55,7 @@ function main( // generation-match precondition using its generation number. const moveOptions = { preconditionOpts: { - ifGenerationMatch: generationMatchPrecondition, + ifGenerationMatch: destinationGenerationMatchPrecondition, }, };