Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix!: breaking change, remove uses of configstore #1870

Closed
wants to merge 3 commits into from
Closed

fix!: breaking change, remove uses of configstore #1870

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
640f7b0
refactor: Refactor Upload Directory for Improved Reliability (#1867)
danielbankhead Apr 13, 2022
5462675
fix: breaking change, remove uses of configstore
shaffeeullah Apr 13, 2022
e34f0ab
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Apr 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@
"arrify": "^2.0.0",
"async-retry": "^1.3.3",
"compressible": "^2.0.12",
"configstore": "^5.0.0",
"date-and-time": "^2.0.0",
"duplexify": "^4.0.0",
"ent": "^2.2.0",
Expand All @@ -82,7 +81,6 @@
"@grpc/proto-loader": "^0.6.0",
"@types/async-retry": "^1.4.3",
"@types/compressible": "^2.0.0",
"@types/configstore": "^5.0.0",
"@types/date-and-time": "^0.13.0",
"@types/ent": "^2.2.1",
"@types/extend": "^3.0.0",
Expand Down
2 changes: 1 addition & 1 deletion samples/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"license": "Apache-2.0",
"author": "Google Inc.",
"engines": {
"node": ">=8"
"node": ">=10"
},
"repository": "googleapis/nodejs-storage",
"private": true,
Expand Down
96 changes: 42 additions & 54 deletions samples/uploadDirectory.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,10 @@
// description: Uploads full hierarchy of a local directory to a bucket.
// usage: node files.js upload-directory <bucketName> <directoryPath>

async function main(bucketName, directoryPath) {
function main(
bucketName = 'your-unique-bucket-name',
directoryPath = './local/path/to/directory'
) {
// [START upload_directory]
/**
* TODO(developer): Uncomment the following lines before running the sample.
Expand All @@ -36,68 +39,53 @@ async function main(bucketName, directoryPath) {
// Creates a client
const storage = new Storage();

const {promisify} = require('util');
const fs = require('fs');
const path = require('path');
const fileList = [];

async function uploadDirectory() {
// Get a list of files from the specified directory
let dirCtr = 1;
let itemCtr = 0;
const pathDirName = path.dirname(directoryPath);

getFiles(directoryPath);

function getFiles(directory) {
fs.readdir(directory, (err, items) => {
dirCtr--;
itemCtr += items.length;
items.forEach(item => {
const fullPath = path.join(directory, item);
fs.stat(fullPath, (err, stat) => {
itemCtr--;
if (stat.isFile()) {
fileList.push(fullPath);
} else if (stat.isDirectory()) {
dirCtr++;
getFiles(fullPath);
}
if (dirCtr === 0 && itemCtr === 0) {
onComplete();
}
});
});
});
const readdir = promisify(fs.readdir);
const stat = promisify(fs.stat);

async function* getFiles(directory = '.') {
for (const file of await readdir(directory)) {
const fullPath = path.join(directory, file);
const stats = await stat(fullPath);

if (stats.isDirectory()) {
yield* getFiles(fullPath);
}

if (stats.isFile()) {
yield fullPath;
}
}
}

async function onComplete() {
const resp = await Promise.all(
fileList.map(filePath => {
let destination = path.relative(pathDirName, filePath);
// If running on Windows
if (process.platform === 'win32') {
destination = destination.replace(/\\/g, '/');
}
return storage
.bucket(bucketName)
.upload(filePath, {destination})
.then(
uploadResp => ({fileName: destination, status: uploadResp[0]}),
err => ({fileName: destination, response: err})
);
})
);

const successfulUploads =
fileList.length - resp.filter(r => r.status instanceof Error).length;
console.log(
`${successfulUploads} files uploaded to ${bucketName} successfully.`
);
async function uploadDirectory() {
const bucket = storage.bucket(bucketName);
let successfulUploads = 0;

for await (const filePath of getFiles(directoryPath)) {
try {
const dirname = path.dirname(directoryPath);
const destination = path.relative(dirname, filePath);

await bucket.upload(filePath, {destination});

console.log(`Successfully uploaded: ${filePath}`);
successfulUploads++;
} catch (e) {
console.error(`Error uploading ${filePath}:`, e);
}
}

console.log(
`${successfulUploads} files uploaded to ${bucketName} successfully.`
);
}

uploadDirectory().catch(console.error);
// [END upload_directory]
}

main(...process.argv.slice(2)).catch(console.error);
main(...process.argv.slice(2));
124 changes: 1 addition & 123 deletions src/file.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,6 @@ export type PredefinedAcl =

export interface CreateResumableUploadOptions {
chunkSize?: number;
configPath?: string;
metadata?: Metadata;
origin?: string;
offset?: number;
Expand Down Expand Up @@ -1545,8 +1544,6 @@ class File extends ServiceObject<File> {
*/
/**
* @typedef {object} CreateResumableUploadOptions
* @property {string} [configPath] A full JSON file path to use with
* `gcs-resumable-upload`. This maps to the {@link https://github.com/yeoman/configstore/tree/0df1ec950d952b1f0dfb39ce22af8e505dffc71a#configpath| configstore option by the same name}.
* @property {object} [metadata] Metadata to set on the file.
* @property {number} [offset] The starting byte of the upload stream for resuming an interrupted upload.
* @property {string} [origin] Origin header to set for the upload.
Expand Down Expand Up @@ -1648,7 +1645,6 @@ class File extends ServiceObject<File> {
authClient: this.storage.authClient,
apiEndpoint: this.storage.apiEndpoint,
bucket: this.bucket.name,
configPath: options.configPath,
customRequestOptions: this.getRequestInterceptors().reduce(
(reqOpts, interceptorFn) => interceptorFn(reqOpts),
{}
Expand All @@ -1674,9 +1670,6 @@ class File extends ServiceObject<File> {

/**
* @typedef {object} CreateWriteStreamOptions Configuration options for File#createWriteStream().
* @property {string} [configPath] **This only applies to resumable
* uploads.** A full JSON file path to use with `gcs-resumable-upload`.
* This maps to the {@link https://github.com/yeoman/configstore/tree/0df1ec950d952b1f0dfb39ce22af8e505dffc71a#configpath| configstore option by the same name}.
* @property {string} [contentType] Alias for
* `options.metadata.contentType`. If set to `auto`, the file name is used
* to determine the contentType.
Expand Down Expand Up @@ -1744,12 +1737,6 @@ class File extends ServiceObject<File> {
* Resumable uploads are automatically enabled and must be shut off explicitly
* by setting `options.resumable` to `false`.
*
* Resumable uploads require write access to the $HOME directory. Through
* {@link https://www.npmjs.com/package/configstore| `config-store`}, some metadata
* is stored. By default, if the directory is not writable, we will fall back
* to a simple upload. However, if you explicitly request a resumable upload,
* and we cannot write to the config directory, we will return a
* `ResumableUploadError`.
*
* <p class="notice">
* There is some overhead when using a resumable upload that can cause
Expand Down Expand Up @@ -1899,71 +1886,7 @@ class File extends ServiceObject<File> {
this.startSimpleUpload_(fileWriteStream, options);
return;
}

if (options.configPath) {
this.startResumableUpload_(fileWriteStream, options);
return;
}

// The logic below attempts to mimic the resumable upload library,
// gcs-resumable-upload. That library requires a writable configuration
// directory in order to work. If we wait for that library to discover any
// issues, we've already started a resumable upload which is difficult to back
// out of. We want to catch any errors first, so we can choose a simple, non-
// resumable upload instead.

// Same as configstore (used by gcs-resumable-upload):
// https://github.com/yeoman/configstore/blob/f09f067e50e6a636cfc648a6fc36a522062bd49d/index.js#L11
const configDir = xdgBasedir.config || os.tmpdir();

fs.access(configDir, fs.constants.W_OK, accessErr => {
if (!accessErr) {
// A configuration directory exists, and it's writable. gcs-resumable-upload
// should have everything it needs to work.
this.startResumableUpload_(fileWriteStream, options);
return;
}

// The configuration directory is either not writable, or it doesn't exist.
// gcs-resumable-upload will attempt to create it for the user, but we'll try
// it now to confirm that it won't have any issues. That way, if we catch the
// issue before we start the resumable upload, we can instead start a simple
// upload.
fs.mkdir(configDir, {mode: 0o0700}, err => {
if (!err) {
// We successfully created a configuration directory that
// gcs-resumable-upload will use.
this.startResumableUpload_(fileWriteStream, options);
return;
}

if (options.resumable) {
// The user wanted a resumable upload, but we couldn't create a
// configuration directory, which means gcs-resumable-upload will fail.

// Determine if the issue is that the directory does not exist or
// if the directory exists, but is not writable.
const error = new ResumableUploadError(
[
'A resumable upload could not be performed. The directory,',
`${configDir}, is not writable. You may try another upload,`,
'this time setting `options.resumable` to `false`.',
].join(' ')
);
fs.access(configDir, fs.constants.R_OK, noReadErr => {
if (noReadErr) {
error.additionalInfo = 'The directory does not exist.';
} else {
error.additionalInfo = 'The directory is read-only.';
}
stream.destroy(error);
});
} else {
// The user didn't care, resumable or not. Fall back to simple upload.
this.startSimpleUpload_(fileWriteStream, options);
}
});
});
this.startResumableUpload_(fileWriteStream, options);
});

fileWriteStream.on('response', stream.emit.bind(stream, 'response'));
Expand Down Expand Up @@ -2030,50 +1953,6 @@ class File extends ServiceObject<File> {
return stream as Writable;
}

/**
* Delete failed resumable upload file cache.
*
* Resumable file upload cache the config file to restart upload in case of
* failure. In certain scenarios, the resumable upload will not works and
* upload file cache needs to be deleted to upload the same file.
*
* Following are some of the scenarios.
*
* Resumable file upload failed even though the file is successfully saved
* on the google storage and need to clean up a resumable file cache to
* update the same file.
*
* Resumable file upload failed due to pre-condition
* (i.e generation number is not matched) and want to upload a same
* file with the new generation number.
*
* @example
* ```
* const {Storage} = require('@google-cloud/storage');
* const storage = new Storage();
* const myBucket = storage.bucket('my-bucket');
*
* const file = myBucket.file('my-file', { generation: 0 });
* const contents = 'This is the contents of the file.';
*
* file.save(contents, function(err) {
* if (err) {
* file.deleteResumableCache();
* }
* });
*
* ```
*/
deleteResumableCache() {
const uploadStream = resumableUpload.upload({
bucket: this.bucket.name,
file: this.name,
generation: this.generation,
retryOptions: this.storage.retryOptions,
});
uploadStream.deleteConfig();
}

download(options?: DownloadOptions): Promise<DownloadResponse>;
download(options: DownloadOptions, callback: DownloadCallback): void;
download(callback: DownloadCallback): void;
Expand Down Expand Up @@ -3971,7 +3850,6 @@ class File extends ServiceObject<File> {
authClient: this.storage.authClient,
apiEndpoint: this.storage.apiEndpoint,
bucket: this.bucket.name,
configPath: options.configPath,
customRequestOptions: this.getRequestInterceptors().reduce(
(reqOpts, interceptorFn) => interceptorFn(reqOpts),
{}
Expand Down
Loading