Skip to content

Commit

Permalink
refactor delete bucket
Browse files Browse the repository at this point in the history 
Signed-off-by: Romy <35330373+romayalon@users.noreply.github.com>
  • Loading branch information
@romayalon
romayalon committed May 13, 2024
1 parent f981264 commit f381ef1
Show file tree
Hide file tree
Showing 10 changed files with 179 additions and 79 deletions.
4 changes: 2 additions & 2 deletions src/cmd/manage_nsfs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,8 +249,8 @@ async function delete_bucket(data, force) {
const fs_context_fs_backend = native_fs_utils.get_process_fs_context(data.fs_backend);
const bucket_config_path = get_config_file_path(buckets_dir_path, data.name);
try {
const temp_dir_name = config.NSFS_TEMP_DIR_NAME + "_" + data._id;
const bucket_temp_dir_path = path.join(data.path, temp_dir_name);
const temp_dir_name = native_fs_utils.get_bucket_tmpdir_name(data._id);
const bucket_temp_dir_path = native_fs_utils.get_bucket_tmpdir_full_path(data.path, data._id);
const entries = await nb_native().fs.readdir(fs_context_fs_backend, data.path);
const object_entries = entries.filter(element => !element.name.endsWith(temp_dir_name));
if (object_entries.length === 0 || force) {
Expand Down
109 changes: 55 additions & 54 deletions src/sdk/bucketspace_fs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ const mongo_utils = require('../util/mongo_utils');
const { CONFIG_SUBDIRS } = require('../manage_nsfs/manage_nsfs_constants');

const KeysSemaphore = require('../util/keys_semaphore');
const native_fs_utils = require('../util/native_fs_utils');
const { get_umasked_mode, isDirectory, validate_bucket_creation,
create_config_file, delete_config_file, get_bucket_tmpdir_full_path, folder_delete } = require('../util/native_fs_utils');
const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent;

const dbg = require('../util/debug_module')(__filename);
Expand Down Expand Up @@ -197,7 +198,6 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
// BUCKET //
////////////

//TODO: we need to add pagination support to list buckets for more than 1000 buckets.
/**
* list_buckets will read all bucket config files, and filter them according to the requesting account's
* permissions
Expand All @@ -224,7 +224,7 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {

const account = object_sdk.requesting_account;
const buckets = await P.map_with_concurrency(10, entries, async entry => {
if (native_fs_utils.isDirectory(entry) || !entry.name.endsWith('.json')) {
if (isDirectory(entry) || !entry.name.endsWith('.json')) {
return;
}
const bucket_name = this.get_bucket_name(entry.name);
Expand Down Expand Up @@ -258,7 +258,7 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
throw new RpcError('MISSING_NSFS_ACCOUNT_CONFIGURATION');
}
const fs_context = prepare_fs_context(sdk);
native_fs_utils.validate_bucket_creation(params);
validate_bucket_creation(params);

const { name } = params;
const bucket_config_path = this._get_bucket_config_path(name);
Expand All @@ -283,15 +283,15 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
const bucket_to_validate = JSON.parse(bucket_config);
dbg.log2("create_bucket: bucket properties before validate_bucket_schema", bucket_to_validate);
nsfs_schema_utils.validate_bucket_schema(bucket_to_validate);
await native_fs_utils.create_config_file(this.fs_context, this.bucket_schema_dir, bucket_config_path, bucket_config);
await create_config_file(this.fs_context, this.bucket_schema_dir, bucket_config_path, bucket_config);
} catch (err) {
new NoobaaEvent(NoobaaEvent.BUCKET_CREATION_FAILED).create_event(name, {bucket_name: name}, err);
throw this._translate_bucket_error_codes(err);
}

// create bucket's underlying storage directory
try {
await nb_native().fs.mkdir(fs_context, bucket_storage_path, native_fs_utils.get_umasked_mode(config.BASE_MODE_DIR));
await nb_native().fs.mkdir(fs_context, bucket_storage_path, get_umasked_mode(config.BASE_MODE_DIR));
new NoobaaEvent(NoobaaEvent.BUCKET_CREATED).create_event(name, {bucket_name: name});
} catch (err) {
dbg.error('BucketSpaceFS: create_bucket could not create underlying directory - nsfs, deleting bucket', err);
Expand Down Expand Up @@ -323,41 +323,44 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
};
}


/**
* delete_bucket will delete the bucket config file and underlying directory if needed based on the requesting account permissions
* 1. if bucket.should_create_underlying_storage - delete the underlying storage directory = the bucket's underlying FS directory in which the objects are stored
* 2. else - check if there are no objects in the bucket, if any - throw err, else - delete export tmp file
* 3. delete bucket config file
* @param {nb.ObjectSDK} object_sdk
* @returns {Promise<void>}
*/
async delete_bucket(params, object_sdk) {
return bucket_semaphore.surround_key(String(params.name), async () => {
const { name } = params;
const bucket_path = this._get_bucket_config_path(name);
const { name } = params;
return bucket_semaphore.surround_key(String(name), async () => {
const bucket_config_path = this._get_bucket_config_path(name);
try {
const namespace_bucket_config = await object_sdk.read_bucket_sdk_namespace_info(params.name);
const { ns, bucket } = await object_sdk.read_bucket_full_info(name);
const namespace_bucket_config = bucket && bucket.namespace;
dbg.log1('BucketSpaceFS.delete_bucket: namespace_bucket_config', namespace_bucket_config);
const ns = await object_sdk._get_bucket_namespace(params.name);
if (namespace_bucket_config && namespace_bucket_config.should_create_underlying_storage) {
// delete underlying storage = the directory which represents the bucket
if (!namespace_bucket_config) throw new RpcError('INTERNAL_ERROR', 'Invalid Bucket configuration');

if (namespace_bucket_config.should_create_underlying_storage) {
// 1. delete underlying storage
dbg.log1('BucketSpaceFS.delete_bucket: deleting uls', this.fs_root, namespace_bucket_config.write_resource.path);
await ns.delete_uls({
name,
full_path: path.join(this.fs_root, namespace_bucket_config.write_resource.path) // includes write_resource.path + bucket name (s3 flow)
}, object_sdk);
} else if (namespace_bucket_config) {
// S3 Delete for NSFS Manage buckets
const bucket_storage_path = path.join(this.fs_root, namespace_bucket_config.write_resource.path); // includes write_resource.path + bucket name (s3 flow)
await ns.delete_uls({ name, full_path: bucket_storage_path }, object_sdk);
} else {
// 2. delete only bucket tmpdir
const list = await ns.list_objects({ ...params, limit: 1 }, object_sdk);
if (list && list.objects && list.objects.length > 0) {
throw new RpcError('NOT_EMPTY', 'underlying directory has files in it');
}
const bucket = await object_sdk.read_bucket_sdk_config_info(params.name);
const bucket_temp_dir_path = path.join(namespace_bucket_config.write_resource.path,
config.NSFS_TEMP_DIR_NAME + "_" + bucket._id);
await native_fs_utils.folder_delete(bucket_temp_dir_path, this.fs_context, true);
if (list && list.objects && list.objects.length > 0) throw new RpcError('NOT_EMPTY', 'underlying directory has files in it');
const bucket_tmpdir_path = get_bucket_tmpdir_full_path(namespace_bucket_config.write_resource.path, bucket._id);
await folder_delete(bucket_tmpdir_path, this.fs_context, true);
}
dbg.log1(`BucketSpaceFS: delete_fs_bucket ${bucket_path}`);
// delete bucket config json file
await native_fs_utils.delete_config_file(this.fs_context, this.bucket_schema_dir, bucket_path);
new NoobaaEvent(NoobaaEvent.BUCKET_DELETE).create_event(name, {bucket_name: name});
// 3. delete bucket config json file
dbg.log1(`BucketSpaceFS: delete_bucket: deleting config file ${bucket_config_path}`);
await delete_config_file(this.fs_context, this.bucket_schema_dir, bucket_config_path);
new NoobaaEvent(NoobaaEvent.BUCKET_DELETE).create_event(name, { bucket_name: name });
} catch (err) {
new NoobaaEvent(NoobaaEvent.BUCKET_DELETE_FAILED).create_event(params.name,
{bucket_name: params.name, bucket_path: bucket_path}, err);
dbg.error('BucketSpaceFS: delete_bucket error', err);
dbg.error('BucketSpaceFS: delete_bucket: error', err);
new NoobaaEvent(NoobaaEvent.BUCKET_DELETE_FAILED).create_event(name,
{ bucket_name: name, bucket_path: bucket_config_path }, err);
throw this._translate_bucket_error_codes(err);
}
});
Expand Down Expand Up @@ -402,7 +405,7 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
this.fs_context,
bucket_config_path,
Buffer.from(update_bucket), {
mode: native_fs_utils.get_umasked_mode(config.BASE_MODE_CONFIG_FILE)
mode: get_umasked_mode(config.BASE_MODE_CONFIG_FILE)
}
);
} catch (err) {
Expand Down Expand Up @@ -447,9 +450,8 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
await nb_native().fs.writeFile(
this.fs_context,
bucket_config_path,
Buffer.from(update_bucket), {
mode: native_fs_utils.get_umasked_mode(config.BASE_MODE_CONFIG_FILE)
}
Buffer.from(update_bucket),
{ mode: get_umasked_mode(config.BASE_MODE_CONFIG_FILE) }
);
} catch (err) {
throw this._translate_bucket_error_codes(err);
Expand Down Expand Up @@ -484,9 +486,8 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
await nb_native().fs.writeFile(
this.fs_context,
bucket_config_path,
Buffer.from(update_bucket), {
mode: native_fs_utils.get_umasked_mode(config.BASE_MODE_CONFIG_FILE)
}
Buffer.from(update_bucket),
{ mode: get_umasked_mode(config.BASE_MODE_CONFIG_FILE) }
);
} catch (err) {
throw this._translate_bucket_error_codes(err);
Expand All @@ -513,9 +514,8 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
await nb_native().fs.writeFile(
this.fs_context,
bucket_config_path,
Buffer.from(update_bucket), {
mode: native_fs_utils.get_umasked_mode(config.BASE_MODE_CONFIG_FILE)
}
Buffer.from(update_bucket),
{ mode: get_umasked_mode(config.BASE_MODE_CONFIG_FILE) }
);
} catch (err) {
throw this._translate_bucket_error_codes(err);
Expand All @@ -537,23 +537,26 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
await nb_native().fs.writeFile(
this.fs_context,
bucket_config_path,
Buffer.from(update_bucket), {
mode: native_fs_utils.get_umasked_mode(config.BASE_MODE_CONFIG_FILE)
}
Buffer.from(update_bucket),
{ mode: get_umasked_mode(config.BASE_MODE_CONFIG_FILE) }
);
} catch (err) {
throw this._translate_bucket_error_codes(err);
}
}

/**
* @param {object} params
* @returns {Promise<object>}
*/
async get_bucket_website(params) {
try {
const { name } = params;
dbg.log0('BucketSpaceFS.get_bucket_website: Bucket name', name);
const bucket_config_path = this._get_bucket_config_path(name);
const { data } = await nb_native().fs.readFile(this.fs_context, bucket_config_path);
const bucket = JSON.parse(data.toString());
return {website: bucket.website};
return { website: bucket.website };
} catch (err) {
throw this._translate_bucket_error_codes(err);
}
Expand Down Expand Up @@ -582,9 +585,8 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
await nb_native().fs.writeFile(
this.fs_context,
bucket_config_path,
Buffer.from(update_bucket), {
mode: native_fs_utils.get_umasked_mode(config.BASE_MODE_CONFIG_FILE)
}
Buffer.from(update_bucket),
{ mode: get_umasked_mode(config.BASE_MODE_CONFIG_FILE) }
);
} catch (err) {
throw this._translate_bucket_error_codes(err);
Expand All @@ -606,9 +608,8 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
await nb_native().fs.writeFile(
this.fs_context,
bucket_config_path,
Buffer.from(update_bucket), {
mode: native_fs_utils.get_umasked_mode(config.BASE_MODE_CONFIG_FILE)
}
Buffer.from(update_bucket),
{ mode: get_umasked_mode(config.BASE_MODE_CONFIG_FILE) }
);
} catch (err) {
throw this._translate_bucket_error_codes(err);
Expand Down
28 changes: 27 additions & 1 deletion src/sdk/bucketspace_simple_fs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,12 @@ class BucketSpaceSimpleFS {
}
}

async delete_bucket(params) {
/**
* @param {object} params
* @param {nb.ObjectSDK} object_sdk
* @returns {Promise<object>}
*/
async delete_bucket(params, object_sdk) {
try {
const { name } = params;
const bucket_path = path.join(this.fs_root, name);
Expand Down Expand Up @@ -203,6 +208,10 @@ class BucketSpaceSimpleFS {
// TODO
}

/**
* @param {object} params
* @returns {Promise<object>}
*/
async get_bucket_website(params) {
// TODO
}
Expand Down Expand Up @@ -236,6 +245,23 @@ class BucketSpaceSimpleFS {
async put_object_lock_configuration(params, object_sdk) {
// TODO
}


/////////////////////
// BUCKET LOGGING //
/////////////////////

async put_bucket_logging(params) {
// TODO
}

async delete_bucket_logging(params) {
// TODO
}

async get_bucket_logging(params) {
// TODO
}
}

module.exports = BucketSpaceSimpleFS;
32 changes: 21 additions & 11 deletions src/sdk/namespace_fs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -491,8 +491,18 @@ class NamespaceFS {
return fs_context;
}

get_bucket_tmpdir() {
return config.NSFS_TEMP_DIR_NAME + '_' + this.bucket_id;
/**
* @returns {string}
*/
get_bucket_tmpdir_name() {
return native_fs_utils.get_bucket_tmpdir_name(this.bucket_id);
}

/**
* @returns {string}
*/
get_bucket_tmpdir_full_path() {
return native_fs_utils.get_bucket_tmpdir_full_path(this.bucket_path, this.bucket_id);
}

get_write_resource() {
Expand Down Expand Up @@ -706,7 +716,7 @@ class NamespaceFS {
// dbg.log0('process_entry', dir_key, ent.name);
if ((!ent.name.startsWith(prefix_ent) ||
ent.name < marker_curr ||
ent.name === this.get_bucket_tmpdir() ||
ent.name === this.get_bucket_tmpdir_name() ||
ent.name === config.NSFS_FOLDER_OBJECT_NAME) &&
!this._is_hidden_version_path(ent.name)) {
return;
Expand Down Expand Up @@ -1163,7 +1173,8 @@ class NamespaceFS {
// upload path is needed only when open_mode is w / for copy
if (open_mode === 'w' || params.copy_source) {
const upload_id = uuidv4();
upload_path = path.join(this.bucket_path, this.get_bucket_tmpdir(), 'uploads', upload_id);
const bucket_tmp_dir_path = this.get_bucket_tmpdir_full_path();
upload_path = path.join(bucket_tmp_dir_path, 'uploads', upload_id);
await native_fs_utils._make_path_dirs(upload_path, fs_context);
}
let open_path = upload_path || file_path;
Expand Down Expand Up @@ -1408,7 +1419,7 @@ class NamespaceFS {
await this._open_files_gpfs(fs_context, new_ver_tmp_path, latest_ver_path, upload_file,
latest_ver_info, open_mode, undefined, versioned_info) :
undefined;
const bucket_tmp_dir_path = path.join(this.bucket_path, this.get_bucket_tmpdir());
const bucket_tmp_dir_path = this.get_bucket_tmpdir_full_path();
dbg.log1('Namespace_fs._move_to_dest_version:', latest_ver_info, new_ver_info, gpfs_options);

if (this._is_versioning_suspended()) {
Expand Down Expand Up @@ -2385,8 +2396,7 @@ class NamespaceFS {

_mpu_root_path() {
return path.join(
this.bucket_path,
this.get_bucket_tmpdir(),
this.get_bucket_tmpdir_full_path(),
'multipart-uploads');
}

Expand Down Expand Up @@ -2715,7 +2725,7 @@ class NamespaceFS {
gpfs_options = is_gpfs ?
await this._open_files_gpfs(fs_context, file_path, undefined, undefined, undefined, undefined, true) :
undefined;
const bucket_tmp_dir_path = path.join(this.bucket_path, this.get_bucket_tmpdir());
const bucket_tmp_dir_path = this.get_bucket_tmpdir_full_path();
await native_fs_utils.safe_unlink(fs_context, file_path, version_info, gpfs_options, bucket_tmp_dir_path);
return { ...version_info, latest: true };
} else {
Expand Down Expand Up @@ -2827,7 +2837,7 @@ class NamespaceFS {
deleted_version_info.mtimeNsBigint < max_past_ver_info.mtimeNsBigint) return;
dbg.log1('Namespace_fs._promote_version_to_latest ', max_past_ver_info.path, latest_ver_path, max_past_ver_info, latest_version_info);
// on concurrent put, safe_move_gpfs might override new coming latest (no fd verification, gpfs linkfileat will override)
const bucket_tmp_dir_path = path.join(this.bucket_path, this.get_bucket_tmpdir());
const bucket_tmp_dir_path = this.get_bucket_tmpdir_full_path();
await native_fs_utils.safe_move_posix(fs_context, max_past_ver_info.path, latest_ver_path,
max_past_ver_info, bucket_tmp_dir_path);
break;
Expand Down Expand Up @@ -2881,7 +2891,7 @@ class NamespaceFS {
if (latest_ver_info) {
const suspended_and_latest_is_not_null = this._is_versioning_suspended() &&
latest_ver_info.version_id_str !== NULL_VERSION_ID;
const bucket_tmp_dir_path = path.join(this.bucket_path, this.get_bucket_tmpdir());
const bucket_tmp_dir_path = this.get_bucket_tmpdir_full_path();
if (this._is_versioning_enabled() || suspended_and_latest_is_not_null) {
await native_fs_utils._make_path_dirs(versioned_path, fs_context);
await native_fs_utils.safe_move(fs_context, latest_ver_path, versioned_path, latest_ver_info,
Expand Down Expand Up @@ -2930,7 +2940,7 @@ class NamespaceFS {
const gpfs_options = is_gpfs ?
await this._open_files_gpfs(fs_context, null_versioned_path, undefined, undefined, undefined, undefined, true) :
undefined;
const bucket_tmp_dir_path = path.join(this.bucket_path, this.get_bucket_tmpdir());
const bucket_tmp_dir_path = this.get_bucket_tmpdir_full_path();
await native_fs_utils.safe_unlink(fs_context, null_versioned_path, null_versioned_path_info,
gpfs_options, bucket_tmp_dir_path);

Expand Down
Loading

0 comments on commit f381ef1

Please sign in to comment.