Permalink
Browse files

drm: Add remove file capability during walk

    - Add remove option to drm
    - Add mfu_walk_opts_t struct to hold
      desired functionality during walk
    - Pass walk_opts struct to
      mfu_flist_walk_param_paths instead of
      individual parameters (dir_perms & remove)
    - Delete files in mfu_flist_walk for the
      walk with stat and without
    - Delete directories by level in drm
  • Loading branch information...
dsikich authored and gonsie committed Dec 12, 2018
1 parent 0fd2084 commit 058fa24d5b1b34b389c6dcaa36da005e33d3b462
@@ -35,8 +35,6 @@

/* TODO: change globals to struct */
static int verbose = 0;
static int walk_stat = 1;
static int dir_perm = 0;

/* keep stats during walk */
uint64_t total_dirs = 0;
@@ -1583,7 +1581,7 @@ static void print_files(mfu_flist flist, mfu_path* path)
return;
}

static int invalid_sortfields(char* sortfields)
static int invalid_sortfields(char* sortfields, mfu_walk_opts_t* walk_opts)
{
/* get our rank */
int rank;
@@ -1595,7 +1593,7 @@ static int invalid_sortfields(char* sortfields)
int maxfields;
int nfields = 0;
char* sortfields_copy = MFU_STRDUP(sortfields);
if (walk_stat) {
if (walk_opts->use_stat) {
maxfields = 7;
char* token = strtok(sortfields_copy, ",");
while (token != NULL) {
@@ -1725,6 +1723,9 @@ int main(int argc, char** argv)
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &ranks);

/* pointer to mfu_walk_opts */
mfu_walk_opts_t* walk_opts = mfu_walk_opts_new();

/* TODO: extend options
* - allow user to cache scan result in file
* - allow user to load cached scan as input
@@ -1779,7 +1780,7 @@ int main(int argc, char** argv)
outputname = MFU_STRDUP(optarg);
break;
case 'l':
walk_stat = 0;
walk_opts->use_stat = 0;
break;
case 'h':
usage = 1;
@@ -1855,7 +1856,7 @@ int main(int argc, char** argv)
* input file */
if (walk) {
/* walk list of input paths */
mfu_flist_walk_param_paths(numpaths, paths, walk_stat, dir_perm, flist);
mfu_flist_walk_param_paths(numpaths, paths, walk_opts, flist);
}
else {
/* read list from file */
@@ -1961,7 +1962,7 @@ int main(int argc, char** argv)
regex = arg_to_regex(ls_args);
#if 0
sortfields = MFU_STRDUP(ls_args);
if (invalid_sortfields(sortfields)) {
if (invalid_sortfields(sortfields, walk_opts)) {
/* disable printing and sorting */
mfu_free(&sortfields);
print = 0;
@@ -2060,6 +2061,9 @@ int main(int argc, char** argv)
}
}

/* free the walk options */
mfu_walk_opts_delete(&walk_opts);

/* free users, groups, and files objects */
mfu_flist_free(&flist);

@@ -95,6 +95,9 @@ int main(int argc, char** argv)
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);

/* pointer to mfu_walk_opts */
mfu_walk_opts_t* walk_opts = mfu_walk_opts_new();

int option_index = 0;
static struct option long_options[] = {
{"create", 0, 0, 'c'},
@@ -249,7 +252,7 @@ int main(int argc, char** argv)

/* walk path to get stats info on all files */
mfu_flist flist = mfu_flist_new();
mfu_flist_walk_param_paths(num_src_params, src_params, 1, 0, flist);
mfu_flist_walk_param_paths(num_src_params, src_params, walk_opts, flist);

/* create the archive file */
mfu_flist_archive_create(flist, opts_tarfile, &archive_opts);
@@ -306,6 +309,9 @@ int main(int argc, char** argv)
}
}

/* free the walk options */
mfu_walk_opts_delete(&walk_opts);

/* free context */
mfu_free(&opts_tarfile);

@@ -50,6 +50,31 @@
#include <sys/ioctl.h>
#endif

/* return a newly allocated walk_opts structure, set default values on its fields */
mfu_walk_opts_t* mfu_walk_opts_new(void)
{
mfu_walk_opts_t* opts = (mfu_walk_opts_t*)MFU_MALLOC(sizeof(mfu_walk_opts_t));

/* Don't set permissions on walk by default */
opts->dir_perms = 0;

/* Remove files in walk by default */
opts->remove = 0;

/* Don't stat files in walk by default */
opts->use_stat = 1;

return opts;
}

void mfu_walk_opts_delete(mfu_walk_opts_t** popts)
{
if (popts != NULL) {
mfu_walk_opts_t* opts = *popts;
mfu_free(popts);
}
}

/****************************************
* Functions on types
***************************************/
@@ -144,7 +144,7 @@ int mfu_perms_parse(const char* modestr, mfu_perms** pperms);
* execute for the "all" bits as well because those can also turn on the user's
* read and execute bits, sets dir_perms to 1 if "rx" should be set on directories
* and set to 0 otherwise */
void mfu_perms_need_dir_rx(const mfu_perms* head, int* dir_perms);
void mfu_perms_need_dir_rx(const mfu_perms* head, mfu_walk_opts_t* walk_opts);

/* free the permissions linked list allocated in mfu_perms_parse,
* sets pointer to NULL on return */
@@ -169,27 +169,24 @@ void mfu_flist_free(mfu_flist* flist);
* set directory permission bits in order to walk into a directory,
* whose permission bits are otherwise restrictive (e.g., useful for recursive unlink) */
void mfu_flist_walk_path(
const char* path, /* IN - path to be walked */
int use_stat, /* IN - whether to stat each item (1) or not (0) */
int dir_permissions, /* IN - whether to set directory permission bits (1) or not (0) during walk */
mfu_flist flist /* OUT - flist to insert walked items into */
const char* path, /* IN - path to be walked */
mfu_walk_opts_t* walk_opts, /* IN - functions to perform during the walk */
mfu_flist flist /* OUT - flist to insert walked items into */
);

/* create file list by walking list of directories */
void mfu_flist_walk_paths(
uint64_t num_paths, /* IN - number of paths in array */
const char** paths, /* IN - array of paths to be walkted */
int use_stat, /* IN - whether to stat each item (1) or not (0) */
int dir_permissions, /* IN - whether to set directory permission bits (1) or not (0) during walk */
mfu_flist flist /* OUT - flist to insert walked items into */
uint64_t num_paths, /* IN - number of paths in array */
const char** paths, /* IN - array of paths to be walkted */
mfu_walk_opts_t* walk_opts, /* IN - functions to perform during the walk */
mfu_flist flist /* OUT - flist to insert walked items into */
);

/* given a list of param_paths, walk each one and add to flist */
void mfu_flist_walk_param_paths(
uint64_t num, /* IN - number of paths in array */
const mfu_param_path* params, /* IN - array of paths to be walkted */
int use_stat, /* IN - whether to stat each item (1) or not (0) */
int dir_perms, /* IN - whether to set directory permission bits (1) or not (0) during walk */
mfu_walk_opts_t* walk_opts, /* IN - functions to perform during the walk */
mfu_flist flist /* OUT - flist to insert walked items into */
);

@@ -444,6 +441,13 @@ int mfu_flist_copy(
mfu_copy_opts_t* mfu_copy_opts /* IN - options to be used during copy */
);

/* allocate a new mfu_walk_opts structure,
* and set its fields with default values */
mfu_walk_opts_t* mfu_walk_opts_new(void);

/* free object allocated in mfu_walk_opts_new */
void mfu_walk_opts_delete(mfu_walk_opts_t** opts);

/* create all directories in flist */
void mfu_flist_mkdir(mfu_flist flist);

@@ -399,11 +399,11 @@ int mfu_perms_parse(const char* modestr, mfu_perms** p_head)
* directories during the walk in this case. Also, check for turning on read and
* execute for the "all" bits as well because those can also turn on the user's
* read and execute bits */
void mfu_perms_need_dir_rx(const mfu_perms* head, int* dir_perms)
void mfu_perms_need_dir_rx(const mfu_perms* head, mfu_walk_opts_t* walk_opts)
{
/* flag to check if the usr read & execute bits are being turned on,
* assume they are not */
*dir_perms = 0;
walk_opts->dir_perms = 0;

/* extra flags to check if usr read and execute are being turned on */
int usr_r = 0;
@@ -457,7 +457,7 @@ void mfu_perms_need_dir_rx(const mfu_perms* head, int* dir_perms)

/* only set the dir_perms flag if both the user execute and user read flags are on */
if (usr_r && usr_x) {
*dir_perms = 1;
walk_opts->dir_perms = 1;
}

return;
@@ -49,6 +49,7 @@ static uint64_t CURRENT_NUM_DIRS;
static const char** CURRENT_DIRS;
static flist_t* CURRENT_LIST;
static int SET_DIR_PERMS;
static int REMOVE_FILES;

/****************************************
* Global counter and callbacks for LIBCIRCLE reductions
@@ -495,10 +496,16 @@ static void walk_readdir_process_dir(char* dir, CIRCLE_handle* handle)
mode_t mode;
int have_mode = 0;
if (entry->d_type != DT_UNKNOWN) {
/* we can read object type from directory entry */
have_mode = 1;
mode = DTTOIF(entry->d_type);
mfu_flist_insert_stat(CURRENT_LIST, newpath, mode, NULL);
/* unlink files here if remove option is on,
* and dtype is known without a stat */
if (REMOVE_FILES && (entry->d_type != DT_DIR)) {
mfu_unlink(newpath);
} else {
/* we can read object type from directory entry */
have_mode = 1;
mode = DTTOIF(entry->d_type);
mfu_flist_insert_stat(CURRENT_LIST, newpath, mode, NULL);
}
}
else {
/* type is unknown, we need to stat it */
@@ -507,7 +514,13 @@ static void walk_readdir_process_dir(char* dir, CIRCLE_handle* handle)
if (status == 0) {
have_mode = 1;
mode = st.st_mode;
mfu_flist_insert_stat(CURRENT_LIST, newpath, mode, &st);
/* unlink files here if remove option is on,
* and stat was necessary to get type */
if (REMOVE_FILES && !S_ISDIR(st.st_mode)) {
mfu_unlink(newpath);
} else {
mfu_flist_insert_stat(CURRENT_LIST, newpath, mode, &st);
}
}
else {
/* error */
@@ -656,8 +669,12 @@ static void walk_stat_process(CIRCLE_handle* handle)

/* TODO: filter items by stat info */

/* record info for item in list */
mfu_flist_insert_stat(CURRENT_LIST, path, st.st_mode, &st);
if (REMOVE_FILES && !S_ISDIR(st.st_mode)) {
mfu_unlink(path);
} else {
/* record info for item in list */
mfu_flist_insert_stat(CURRENT_LIST, path, st.st_mode, &st);
}

/* recurse into directory */
if (S_ISDIR(st.st_mode)) {
@@ -682,24 +699,30 @@ static void walk_stat_process(CIRCLE_handle* handle)
}

/* Set up and execute directory walk */
void mfu_flist_walk_path(const char* dirpath, int use_stat, int dir_permissions, mfu_flist bflist)
void mfu_flist_walk_path(const char* dirpath, mfu_walk_opts_t* walk_opts,
mfu_flist bflist)
{
mfu_flist_walk_paths(1, &dirpath, use_stat, dir_permissions, bflist);
mfu_flist_walk_paths(1, &dirpath, walk_opts, bflist);
return;
}

/* Set up and execute directory walk */
void mfu_flist_walk_paths(uint64_t num_paths, const char** paths, int use_stat, int dir_permissions, mfu_flist bflist)
void mfu_flist_walk_paths(uint64_t num_paths, const char** paths,
mfu_walk_opts_t* walk_opts, mfu_flist bflist)
{
/* report walk count, time, and rate */
double start_walk = MPI_Wtime();

/* if dir_permission is set to 1 then set global variable */
if (dir_permissions) {
SET_DIR_PERMS = 0;
if (walk_opts->dir_perms) {
SET_DIR_PERMS = 1;
}
else {
SET_DIR_PERMS = 0;

/* if remove is set to 1 then set global variable */
REMOVE_FILES = 0;
if (walk_opts->remove) {
REMOVE_FILES = 1;
}

/* convert handle to flist_t */
@@ -736,7 +759,7 @@ void mfu_flist_walk_paths(uint64_t num_paths, const char** paths, int use_stat,
/* we lookup users and groups first in case we can use
* them to filter the walk */
flist->detail = 0;
if (use_stat) {
if (walk_opts->use_stat) {
flist->detail = 1;
if (flist->have_users == 0) {
mfu_flist_usrgrp_get_users(flist);
@@ -747,7 +770,7 @@ void mfu_flist_walk_paths(uint64_t num_paths, const char** paths, int use_stat,
}

/* register callbacks */
if (use_stat) {
if (walk_opts->use_stat) {
/* walk directories by calling stat on every item */
CIRCLE_cb_create(&walk_stat_create);
CIRCLE_cb_process(&walk_stat_process);
@@ -797,7 +820,10 @@ void mfu_flist_walk_paths(uint64_t num_paths, const char** paths, int use_stat,
}

/* given a list of param_paths, walk each one and add to flist */
void mfu_flist_walk_param_paths(uint64_t num, const mfu_param_path* params, int walk_stat, int dir_perms, mfu_flist flist)
void mfu_flist_walk_param_paths(uint64_t num,
const mfu_param_path* params,
mfu_walk_opts_t* walk_opts,
mfu_flist flist)
{
/* allocate memory to hold a list of paths */
const char** path_list = (const char**) MFU_MALLOC(num * sizeof(char*));
@@ -810,7 +836,7 @@ void mfu_flist_walk_param_paths(uint64_t num, const mfu_param_path* params, int
}

/* walk file tree and record stat data for each file */
mfu_flist_walk_paths((uint64_t) num, path_list, walk_stat, dir_perms, flist);
mfu_flist_walk_paths((uint64_t) num, path_list, walk_opts, flist);

/* free the list */
mfu_free(&path_list);
@@ -79,7 +79,14 @@ void mfu_param_path_check_copy(
int* flag_copy_into_dir /* OUT - flag indicating whether source items should be copied into destination directory (1) or not (0) */
);

/* options passed to mfu_flist_copy that affect how a copy is executed */
/* options passed to walk that effect how the walk is executed */
typedef struct {
int dir_perms; /* flag option to update dir perms during walk */
int remove; /* flag option to remove files during walk */
int use_stat; /* flag option on whether or not to stat files during walk */
} mfu_walk_opts_t;

/* options passed to mfu_ */
typedef struct {
int copy_into_dir; /* flag indicating whether copying into existing dir */
int do_sync; /* flag option to sync src dir with dest dir */
Oops, something went wrong.

0 comments on commit 058fa24

Please sign in to comment.