Skip to content

Commit

Permalink
Merge pull request #102 from KnightKu/dcmp-dev
Browse files Browse the repository at this point in the history 
Dcmp improve
  • Loading branch information
dsikich committed Apr 19, 2018
2 parents 4c3550c + bf0eb9c commit 8f8121f
Show file tree
Hide file tree
Showing 3 changed files with 88 additions and 39 deletions.
12 changes: 12 additions & 0 deletions src/common/mfu_flist.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,18 @@ static size_t list_elem_unpack2(const void* buf, elem_t* elem)
return bytes;
}

/* fake insert, just increase the count, used for counting */
void mfu_flist_increase(mfu_flist* pbflist)
{
/* convert handle to flist_t */
flist_t* flist = *(flist_t**)pbflist;

/* increase list count by one */
flist->list_count++;

return;
}

/* append element to tail of linked list */
void mfu_flist_insert_elem(flist_t* flist, elem_t* elem)
{
Expand Down
3 changes: 3 additions & 0 deletions src/common/mfu_flist.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,9 @@ void mfu_flist_write_cache(
/* free resouces in file list */
void mfu_flist_free(mfu_flist* flist);

/* fake insert, just increase the count, used for counting */
void mfu_flist_increase(mfu_flist* pblist);

/* given an input list, split items into separate lists depending
* on their depth, returns number of levels, minimum depth, and
* array of lists as output */
Expand Down
112 changes: 73 additions & 39 deletions src/dcmp/dcmp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ static void print_usage(void)
printf(" -b, --base - do base comparison\n");
printf(" -o, --output field0=state0@field1=state1,field2=state2:file "
"- write list to file\n");
printf(" multiple output files supports, e.g.\n");
printf(" -o EXIST=ONLY_SRC,TYPE=DIFFER,PERM=DIFFER,MTIME=DIFFER:"
"fileA -o MTIME=DIFFER:fileB\n");
printf(" -v, --verbose\n");
printf(" -h, --help - print usage\n");
printf("\n");
Expand Down Expand Up @@ -130,12 +133,15 @@ struct dcmp_expression {

struct dcmp_conjunction {
struct list_head linkage; /* linkage to struct dcmp_disjunction */
struct list_head expressions; /* list of logical conjunction */
struct list_head expressions; /* list of logical conjunction */
mfu_flist src_matched_list; /* matched src items in this conjunction */
mfu_flist dst_matched_list; /* matched dst items in this conjunction */
};

struct dcmp_disjunction {
struct list_head linkage; /* linkage to struct dcmp_output */
struct list_head conjunctions; /* list of logical conjunction */
unsigned count; /* logical conjunctions count */
};

struct dcmp_output {
Expand Down Expand Up @@ -658,7 +664,13 @@ static int dcmp_compare_metadata(
int diff = 0;

if (dcmp_option_need_compare(DCMPF_SIZE)) {
dcmp_compare_field(size, DCMPF_SIZE);
mfu_filetype type = mfu_flist_file_get_type(src_list, src_index);
if (type != MFU_TYPE_DIR) {
dcmp_compare_field(size, DCMPF_SIZE);
} else {
dcmp_strmap_item_update(src_map, key, DCMPF_SIZE, DCMPS_COMMON);
dcmp_strmap_item_update(dst_map, key, DCMPF_SIZE, DCMPS_COMMON);
}
}
if (dcmp_option_need_compare(DCMPF_GID)) {
dcmp_compare_field(gid, DCMPF_GID);
Expand Down Expand Up @@ -943,7 +955,8 @@ static void dcmp_strmap_compare_data(
}

static void time_strmap_compare(mfu_flist src_list, double start_compare,
double end_compare, uint64_t total_bytes_read) {
double end_compare, time_t *time_started,
time_t *time_ended, uint64_t total_bytes_read) {

/* if the verbose option is set print the timing data
report compare count, time, and rate */
Expand All @@ -966,23 +979,11 @@ static void time_strmap_compare(mfu_flist src_list, double start_compare,
char starttime_str[256];
char endtime_str[256];

/* snprintf takes in a char* restrict type as a first argument
* so convert to this type before passing it to avoid the
* compiler warning */
snprintf(starttime_str, 256, "%f", start_compare);
snprintf(endtime_str, 256, "%f", end_compare);

/* convert time to time_t */
time_t start_rawtime = (time_t)start_compare;
time_t end_rawtime = (time_t)end_compare;

/* format start & end time string I made copies of the localstart
* & localend because the next call to localtime was overriding
* the second call */
struct tm* localstart = localtime(&start_rawtime);
struct tm* localstart = localtime(time_started);
struct tm cp_localstart = *localstart;
struct tm* localend = localtime(&end_rawtime);
struct tm* localend = localtime(time_ended);
struct tm cp_localend = *localend;

strftime(starttime_str, 256, "%b-%d-%Y, %H:%M:%S", &cp_localstart);
strftime(endtime_str, 256, "%b-%d-%Y, %H:%M:%S", &cp_localend);

Expand Down Expand Up @@ -1090,7 +1091,12 @@ static void dcmp_strmap_compare(mfu_flist src_list,
{
/* wait for all tasks and start timer */
MPI_Barrier(MPI_COMM_WORLD);

time_t time_started;
time_t time_ended;

double start_compare = MPI_Wtime();
time(&time_started);

/* create compare_lists */
mfu_flist src_compare_list = mfu_flist_subset(src_list);
Expand Down Expand Up @@ -1239,6 +1245,7 @@ static void dcmp_strmap_compare(mfu_flist src_list,
/* wait for all procs to finish before stopping timer */
MPI_Barrier(MPI_COMM_WORLD);
double end_compare = MPI_Wtime();
time(&time_ended);

/* initalize total_bytes_read to zero */
uint64_t total_bytes_read = 0;
Expand All @@ -1248,7 +1255,8 @@ static void dcmp_strmap_compare(mfu_flist src_list,
total_bytes_read = get_total_bytes_read(src_compare_list);
}

time_strmap_compare(src_list, start_compare, end_compare, total_bytes_read);
time_strmap_compare(src_list, start_compare, end_compare, &time_started,
&time_ended, total_bytes_read);

/* if the sync option is on then we need to remove the files
* from the destination list that are not in the src list. Then,
Expand Down Expand Up @@ -1627,6 +1635,8 @@ static struct dcmp_conjunction* dcmp_conjunction_alloc(void)
MFU_MALLOC(sizeof(struct dcmp_conjunction));
INIT_LIST_HEAD(&conjunction->linkage);
INIT_LIST_HEAD(&conjunction->expressions);
conjunction->src_matched_list = mfu_flist_new();
conjunction->dst_matched_list = mfu_flist_new();

return conjunction;
}
Expand All @@ -1653,6 +1663,8 @@ static void dcmp_conjunction_free(struct dcmp_conjunction *conjunction)
dcmp_expression_free(expression);
}
assert(list_empty(&conjunction->expressions));
mfu_flist_free(&conjunction->src_matched_list);
mfu_flist_free(&conjunction->dst_matched_list);
mfu_free(&conjunction);
}

Expand Down Expand Up @@ -1710,6 +1722,7 @@ static struct dcmp_disjunction* dcmp_disjunction_alloc(void)
MFU_MALLOC(sizeof(struct dcmp_disjunction));
INIT_LIST_HEAD(&disjunction->linkage);
INIT_LIST_HEAD(&disjunction->conjunctions);
disjunction->count = 0;

return disjunction;
}
Expand All @@ -1720,6 +1733,7 @@ static void dcmp_disjunction_add_conjunction(
{
assert(list_empty(&conjunction->linkage));
list_add_tail(&conjunction->linkage, &disjunction->conjunctions);
disjunction->count++;
}

static void dcmp_disjunction_free(struct dcmp_disjunction* disjunction)
Expand Down Expand Up @@ -1750,7 +1764,11 @@ static void dcmp_disjunction_print(
&disjunction->conjunctions,
linkage) {
dcmp_conjunction_print(conjunction, simple);

printf(": [%lu/%lu]", mfu_flist_global_size(conjunction->src_matched_list),
mfu_flist_global_size(conjunction->dst_matched_list));
if (conjunction->linkage.next != &disjunction->conjunctions) {

if (simple) {
printf("||");
} else {
Expand All @@ -1768,7 +1786,8 @@ static void dcmp_disjunction_print(
static int dcmp_disjunction_match(
struct dcmp_disjunction* disjunction,
strmap* map,
const char* key)
const char* key,
int is_src)
{
struct dcmp_conjunction *conjunction;
int matched;
Expand All @@ -1778,6 +1797,10 @@ static int dcmp_disjunction_match(
linkage) {
matched = dcmp_conjunction_match(conjunction, map, key);
if (matched) {
if (is_src)
mfu_flist_increase(&conjunction->src_matched_list);
else
mfu_flist_increase(&conjunction->dst_matched_list);
return 1;
}
}
Expand Down Expand Up @@ -1856,11 +1879,12 @@ static int dcmp_output_flist_match(
strmap* map,
mfu_flist flist,
mfu_flist new_flist,
uint64_t *number)
mfu_flist *matched_flist,
int is_src)
{
const strmap_node* node;
struct dcmp_conjunction *conjunction;

*number = 0;
/* iterate over each item in map */
strmap_foreach(map, node) {
/* get file name */
Expand All @@ -1871,11 +1895,22 @@ static int dcmp_output_flist_match(
int ret = dcmp_strmap_item_index(map, key, &idx);
assert(ret == 0);

if (dcmp_disjunction_match(output->disjunction, map, key)) {
(*number)++;
if (dcmp_disjunction_match(output->disjunction, map, key, is_src)) {
mfu_flist_increase(matched_flist);
mfu_flist_file_copy(flist, idx, new_flist);
}
}

list_for_each_entry(conjunction,
&output->disjunction->conjunctions,
linkage) {
if (is_src) {
mfu_flist_summarize(conjunction->src_matched_list);
} else {
mfu_flist_summarize(conjunction->dst_matched_list);
}
}

return 0;
}

Expand All @@ -1892,46 +1927,45 @@ static int dcmp_output_write(
mfu_flist new_flist = mfu_flist_subset(src_flist);

/* find matched file in source map */
uint64_t src_matched = 0;
mfu_flist src_matched = mfu_flist_new();
ret = dcmp_output_flist_match(output, src_map, src_flist,
new_flist, &src_matched);
new_flist, &src_matched, 1);
assert(ret == 0);

/* find matched file in dest map */
uint64_t dst_matched = 0;
mfu_flist dst_matched = mfu_flist_new();
ret = dcmp_output_flist_match(output, dst_map, dst_flist,
new_flist, &dst_matched);
new_flist, &dst_matched, 0);
assert(ret == 0);

mfu_flist_summarize(new_flist);
mfu_flist_summarize(src_matched);
mfu_flist_summarize(dst_matched);
if (output->file_name != NULL) {
mfu_flist_write_cache(output->file_name, new_flist);
}

uint64_t src_matched_total;
MPI_Allreduce(&src_matched, &src_matched_total, 1,
MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);

uint64_t dst_matched_total;
MPI_Allreduce(&dst_matched, &dst_matched_total, 1,
MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD);

int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (rank == 0) {
printf(DCMP_OUTPUT_PREFIX);
dcmp_disjunction_print(output->disjunction, 0,
strlen(DCMP_OUTPUT_PREFIX));
printf(", number: %lu/%lu",
src_matched_total,
dst_matched_total);

if (output->disjunction->count > 1)
printf(", total number: %lu/%lu",
mfu_flist_global_size(src_matched),
mfu_flist_global_size(dst_matched));

if (output->file_name != NULL) {
printf(", dumped to \"%s\"",
output->file_name);
}
printf("\n");
}
mfu_flist_free(&new_flist);
mfu_flist_free(&src_matched);
mfu_flist_free(&dst_matched);

return 0;
}
Expand Down

0 comments on commit 8f8121f

Please sign in to comment.