From 11772bb3a787b886ce0f8705d7a25a8e6313a8c8 Mon Sep 17 00:00:00 2001 From: ellie timoney Date: Mon, 15 Aug 2016 16:53:51 +1000 Subject: [PATCH] reconstruct: try harder to not revisit mailboxes --- imap/reconstruct.c | 66 +++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/imap/reconstruct.c b/imap/reconstruct.c index 54cb789578..294dde8b10 100644 --- a/imap/reconstruct.c +++ b/imap/reconstruct.c @@ -114,6 +114,11 @@ hash_table unqid_table; /* current namespace */ static struct namespace recon_namespace; +struct reconstruct_rock { + strarray_t *discovered; + hash_table visited; +}; + /* forward declarations */ static void do_mboxlist(void); static int do_reconstruct_p(const mbentry_t *mbentry, void *rock); @@ -134,9 +139,9 @@ int main(int argc, char **argv) int fflag = 0; int xflag = 0; char buf[MAX_MAILBOX_PATH+1]; - strarray_t discovered = STRARRAY_INITIALIZER; char *alt_config = NULL; char *start_part = NULL; + struct reconstruct_rock rrock = { NULL, HASH_TABLE_INITIALIZER }; if ((geteuid()) == 0 && (become_cyrus(/*is_master*/0) != 0)) { fatal("must run as the Cyrus user", EC_USAGE); @@ -311,6 +316,10 @@ int main(int argc, char **argv) } } + /* set up reconstruct rock */ + if (fflag) rrock.discovered = strarray_new(); + construct_hash_table(&rrock.visited, 2047, 1); /* XXX magic numbers */ + /* Normal Operation */ if (optind == argc) { if (rflag || dousers) { @@ -321,7 +330,7 @@ int main(int argc, char **argv) assert(!rflag); strlcpy(buf, "*", sizeof(buf)); mboxlist_findall(&recon_namespace, buf, 1, 0, 0, - do_reconstruct, NULL); + do_reconstruct, &rrock); } for (i = optind; i < argc; i++) { @@ -337,9 +346,8 @@ int main(int argc, char **argv) strlcpy(buf, argv[i], sizeof(buf)); /* reconstruct the first mailbox/pattern */ - mboxlist_findall(&recon_namespace, buf, 1, 0, - 0, do_reconstruct, - fflag ? &discovered : NULL); + mboxlist_findall(&recon_namespace, buf, 1, 0, 0, do_reconstruct, &rrock); + if (rflag) { /* build a pattern for submailboxes */ char *p = strchr(buf, '@'); @@ -350,15 +358,13 @@ int main(int argc, char **argv) if (domain) strlcat(buf, domain, sizeof(buf)); /* reconstruct the submailboxes */ - mboxlist_findall(&recon_namespace, buf, 1, 0, - 0, do_reconstruct, - fflag ? &discovered : NULL); + mboxlist_findall(&recon_namespace, buf, 1, 0, 0, do_reconstruct, &rrock); } } /* examine our list to see if we discovered anything */ - while (discovered.count) { - char *name = strarray_shift(&discovered); + while (rrock.discovered && rrock.discovered->count) { + char *name = strarray_shift(rrock.discovered); int r = 0; /* create p (database only) and reconstruct it */ @@ -370,15 +376,16 @@ int main(int argc, char **argv) fprintf(stderr, "createmailbox %s: %s\n", name, error_message(r)); } else { - mboxlist_findone(&recon_namespace, name, 1, 0, - 0, do_reconstruct, - &discovered); + mboxlist_findone(&recon_namespace, name, 1, 0, 0, do_reconstruct, &rrock); } /* may have added more things into our list */ free(name); } + if (rrock.discovered) strarray_free(rrock.discovered); + free_hash_table(&rrock.visited, NULL); + free_hash_table(&unqid_table, free); sync_log_done(); @@ -398,8 +405,6 @@ int main(int argc, char **argv) cyrus_done(); - strarray_fini(&discovered); - return 0; } @@ -431,9 +436,8 @@ static int do_reconstruct_p(const mbentry_t *mbentry, void *rock) static int do_reconstruct(struct findall_data *data, void *rock) { if (!data) return 0; - strarray_t *discovered = (strarray_t *)rock; + struct reconstruct_rock *rrock = (struct reconstruct_rock *) rock; int r; - static char lastname[MAX_MAILBOX_NAME] = ""; char *other; struct mailbox *mailbox = NULL; char outpath[MAX_MAILBOX_PATH]; @@ -442,21 +446,18 @@ static int do_reconstruct(struct findall_data *data, void *rock) signals_poll(); /* don't repeat */ - if (!strcmp(name, lastname)) return 0; - - strncpy(lastname, name, sizeof(lastname)); - lastname[sizeof(lastname)-1] = '\0'; + if (hash_lookup(name, &rrock->visited)) return 0; - r = mailbox_reconstruct(lastname, reconstruct_flags); + r = mailbox_reconstruct(name, reconstruct_flags); if (r) { - com_err(lastname, r, "%s", - (r == IMAP_IOERROR) ? error_message(errno) : "Failed to reconstruct mailbox"); - return 0; + com_err(name, r, "%s", + (r == IMAP_IOERROR) ? error_message(errno) : "Failed to reconstruct mailbox"); + return 0; } - r = mailbox_open_iwl(lastname, &mailbox); + r = mailbox_open_iwl(name, &mailbox); if (r) { - com_err(lastname, r, "Failed to open after reconstruct"); + com_err(name, r, "Failed to open after reconstruct"); return 0; } @@ -471,7 +472,7 @@ static int do_reconstruct(struct findall_data *data, void *rock) hash_insert(mailbox->uniqueid, xstrdup(mailbox->name), &unqid_table); /* Convert internal name to external */ - char *extname = mboxname_to_external(lastname, &recon_namespace, NULL); + char *extname = mboxname_to_external(name, &recon_namespace, NULL); if (!(reconstruct_flags & RECONSTRUCT_QUIET)) printf("%s\n", extname); @@ -490,7 +491,7 @@ static int do_reconstruct(struct findall_data *data, void *rock) mailbox_close(&mailbox); free(extname); - if (discovered) { + if (rrock->discovered) { char fnamebuf[MAX_MAILBOX_PATH]; char *ptr; DIR *dirp; @@ -537,11 +538,16 @@ static int do_reconstruct(struct findall_data *data, void *rock) else r = 0; /* reset error condition */ printf("discovered %s\n", buf); - strarray_append(discovered, buf); + strarray_append(rrock->discovered, buf); } closedir(dirp); } + /* mark it as visited + * we don't care about the value, it just needs to be a non-NULL pointer + */ + hash_insert(name, &rrock, &rrock->visited); + return 0; }