Skip to content

Commit

Permalink
Drop the duplicate hash table implementation.
Browse files Browse the repository at this point in the history 
Findlib has a hash table implementation for keeping hardlinked files
its cleaner to use the htable class we already have in the generic
library. So this patch adds support for binary keys in the htable class
so we can store there the combined inode/device key. We now also use the
hash_malloc of the htable class which allocates memory from a big memory
block so it should also scale a bit better.
  • Loading branch information
Marco van Wieringen committed Feb 17, 2015
1 parent 3b7173f commit 02fe8a9
Show file tree
Hide file tree
Showing 9 changed files with 503 additions and 264 deletions.
74 changes: 67 additions & 7 deletions src/filed/fd_plugins.c
View file
Expand Up @@ -564,17 +564,19 @@ bRC plugin_option_handle_file(JCR *jcr, FF_PKT *ff_pkt, struct save_pkt *sp)
*/
int plugin_save(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
{
Plugin *plugin;
int len;
int i;
char *cmd = ff_pkt->top_fname;
struct save_pkt sp;
int len;
bRC ret;
char *cmd;
bEvent event;
Plugin *plugin;
struct save_pkt sp;
bEventType eventType;
POOL_MEM fname(PM_FNAME);
POOL_MEM link(PM_FNAME);
bpContext *plugin_ctx_list;

cmd = ff_pkt->top_fname;
plugin_ctx_list = (bpContext *)jcr->plugin_ctx_list;
if (!fd_plugin_list || !plugin_ctx_list || jcr->is_job_canceled()) {
Jmsg1(jcr, M_FATAL, 0, "Command plugin \"%s\" requested, but is not loaded.\n", cmd);
Expand Down Expand Up @@ -644,11 +646,13 @@ int plugin_save(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
if (plug_func(plugin)->startBackupFile(ctx, &sp) != bRC_OK) {
goto bail_out;
}

if (sp.type == 0) {
Jmsg1(jcr, M_FATAL, 0, _("Command plugin \"%s\": no type in startBackupFile packet.\n"),
cmd);
goto bail_out;
}

jcr->plugin_sp = &sp;
ff_pkt = jcr->ff;

Expand Down Expand Up @@ -690,11 +694,67 @@ int plugin_save(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
Dmsg2(dbglvl, "index=%d object=%s\n", sp.index, sp.object);
}

/*
* Handle hard linked files
*
* Maintain a list of hard linked files already backed up. This allows us to ensure
* that the data of each file gets backed up only once.
*/
ff_pkt->LinkFI = 0;
if (!(ff_pkt->flags & FO_NO_HARDLINK) && ff_pkt->statp.st_nlink > 1) {
CurLink *hl;

switch (ff_pkt->statp.st_mode & S_IFMT) {
case S_IFREG:
case S_IFCHR:
case S_IFBLK:
case S_IFIFO:
#ifdef S_IFSOCK
case S_IFSOCK:
#endif
hl = lookup_hardlink(jcr, ff_pkt, ff_pkt->statp.st_ino, ff_pkt->statp.st_dev);
if (hl) {
/*
* If we have already backed up the hard linked file don't do it again
*/
if (bstrcmp(hl->name, sp.fname)) {
Dmsg2(400, "== Name identical skip FI=%d file=%s\n", hl->FileIndex, fname.c_str());
ff_pkt->no_read = true;
} else {
ff_pkt->link = hl->name;
ff_pkt->type = FT_LNKSAVED; /* Handle link, file already saved */
ff_pkt->LinkFI = hl->FileIndex;
ff_pkt->linked = NULL;
ff_pkt->digest = hl->digest;
ff_pkt->digest_stream = hl->digest_stream;
ff_pkt->digest_len = hl->digest_len;

Dmsg3(400, "FT_LNKSAVED FI=%d LinkFI=%d file=%s\n", ff_pkt->FileIndex, hl->FileIndex, hl->name);

ff_pkt->no_read = true;
}
} else {
/*
* File not previously dumped. Chain it into our list.
*/
hl = new_hardlink(jcr, ff_pkt, sp.fname, ff_pkt->statp.st_ino, ff_pkt->statp.st_dev);
ff_pkt->linked = hl; /* Mark saved link */
Dmsg2(400, "Added to hash FI=%d file=%s\n", ff_pkt->FileIndex, hl->name);
}
break;
default:
ff_pkt->linked = NULL;
break;
}
} else {
ff_pkt->linked = NULL;
}

/*
* Call Bareos core code to backup the plugin's file
*/
save_file(jcr, ff_pkt, true);
bRC ret = plug_func(plugin)->endBackupFile(ctx);
ret = plug_func(plugin)->endBackupFile(ctx);
if (ret == bRC_More || ret == bRC_OK) {
accurate_mark_file_as_seen(jcr, fname.c_str());
}
Expand Down Expand Up @@ -1096,7 +1156,7 @@ int plugin_create_file(JCR *jcr, ATTR *attr, BFILE *bfd, int replace)
rp.file_index = attr->file_index;
rp.LinkFI = attr->LinkFI;
rp.uid = attr->uid;
rp.statp = attr->statp; /* structure assignment */
memcpy(&rp.statp, &attr->statp, sizeof(rp.statp));
rp.attrEx = attr->attrEx;
rp.ofname = attr->ofname;
rp.olname = attr->olname;
Expand Down Expand Up @@ -1191,7 +1251,7 @@ bool plugin_set_attributes(JCR *jcr, ATTR *attr, BFILE *ofd)
rp.file_index = attr->file_index;
rp.LinkFI = attr->LinkFI;
rp.uid = attr->uid;
rp.statp = attr->statp; /* structure assignment */
memcpy(&rp.statp, &attr->statp, sizeof(rp.statp));
rp.attrEx = attr->attrEx;
rp.ofname = attr->ofname;
rp.olname = attr->olname;
Expand Down
6 changes: 3 additions & 3 deletions src/findlib/Makefile.in
View file
Expand Up @@ -26,9 +26,9 @@ INCLUDE_FILES = bfile.h find.h protos.h

#
LIBBAREOSFIND_SRCS = acl.c attribs.c bfile.c create_file.c \
drivetype.c enable_priv.c find_one.c \
find.c fstype.c match.c mkpath.c \
savecwd.c shadowing.c xattr.c
drivetype.c enable_priv.c find_one.c \
find.c fstype.c hardlink.c match.c mkpath.c \
savecwd.c shadowing.c xattr.c

LIBBAREOSFIND_OBJS = $(LIBBAREOSFIND_SRCS:.c=.o)
LIBBAREOSFIND_LOBJS = $(LIBBAREOSFIND_SRCS:.c=.lo)
Expand Down
59 changes: 35 additions & 24 deletions src/findlib/find.c
View file
Expand Up @@ -99,9 +99,9 @@ void set_find_changed_function(FF_PKT *ff, bool check_fct(JCR *jcr, FF_PKT *ff))
* argument and a packet as the second argument, this packet
* will be passed back to the callback subroutine as the last
* argument.
*
*/
int find_files(JCR *jcr, FF_PKT *ff, int file_save(JCR *jcr, FF_PKT *ff_pkt, bool top_level),
int find_files(JCR *jcr, FF_PKT *ff,
int file_save(JCR *jcr, FF_PKT *ff_pkt, bool top_level),
int plugin_save(JCR *jcr, FF_PKT *ff_pkt, bool top_level))
{
ff->file_save = file_save;
Expand All @@ -111,25 +111,28 @@ int find_files(JCR *jcr, FF_PKT *ff, int file_save(JCR *jcr, FF_PKT *ff_pkt, boo
findFILESET *fileset = ff->fileset;
if (fileset) {
int i, j;
/* TODO: We probably need be move the initialization in the fileset loop,
/*
* TODO: We probably need be move the initialization in the fileset loop,
* at this place flags options are "concatenated" accross Include {} blocks
* (not only Options{} blocks inside a Include{})
*/
ff->flags = 0;
for (i = 0; i < fileset->include_list.size(); i++) {
dlistString *node;
findINCEXE *incexe = (findINCEXE *)fileset->include_list.get(i);
fileset->incexe = incexe;

/* Here, we reset some values between two different Include{} */
/*
* Here, we reset some values between two different Include{}
*/
strcpy(ff->VerifyOpts, "V");
strcpy(ff->AccurateOpts, "Cmcs"); /* mtime+ctime+size by default */
strcpy(ff->BaseJobOpts, "Jspug5"); /* size+perm+user+group+chk */
ff->plugin = NULL;
ff->opt_plugin = false;

/*
* By setting all options, we in effect OR the global options
* which is what we want.
* By setting all options, we in effect OR the global options which is what we want.
*/
for (j = 0; j < incexe->opts_list.size(); j++) {
findFOPTS *fo = (findFOPTS *)incexe->opts_list.get(j);
Expand All @@ -154,9 +157,10 @@ int find_files(JCR *jcr, FF_PKT *ff, int file_save(JCR *jcr, FF_PKT *ff_pkt, boo
}
Dmsg4(50, "Verify=<%s> Accurate=<%s> BaseJob=<%s> flags=<%d>\n",
ff->VerifyOpts, ff->AccurateOpts, ff->BaseJobOpts, ff->flags);
dlistString *node;

foreach_dlist(node, &incexe->name_list) {
char *fname = node->c_str();

Dmsg1(dbglvl, "F %s\n", fname);
ff->top_fname = fname;
if (find_one_file(jcr, ff, our_callback, ff->top_fname, (dev_t)-1, true) == 0) {
Expand All @@ -166,8 +170,10 @@ int find_files(JCR *jcr, FF_PKT *ff, int file_save(JCR *jcr, FF_PKT *ff_pkt, boo
return 0;
}
}

foreach_dlist(node, &incexe->plugin_list) {
char *fname = node->c_str();

if (!plugin_save) {
Jmsg(jcr, M_FATAL, 0, _("Plugin: \"%s\" not found.\n"), fname);
return 0;
Expand All @@ -188,16 +194,16 @@ int find_files(JCR *jcr, FF_PKT *ff, int file_save(JCR *jcr, FF_PKT *ff_pkt, boo

/*
* Test if the currently selected directory (in ff->fname) is
* explicitly in the Include list or explicitly in the Exclude
* list.
* explicitly in the Include list or explicitly in the Exclude list.
*/
bool is_in_fileset(FF_PKT *ff)
{
dlistString *node;
char *fname;
int i;
char *fname;
dlistString *node;
findINCEXE *incexe;
findFILESET *fileset = ff->fileset;

if (fileset) {
for (i = 0; i < fileset->include_list.size(); i++) {
incexe = (findINCEXE *)fileset->include_list.get(i);
Expand All @@ -220,16 +226,17 @@ bool is_in_fileset(FF_PKT *ff)
}
}
}

return false;
}

bool accept_file(FF_PKT *ff)
{
int i, j, k;
int fnm_flags;
const char *basename;
findFILESET *fileset = ff->fileset;
findINCEXE *incexe = fileset->incexe;
const char *basename;
int (*match_func)(const char *pattern, const char *string, int flags);

Dmsg1(dbglvl, "enter accept_file: fname=%s\n", ff->fname);
Expand Down Expand Up @@ -257,7 +264,7 @@ bool accept_file(FF_PKT *ff)

if (S_ISDIR(ff->statp.st_mode)) {
for (k = 0; k < fo->wilddir.size(); k++) {
if (match_func((char *)fo->wilddir.get(k), ff->fname, fnmode|fnm_flags) == 0) {
if (match_func((char *)fo->wilddir.get(k), ff->fname, fnmode | fnm_flags) == 0) {
if (ff->flags & FO_EXCLUDE) {
Dmsg2(dbglvl, "Exclude wilddir: %s file=%s\n", (char *)fo->wilddir.get(k),
ff->fname);
Expand All @@ -268,7 +275,7 @@ bool accept_file(FF_PKT *ff)
}
} else {
for (k = 0; k < fo->wildfile.size(); k++) {
if (match_func((char *)fo->wildfile.get(k), ff->fname, fnmode|fnm_flags) == 0) {
if (match_func((char *)fo->wildfile.get(k), ff->fname, fnmode | fnm_flags) == 0) {
if (ff->flags & FO_EXCLUDE) {
Dmsg2(dbglvl, "Exclude wildfile: %s file=%s\n", (char *)fo->wildfile.get(k),
ff->fname);
Expand All @@ -279,7 +286,7 @@ bool accept_file(FF_PKT *ff)
}

for (k = 0; k < fo->wildbase.size(); k++) {
if (match_func((char *)fo->wildbase.get(k), basename, fnmode|fnm_flags) == 0) {
if (match_func((char *)fo->wildbase.get(k), basename, fnmode | fnm_flags) == 0) {
if (ff->flags & FO_EXCLUDE) {
Dmsg2(dbglvl, "Exclude wildbase: %s file=%s\n", (char *)fo->wildbase.get(k),
basename);
Expand All @@ -289,8 +296,9 @@ bool accept_file(FF_PKT *ff)
}
}
}

for (k = 0; k < fo->wild.size(); k++) {
if (match_func((char *)fo->wild.get(k), ff->fname, fnmode|fnm_flags) == 0) {
if (match_func((char *)fo->wild.get(k), ff->fname, fnmode | fnm_flags) == 0) {
if (ff->flags & FO_EXCLUDE) {
Dmsg2(dbglvl, "Exclude wild: %s file=%s\n", (char *)fo->wild.get(k),
ff->fname);
Expand All @@ -299,6 +307,7 @@ bool accept_file(FF_PKT *ff)
return true; /* accept file */
}
}

if (S_ISDIR(ff->statp.st_mode)) {
for (k = 0; k < fo->regexdir.size(); k++) {
const int nmatch = 30;
Expand All @@ -322,6 +331,7 @@ bool accept_file(FF_PKT *ff)
}
}
}

for (k = 0; k < fo->regex.size(); k++) {
const int nmatch = 30;
regmatch_t pmatch[nmatch];
Expand All @@ -337,8 +347,8 @@ bool accept_file(FF_PKT *ff)
* exclude the file
*/
if (ff->flags & FO_EXCLUDE &&
fo->regex.size() == 0 && fo->wild.size() == 0 &&
fo->regexdir.size() == 0 && fo->wilddir.size() == 0 &&
fo->regex.size() == 0 && fo->wild.size() == 0 &&
fo->regexdir.size() == 0 && fo->wilddir.size() == 0 &&
fo->regexfile.size() == 0 && fo->wildfile.size() == 0 &&
fo->wildbase.size() == 0) {
return false; /* reject file */
Expand All @@ -349,22 +359,24 @@ bool accept_file(FF_PKT *ff)
* Now apply the Exclude { } directive
*/
for (i = 0; i < fileset->exclude_list.size(); i++) {
dlistString *node;
findINCEXE *incexe = (findINCEXE *)fileset->exclude_list.get(i);

for (j = 0; j < incexe->opts_list.size(); j++) {
findFOPTS *fo = (findFOPTS *)incexe->opts_list.get(j);
fnm_flags = (fo->flags & FO_IGNORECASE) ? FNM_CASEFOLD : 0;
for (k = 0; k < fo->wild.size(); k++) {
if (fnmatch((char *)fo->wild.get(k), ff->fname, fnmode|fnm_flags) == 0) {
if (fnmatch((char *)fo->wild.get(k), ff->fname, fnmode | fnm_flags) == 0) {
Dmsg1(dbglvl, "Reject wild1: %s\n", ff->fname);
return false; /* reject file */
}
}
}
fnm_flags = (incexe->current_opts != NULL &&
incexe->current_opts->flags & FO_IGNORECASE) ? FNM_CASEFOLD : 0;
dlistString *node;
foreach_dlist(node, &incexe->name_list) {
char *fname = node->c_str();

if (fnmatch(fname, ff->fname, fnmode|fnm_flags) == 0) {
Dmsg1(dbglvl, "Reject wild2: %s\n", ff->fname);
return false; /* reject file */
Expand All @@ -377,8 +389,7 @@ bool accept_file(FF_PKT *ff)

/*
* The code comes here for each file examined.
* We filter the files, then call the user's callback if
* the file is included.
* We filter the files, then call the user's callback if the file is included.
*/
static int our_callback(JCR *jcr, FF_PKT *ff, bool top_level)
{
Expand Down Expand Up @@ -425,8 +436,7 @@ static int our_callback(JCR *jcr, FF_PKT *ff, bool top_level)
}

/*
* Terminate find_files() and release
* all allocated memory
* Terminate find_files() and release all allocated memory
*/
int term_find_files(FF_PKT *ff)
{
Expand All @@ -444,6 +454,7 @@ int term_find_files(FF_PKT *ff)
}
hard_links = term_find_one(ff);
free(ff);

return hard_links;
}

Expand Down

0 comments on commit 02fe8a9

Please sign in to comment.