From 11e6db960f22660efcbf65336c286659d4737c09 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Fri, 22 Jul 2016 10:10:37 +0200 Subject: [PATCH 01/10] Fix printing wildbase entries Wild Base settings were not correctly printed when showing a fileset. This is now fixed --- src/dird/dird_conf.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/dird/dird_conf.c b/src/dird/dird_conf.c index f93d7fe9540..700de4ad2ea 100644 --- a/src/dird/dird_conf.c +++ b/src/dird/dird_conf.c @@ -2051,8 +2051,13 @@ bool FILESETRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) indent_config_item(cfg_str, 3, temp.c_str()); } + /* + * Wildbase is WildFile not containing a / or \\ + * see void store_wild() in inc_conf.c + * so we need to translate it back to a Wild File entry + */ for (int k = 0; k < fo->wildbase.size(); k++) { - Mmsg(temp, "Wild Base = \"%c %s\"\n", enhanced_wild ? 'B' : 'F', fo->wildbase.get(k)); + Mmsg(temp, "Wild File = \"%s\"\n", fo->wildbase.get(k)); indent_config_item(cfg_str, 3, temp.c_str()); } From c5c876ca65d87a9846d4d89ed3bc404c6fc85e0a Mon Sep 17 00:00:00 2001 From: Marco van Wieringen Date: Sat, 23 Jul 2016 10:21:11 +0200 Subject: [PATCH 02/10] Tweak layout. --- src/dird/dird_conf.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dird/dird_conf.c b/src/dird/dird_conf.c index 9d564eaaff5..cb054e67eaf 100644 --- a/src/dird/dird_conf.c +++ b/src/dird/dird_conf.c @@ -422,11 +422,11 @@ RES_ITEM job_items[] = { { "MaxConcurrentCopies", CFG_TYPE_PINT32, ITEM(res_job.MaxConcurrentCopies), 0, CFG_ITEM_DEFAULT, "100", NULL, NULL }, /* Settings for always incremental */ { "AlwaysIncremental", CFG_TYPE_BOOL, ITEM(res_job.AlwaysIncremental), 0, CFG_ITEM_DEFAULT, "false", "16.2.4-", - "Enable/disable always incremental backup scheme." }, + "Enable/disable always incremental backup scheme." }, { "AlwaysIncrementalJobRetention", CFG_TYPE_TIME, ITEM(res_job.AlwaysIncrementalJobRetention), 0, CFG_ITEM_DEFAULT, "0", "16.2.4-", - "Backup Jobs older than the specified time duration will be merged into a new Virtual Full." }, + "Backup Jobs older than the specified time duration will be merged into a new Virtual backup." }, { "AlwaysIncrementalKeepNumber", CFG_TYPE_PINT32, ITEM(res_job.AlwaysIncrementalKeepNumber), 0, CFG_ITEM_DEFAULT, "0", "16.2.4-", - "Guarantee that at least the specified number of Backup Jobs will persist, even if they are older than \"Always Incremental Job Retention\"."}, + "Guarantee that at least the specified number of Backup Jobs will persist, even if they are older than \"Always Incremental Job Retention\"."}, { NULL, 0, { 0 }, 0, 0, NULL, NULL, NULL } }; From e1239a573ac1e023d1259696d5b61086367b7219 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Sat, 23 Jul 2016 18:09:57 +0200 Subject: [PATCH 03/10] Fix wrong reuse of ed1 in update_job() --- src/dird/ua_update.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/dird/ua_update.c b/src/dird/ua_update.c index 8cd8c346ad3..5772cf63eb2 100644 --- a/src/dird/ua_update.c +++ b/src/dird/ua_update.c @@ -957,7 +957,7 @@ static bool update_pool(UAContext *ua) static bool update_job(UAContext *ua) { int i; - char ed1[50], ed2[50], ed3[50]; + char ed1[50], ed2[50], ed3[50], ed4[50]; POOL_MEM cmd(PM_MESSAGE); JOB_DBR jr; CLIENT_DBR cr; @@ -1058,10 +1058,10 @@ static bool update_job(UAContext *ua) jr.cStartTime, jr.cSchedTime, jr.cEndTime, - edit_uint64(jr.JobTDate, ed1), - edit_uint64(jr.FileSetId, ed2), + edit_uint64(jr.JobTDate, ed2), + edit_uint64(jr.FileSetId, ed3), jr.JobType, - edit_int64(jr.JobId, ed3)); + edit_int64(jr.JobId, ed4)); if (!db_sql_query(ua->db, cmd.c_str())) { ua->error_msg("%s", db_strerror(ua->db)); return false; From 01cae663a5214112fb28bc23dc71199d8740c67a Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Tue, 26 Jul 2016 09:29:13 +0200 Subject: [PATCH 04/10] config: escape File= and Plugin= directives Plugin= and Fileset= directives of Filesets were not correctly escaped in FILESETRES::print_config. Additionally, the Plugin= entries were not printed with quotes. --- src/dird/dird_conf.c | 23 ++++++++++++++++++++--- src/lib/util.c | 5 +++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/dird/dird_conf.c b/src/dird/dird_conf.c index 700de4ad2ea..849fcdb2773 100644 --- a/src/dird/dird_conf.c +++ b/src/dird/dird_conf.c @@ -1786,6 +1786,12 @@ bool FILESETRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) POOL_MEM temp; const char *p; + /* escape strings */ + POOLMEM *buf; + int len; + buf = get_pool_memory(PM_NAME); + + Dmsg0(200,"FILESETRES::print_config\n"); Mmsg(temp, "FileSet {\n"); @@ -2105,7 +2111,10 @@ bool FILESETRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) */ if (incexe->name_list.size()) { for (int l = 0; l < incexe->name_list.size(); l++) { - Mmsg(temp, "File = \"%s\"\n", incexe->name_list.get(l)); + len = strlen((char*)incexe->name_list.get(l)); + buf = check_pool_memory_size(buf, len * 2); + escape_string(buf, (char*)incexe->name_list.get(l), len); + Mmsg(temp, "File = \"%s\"\n", buf); indent_config_item(cfg_str, 2, temp.c_str()); } } @@ -2115,7 +2124,10 @@ bool FILESETRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) */ if (incexe->plugin_list.size()) { for (int l = 0; l < incexe->plugin_list.size(); l++) { - Mmsg(temp, "Plugin = %s\n", incexe->plugin_list.get(l)); + len = strlen((char*)incexe->plugin_list.get(l)); + buf = check_pool_memory_size(buf, len * 2); + escape_string(buf, (char*)incexe->plugin_list.get(l), len); + Mmsg(temp, "Plugin = \"%s\"\n", buf); indent_config_item(cfg_str, 2, temp.c_str()); } } @@ -2149,7 +2161,10 @@ bool FILESETRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) indent_config_item(cfg_str, 1, "Exclude {\n"); for (int k = 0; k < incexe->name_list.size(); k++) { - Mmsg(temp, "File = \"%s\"\n", incexe->name_list.get(k)); + len = strlen((char*)incexe->name_list.get(k)); + buf = check_pool_memory_size(buf, len * 2); + escape_string(buf, (char*)incexe->name_list.get(k), len); + Mmsg(temp, "File = \"%s\"\n", buf); indent_config_item(cfg_str, 2, temp.c_str()); } @@ -2158,6 +2173,8 @@ bool FILESETRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) } /* loop over all exclude blocks */ } + free_pool_memory(buf); + pm_strcat(cfg_str, "}\n\n"); pm_strcat(buff, cfg_str.c_str()); diff --git a/src/lib/util.c b/src/lib/util.c index ae9ecc7cf36..5c064a980db 100644 --- a/src/lib/util.c +++ b/src/lib/util.c @@ -49,6 +49,11 @@ void escape_string(char *snew, char *old, int len) *n++ = '\''; o++; break; + case '\\': + *n++ = '\\'; + *n++ = '\\'; + o++; + break; case 0: *n++ = '\\'; *n++ = 0; From cd2433e45bf8e06ad777ca0c32810e517025f356 Mon Sep 17 00:00:00 2001 From: Marco van Wieringen Date: Tue, 26 Jul 2016 15:25:42 +0200 Subject: [PATCH 05/10] config: Refactor escape_string function. The escape_string function used for pretty printing from now on has a POOL_MEM as argument to store the escaped data in. It also now sizes the buffer right based on the len argument given. This way the calling code can be cleaned up quite a bit as it now does the same action multiple times which leads to much more code maintenance. --- src/dird/dird_conf.c | 61 ++++++++++++--------------- src/lib/protos.h | 2 +- src/lib/res.c | 28 ++++++------- src/lib/util.c | 99 ++++++++++++++++++++++++-------------------- 4 files changed, 94 insertions(+), 96 deletions(-) diff --git a/src/dird/dird_conf.c b/src/dird/dird_conf.c index 849fcdb2773..89f08f10882 100644 --- a/src/dird/dird_conf.c +++ b/src/dird/dird_conf.c @@ -1231,13 +1231,9 @@ static inline void print_config_runscript(RES_ITEM *item, POOL_MEM &cfg_str) if (bstrcasecmp(item->name, "runscript")) { if (list != NULL) { foreach_alist(runscript, list) { - int len; - POOLMEM *cmdbuf; + POOL_MEM esc; - len = strlen(runscript->command); - cmdbuf = get_pool_memory(PM_NAME); - cmdbuf = check_pool_memory_size(cmdbuf, len * 2); - escape_string(cmdbuf, runscript->command, len); + escape_string(esc, runscript->command, strlen(runscript->command)); /* * Don't print runscript when its inherited from a JobDef. @@ -1252,28 +1248,28 @@ static inline void print_config_runscript(RES_ITEM *item, POOL_MEM &cfg_str) if (runscript->short_form) { if (runscript->when == SCRIPT_Before && /* runbeforejob */ (bstrcmp(runscript->target, ""))) { - Mmsg(temp, "run before job = \"%s\"\n", cmdbuf); + Mmsg(temp, "run before job = \"%s\"\n", esc.c_str()); } else if (runscript->when == SCRIPT_After && /* runafterjob */ runscript->on_success && !runscript->on_failure && !runscript->fail_on_error && bstrcmp(runscript->target, "")) { - Mmsg(temp, "run after job = \"%s\"\n", cmdbuf); + Mmsg(temp, "run after job = \"%s\"\n", esc.c_str()); } else if (runscript->when == SCRIPT_After && /* client run after job */ runscript->on_success && !runscript->on_failure && !runscript->fail_on_error && !bstrcmp(runscript->target, "")) { - Mmsg(temp, "client run after job = \"%s\"\n", cmdbuf); + Mmsg(temp, "client run after job = \"%s\"\n", esc.c_str()); } else if (runscript->when == SCRIPT_Before && /* client run before job */ !bstrcmp(runscript->target, "")) { - Mmsg(temp, "client run before job = \"%s\"\n", cmdbuf); + Mmsg(temp, "client run before job = \"%s\"\n", esc.c_str()); } else if (runscript->when == SCRIPT_After && /* run after failed job */ runscript->on_failure && !runscript->on_success && !runscript->fail_on_error && bstrcmp(runscript->target, "")) { - Mmsg(temp, "run after failed job = \"%s\"\n", cmdbuf); + Mmsg(temp, "run after failed job = \"%s\"\n", esc.c_str()); } indent_config_item(cfg_str, 1, temp.c_str()); } else { @@ -1285,7 +1281,7 @@ static inline void print_config_runscript(RES_ITEM *item, POOL_MEM &cfg_str) cmdstring = (char *)"console"; } - Mmsg(temp, "%s = \"%s\"\n", cmdstring, cmdbuf); + Mmsg(temp, "%s = \"%s\"\n", cmdstring, esc.c_str()); indent_config_item(cfg_str, 2, temp.c_str()); /* @@ -1358,8 +1354,6 @@ static inline void print_config_runscript(RES_ITEM *item, POOL_MEM &cfg_str) indent_config_item(cfg_str, 1, "}\n"); } - - free_pool_memory(cmdbuf); } } /* foreach runscript */ } @@ -1786,12 +1780,6 @@ bool FILESETRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) POOL_MEM temp; const char *p; - /* escape strings */ - POOLMEM *buf; - int len; - buf = get_pool_memory(PM_NAME); - - Dmsg0(200,"FILESETRES::print_config\n"); Mmsg(temp, "FileSet {\n"); @@ -2110,11 +2098,13 @@ bool FILESETRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) * File = entries. */ if (incexe->name_list.size()) { + char *entry; + POOL_MEM esc; + for (int l = 0; l < incexe->name_list.size(); l++) { - len = strlen((char*)incexe->name_list.get(l)); - buf = check_pool_memory_size(buf, len * 2); - escape_string(buf, (char*)incexe->name_list.get(l), len); - Mmsg(temp, "File = \"%s\"\n", buf); + entry = (char *)incexe->name_list.get(l); + escape_string(esc, entry, strlen(entry)); + Mmsg(temp, "File = \"%s\"\n", esc.c_str()); indent_config_item(cfg_str, 2, temp.c_str()); } } @@ -2123,11 +2113,13 @@ bool FILESETRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) * Plugin = entries. */ if (incexe->plugin_list.size()) { + char *entry; + POOL_MEM esc; + for (int l = 0; l < incexe->plugin_list.size(); l++) { - len = strlen((char*)incexe->plugin_list.get(l)); - buf = check_pool_memory_size(buf, len * 2); - escape_string(buf, (char*)incexe->plugin_list.get(l), len); - Mmsg(temp, "Plugin = \"%s\"\n", buf); + entry = (char *)incexe->plugin_list.get(l); + escape_string(esc, entry, strlen(entry)); + Mmsg(temp, "Plugin = \"%s\"\n", esc.c_str()); indent_config_item(cfg_str, 2, temp.c_str()); } } @@ -2158,13 +2150,14 @@ bool FILESETRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) INCEXE *incexe = exclude_items[j]; if (incexe->name_list.size()) { - indent_config_item(cfg_str, 1, "Exclude {\n"); + char *entry; + POOL_MEM esc; + indent_config_item(cfg_str, 1, "Exclude {\n"); for (int k = 0; k < incexe->name_list.size(); k++) { - len = strlen((char*)incexe->name_list.get(k)); - buf = check_pool_memory_size(buf, len * 2); - escape_string(buf, (char*)incexe->name_list.get(k), len); - Mmsg(temp, "File = \"%s\"\n", buf); + entry = (char *)incexe->name_list.get(k); + escape_string(esc, entry, strlen(entry)); + Mmsg(temp, "File = \"%s\"\n", esc.c_str()); indent_config_item(cfg_str, 2, temp.c_str()); } @@ -2173,8 +2166,6 @@ bool FILESETRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) } /* loop over all exclude blocks */ } - free_pool_memory(buf); - pm_strcat(cfg_str, "}\n\n"); pm_strcat(buff, cfg_str.c_str()); diff --git a/src/lib/protos.h b/src/lib/protos.h index cb3fb88ce84..182471a99cd 100644 --- a/src/lib/protos.h +++ b/src/lib/protos.h @@ -384,7 +384,7 @@ void set_tls_enable(TLS_CONTEXT *ctx, bool value); bool get_tls_verify_peer(TLS_CONTEXT *ctx); /* util.c */ -void escape_string(char *snew, char *old, int len); +void escape_string(POOL_MEM &snew, char *old, int len); bool is_buf_zero(char *buf, int len); void lcase(char *str); void bash_spaces(char *str); diff --git a/src/lib/res.c b/src/lib/res.c index 165bd827689..43f3102077d 100644 --- a/src/lib/res.c +++ b/src/lib/res.c @@ -1388,8 +1388,6 @@ static inline void print_config_time(RES_ITEM *item, POOL_MEM &cfg_str) bool MSGSRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) { - int len; - POOLMEM *cmdbuf; POOL_MEM cfg_str; /* configuration as string */ POOL_MEM temp; MSGSRES *msgres; @@ -1401,31 +1399,29 @@ bool MSGSRES::print_config(POOL_MEM &buff, bool hide_sensitive_data) Mmsg(temp, " %s = \"%s\"\n", "Name", msgres->name()); pm_strcat(cfg_str, temp.c_str()); - cmdbuf = get_pool_memory(PM_NAME); if (msgres->mail_cmd) { - len = strlen(msgres->mail_cmd); - cmdbuf = check_pool_memory_size(cmdbuf, len * 2); - escape_string(cmdbuf, msgres->mail_cmd, len); - Mmsg(temp, " MailCommand = \"%s\"\n", cmdbuf); + POOL_MEM esc; + + escape_string(esc, msgres->mail_cmd, strlen(msgres->mail_cmd)); + Mmsg(temp, " MailCommand = \"%s\"\n", esc.c_str()); pm_strcat(cfg_str, temp.c_str()); } if (msgres->operator_cmd) { - len = strlen(msgres->operator_cmd); - cmdbuf = check_pool_memory_size(cmdbuf, len * 2); - escape_string(cmdbuf, msgres->operator_cmd, len); - Mmsg(temp, " OperatorCommand = \"%s\"\n", cmdbuf); + POOL_MEM esc; + + escape_string(esc, msgres->operator_cmd, strlen(msgres->operator_cmd)); + Mmsg(temp, " OperatorCommand = \"%s\"\n", esc.c_str()); pm_strcat(cfg_str, temp.c_str()); } if (msgres->timestamp_format) { - len = strlen(msgres->timestamp_format); - cmdbuf = check_pool_memory_size(cmdbuf, len * 2); - escape_string(cmdbuf, msgres->timestamp_format, len); - Mmsg(temp, " TimestampFormat = \"%s\"\n", cmdbuf); + POOL_MEM esc; + + escape_string(esc, msgres->timestamp_format, strlen(msgres->timestamp_format)); + Mmsg(temp, " TimestampFormat = \"%s\"\n", esc.c_str()); pm_strcat(cfg_str, temp.c_str()); } - free_pool_memory(cmdbuf); for (d = msgres->dest_chain; d; d = d->next) { int nr_set = 0; diff --git a/src/lib/util.c b/src/lib/util.c index 5c064a980db..12568fd2070 100644 --- a/src/lib/util.c +++ b/src/lib/util.c @@ -29,18 +29,18 @@ /* * Various BAREOS Utility subroutines - * */ /* - * Escape special characters in bareos configuration strings - * needed for dumping config strings + * Escape special characters in bareos configuration strings + * needed for dumping config strings */ -void escape_string(char *snew, char *old, int len) +void escape_string(POOL_MEM &snew, char *old, int len) { char *n, *o; - n = snew; + snew.check_size(len * 2); + n = snew.c_str(); o = old; while (len--) { switch (*o) { @@ -75,7 +75,9 @@ void escape_string(char *snew, char *old, int len) *n = 0; } -/* Return true of buffer has all zero bytes */ +/* + * Return true of buffer has all zero bytes + */ bool is_buf_zero(char *buf, int len) { uint64_t *ip; @@ -86,7 +88,10 @@ bool is_buf_zero(char *buf, int len) return false; } ip = (uint64_t *)buf; - /* Optimize by checking uint64_t for zero */ + + /* + * Optimize by checking uint64_t for zero + */ len64 = len / sizeof(uint64_t); for (i=0; i < len64; i++) { if (ip[i] != 0) { @@ -105,7 +110,9 @@ bool is_buf_zero(char *buf, int len) } -/* Convert a string in place to lower case */ +/* + * Convert a string in place to lower case + */ void lcase(char *str) { while (*str) { @@ -116,11 +123,11 @@ void lcase(char *str) } } -/* Convert spaces to non-space character. +/* + * Convert spaces to non-space character. * This makes scanf of fields containing spaces easier. */ -void -bash_spaces(char *str) +void bash_spaces(char *str) { while (*str) { if (*str == ' ') @@ -129,11 +136,11 @@ bash_spaces(char *str) } } -/* Convert spaces to non-space character. +/* + * Convert spaces to non-space character. * This makes scanf of fields containing spaces easier. */ -void -bash_spaces(POOL_MEM &pm) +void bash_spaces(POOL_MEM &pm) { char *str = pm.c_str(); while (*str) { @@ -144,9 +151,10 @@ bash_spaces(POOL_MEM &pm) } -/* Convert non-space characters (0x1) back into spaces */ -void -unbash_spaces(char *str) +/* + * Convert non-space characters (0x1) back into spaces + */ +void unbash_spaces(char *str) { while (*str) { if (*str == 0x1) @@ -155,9 +163,10 @@ unbash_spaces(char *str) } } -/* Convert non-space characters (0x1) back into spaces */ -void -unbash_spaces(POOL_MEM &pm) +/* + * Convert non-space characters (0x1) back into spaces + */ +void unbash_spaces(POOL_MEM &pm) { char *str = pm.c_str(); while (*str) { @@ -199,7 +208,6 @@ char *encode_time(utime_t utime, char *buf) } - /* * Convert a JobStatus code into a human readable form */ @@ -345,7 +353,6 @@ void jobstatus_to_ascii_gui(int JobStatus, char *msg, int maxlen) } } - /* * Convert Job Termination Status into a string */ @@ -380,7 +387,6 @@ const char *job_status_to_str(int stat) return str; } - /* * Convert Job Type into a string */ @@ -432,7 +438,8 @@ const char *job_type_to_str(int type) return str; } -/* Convert ActionOnPurge to string (Truncate, Erase, Destroy) +/* + * Convert ActionOnPurge to string (Truncate, Erase, Destroy) */ char *action_on_purge_to_string(int aop, POOL_MEM &ret) { @@ -524,10 +531,9 @@ const char *volume_status_to_str(const char *status) } -/*********************************************************************** +/* * Encode the mode bits into a 10 character string like LS does - ***********************************************************************/ - + */ char *encode_mode(mode_t mode, char *buf) { char *cp = buf; @@ -575,7 +581,9 @@ int do_shell_expansion(char *name, int name_len) BPIPE *bpipe; const char *shellcmd; - /* Check if any meta characters are present */ + /* + * Check if any meta characters are present + */ len = strlen(meta); for (i = 0; i < len; i++) { if (strchr(name, meta[i])) { @@ -586,7 +594,9 @@ int do_shell_expansion(char *name, int name_len) if (found) { cmd = get_pool_memory(PM_FNAME); line = get_pool_memory(PM_FNAME); - /* look for shell */ + /* + * Look for shell + */ if ((shellcmd = getenv("SHELL")) == NULL) { shellcmd = "/bin/sh"; } @@ -613,14 +623,14 @@ int do_shell_expansion(char *name, int name_len) } #endif - -/* MAKESESSIONKEY -- Generate session key with optional start - key. If mode is TRUE, the key will be - translated to a string, otherwise it is - returned as 16 binary bytes. - - from SpeakFreely by John Walker */ - +/* + * MAKESESSIONKEY -- Generate session key with optional start + * key. If mode is TRUE, the key will be + * translated to a string, otherwise it is + * returned as 16 binary bytes. + * + * from SpeakFreely by John Walker + */ void make_session_key(char *key, char *seed, int mode) { int j, k; @@ -635,13 +645,14 @@ void make_session_key(char *key, char *seed, int mode) bstrncat(s, seed, sizeof(s)); } - /* The following creates a seed for the session key generator - based on a collection of volatile and environment-specific - information unlikely to be vulnerable (as a whole) to an - exhaustive search attack. If one of these items isn't - available on your machine, replace it with something - equivalent or, if you like, just delete it. */ - + /* + * The following creates a seed for the session key generator + * based on a collection of volatile and environment-specific + * information unlikely to be vulnerable (as a whole) to an + * exhaustive search attack. If one of these items isn't + * available on your machine, replace it with something + * equivalent or, if you like, just delete it. + */ #if defined(HAVE_WIN32) { LARGE_INTEGER li; From 4583bfdcc99d152aa35f292aeee558408a71e506 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Mon, 25 Jul 2016 16:31:51 +0200 Subject: [PATCH 06/10] consolidate inherits accurate setting to started jobs Now the accurate setting from the consolidate job is also set when starting the virtualfull consolidation jobs --- src/dird/backup.c | 6 ++++-- src/dird/consolidate.c | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/dird/backup.c b/src/dird/backup.c index ad8c8a9bd5f..edc65741db5 100644 --- a/src/dird/backup.c +++ b/src/dird/backup.c @@ -1042,9 +1042,11 @@ void generate_backup_summary(JCR *jcr, CLIENT_DBR *cr, int msg_type, const char if (jcr->is_JobLevel(L_VIRTUAL_FULL)) { Mmsg(daemon_status, _( " SD Errors: %d\n" - " SD termination status: %s\n"), + " SD termination status: %s\n" + " Accurate: %s\n"), jcr->SDErrors, - sd_term_msg); + sd_term_msg, + jcr->accurate ? _("yes") : _("no")); } else { if (jcr->HasBase) { Mmsg(client_options, _( diff --git a/src/dird/consolidate.c b/src/dird/consolidate.c index df7f1de33c8..44978da911d 100644 --- a/src/dird/consolidate.c +++ b/src/dird/consolidate.c @@ -50,8 +50,7 @@ static inline void start_new_consolidation_job(JCR *jcr, char *jobname) ua = new_ua_context(jcr); ua->batch = true; - - Mmsg(ua->cmd, "run job=\"%s\" jobid=%s level=VirtualFull", jobname, jcr->vf_jobids); + Mmsg(ua->cmd, "run job=\"%s\" jobid=%s level=VirtualFull %s", jobname, jcr->vf_jobids, jcr->accurate ? "accurate=yes" : "accurate=no"); Dmsg1(dbglvl, "=============== consolidate cmd=%s\n", ua->cmd); parse_ua_args(ua); /* parse command */ From c8f8b104c5e56faedf27c426a43f6977221ebb7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Gro=C3=9F?= Date: Wed, 27 Jul 2016 17:58:25 +0200 Subject: [PATCH 07/10] cats: Fix non performant SQL query for 'list jobs last' On at least MYSQL this query leads to long running queries improvements are seen from 10 minutes down do 0.03 second. Signed-off-by: Marco van Wieringen --- src/cats/sql_cmds.c | 60 ++++++++++++++++++++++++++++++++++++++------- src/cats/sql_cmds.h | 1 + src/cats/sql_list.c | 28 +++++---------------- 3 files changed, 58 insertions(+), 31 deletions(-) diff --git a/src/cats/sql_cmds.c b/src/cats/sql_cmds.c index 339b3517cb1..7c0dc85f3f8 100644 --- a/src/cats/sql_cmds.c +++ b/src/cats/sql_cmds.c @@ -96,6 +96,31 @@ const char *list_jobs = "%s " "ORDER BY StartTime%s"; +const char *list_jobs_last = + "SELECT DISTINCT " + "Job.JobId,Job.Name, " + "Client.Name as Client, " + "Job.StartTime,Job.Type,Job.Level,Job.JobFiles,Job.JobBytes,Job.JobStatus " + "FROM Job " + "LEFT JOIN Client ON Client.ClientId=Job.ClientId " + "LEFT JOIN JobMedia ON JobMedia.JobId=Job.JobId " + "LEFT JOIN Media ON JobMedia.MediaId=Media.MediaId " + "LEFT JOIN FileSet ON FileSet.FileSetId=Job.FileSetId " + "INNER JOIN ( " + "SELECT MAX(Job.JobId) as MaxJobId " + "FROM Job " + "LEFT JOIN Client ON Client.ClientId=Job.ClientId " + "LEFT JOIN Pool ON Pool.PoolId=Job.PoolId " + "LEFT JOIN JobMedia ON JobMedia.JobId=Job.JobId " + "LEFT JOIN Media ON JobMedia.MediaId=Media.MediaId " + "LEFT JOIN FileSet ON FileSet.FileSetId=Job.FileSetId " + "WHERE Job.JobId > 0 " + "%s " + "GROUP BY Job.Name " + ") LastJob " + "ON Job.JobId = LastJob.MaxJobId " + "ORDER BY StartTime%s"; + const char *list_jobs_count = "SELECT DISTINCT " "COUNT(DISTINCT Job.JobId) as count " @@ -130,21 +155,38 @@ const char *list_jobs_long = "%s " "ORDER BY StartTime%s"; -/* - * Get the last JobId of each Job.Name matching the given criteria. - */ -const char *list_jobs_last = +const char *list_jobs_long_last = "SELECT DISTINCT " - "MAX(DISTINCT Job.JobId) as MaxJobId " + "Job.JobId, Job.Job, Job.Name, " + "Job.PurgedFiles, Job.Type, Job.Level, " + "Job.ClientId, Client.Name as Client, " + "Job.JobStatus," + "Job.SchedTime, Job.StartTime, Job.EndTime, Job.RealEndTime, Job.JobTDate, " + "Job.VolSessionId, Job.VolSessionTime, " + "Job.JobFiles, Job.JobBytes, Job.JobErrors, Job.JobMissingFiles, " + "Job.PoolId, Pool.Name as PoolName," + "Job.PriorJobId, " + "Job.FileSetId, FileSet.FileSet " "FROM Job " "LEFT JOIN Client ON Client.ClientId=Job.ClientId " + "LEFT JOIN Pool ON Pool.PoolId=Job.PoolId " "LEFT JOIN JobMedia ON JobMedia.JobId=Job.JobId " "LEFT JOIN Media ON JobMedia.MediaId=Media.MediaId " "LEFT JOIN FileSet ON FileSet.FileSetId=Job.FileSetId " - "WHERE Job.JobId > 0 " - "%s " - "GROUP BY Job.Name " - "%s"; + "INNER JOIN ( " + "SELECT MAX(Job.JobId) as MaxJobId " + "FROM Job " + "LEFT JOIN Client ON Client.ClientId=Job.ClientId " + "LEFT JOIN Pool ON Pool.PoolId=Job.PoolId " + "LEFT JOIN JobMedia ON JobMedia.JobId=Job.JobId " + "LEFT JOIN Media ON JobMedia.MediaId=Media.MediaId " + "LEFT JOIN FileSet ON FileSet.FileSetId=Job.FileSetId " + "WHERE Job.JobId > 0 " + "%s " + "GROUP BY Job.Name " + ") LastJob " + "ON Job.JobId = LastJob.MaxJobId " + "ORDER BY StartTime%s"; /* ====== ua_prune.c */ diff --git a/src/cats/sql_cmds.h b/src/cats/sql_cmds.h index 6ff92459c77..af7ffce6dee 100644 --- a/src/cats/sql_cmds.h +++ b/src/cats/sql_cmds.h @@ -27,6 +27,7 @@ extern const char CATS_IMP_EXP *list_jobs; extern const char CATS_IMP_EXP *list_jobs_count; extern const char CATS_IMP_EXP *list_jobs_long; extern const char CATS_IMP_EXP *list_jobs_last; +extern const char CATS_IMP_EXP *list_jobs_long_last; extern const char CATS_IMP_EXP *list_pool; extern const char CATS_IMP_EXP *drop_deltabs[]; extern const char CATS_IMP_EXP *create_delindex; diff --git a/src/cats/sql_list.c b/src/cats/sql_list.c index 92e315da1ca..6e14e4e8611 100644 --- a/src/cats/sql_list.c +++ b/src/cats/sql_list.c @@ -450,31 +450,15 @@ void db_list_job_records(JCR *jcr, B_DB *mdb, JOB_DBR *jr, const char *range, pm_strcat(selection, temp.c_str()); } - if (last > 0) { - /* - * Show only the last run of a job (Job.Name). - * Do a subquery to get a list of matching JobIds - * to be used in the main query later. - * - * range: while it might be more efficient, - * to apply the range to the subquery, - * at least mariadb 10 does not support this. - * Therefore range is handled in the main query. - */ - temp.bsprintf("AND Job.JobId IN (%s) ", list_jobs_last); - selection_last.bsprintf(temp.c_str(), selection.c_str(), ""); - - /* - * As the existing selection is handled in the subquery, - * overwrite the main query selection - * by the newly created selection_last. - */ - pm_strcpy(selection, selection_last.c_str()); - } - db_lock(mdb); if (count > 0) { Mmsg(mdb->cmd, list_jobs_count, selection.c_str(), range); + } else if (last > 0) { + if (type == VERT_LIST) { + Mmsg(mdb->cmd, list_jobs_long_last, selection.c_str(), range); + } else { + Mmsg(mdb->cmd, list_jobs_last, selection.c_str(), range); + } } else { if (type == VERT_LIST) { Mmsg(mdb->cmd, list_jobs_long, selection.c_str(), range); From 494948e84c491f6e18458d911f372ff786bdea8f Mon Sep 17 00:00:00 2001 From: Marco van Wieringen Date: Thu, 28 Jul 2016 14:01:02 +0200 Subject: [PATCH 08/10] cats: Tweak layout. --- src/cats/sql_cmds.c | 43 +++++++++++++------------------------------ src/cats/sql_list.c | 1 - 2 files changed, 13 insertions(+), 31 deletions(-) diff --git a/src/cats/sql_cmds.c b/src/cats/sql_cmds.c index 7c0dc85f3f8..6ce0cfebf63 100644 --- a/src/cats/sql_cmds.c +++ b/src/cats/sql_cmds.c @@ -83,9 +83,7 @@ const char *list_pool = "SELECT * FROM Pool WHERE PoolId=%s"; * JOIN with Media is required for filter to Media.Volumename. */ const char *list_jobs = - "SELECT DISTINCT " - "Job.JobId,Job.Name, " - "Client.Name as Client, " + "SELECT DISTINCT Job.JobId,Job.Name, Client.Name as Client, " "Job.StartTime,Job.Type,Job.Level,Job.JobFiles,Job.JobBytes,Job.JobStatus " "FROM Job " "LEFT JOIN Client ON Client.ClientId=Job.ClientId " @@ -97,9 +95,7 @@ const char *list_jobs = "ORDER BY StartTime%s"; const char *list_jobs_last = - "SELECT DISTINCT " - "Job.JobId,Job.Name, " - "Client.Name as Client, " + "SELECT DISTINCT Job.JobId,Job.Name, Client.Name as Client, " "Job.StartTime,Job.Type,Job.Level,Job.JobFiles,Job.JobBytes,Job.JobStatus " "FROM Job " "LEFT JOIN Client ON Client.ClientId=Job.ClientId " @@ -122,8 +118,7 @@ const char *list_jobs_last = "ORDER BY StartTime%s"; const char *list_jobs_count = - "SELECT DISTINCT " - "COUNT(DISTINCT Job.JobId) as count " + "SELECT DISTINCT COUNT(DISTINCT Job.JobId) as count " "FROM Job " "LEFT JOIN Client ON Client.ClientId=Job.ClientId " "LEFT JOIN JobMedia ON JobMedia.JobId=Job.JobId " @@ -134,17 +129,11 @@ const char *list_jobs_count = "%s"; const char *list_jobs_long = - "SELECT DISTINCT " - "Job.JobId, Job.Job, Job.Name, " - "Job.PurgedFiles, Job.Type, Job.Level, " - "Job.ClientId, Client.Name as Client, " - "Job.JobStatus," - "Job.SchedTime, Job.StartTime, Job.EndTime, Job.RealEndTime, Job.JobTDate, " - "Job.VolSessionId, Job.VolSessionTime, " - "Job.JobFiles, Job.JobBytes, Job.JobErrors, Job.JobMissingFiles, " - "Job.PoolId, Pool.Name as PoolName," - "Job.PriorJobId, " - "Job.FileSetId, FileSet.FileSet " + "SELECT DISTINCT Job.JobId, Job.Job, Job.Name, Job.PurgedFiles, Job.Type, Job.Level, " + "Job.ClientId, Client.Name as Client, Job.JobStatus, Job.SchedTime, Job.StartTime, " + "Job.EndTime, Job.RealEndTime, Job.JobTDate, Job.VolSessionId, Job.VolSessionTime, " + "Job.JobFiles, Job.JobBytes, Job.JobErrors, Job.JobMissingFiles, Job.PoolId, " + "Pool.Name as PoolName, Job.PriorJobId, Job.FileSetId, FileSet.FileSet " "FROM Job " "LEFT JOIN Client ON Client.ClientId=Job.ClientId " "LEFT JOIN Pool ON Pool.PoolId=Job.PoolId " @@ -156,17 +145,11 @@ const char *list_jobs_long = "ORDER BY StartTime%s"; const char *list_jobs_long_last = - "SELECT DISTINCT " - "Job.JobId, Job.Job, Job.Name, " - "Job.PurgedFiles, Job.Type, Job.Level, " - "Job.ClientId, Client.Name as Client, " - "Job.JobStatus," - "Job.SchedTime, Job.StartTime, Job.EndTime, Job.RealEndTime, Job.JobTDate, " - "Job.VolSessionId, Job.VolSessionTime, " - "Job.JobFiles, Job.JobBytes, Job.JobErrors, Job.JobMissingFiles, " - "Job.PoolId, Pool.Name as PoolName," - "Job.PriorJobId, " - "Job.FileSetId, FileSet.FileSet " + "SELECT DISTINCT Job.JobId, Job.Job, Job.Name, Job.PurgedFiles, Job.Type, Job.Level, " + "Job.ClientId, Client.Name as Client, Job.JobStatus, Job.SchedTime, Job.StartTime, " + "Job.EndTime, Job.RealEndTime, Job.JobTDate, Job.VolSessionId, Job.VolSessionTime, " + "Job.JobFiles, Job.JobBytes, Job.JobErrors, Job.JobMissingFiles, Job.PoolId, " + "Pool.Name as PoolName, Job.PriorJobId, Job.FileSetId, FileSet.FileSet " "FROM Job " "LEFT JOIN Client ON Client.ClientId=Job.ClientId " "LEFT JOIN Pool ON Pool.PoolId=Job.PoolId " diff --git a/src/cats/sql_list.c b/src/cats/sql_list.c index 6e14e4e8611..5d515a7e930 100644 --- a/src/cats/sql_list.c +++ b/src/cats/sql_list.c @@ -415,7 +415,6 @@ void db_list_job_records(JCR *jcr, B_DB *mdb, JOB_DBR *jr, const char *range, POOL_MEM temp(PM_MESSAGE), selection(PM_MESSAGE), criteria(PM_MESSAGE); - POOL_MEM selection_last(PM_MESSAGE); char dt[MAX_TIME_LENGTH]; if (jr->JobId > 0) { From ca56bea244f6468dca8485b8c23c0323519fa783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wanderlei=20H=C3=BCttel?= Date: Thu, 28 Jul 2016 10:26:43 -0300 Subject: [PATCH 09/10] Add error message when action on purge is not set in the pool resource --- src/dird/ua_purge.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/dird/ua_purge.c b/src/dird/ua_purge.c index a8ef6125e94..e5137d27a55 100644 --- a/src/dird/ua_purge.c +++ b/src/dird/ua_purge.c @@ -653,6 +653,8 @@ static void do_truncate_on_purge(UAContext *ua, MEDIA_DBR *mr, * Do it only if action on purge = truncate is set */ if (!(mr->ActionOnPurge & ON_PURGE_TRUNCATE)) { + ua->error_msg(_("\nThe option \"Action On Purge = Truncate\" was not defined in the Pool resource.\n" + "Unable to truncate volume \"%s\"\n"), mr->VolumeName); return; } From 51f2e38bf80f13416ef0bd869871a93139616ab0 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Mon, 25 Jul 2016 11:40:15 +0200 Subject: [PATCH 10/10] add .consoles command --- src/dird/ua_cmds.c | 3 +++ src/dird/ua_dotcmds.c | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/dird/ua_cmds.c b/src/dird/ua_cmds.c index 38aef31be90..74d235f57b0 100644 --- a/src/dird/ua_cmds.c +++ b/src/dird/ua_cmds.c @@ -65,6 +65,7 @@ extern bool dot_jobs_cmd(UAContext *ua, const char *cmd); extern bool dot_jobstatus_cmd(UAContext *ua, const char *cmd); extern bool dot_filesets_cmd(UAContext *ua, const char *cmd); extern bool dot_clients_cmd(UAContext *ua, const char *cmd); +extern bool dot_consoles_cmd(UAContext *ua, const char *cmd); extern bool dot_msgs_cmd(UAContext *ua, const char *cmd); extern bool dot_pools_cmd(UAContext *ua, const char *cmd); extern bool dot_schedule_cmd(UAContext *ua, const char *cmd); @@ -158,6 +159,8 @@ static struct cmdstruct commands[] = { NULL, false, false }, { NT_(".clients"), dot_clients_cmd, _("List all client resources"), NULL, true, false }, + { NT_(".consoles"), dot_consoles_cmd, _("List all console resources"), + NULL, true, false }, { NT_(".defaults"), dot_defaults_cmd, _("Get default settings"), NT_("job= | client= | storage="), false, false }, #ifdef DEVELOPER diff --git a/src/dird/ua_dotcmds.c b/src/dird/ua_dotcmds.c index 71a90ac01b6..bfe3e2721e4 100644 --- a/src/dird/ua_dotcmds.c +++ b/src/dird/ua_dotcmds.c @@ -923,6 +923,23 @@ bool dot_clients_cmd(UAContext *ua, const char *cmd) return true; } +bool dot_consoles_cmd(UAContext *ua, const char *cmd) +{ + CONRES *console; + + LockRes(); + ua->send->array_start("consoles"); + foreach_res(console, R_CONSOLE) { + ua->send->object_start(); + ua->send->object_key_value("name", console->name(), "%s\n"); + ua->send->object_end(); + } + ua->send->array_end("consoles"); + UnlockRes(); + + return true; +} + bool dot_msgs_cmd(UAContext *ua, const char *cmd) { MSGSRES *msgs = NULL;