Consolidation jobs: don't count for client concurrency

src/dird/consolidate.c

@@ -76,6 +76,9 @@ bool do_consolidate(JCR *jcr)
    bool retval = true;
    time_t now = time(NULL);
 
+   char *jobids;
+   char *p;
+
    tmpjob = jcr->res.job; /* Memorize job */
 
    jcr->jr.JobId = jcr->JobId;
@@ -96,7 +99,6 @@ bool do_consolidate(JCR *jcr)
          /*
           * Fake always incremental job as job of current jcr.
           */
-         init_jcr_job_record(jcr);
          jcr->res.job = job;
          jcr->res.fileset = job->fileset;
          jcr->res.client = job->client;
@@ -146,11 +148,13 @@ bool do_consolidate(JCR *jcr)
          /*
           * Consolidation of zero or one job does not make sense, we leave it like it is
           */
-         if (jobids_ctx.count < 2) {
+         if (incrementals_total < 1) {
             Jmsg(jcr, M_INFO, 0, _("%s: less than two jobs to consolidate found, doing nothing.\n"), job->name());
             continue;
          }
+
          int32_t max_incrementals_to_consolidate;
+         int32_t incrementals_to_consolidate;
 
          /*
           * Calculate limit for query. We specify how many incrementals should be left.
@@ -164,19 +168,50 @@ bool do_consolidate(JCR *jcr)
             Dmsg3(10, "total: %d, to_consolidate: %d, limit: %d.\n", incrementals_total, max_incrementals_to_consolidate, jcr->jr.limit);
             jobids_ctx.reset();
             db_accurate_get_jobids(jcr, jcr->db, &jcr->jr, &jobids_ctx);
-            Dmsg1(10, "consolidate ids after limit: %s.\n", jobids_ctx.list);
+            incrementals_to_consolidate = jobids_ctx.count - 1;
+            Dmsg2(10, "%d consolidate ids after limit: %s.\n", jobids_ctx.count, jobids_ctx.list);
+            if (incrementals_to_consolidate < 1) {
+               Jmsg(jcr, M_INFO, 0, _("%s: After limited query: less incrementals than required, not consolidating\n"), job->name());
+               continue;
+            }
          } else {
             Jmsg(jcr, M_INFO, 0, _("%s: less incrementals than required, not consolidating\n"), job->name());
             continue;
          }
 
+         jobids = bstrdup(jobids_ctx.list);
+         p = jobids;
+
+         /*
+          * Check if we need to skip the first (full) job from consolidation
+          */
+         if (!job->AlwaysIncrementalConsolidateFull) {
+            if (incrementals_to_consolidate < 2) {
+               Jmsg(jcr, M_INFO, 0, _("%s: less incrementals than required to consolidate without full, not consolidating\n"), job->name());
+               continue;
+            }
+            Jmsg(jcr, M_INFO, 0, _("before ConsolidateFull: jobids: %s\n"), jobids);
+            p = strchr(jobids, ',');                /* find oldest jobid and skip it */
+            if (p) {
+               *p = '\0';
+            }
+            Jmsg(jcr, M_INFO, 0, _("AlwaysIncrementalConsolidateFull is set to no, skipping first jobid %s\n"), jobids);
+
+            if (p) {
+               *p = ','; /* restore ,*/
+               p++;      /* point to rest of list */
+            }
+            Jmsg(jcr, M_INFO, 0, _("after ConsolidateFull: jobids: %s\n"), p);
+         }
+
+
          /*
           * Set the virtualfull jobids to be consolidated
           */
          if (!jcr->vf_jobids) {
             jcr->vf_jobids = get_pool_memory(PM_MESSAGE);
          }
-         pm_strcpy(jcr->vf_jobids, jobids_ctx.list);
+         pm_strcpy(jcr->vf_jobids, p);
 
          Jmsg(jcr, M_INFO, 0, _("%s: Start new consolidation\n"), job->name());
          start_new_consolidation_job(jcr, job->name());
@@ -190,6 +225,9 @@ bool do_consolidate(JCR *jcr)
    jcr->res.job = tmpjob;
    jcr->setJobStatus(JS_Terminated);
    consolidate_cleanup(jcr, JS_Terminated);
+   if (jobids) {
+      free(jobids);
+   }
 
    return retval;
 }

src/dird/dird_conf.c

@@ -427,6 +427,8 @@ RES_ITEM job_items[] = {
      "Backup Jobs older than the specified time duration will be merged into a new Virtual backup." },
    { "AlwaysIncrementalKeepNumber", CFG_TYPE_PINT32, ITEM(res_job.AlwaysIncrementalKeepNumber), 0, CFG_ITEM_DEFAULT, "0", "16.2.4-",
      "Guarantee that at least the specified number of Backup Jobs will persist, even if they are older than \"Always Incremental Job Retention\"."},
+   { "AlwaysIncrementalConsolidateFull", CFG_TYPE_BOOL, ITEM(res_job.AlwaysIncrementalConsolidateFull), 0, CFG_ITEM_DEFAULT, "true", "16.2.4-",
+     "Consolidate jobs also consolidate the full backup. If disabled, only the latest incremental will be consolidated to reduce consolidation volume." },
    { NULL, 0, { 0 }, 0, 0, NULL, NULL, NULL }
 };
 

src/dird/dird_conf.h

@@ -438,6 +438,7 @@ class JOBRES : public BRSRES {
    bool IgnoreDuplicateJobChecking;   /* Ignore Duplicate Job Checking */
    bool SaveFileHist;                 /* Ability to disable File history saving for certain protocols */
    bool AlwaysIncremental;            /* Always incremental with regular consolidation */
+   bool AlwaysIncrementalConsolidateFull; /* Always incremental only consolidates incremental backups and full will be left to reduce volume */
 
    runtime_job_status_t *rjs;         /* Runtime Job Status */
 

src/dird/jobq.c

@@ -807,8 +807,9 @@ static bool acquire_resources(JCR *jcr)
    switch (jcr->getJobType()) {
    case JT_MIGRATE:
    case JT_COPY:
+   case JT_CONSOLIDATE:
       /*
-       * Migration/Copy jobs are not counted for client concurrency
+       * Migration/Copy and Consolidation jobs are not counted for client concurrency
        * as they do not touch the client at all
        */
       jcr->IgnoreClientConcurrency = true;

src/lib/parse_conf.h

@@ -178,7 +178,7 @@ struct RES_ITEM {
 /* For storing name_addr items in res_items table */
 #define ITEM(x) {(char **)&res_all.x}
 
-#define MAX_RES_ITEMS 88                /* maximum resource items per RES */
+#define MAX_RES_ITEMS 89                /* maximum resource items per RES */
 
 /*
  * This is the universal header that is at the beginning of every resource record.