Fix some bugs in the NDMP implementation.

Up until now we were not fully RFC compliant. You need to restore the full NDMP
environment returned during a NDMP backup when performing a NDMP restore. We more
or less re-created the environment when doing a restore but were missing some important
elements. We now save the full environment in the database and use that to restore
the whole environment when doing a restore.

src/cats/cats.h

@@ -452,7 +452,7 @@ typedef int (DB_RESULT_HANDLER)(void *, int, char **);
 #define db_unlock(mdb) mdb->_db_unlock(__FILE__, __LINE__)
 
 /* Current database version number for all drivers */
-#define BDB_VERSION 2001
+#define BDB_VERSION 2002
 
 class B_DB: public SMARTALLOC {
 protected:

src/cats/grant_ingres_privileges.in

@@ -41,6 +41,7 @@ GRANT ALL ON TABLE PathVisibility TO ${db_user};
 GRANT ALL ON TABLE Version TO ${db_user};
 GRANT ALL ON TABLE Status TO ${db_user};
 GRANT ALL ON TABLE NDMPLevelMap TO ${db_user};
+GRANT ALL ON TABLE NDMPJobEnvironment TO  ${db_user};
 
 -- for sequences ON those tables
 GRANT NEXT ON SEQUENCE Filename_Seq TO ${db_user};

src/cats/grant_postgresql_privileges.in

@@ -43,6 +43,7 @@ GRANT ALL ON PathVisibility TO ${db_user};
 GRANT ALL ON RestoreObject TO ${db_user};
 GRANT ALL ON Quota TO ${db_user};
 GRANT ALL ON NDMPLevelMap TO ${db_user};
+GRANT ALL ON NDMPJobEnvironment TO ${db_user};
 
 -- For sequences ON those tables
 GRANT SELECT, UPDATE ON filename_filenameid_seq TO ${db_user};

src/cats/make_ingres_tables.in

@@ -465,6 +465,15 @@ CREATE TABLE NDMPLevelMap
    DumpLevel		INTEGER 	DEFAULT 0,
    CONSTRAINT NDMPLevelMap_pkey PRIMARY KEY (ClientId, FilesetId, FileSystem)
 );
+
+CREATE TABLE NDMPJobEnvironment
+(
+   JobId		INTEGER 	NOT NULL,
+   FileIndex		INTEGER 	NOT NULL,
+   EnvName    		VARBYTE(256)	NOT NULL,
+   EnvValue   		VARBYTE(256)	NOT NULL,
+   CONSTRAINT NDMPJobEnvironment_pkey PRIMARY KEY (JobId, FileIndex, EnvName)
+);
 \g
 
 INSERT INTO Status (JobStatus,JobStatusLong,Severity) VALUES

src/cats/make_mysql_tables.in

@@ -407,6 +407,14 @@ CREATE TABLE NDMPLevelMap (
    CONSTRAINT NDMPLevelMap_pkey PRIMARY KEY (ClientId, FilesetId, FileSystem(256))
 );
 
+CREATE TABLE NDMPJobEnvironment (
+   JobId INTEGER UNSIGNED NOT NULL REFERENCES Job,
+   FileIndex INTEGER UNSIGNED NOT NULL,
+   EnvName TINYBLOB NOT NULL,
+   EnvValue TINYBLOB NOT NULL,
+   CONSTRAINT NDMPJobEnvironment_pkey PRIMARY KEY (JobId, FileIndex, EnvName(256))
+);
+
 INSERT INTO Status (JobStatus,JobStatusLong,Severity) VALUES
    ('C', 'Created, not yet running', 15),
    ('R', 'Running', 15),

src/cats/make_postgresql_tables.in

@@ -409,6 +409,14 @@ CREATE TABLE NDMPLevelMap (
     CONSTRAINT NDMPLevelMap_pkey PRIMARY KEY (ClientId, FilesetId, FileSystem)
 );
 
+CREATE TABLE NDMPJobEnvironment (
+    JobId             INTEGER     NOT NULL,
+    FileIndex         INTEGER     NOT NULL,
+    EnvName           TEXT        NOT NULL,
+    EnvValue          TEXT        NOT NULL,
+    CONSTRAINT NDMPJobEnvironment_pkey PRIMARY KEY (JobId, FileIndex, EnvName)
+);
+
 INSERT INTO Status (JobStatus,JobStatusLong,Severity) VALUES
    ('C', 'Created, not yet running',15);
 INSERT INTO Status (JobStatus,JobStatusLong,Severity) VALUES

src/cats/make_sqlite3_tables.in

@@ -393,6 +393,14 @@ CREATE TABLE NDMPLevelMap (
    CONSTRAINT NDMPLevelMap_pkey PRIMARY KEY (ClientId, FilesetId, FileSystem)
 );
 
+CREATE TABLE NDMPJobEnvironment (
+   JobId integer NOT NULL,
+   FileIndex INTEGER UNSIGNED NOT NULL,
+   EnvName TEXT NOT NULL,
+   EnvValue TEXT NOT NULL,
+   CONSTRAINT NDMPJobEnvironment_pkey PRIMARY KEY (JobId, FileIndex, EnvName)
+);
+
 INSERT INTO Status (JobStatus,JobStatusLong,Severity) VALUES
    ('C', 'Created, not yet running',15);
 INSERT INTO Status (JobStatus,JobStatusLong,Severity) VALUES

src/cats/protos.h

@@ -62,6 +62,7 @@ bool db_commit_base_file_attributes_record(JCR *jcr, B_DB *mdb);
 bool db_create_base_file_list(JCR *jcr, B_DB *mdb, char *jobids);
 bool db_create_quota_record(JCR *jcr, B_DB *mdb, CLIENT_DBR *cr);
 bool db_create_ndmp_level_mapping(JCR *jcr, B_DB *mdb, JOB_DBR *jr, char *filesystem);
+bool db_create_ndmp_environment_string(JCR *jcr, B_DB *mdb, JOB_DBR *jr, char *name, char *value);
 
 /* sql_delete.c */
 bool db_delete_pool_record(JCR *jcr, B_DB *db, POOL_DBR *pool_dbr);
@@ -105,6 +106,7 @@ bool db_get_quota_record(JCR *jcr, B_DB *mdb, CLIENT_DBR *cr);
 bool db_get_quota_jobbytes(JCR *jcr, B_DB *mdb, JOB_DBR *jr, utime_t JobRetention);
 bool db_get_quota_jobbytes_nofailed(JCR *jcr, B_DB *mdb, JOB_DBR *jr, utime_t JobRetention);
 int db_get_ndmp_level_mapping(JCR *jcr, B_DB *mdb, JOB_DBR *jr, char *filesystem);
+bool db_get_ndmp_environment_string(JCR *jcr, B_DB *mdb, JOB_DBR *jr, DB_RESULT_HANDLER *result_handler, void *ctx);
 
 /* sql_list.c */
 void db_list_pool_records(JCR *jcr, B_DB *db, POOL_DBR *pr, DB_LIST_HANDLER sendit, void *ctx, e_list_type type);

src/cats/sql_create.c

@@ -63,7 +63,7 @@ bool db_create_job_record(JCR *jcr, B_DB *mdb, JOB_DBR *jr)
    struct tm tm;
    int len;
    utime_t JobTDate;
-   char ed1[30],ed2[30];
+   char ed1[30], ed2[30];
    char esc_job[MAX_ESCAPE_NAME_LENGTH];
    char esc_name[MAX_ESCAPE_NAME_LENGTH];
 
@@ -1337,4 +1337,35 @@ bool db_create_ndmp_level_mapping(JCR *jcr, B_DB *mdb, JOB_DBR *jr, char *filesy
    db_unlock(mdb);
    return retval;
 }
+
+/**
+ * Create a NDMP Job Environment String
+ * Returns: false on failure
+ *          true on success
+ */
+bool db_create_ndmp_environment_string(JCR *jcr, B_DB *mdb, JOB_DBR *jr, char *name, char *value)
+{
+   bool retval = false;
+   char ed1[50], ed2[50];
+   char esc_name[MAX_ESCAPE_NAME_LENGTH];
+   char esc_value[MAX_ESCAPE_NAME_LENGTH];
+
+   db_lock(mdb);
+
+   mdb->db_escape_string(jcr, esc_name, name, strlen(name));
+   mdb->db_escape_string(jcr, esc_value, value, strlen(value));
+   Mmsg(mdb->cmd, "INSERT INTO NDMPJobEnvironment (JobId, FileIndex, EnvName, EnvValue)"
+                  " VALUES ('%s', '%s', '%s', '%s')",
+        edit_int64(jr->JobId, ed1), edit_uint64(jr->FileIndex, ed2), esc_name, esc_value);
+   if (!INSERT_DB(jcr, mdb, mdb->cmd)) {
+      Mmsg2(&mdb->errmsg, _("Create DB NDMP Job Environment record %s failed. ERR=%s\n"),
+            mdb->cmd, sql_strerror(mdb));
+      Jmsg(jcr, M_ERROR, 0, "%s", mdb->errmsg);
+   } else {
+      retval = true;
+   }
+
+   db_unlock(mdb);
+   return retval;
+}
 #endif /* HAVE_SQLITE3 || HAVE_MYSQL || HAVE_POSTGRESQL || HAVE_INGRES || HAVE_DBI */

src/cats/sql_get.c

@@ -1127,44 +1127,47 @@ bool db_get_file_list(JCR *jcr, B_DB *mdb, char *jobids,
                       bool use_md5, bool use_delta,
                       DB_RESULT_HANDLER *result_handler, void *ctx)
 {
+   POOL_MEM query(PM_FNAME);
+   POOL_MEM query2(PM_FNAME);
+
    if (!*jobids) {
       db_lock(mdb);
       Mmsg(mdb->errmsg, _("ERR=JobIds are empty\n"));
       db_unlock(mdb);
       return false;
    }
-   POOL_MEM buf(PM_MESSAGE);
-   POOL_MEM buf2(PM_MESSAGE);
+
    if (use_delta) {
-      Mmsg(buf2, select_recent_version_with_basejob_and_delta[db_get_type_index(mdb)],
+      Mmsg(query2, select_recent_version_with_basejob_and_delta[db_get_type_index(mdb)],
            jobids, jobids, jobids, jobids);
 
    } else {
-      Mmsg(buf2, select_recent_version_with_basejob[db_get_type_index(mdb)],
+      Mmsg(query2, select_recent_version_with_basejob[db_get_type_index(mdb)],
            jobids, jobids, jobids, jobids);
    }
 
-   /* bsr code is optimized for JobId sorted, with Delta, we need to get
+   /*
+    * BSR code is optimized for JobId sorted, with Delta, we need to get
     * them ordered by date. JobTDate and JobId can be mixed if using Copy
     * or Migration
     */
-   Mmsg(buf,
+   Mmsg(query,
 "SELECT Path.Path, Filename.Name, T1.FileIndex, T1.JobId, LStat, DeltaSeq, MD5 "
  "FROM ( %s ) AS T1 "
  "JOIN Filename ON (Filename.FilenameId = T1.FilenameId) "
  "JOIN Path ON (Path.PathId = T1.PathId) "
 "WHERE FileIndex > 0 "
 "ORDER BY T1.JobTDate, FileIndex ASC",/* Return sorted by JobTDate */
                                       /* FileIndex for restore code */
-        buf2.c_str());
+        query2.c_str());
 
    if (!use_md5) {
-      strip_md5(buf.c_str());
+      strip_md5(query.c_str());
    }
 
-   Dmsg1(100, "q=%s\n", buf.c_str());
+   Dmsg1(100, "q=%s\n", query.c_str());
 
-   return db_big_sql_query(mdb, buf.c_str(), result_handler, ctx);
+   return db_big_sql_query(mdb, query.c_str(), result_handler, ctx);
 }
 
 /**
@@ -1173,13 +1176,14 @@ bool db_get_file_list(JCR *jcr, B_DB *mdb, char *jobids,
 bool db_get_used_base_jobids(JCR *jcr, B_DB *mdb,
                              POOLMEM *jobids, db_list_ctx *result)
 {
-   POOL_MEM buf;
-   Mmsg(buf,
+   POOL_MEM query(PM_FNAME);
+
+   Mmsg(query,
  "SELECT DISTINCT BaseJobId "
  "  FROM Job JOIN BaseFiles USING (JobId) "
  " WHERE Job.HasBase = 1 "
  "   AND Job.JobId IN (%s) ", jobids);
-   return db_sql_query(mdb, buf.c_str(), db_list_handler, result);
+   return db_sql_query(mdb, query.c_str(), db_list_handler, result);
 }
 
 /**
@@ -1277,17 +1281,17 @@ bool db_accurate_get_jobids(JCR *jcr, B_DB *mdb,
 bool db_get_base_file_list(JCR *jcr, B_DB *mdb, bool use_md5,
                            DB_RESULT_HANDLER *result_handler, void *ctx)
 {
-   POOL_MEM buf(PM_MESSAGE);
+   POOL_MEM query(PM_FNAME);
 
-   Mmsg(buf,
+   Mmsg(query,
  "SELECT Path, Name, FileIndex, JobId, LStat, 0 As DeltaSeq, MD5 "
    "FROM new_basefile%lld ORDER BY JobId, FileIndex ASC",
         (uint64_t) jcr->JobId);
 
    if (!use_md5) {
-      strip_md5(buf.c_str());
+      strip_md5(query.c_str());
    }
-   return db_big_sql_query(mdb, buf.c_str(), result_handler, ctx);
+   return db_big_sql_query(mdb, query.c_str(), result_handler, ctx);
 }
 
 bool db_get_base_jobid(JCR *jcr, B_DB *mdb, JOB_DBR *jr, JobId_t *jobid)
@@ -1299,6 +1303,7 @@ bool db_get_base_jobid(JCR *jcr, B_DB *mdb, JOB_DBR *jr, JobId_t *jobid)
    char esc[MAX_ESCAPE_NAME_LENGTH];
    bool retval = false;
 // char clientid[50], filesetid[50];
+
    *jobid = 0;
    lctx.count = 0;
    lctx.value = 0;
@@ -1414,6 +1419,7 @@ bool db_get_quota_jobbytes(JCR *jcr, B_DB *mdb, JOB_DBR *jr, utime_t JobRetentio
 
 /**
  * This function returns the sum of all the Clients JobBytes of non failed jobs.
+ *
  * Returns false: on failure
  *         true: on success
  */
@@ -1562,4 +1568,50 @@ int db_get_ndmp_level_mapping(JCR *jcr, B_DB *mdb, JOB_DBR *jr, char *filesystem
    db_unlock(mdb);
    return dumplevel;
 }
+
+/**
+ * Fetch the NDMP Job Environment Strings
+ *
+ * Returns false: on failure
+ *         true: on success
+ */
+bool db_get_ndmp_environment_string(JCR *jcr, B_DB *mdb, JOB_DBR *jr, DB_RESULT_HANDLER *result_handler, void *ctx)
+{
+   POOL_MEM query(PM_FNAME);
+   char ed1[50], ed2[50];
+   db_int64_ctx lctx;
+   JobId_t JobId;
+   bool retval = false;
+
+   lctx.count = 0;
+   lctx.value = 0;
+
+   /*
+    * Lookup the JobId
+    */
+   Mmsg(query, "SELECT JobId FROM Job "
+               "WHERE VolSessionId = '%s' "
+               "AND VolSessionTime = '%s'",
+               edit_uint64(jr->VolSessionId, ed1),
+               edit_uint64(jr->VolSessionTime, ed2));
+   if (!db_sql_query(mdb, query.c_str(), db_int64_handler, &lctx)) {
+      goto bail_out;
+   }
+
+   JobId = (JobId_t) lctx.value;
+
+   /*
+    * Lookup all environment settings belonging to this JobId and FileIndex.
+    */
+   Mmsg(query, "SELECT EnvName, EnvValue FROM NDMPJobEnvironment "
+               "WHERE JobId='%s' "
+               "AND FileIndex='%s'",
+               edit_uint64(JobId, ed1),
+               edit_uint64(jr->FileIndex, ed2));
+
+   retval = db_sql_query(mdb, query.c_str(), result_handler, ctx);
+
+bail_out:
+   return retval;
+}
 #endif /* HAVE_SQLITE3 || HAVE_MYSQL || HAVE_POSTGRESQL || HAVE_INGRES || HAVE_DBI */