systemtests: add python-modules and rename files

core/src/dird/dbcopy/column_description.cc

@@ -23,6 +23,7 @@
 #include "dird/dbcopy/row_data.h"
 
 #include <cstring>
+#include <iostream>
 #include <vector>
 
 ColumnDescription::ColumnDescription(const char* column_name_in,
@@ -46,12 +47,12 @@ ColumnDescription::ColumnDescription(const char* column_name_in,
   }
 }
 
-static void no_conversion(BareosDb* db, FieldData& fd)
+static void no_conversion(BareosDb* db, ColumnData& fd)
 {
   fd.data_pointer = fd.data_pointer ? fd.data_pointer : "";
 }
 
-static void timestamp_conversion_postgresql(BareosDb* db, FieldData& fd)
+static void timestamp_conversion_postgresql(BareosDb* db, ColumnData& fd)
 {
   static const char* dummy_timepoint = "1970-01-01 00:00:00";
   if (!fd.data_pointer) {
@@ -63,60 +64,55 @@ static void timestamp_conversion_postgresql(BareosDb* db, FieldData& fd)
   }
 }
 
-static void string_conversion_postgresql(BareosDb* db, FieldData& fd)
+static void string_conversion_postgresql(BareosDb* db, ColumnData& fd)
 {
   if (fd.data_pointer) {
     std::size_t len{strlen(fd.data_pointer)};
     fd.converted_data.resize(len * 2 + 1);
-#if 1
     db->EscapeString(nullptr, fd.converted_data.data(), fd.data_pointer, len);
-#else
-    char* n = fd.converted_data.data();
-    const char* o = fd.data_pointer;
-
-    while (len--) {
-      switch (*o) {
-        case '\'':
-          *n++ = '\'';
-          *n++ = '\'';
-          o++;
-          break;
-        case 0:
-          *n++ = '\\';
-          *n++ = 0;
-          o++;
-          break;
-        default:
-          *n++ = *o++;
-          break;
-      }
-    }
-    *n = 0;
-#endif
     fd.data_pointer = fd.converted_data.data();
   } else {
     fd.data_pointer = "";
   }
 }
 
-static void bytea_conversion_postgresql(BareosDb* db, FieldData& fd)
+static void bytea_conversion_postgresql(BareosDb* db, ColumnData& fd)
 {
   std::size_t new_len{};
+  std::size_t old_len = fd.size;
+
   auto old = reinterpret_cast<const unsigned char*>(fd.data_pointer);
-  auto obj = db->EscapeObject(old, strlen(fd.data_pointer), new_len);
+
+  auto obj = db->EscapeObject(old, old_len, new_len);
+
   fd.converted_data.resize(new_len + 1);
   memcpy(fd.converted_data.data(), obj, new_len + 1);
+
   db->FreeEscapedObjectMemory(obj);
+
+  fd.data_pointer = fd.converted_data.data();
+}
+
+static void longblob_conversion_mysql(BareosDb* db, ColumnData& fd)
+{
+  no_conversion(db, fd);
 }
 
 const DataTypeConverterMap ColumnDescriptionMysql::db_import_converter_map{
-    {"bigint", no_conversion},   {"binary", no_conversion},
-    {"blob", no_conversion},     {"char", no_conversion},
-    {"datetime", no_conversion}, {"decimal", no_conversion},
-    {"enum", no_conversion},     {"int", no_conversion},
-    {"longblob", no_conversion}, {"smallint", no_conversion},
-    {"text", no_conversion},     {"timestamp", no_conversion},
-    {"tinyblob", no_conversion}, {"tinyint", no_conversion},
+    {"bigint", no_conversion},
+    {"binary", no_conversion},
+    {"blob", no_conversion},
+    {"char", no_conversion},
+    {"datetime", no_conversion},
+    {"decimal", no_conversion},
+    {"enum", no_conversion},
+    {"int", no_conversion},
+    {"longblob", longblob_conversion_mysql},
+    {"smallint", no_conversion},
+    {"text", no_conversion},
+    {"timestamp", no_conversion},
+    {"tinyblob", no_conversion},
+    {"tinyint", no_conversion},
     {"varchar", no_conversion}};
 
 ColumnDescriptionMysql::ColumnDescriptionMysql(const char* column_name_in,

core/src/dird/dbcopy/column_description.h

@@ -26,7 +26,7 @@
 #include <map>
 #include <string>
 
-class FieldData;
+class ColumnData;
 class BareosDb;
 
 class ColumnDescription {
@@ -40,7 +40,7 @@ class ColumnDescription {
   std::string data_type;
   std::size_t character_maximum_length{};
 
-  using ConverterCallback = std::function<void(BareosDb*, FieldData&)>;
+  using ConverterCallback = std::function<void(BareosDb*, ColumnData&)>;
   ConverterCallback db_export_converter{};
   ConverterCallback db_import_converter{};
 };

core/src/dird/dbcopy/database_export_postgresql.cc

@@ -59,33 +59,33 @@ DatabaseExportPostgresql::~DatabaseExportPostgresql()
   if (transaction_) { db_->SqlQuery("ROLLBACK"); }
 }
 
-void DatabaseExportPostgresql::CopyRow(RowData& origin_data)
+void DatabaseExportPostgresql::CopyRow(RowData& source_data_row)
 {
   std::string query_into{"INSERT INTO "};
-  query_into += origin_data.table_name;
+  query_into += source_data_row.table_name;
   query_into += " (";
 
   std::string query_values = " VALUES (";
 
-  for (std::size_t i = 0; i < origin_data.column_descriptions.size(); i++) {
-    query_into += origin_data.column_descriptions[i]->column_name;
-    query_into += ", ";
-
+  for (std::size_t i = 0; i < source_data_row.column_descriptions.size(); i++) {
     const ColumnDescription* column_description =
         table_descriptions_->GetColumnDescription(
-            origin_data.table_name,
-            origin_data.column_descriptions[i]->column_name);
+            source_data_row.table_name,
+            source_data_row.column_descriptions[i]->column_name);
 
     if (!column_description) {
       std::string err{"Could not get column description for: "};
-      err += origin_data.column_descriptions[i]->column_name;
+      err += source_data_row.column_descriptions[i]->column_name;
       throw std::runtime_error(err);
     }
 
-    if (i < origin_data.row.size()) {
+    query_into += column_description->column_name;
+    query_into += ", ";
+
+    if (i < source_data_row.columns.size()) {
       query_values += "'";
-      column_description->db_export_converter(db_, origin_data.row[i]);
-      query_values += origin_data.row[i].data_pointer;
+      column_description->db_export_converter(db_, source_data_row.columns[i]);
+      query_values += source_data_row.columns[i].data_pointer;
       query_values += "',";
     } else {
       throw std::runtime_error("Row number does not match column description");
@@ -250,8 +250,8 @@ int DatabaseExportPostgresql::ResultHandlerCompare(void* ctx,
 
   for (int i = 0; i < fields; i++) {
     std::string r1{row[i]};
-    std::string r2{rd->row[i].data_pointer};
-    if (r1 != r2) { throw std::runtime_error("What??"); }
+    std::string r2{rd->columns[i].data_pointer};
+    if (r1 != r2) { throw std::runtime_error("Data is not same"); }
   }
   return 0;
 }

core/src/dird/dbcopy/database_import_mysql.cc

@@ -30,6 +30,7 @@
 #include <chrono>
 #include <iomanip>
 #include <iostream>
+#include <limits>
 #include <sstream>
 #include <vector>
 
@@ -70,16 +71,22 @@ struct ResultHandlerContext {
       const DatabaseColumnDescriptions::VectorOfColumnDescriptions& c,
       RowData& d,
       DatabaseExport& e,
-      BareosDb* db_in)
-      : column_descriptions(c), row_data(d), exporter(e), db(db_in)
+      BareosDb* db_in,
+      bool is_restore_object_in)
+      : column_descriptions(c)
+      , row_data(d)
+      , exporter(e)
+      , db(db_in)
+      , is_restore_object(is_restore_object_in)
   {
   }
   const DatabaseColumnDescriptions::VectorOfColumnDescriptions&
       column_descriptions;
   RowData& row_data;
   DatabaseExport& exporter;
   uint64_t counter{};
-  BareosDb* db;
+  BareosDb* db{};
+  bool is_restore_object{};
 };
 
 void DatabaseImportMysql::RunQuerySelectAllRows(
@@ -100,6 +107,13 @@ void DatabaseImportMysql::RunQuerySelectAllRows(
       query += "`, `";
     }
     query.resize(query.size() - 3);
+
+    bool is_restore_object = false;
+    if (t.table_name == "RestoreObject") {
+      query += ", length(`RestoreObject`) as `length_of_restore_object`";
+      is_restore_object = true;
+    }
+
     query += " FROM ";
     query += t.table_name;
 
@@ -108,8 +122,9 @@ void DatabaseImportMysql::RunQuerySelectAllRows(
       query += std::to_string(maximum_amount_of_rows);
     }
 
-    RowData row_data(t.column_descriptions, t.table_name);
-    ResultHandlerContext ctx(t.column_descriptions, row_data, exporter, db_);
+    RowData row_data(t.column_descriptions, t.table_name, is_restore_object);
+    ResultHandlerContext ctx(t.column_descriptions, row_data, exporter, db_,
+                             is_restore_object);
 
     if (!db_->SqlQuery(query.c_str(), ResultHandler, &ctx)) {
       std::cout << "Could not import table: " << t.table_name << std::endl;
@@ -144,19 +159,55 @@ void DatabaseImportMysql::CompareWith(DatabaseExport& exporter)
   RunQuerySelectAllRows(ResultHandlerCompare, exporter);
 }
 
+static void ReadoutSizeOfRestoreObject(ResultHandlerContext* r,
+                                       RowData& row_data,
+                                       int field_index_longblob,
+                                       char** row)
+{
+  auto invalid = std::numeric_limits<std::size_t>::max();
+  std::size_t index_of_restore_object = invalid;
+
+  for (std::size_t i = 0; i < r->column_descriptions.size(); i++) {
+    if (r->column_descriptions[i]->data_type == "longblob") {
+      index_of_restore_object = i;
+      break;
+    }
+  }
+
+  if (index_of_restore_object == invalid) {
+    throw std::runtime_error("No longblob object found as restore object");
+  }
+
+  std::istringstream(row[field_index_longblob]) >>
+      row_data.columns[index_of_restore_object].size;
+}
+
+
 void DatabaseImportMysql::FillRowWithDatabaseResult(ResultHandlerContext* r,
                                                     int fields,
                                                     char** row)
 {
-  if (fields != static_cast<int>(r->column_descriptions.size())) {
+  std::size_t number_of_fields = r->column_descriptions.size();
+
+  if (r->is_restore_object) {
+    ++number_of_fields;  // one more for length_of_restore_object
+  }
+
+  if (fields != static_cast<int>(number_of_fields)) {
     throw std::runtime_error(
         "Number of database fields does not match description");
   }
 
-  for (int i = 0; i < fields; i++) {
-    RowData& row_data = r->row_data;
-    row_data.row[i].data_pointer = row[i];
-    r->column_descriptions[i]->db_import_converter(r->db, row_data.row[i]);
+  RowData& row_data = r->row_data;
+
+  if (r->is_restore_object) {
+    std::size_t field_index_longblob = fields - 1;
+    ReadoutSizeOfRestoreObject(r, row_data, field_index_longblob, row);
+  }
+
+  for (std::size_t i = 0; i < r->column_descriptions.size(); i++) {
+    row_data.columns[i].data_pointer = row[i];
+    r->column_descriptions[i]->db_import_converter(r->db, row_data.columns[i]);
   }
 }
 

core/src/dird/dbcopy/row_data.h

@@ -28,25 +28,30 @@
 
 class BareosDb;
 
-struct FieldData {
+struct ColumnData {
+  std::size_t size{};  // length_of_restore_object
   const char* data_pointer{};
   std::vector<char> converted_data;
 };
 
 struct RowData {
   RowData(const DatabaseColumnDescriptions::VectorOfColumnDescriptions&
               column_descriptions_in,
-          const std::string& table_name_in)
-      : table_name(table_name_in), column_descriptions(column_descriptions_in)
+          const std::string& table_name_in,
+          bool is_restore_object_in)
+      : table_name(table_name_in)
+      , column_descriptions(column_descriptions_in)
+      , is_restore_object(is_restore_object_in)
   {
-    row.resize(column_descriptions.size());
+    columns.resize(column_descriptions.size());
   }
 
   std::string table_name;
 
-  std::vector<FieldData> row;  // same index as column_descriptions
+  std::vector<ColumnData> columns;  // same index as column_descriptions
   const DatabaseColumnDescriptions::VectorOfColumnDescriptions&
       column_descriptions;
+  bool is_restore_object{};
 
  public:
   ~RowData() = default;