systemtest: fix bareos dbcopy test

bareos · Jan 31, 2020 · 08a6bb6 · 08a6bb6
1 parent 6fd4bf6
commit 08a6bb6
Show file tree

Hide file tree

Showing 11 changed files with 54 additions and 63 deletions.
diff --git a/core/src/dird/dbcopy/column_description.cc b/core/src/dird/dbcopy/column_description.cc
@@ -93,26 +93,14 @@ static void bytea_conversion_postgresql(BareosDb* db, ColumnData& fd)
   fd.data_pointer = fd.converted_data.data();
 }
 
-static void longblob_conversion_mysql(BareosDb* db, ColumnData& fd)
-{
-  no_conversion(db, fd);
-}
-
 const DataTypeConverterMap ColumnDescriptionMysql::db_import_converter_map{
-    {"bigint", no_conversion},
-    {"binary", no_conversion},
-    {"blob", no_conversion},
-    {"char", no_conversion},
-    {"datetime", no_conversion},
-    {"decimal", no_conversion},
-    {"enum", no_conversion},
-    {"int", no_conversion},
-    {"longblob", longblob_conversion_mysql},
-    {"smallint", no_conversion},
-    {"text", no_conversion},
-    {"timestamp", no_conversion},
-    {"tinyblob", no_conversion},
-    {"tinyint", no_conversion},
+    {"bigint", no_conversion},   {"binary", no_conversion},
+    {"blob", no_conversion},     {"char", no_conversion},
+    {"datetime", no_conversion}, {"decimal", no_conversion},
+    {"enum", no_conversion},     {"int", no_conversion},
+    {"longblob", no_conversion}, {"smallint", no_conversion},
+    {"text", no_conversion},     {"timestamp", no_conversion},
+    {"tinyblob", no_conversion}, {"tinyint", no_conversion},
     {"varchar", no_conversion}};
 
 ColumnDescriptionMysql::ColumnDescriptionMysql(const char* column_name_in,

diff --git a/core/src/dird/dbcopy/database_export_postgresql.cc b/core/src/dird/dbcopy/database_export_postgresql.cc
@@ -56,7 +56,7 @@ DatabaseExportPostgresql::DatabaseExportPostgresql(
 
 DatabaseExportPostgresql::~DatabaseExportPostgresql()
 {
-  if (transaction_) { db_->SqlQuery("ROLLBACK"); }
+  if (transaction_open_) { db_->SqlQuery("ROLLBACK"); }
 }
 
 void DatabaseExportPostgresql::CopyRow(RowData& source_data_row)
@@ -124,21 +124,21 @@ static void UpdateSequences(
     std::transform(table_name.cbegin(), table_name.cend(),
                    std::back_inserter(table_name_lower_case), ::tolower);
     if (s.table_name == table_name_lower_case) {
-    std::string sequence_schema_query{"select setval(' "};
-    sequence_schema_query += s.sequence_name;
-    sequence_schema_query += "', (select max(";
-    sequence_schema_query += s.column_name;
-    sequence_schema_query += ") from ";
+      std::string sequence_schema_query{"select setval(' "};
+      sequence_schema_query += s.sequence_name;
+      sequence_schema_query += "', (select max(";
+      sequence_schema_query += s.column_name;
+      sequence_schema_query += ") from ";
       sequence_schema_query += table_name_lower_case;
-    sequence_schema_query += "))";
+      sequence_schema_query += "))";
       std::cout << "--> updating sequence" << std::endl;
-    if (!db->SqlQuery(sequence_schema_query.c_str())) {
-      throw std::runtime_error(
-          "DatabaseExportPostgresql: Could not set sequence");
+      if (!db->SqlQuery(sequence_schema_query.c_str())) {
+        throw std::runtime_error(
+            "DatabaseExportPostgresql: Could not set sequence");
+      }
     }
   }
 }
-}
 
 void DatabaseExportPostgresql::CopyEnd() {}
 
@@ -211,8 +211,8 @@ bool DatabaseExportPostgresql::StartTable(const std::string& table_name)
   }
 
   if (db_->SqlQuery("BEGIN")) {
-    transaction_ = true;
-    start_new_table = true;
+    transaction_open_ = true;
+    cursor_start_new_table_ = true;
   }
 
   return true;
@@ -232,17 +232,17 @@ void DatabaseExportPostgresql::EndTable(const std::string& table_name)
 
   UpdateSequences(db_, sequence_schema_vector_, table_name);
 
-  if (transaction_) {
+  if (transaction_open_) {
     db_->SqlQuery("COMMIT");
-    transaction_ = false;
+    transaction_open_ = false;
   }
 }
 
 void DatabaseExportPostgresql::CompareRow(const RowData& data)
 {
-  if (start_new_table) {
+  if (cursor_start_new_table_) {
     CursorStartTable(data.table_name);
-    start_new_table = false;
+    cursor_start_new_table_ = false;
   }
 
   std::string query{"FETCH NEXT FROM curs1"};

diff --git a/core/src/dird/dbcopy/database_export_postgresql.h b/core/src/dird/dbcopy/database_export_postgresql.h
@@ -50,8 +50,8 @@ class DatabaseExportPostgresql : public DatabaseExport {
   using SequenceSchemaVector = std::vector<SequenceSchema>;
 
  private:
-  bool transaction_{false};
-  bool start_new_table{false};
+  bool transaction_open_{false};
+  bool cursor_start_new_table_{false};
   SequenceSchemaVector sequence_schema_vector_;
 
   void SelectSequenceSchema();

diff --git a/core/src/dird/dbcopy/database_import_mysql.cc b/core/src/dird/dbcopy/database_import_mysql.cc
@@ -38,7 +38,7 @@ DatabaseImportMysql::DatabaseImportMysql(
     const DatabaseConnection& db_connection,
     std::size_t maximum_amount_of_rows_in)
     : DatabaseImport(db_connection)
-    , maximum_amount_of_rows(maximum_amount_of_rows_in)
+    , maximum_amount_of_rows_(maximum_amount_of_rows_in)
 {
   return;
 }
@@ -84,7 +84,7 @@ struct ResultHandlerContext {
       column_descriptions;
   RowData& row_data;
   DatabaseExport& exporter;
-  uint64_t counter{};
+  uint64_t row_counter{};
   BareosDb* db{};
   bool is_restore_object{};
 };
@@ -117,12 +117,12 @@ void DatabaseImportMysql::RunQuerySelectAllRows(
     query += " FROM ";
     query += t.table_name;
 
-    if (maximum_amount_of_rows) {
+    if (maximum_amount_of_rows_) {
       query += " LIMIT ";
-      query += std::to_string(maximum_amount_of_rows);
+      query += std::to_string(maximum_amount_of_rows_);
     }
 
-    RowData row_data(t.column_descriptions, t.table_name, is_restore_object);
+    RowData row_data(t.column_descriptions, t.table_name);
     ResultHandlerContext ctx(t.column_descriptions, row_data, exporter, db_,
                              is_restore_object);
 
@@ -220,7 +220,7 @@ int DatabaseImportMysql::ResultHandlerCopy(void* ctx, int fields, char** row)
 
   r->exporter.CopyRow(r->row_data);
 
-  if (!(++r->counter % 10000)) { std::cout << "." << std::flush; }
+  if (!(++r->row_counter % 10000)) { std::cout << "." << std::flush; }
   return 0;
 }
 
@@ -231,6 +231,6 @@ int DatabaseImportMysql::ResultHandlerCompare(void* ctx, int fields, char** row)
 
   r->exporter.CompareRow(r->row_data);
 
-  if (!(++r->counter % 10000)) { std::cout << "." << std::flush; }
+  if (!(++r->row_counter % 10000)) { std::cout << "." << std::flush; }
   return 0;
 }
diff --git a/core/src/dird/dbcopy/database_import_mysql.h b/core/src/dird/dbcopy/database_import_mysql.h
@@ -48,7 +48,7 @@ class DatabaseImportMysql : public DatabaseImport {
 
   void RunQuerySelectAllRows(DB_RESULT_HANDLER* result_handler,
                              DatabaseExport& exporter);
-  std::size_t maximum_amount_of_rows{};
+  std::size_t maximum_amount_of_rows_{};
 };
 
 

diff --git a/core/src/dird/dbcopy/dbcopy.cc b/core/src/dird/dbcopy/dbcopy.cc
@@ -55,11 +55,6 @@ class DbCopy {
     SetWorkingDir();
     cl.ParseCommandLine(argc, argv);
 
-    if (cl.source_db_resource_name != "mysql")
-      throw std::runtime_error("source database is not mysql");
-    if (cl.destination_db_resource_name != "postgresql")
-      throw std::runtime_error("destination database is not postgresql");
-
     ParseConfig();
     std::cout << "Copying tables from \"" << cl.source_db_resource_name
               << "\" to \"" << cl.destination_db_resource_name << "\""
@@ -69,12 +64,12 @@ class DbCopy {
 
   void DoDatabaseCopy()
   {
-    std::cout << "gathering info about source catalog \""
+    std::cout << "gathering information about source catalog \""
               << cl.source_db_resource_name << "\"..." << std::endl;
     std::unique_ptr<DatabaseImport> imp(
         DatabaseImport::Create(*source_db_, cl.maximum_number_of_rows));
 
-    std::cout << "gathering info about destination catalog \""
+    std::cout << "gathering information about destination catalog \""
               << cl.destination_db_resource_name << "\"..." << std::endl;
     std::unique_ptr<DatabaseExport> exp(
         DatabaseExport::Create(*destination_db_, cl.empty_destination_tables));
@@ -127,6 +122,13 @@ class DbCopy {
       destination_db_ = std::make_unique<DatabaseConnection>(
           cl.destination_db_resource_name, directordaemon::my_config);
 
+      if (source_db_->db_type != DatabaseType::Enum::kMysql)
+        throw std::runtime_error("Error: Source database is not mysql");
+
+      if (destination_db_->db_type != DatabaseType::Enum::kPostgresql)
+        throw std::runtime_error(
+            "Error: Destination database is not postgresql");
+
     } catch (const std::runtime_error& e) {
       throw e;
     }
@@ -215,7 +217,7 @@ class DbCopy {
   std::unique_ptr<DatabaseConnection> source_db_;
   std::unique_ptr<DatabaseConnection> destination_db_;
   std::unique_ptr<ConfigurationParser> my_config_;
-  std::array<char, 1024> current_working_directory_;
+  std::array<char, 2048> current_working_directory_;
 };
 
 class Cleanup {

diff --git a/core/src/dird/dbcopy/row_data.h b/core/src/dird/dbcopy/row_data.h
@@ -37,11 +37,8 @@ struct ColumnData {
 struct RowData {
   RowData(const DatabaseColumnDescriptions::VectorOfColumnDescriptions&
               column_descriptions_in,
-          const std::string& table_name_in,
-          bool is_restore_object_in)
-      : table_name(table_name_in)
-      , column_descriptions(column_descriptions_in)
-      , is_restore_object(is_restore_object_in)
+          const std::string& table_name_in)
+      : table_name(table_name_in), column_descriptions(column_descriptions_in)
   {
     columns.resize(column_descriptions.size());
   }
@@ -51,7 +48,6 @@ struct RowData {
   std::vector<ColumnData> columns;  // same index as column_descriptions
   const DatabaseColumnDescriptions::VectorOfColumnDescriptions&
       column_descriptions;
-  bool is_restore_object{};
 
  public:
   ~RowData() = default;

diff --git a/systemtests/CMakeLists.txt b/systemtests/CMakeLists.txt
@@ -218,7 +218,7 @@ macro(link_binaries_to_test_to_current_sbin_dir_with_individual_filename)
     string(REPLACE "-" "_" binary_name ${binary_name})
     string(TOUPPER ${binary_name} binary_name_upcase)
     string(CONCAT bareos_XXX_binary ${binary_name_upcase} "_BINARY")
-    # message (STATUS "${bareos_XXX_binary}")
+    #message (STATUS "${bareos_XXX_binary}")
     set(${bareos_XXX_binary} ${CURRENT_SBIN_DIR}/${binary_name}-${TEST_NAME})
     # message( "creating symlink ${${bareos_XXX_binary}}  ->
     # ${${binary_name_to_test_upcase}}" )
@@ -332,6 +332,7 @@ set(ALL_BINARIES_BEING_USED_BY_SYSTEMTESTS
     bareos-dbcheck
     bareos-fd
     bareos-sd
+    bareos-dbcopy
     bcopy
     btape
     bextract

diff --git a/systemtests/environment.in b/systemtests/environment.in
@@ -59,6 +59,7 @@ export BAREOS_STORAGEDAEMON_BINARY=@BAREOS_SD_BINARY@
 export BAREOS_BSCAN_BINARY=@BSCAN_BINARY@
 export BAREOS_BLS_BINARY=@BLS_BINARY@
 export BAREOS_BCONSOLE_BINARY=@BCONSOLE_BINARY@
+export BAREOS_DBCOPY_BINARY=@BAREOS_DBCOPY_BINARY@
 
 
 

diff --git a/systemtests/tests/dbcopy-mysql-postgresql-test/testrunner b/systemtests/tests/dbcopy-mysql-postgresql-test/testrunner
@@ -59,7 +59,7 @@ stop_bareos
 cp "${conf}"/bareos-dir.d/catalog/postgresql.conf.template "${conf}"/bareos-dir.d/catalog/postgresql.conf
 
 #do the database copy
-"${sbin}"/bareos-dbcopy -c ${current_test_directory}/etc/bareos mysql postgresql
+"${BAREOS_DBCOPY_BINARY}" -c "${current_test_directory}"/etc/bareos mysql postgresql
 
 #enable only postgresql
 rm "${conf}"/bareos-dir.d/catalog/mysql.conf
@@ -84,7 +84,7 @@ check_for_zombie_jobs storage=File client=bareos-fd
 stop_bareos
 
 check_two_logs
-check_restore_diff ${BackupDirectory}
+check_restore_diff "${BackupDirectory}"
 
 sed -i '/Using Catalog.*/d' "${tmp}"/files.postgresql
 sed -i '/Using Catalog.*/d' "${tmp}"/files.mysql

diff --git a/systemtests/tests/python-fd-plugin-local-fileset-test/testrunner b/systemtests/tests/python-fd-plugin-local-fileset-test/testrunner
@@ -33,6 +33,9 @@ BackupDirectory="${tmp}/data"
 # Data will be placed at "${tmp}/data/".
 setup_data "$@"
 
+# this test does not work with links because of the restore objects
+rm -r "${tmp}"/data/weird-files >/dev/null 2>&1
+
 start_test
 
 cat <<END_OF_DATA >$tmp/bconcmds