Skip to content

Commit

Permalink
Merge pull request #524 from bareos/dev/franku/master/dbcopy-speedup
Browse files Browse the repository at this point in the history 
dbcopy: improve copy speed a lot
  • Loading branch information
@franku
franku committed May 18, 2020
1 parent 35de72c commit 4fb297a
Show file tree
Hide file tree
Showing 12 changed files with 217 additions and 186 deletions.
12 changes: 8 additions & 4 deletions core/src/dird/dbcopy/column_description.cc
View file
Expand Up @@ -27,10 +27,14 @@
#include <string>
#include <vector>

ColumnDescription::ColumnDescription(const char* column_name_in,
const char* data_type_in,
const char* max_length_in)
: column_name(column_name_in), data_type(data_type_in)
ColumnDescription::ColumnDescription(
const std::string& column_name_in,
const std::string& column_name_lower_case_in,
const char* data_type_in,
const char* max_length_in)
: column_name(column_name_in)
, column_name_lower_case(column_name_lower_case_in)
, data_type(data_type_in)
{
std::string field;
try {
Expand Down
4 changes: 3 additions & 1 deletion core/src/dird/dbcopy/column_description.h
View file
Expand Up @@ -31,7 +31,8 @@ class BareosDb;

class ColumnDescription {
public:
ColumnDescription(const char* column_name_in,
ColumnDescription(const std::string& column_name_in,
const std::string& column_name_lower_case_in,
const char* data_type_in,
const char* max_length_in);
virtual ~ColumnDescription() = default;
Expand All @@ -43,6 +44,7 @@ class ColumnDescription {
void RegisterConverterCallbackFromMap(const DataTypeConverterMap&);

std::string column_name;
std::string column_name_lower_case;
std::string data_type;
std::size_t character_maximum_length{};

Expand Down
63 changes: 27 additions & 36 deletions core/src/dird/dbcopy/database_column_descriptions.cc
View file
Expand Up @@ -28,41 +28,44 @@
#include <algorithm>
#include <iostream>

DatabaseColumnDescriptions::DatabaseColumnDescriptions(BareosDb* db) : db_{db}
void DatabaseColumnDescriptions::SelectColumnDescriptionsAndAddToMap(
const std::string& sql_query)
{
}

void DatabaseColumnDescriptions::SelectColumnDescriptions(
const std::string& sql_query,
DB_RESULT_HANDLER* ResultHandler)
{
if (!db_->SqlQuery(sql_query.c_str(), ResultHandler, this)) {
if (!db_->SqlQuery(sql_query.c_str(), ResultHandler_, this)) {
std::string err{"Could not select table names: "};
err += sql_query;
throw std::runtime_error(err);
}
std::sort(column_descriptions.begin(), column_descriptions.end(),
[](const std::unique_ptr<ColumnDescription>& v1,
const std::unique_ptr<ColumnDescription>& v2) {
std::string l1, l2;
ToLowerCase(v1->column_name, v2->column_name, l1, l2);
return l1 < l2;
});
}

int DatabaseColumnDescriptionsPostgresql::ResultHandler(void* ctx,
int /*fields*/,
char** row)
int DatabaseColumnDescriptions::ResultHandler_(void* ctx,
int /*fields*/,
char** row)
{
auto t = static_cast<DatabaseColumnDescriptions*>(ctx);

t->column_descriptions.emplace_back(std::make_unique<ColumnDescription>(
row[RowIndex::kColumnName], row[RowIndex::kDataType],
row[RowIndex::kCharMaxLenght]));
t->AddToMap(row[RowIndex::kColumnName], row[RowIndex::kDataType],
row[RowIndex::kCharMaxLenght]);

return 0;
}

void DatabaseColumnDescriptions::AddToMap(const char* column_name_in,
const char* data_type_in,
const char* max_length_in)

{
std::string column_name_lower_case;
std::string column_name{column_name_in};

StringToLowerCase(column_name_lower_case, column_name_in);

column_descriptions.emplace( // std::map sorts keys ascending by default
std::piecewise_construct, std::forward_as_tuple(column_name_lower_case),
std::forward_as_tuple(column_name, column_name_lower_case, data_type_in,
max_length_in));
}

DatabaseColumnDescriptionsPostgresql::DatabaseColumnDescriptionsPostgresql(
BareosDb* db,
const std::string& table_name)
Expand All @@ -77,20 +80,7 @@ DatabaseColumnDescriptionsPostgresql::DatabaseColumnDescriptionsPostgresql(

std::cout << "--> " << table_name << std::endl;

SelectColumnDescriptions(query, ResultHandler);
}

int DatabaseColumnDescriptionsMysql::ResultHandler(void* ctx,
int /*fields*/,
char** row)
{
auto t = static_cast<DatabaseColumnDescriptions*>(ctx);

t->column_descriptions.emplace_back(std::make_unique<ColumnDescription>(
row[RowIndex::kColumnName], row[RowIndex::kDataType],
row[RowIndex::kCharMaxLenght]));

return 0;
SelectColumnDescriptionsAndAddToMap(query);
}

DatabaseColumnDescriptionsMysql::DatabaseColumnDescriptionsMysql(
Expand All @@ -108,5 +98,6 @@ DatabaseColumnDescriptionsMysql::DatabaseColumnDescriptionsMysql(
query += " table_schema='";
query += db->get_db_name();
query += "'";
SelectColumnDescriptions(query, ResultHandler);

SelectColumnDescriptionsAndAddToMap(query);
}
18 changes: 9 additions & 9 deletions core/src/dird/dbcopy/database_column_descriptions.h
View file
Expand Up @@ -32,13 +32,15 @@ class BareosDb;

class DatabaseColumnDescriptions {
public:
DatabaseColumnDescriptions(BareosDb* db);
DatabaseColumnDescriptions(BareosDb* db) : db_{db} {};
virtual ~DatabaseColumnDescriptions() = default;
void AddToMap(const char* column_name_in,
const char* data_type_in,
const char* max_length_in);

using VectorOfColumnDescriptions =
std::vector<std::unique_ptr<ColumnDescription>>;
using ColumnDescriptions = std::map<std::string, ColumnDescription>;

VectorOfColumnDescriptions column_descriptions;
ColumnDescriptions column_descriptions;

protected:
enum RowIndex : int
Expand All @@ -47,24 +49,22 @@ class DatabaseColumnDescriptions {
kDataType = 1,
kCharMaxLenght = 2
};
void SelectColumnDescriptions(const std::string& sql_query,
DB_RESULT_HANDLER* ResultHandler);
void SelectColumnDescriptionsAndAddToMap(const std::string& sql_query);

private:
BareosDb* db_;
BareosDb* db_{};
static int ResultHandler_(void* ctx, int fields, char** row);
};

class DatabaseColumnDescriptionsPostgresql : public DatabaseColumnDescriptions {
public:
DatabaseColumnDescriptionsPostgresql(BareosDb* db,
const std::string& table_name);
static int ResultHandler(void* ctx, int fields, char** row);
};

class DatabaseColumnDescriptionsMysql : public DatabaseColumnDescriptions {
public:
DatabaseColumnDescriptionsMysql(BareosDb* db, const std::string& table_name);
static int ResultHandler(void* ctx, int fields, char** row);
};

#endif // BAREOS_SRC_DIRD_DBCOPY_DATABASE_COLUMN_DESCRIPTIONS_H_
26 changes: 14 additions & 12 deletions core/src/dird/dbcopy/database_export_postgresql.cc
View file
Expand Up @@ -148,7 +148,7 @@ DatabaseExportPostgresql::DatabaseExportPostgresql(

if (clear_tables) {
for (const auto& t : table_descriptions_->tables) {
ClearTable(db_, t.table_name);
ClearTable(db_, t.second.table_name);
}
}
}
Expand All @@ -160,15 +160,15 @@ DatabaseExportPostgresql::~DatabaseExportPostgresql()

void DatabaseExportPostgresql::DoCopyInsertion(RowData& source_data_row)
{
for (std::size_t i = 0; i < source_data_row.column_descriptions.size(); i++) {
size_t i = 0;
for (const auto& c : source_data_row.column_descriptions) {
const ColumnDescription* column_description =
table_descriptions_->GetColumnDescription(
source_data_row.table_name,
source_data_row.column_descriptions[i]->column_name);
source_data_row.table_name, c.second.column_name_lower_case);

if (column_description == nullptr) {
std::string err{"Could not get column description for: "};
err += source_data_row.column_descriptions[i]->column_name;
err += c.second.column_name;
throw std::runtime_error(err);
}

Expand All @@ -177,6 +177,7 @@ void DatabaseExportPostgresql::DoCopyInsertion(RowData& source_data_row)
} else {
throw std::runtime_error("Row number does not match column description");
}
++i;
}
if (!db_->SqlCopyInsert(source_data_row.data_fields)) {
std::string err{"DatabaseExportPostgresql: Could not execute query: "};
Expand All @@ -193,15 +194,15 @@ void DatabaseExportPostgresql::DoInsertInsertion(RowData& source_data_row)

std::string query_values = " VALUES (";

for (std::size_t i = 0; i < source_data_row.column_descriptions.size(); i++) {
size_t i = 0;
for (const auto& c : source_data_row.column_descriptions) {
const ColumnDescription* column_description =
table_descriptions_->GetColumnDescription(
source_data_row.table_name,
source_data_row.column_descriptions[i]->column_name);
source_data_row.table_name, c.second.column_name_lower_case);

if (column_description == nullptr) {
std::string err{"Could not get column description for: "};
err += source_data_row.column_descriptions[i]->column_name;
err += c.second.column_name_lower_case;
throw std::runtime_error(err);
}

Expand All @@ -216,6 +217,7 @@ void DatabaseExportPostgresql::DoInsertInsertion(RowData& source_data_row)
} else {
throw std::runtime_error("Row number does not match column description");
}
++i;
}

query_values.resize(query_values.size() - 1);
Expand Down Expand Up @@ -295,15 +297,15 @@ static bool TableIsEmtpy(BareosDb* db, const std::string& table_name)
return db->SqlNumRows() <= 0;
}

bool DatabaseExportPostgresql::TableExists(
bool DatabaseExportPostgresql::TableExistsAndGetColumns(
const std::string& table_name,
std::vector<std::string>& column_names)
{
auto found = table_descriptions_->GetTableDescription(table_name);
if (found == nullptr) { return false; }

for (const auto& col : found->column_descriptions) {
column_names.push_back(col->column_name);
column_names.push_back(col.second.column_name);
}

return true;
Expand All @@ -316,7 +318,7 @@ bool DatabaseExportPostgresql::StartTable(const std::string& table_name)

std::vector<std::string> column_names;

if (!TableExists(table_name, column_names)) {
if (!TableExistsAndGetColumns(table_name, column_names)) {
std::cout << "--> destination table does not exist" << std::endl;
return false;
}
Expand Down
4 changes: 2 additions & 2 deletions core/src/dird/dbcopy/database_export_postgresql.h
View file
Expand Up @@ -59,8 +59,8 @@ class DatabaseExportPostgresql : public DatabaseExport {
SequenceSchemaVector sequence_schema_vector_;

void SelectSequenceSchema();
bool TableExists(const std::string& table_name,
std::vector<std::string>& column_names);
bool TableExistsAndGetColumns(const std::string& table_name,
std::vector<std::string>& column_names);
bool UseCopyInsertion();
void DoCopyInsertion(RowData& source_data_row);
void DoInsertInsertion(RowData& source_data_row);
Expand Down

0 comments on commit 4fb297a

Please sign in to comment.