Skip to content

Commit

Permalink
dbcopy: use insert mode with -i from command line interface
Browse files Browse the repository at this point in the history 
- moved the data converters from column description to the
  regarding database export/import class module
- add command line option -i
  • Loading branch information
@franku
franku committed Feb 27, 2020
1 parent 622b15a commit 66d9272
Show file tree
Hide file tree
Showing 12 changed files with 280 additions and 175 deletions.
9 changes: 7 additions & 2 deletions core/src/cats/postgresql_batch.cc
View file
Expand Up @@ -238,6 +238,11 @@ bool BareosDbPostgresql::SqlBatchInsertFileTable(JobControlRecord* jcr,
return true;
}


/* ************************************* *
* ** Generic SQL Copy used by dbcopy ** *
* ************************************* */

class CleanupResult {
public:
CleanupResult(PGresult** r, int* s) : result(r), status(s) {}
Expand Down Expand Up @@ -279,7 +284,7 @@ bool BareosDbPostgresql::SqlCopyStart(
query.resize(query.size() - 2);
query +=
") FROM STDIN WITH ("
" FORMAT csv"
" FORMAT text"
", DELIMITER '\t'"
")";

Expand Down Expand Up @@ -318,7 +323,7 @@ bool BareosDbPostgresql::SqlCopyInsert(const std::vector<ColumnData>& columns)
std::vector<char> buffer;
for (const auto& column : columns) {
if (strlen(column.data_pointer) == 0) {
query += "\"\"NULL\"\"";
query += "";
} else {
buffer.resize(strlen(column.data_pointer) * 2 + 1);
pgsql_copy_escape(buffer.data(), column.data_pointer, buffer.size());
Expand Down
91 changes: 4 additions & 87 deletions core/src/dird/dbcopy/column_description.cc
View file
Expand Up @@ -47,97 +47,14 @@ ColumnDescription::ColumnDescription(const char* column_name_in,
}
}

static void no_conversion(BareosDb* db, ColumnData& fd)
{
fd.data_pointer = fd.data_pointer ? fd.data_pointer : "";
}

static void timestamp_conversion_postgresql(BareosDb* db, ColumnData& fd)
{
static const char* dummy_timepoint = "1970-01-01 00:00:00";
if (!fd.data_pointer) {
fd.data_pointer = dummy_timepoint;
} else if (fd.data_pointer[0] == '0') {
fd.data_pointer = dummy_timepoint;
} else if (strlen(fd.data_pointer) == 0) {
fd.data_pointer = dummy_timepoint;
}
}

static void string_conversion_postgresql(BareosDb* db, ColumnData& fd)
{
if (fd.data_pointer) {
std::size_t len{strlen(fd.data_pointer)};
fd.converted_data.resize(len * 2 + 1);
db->EscapeString(nullptr, fd.converted_data.data(), fd.data_pointer, len);
fd.data_pointer = fd.converted_data.data();
} else {
fd.data_pointer = "";
}
}

static void bytea_conversion_postgresql(BareosDb* db, ColumnData& fd)
{
std::size_t new_len{};
std::size_t old_len = fd.size;

auto old = reinterpret_cast<const unsigned char*>(fd.data_pointer);

auto obj = db->EscapeObject(old, old_len, new_len);

fd.converted_data.resize(new_len + 1);
memcpy(fd.converted_data.data(), obj, new_len + 1);

db->FreeEscapedObjectMemory(obj);

fd.data_pointer = fd.converted_data.data();
}

const DataTypeConverterMap ColumnDescriptionMysql::db_import_converter_map{
{"bigint", no_conversion}, {"binary", no_conversion},
{"blob", no_conversion}, {"char", no_conversion},
{"datetime", no_conversion}, {"decimal", no_conversion},
{"enum", no_conversion}, {"int", no_conversion},
{"longblob", no_conversion}, {"smallint", no_conversion},
{"text", no_conversion}, {"timestamp", no_conversion},
{"tinyblob", no_conversion}, {"tinyint", no_conversion},
{"varchar", no_conversion}};

ColumnDescriptionMysql::ColumnDescriptionMysql(const char* column_name_in,
const char* data_type_in,
const char* max_length_in)
: ColumnDescription(column_name_in, data_type_in, max_length_in)
{
try {
db_import_converter = db_import_converter_map.at(data_type_in);
} catch (const std::out_of_range& e) {
std::string err{"Mysql: Data type not found in conversion map: "};
err += data_type_in;
throw std::runtime_error(err);
}
}

const DataTypeConverterMap ColumnDescriptionPostgresql::db_export_converter_map{
{"bigint", no_conversion},
{"bytea", bytea_conversion_postgresql},
{"character", string_conversion_postgresql},
{"integer", no_conversion},
{"numeric", no_conversion},
{"smallint", no_conversion},
{"text", string_conversion_postgresql},
{"timestamp without time zone", timestamp_conversion_postgresql}};

ColumnDescriptionPostgresql::ColumnDescriptionPostgresql(
const char* column_name_in,
const char* data_type_in,
const char* max_length_in)
: ColumnDescription(column_name_in, data_type_in, max_length_in)
void ColumnDescription::RegisterConverterCallbackFromMap(
const DataTypeConverterMap& converter_map)
{
try {
db_export_converter = db_export_converter_map.at(data_type_in);
converter = converter_map.at(data_type);
} catch (const std::out_of_range& e) {
std::string err{"Postgresql: Data type not found in conversion map: "};
err += data_type_in;
err += data_type;
throw std::runtime_error(err);
}
}
34 changes: 7 additions & 27 deletions core/src/dird/dbcopy/column_description.h
View file
Expand Up @@ -35,38 +35,18 @@ class ColumnDescription {
const char* data_type_in,
const char* max_length_in);
virtual ~ColumnDescription() = default;

std::string column_name;
std::string data_type;
std::size_t character_maximum_length{};
ColumnDescription() = delete;

using ConverterCallback = std::function<void(BareosDb*, ColumnData&)>;
ConverterCallback db_export_converter{};
ConverterCallback db_import_converter{};
};

using DataTypeConverterMap =
std::map<std::string, ColumnDescription::ConverterCallback>;

class ColumnDescriptionMysql : public ColumnDescription {
public:
ColumnDescriptionMysql(const char* column_name_in,
const char* data_type_in,
const char* max_length_in);
using DataTypeConverterMap = std::map<std::string, ConverterCallback>;

private:
static const DataTypeConverterMap db_import_converter_map;
};
void RegisterConverterCallbackFromMap(const DataTypeConverterMap&);

class ColumnDescriptionPostgresql : public ColumnDescription {
public:
ColumnDescriptionPostgresql(const char* column_name_in,
const char* data_type_in,
const char* max_length_in);
std::string column_name;
std::string data_type;
std::size_t character_maximum_length{};

private:
static const DataTypeConverterMap db_export_converter_map;
ConverterCallback converter{};
};


#endif // BAREOS_SRC_DIRD_DBCOPY_COLUMN_DESCRIPTION_H_
9 changes: 4 additions & 5 deletions core/src/dird/dbcopy/database_column_descriptions.cc
View file
Expand Up @@ -56,10 +56,9 @@ int DatabaseColumnDescriptionsPostgresql::ResultHandler(void* ctx,
{
auto t = static_cast<DatabaseColumnDescriptions*>(ctx);

t->column_descriptions.emplace_back(
std::make_unique<ColumnDescriptionPostgresql>(
row[RowIndex::kColumnName], row[RowIndex::kDataType],
row[RowIndex::kCharMaxLenght]));
t->column_descriptions.emplace_back(std::make_unique<ColumnDescription>(
row[RowIndex::kColumnName], row[RowIndex::kDataType],
row[RowIndex::kCharMaxLenght]));

return 0;
}
Expand Down Expand Up @@ -87,7 +86,7 @@ int DatabaseColumnDescriptionsMysql::ResultHandler(void* ctx,
{
auto t = static_cast<DatabaseColumnDescriptions*>(ctx);

t->column_descriptions.emplace_back(std::make_unique<ColumnDescriptionMysql>(
t->column_descriptions.emplace_back(std::make_unique<ColumnDescription>(
row[RowIndex::kColumnName], row[RowIndex::kDataType],
row[RowIndex::kCharMaxLenght]));

Expand Down
6 changes: 3 additions & 3 deletions core/src/dird/dbcopy/database_export.cc
View file
Expand Up @@ -26,8 +26,7 @@

#include <memory>

DatabaseExport::DatabaseExport(const DatabaseConnection& db_connection,
bool clear_tables)
DatabaseExport::DatabaseExport(const DatabaseConnection& db_connection)
: db_(db_connection.db)
, table_descriptions_(DatabaseTableDescriptions::Create(db_connection))
{
Expand All @@ -38,11 +37,12 @@ DatabaseExport::~DatabaseExport() = default;

std::unique_ptr<DatabaseExport> DatabaseExport::Create(
const DatabaseConnection& db_connection,
DatabaseExport::InsertMode mode,
bool clear_tables)
{
switch (db_connection.db_type) {
case DatabaseType::Enum::kPostgresql:
return std::make_unique<DatabaseExportPostgresql>(db_connection,
return std::make_unique<DatabaseExportPostgresql>(db_connection, mode,
clear_tables);
default:
case DatabaseType::Enum::kMysql:
Expand Down
10 changes: 8 additions & 2 deletions core/src/dird/dbcopy/database_export.h
View file
Expand Up @@ -32,12 +32,18 @@ class DatabaseTableDescriptions;

class DatabaseExport {
public:
DatabaseExport(const DatabaseConnection& db_connection,
bool clear_tables = false);
DatabaseExport(const DatabaseConnection& db_connection);
virtual ~DatabaseExport();

enum class InsertMode
{
kSqlInsert,
kSqlCopy
};

static std::unique_ptr<DatabaseExport> Create(
const DatabaseConnection& db_connection,
InsertMode mode = InsertMode::kSqlCopy,
bool clear_tables = false);

virtual bool StartTable(const std::string& table_name) = 0;
Expand Down

0 comments on commit 66d9272

Please sign in to comment.