From e2b880aa5f08c3e0d63ebdaa8ca92d5f512ab995 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Tue, 16 Sep 2025 21:50:17 +0200 Subject: [PATCH 01/48] Added the DuckDB dependency --- CMakeLists.txt | 16 ++++++++++++++++ vcpkg.json | 9 +++++++++ 2 files changed, 25 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index f674381b..87b58a40 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,6 +2,8 @@ cmake_minimum_required(VERSION 3.23) option(SQLGEN_BUILD_SHARED "Build shared library" ${BUILD_SHARED_LIBS}) +option(SQLGEN_DUCKDB "Enable DuckDB support" OFF) + option(SQLGEN_MYSQL "Enable MySQL support" OFF) option(SQLGEN_POSTGRES "Enable PostgreSQL support" ON) # enabled by default @@ -27,6 +29,10 @@ if (SQLGEN_USE_VCPKG) list(APPEND VCPKG_MANIFEST_FEATURES "tests") endif() + if (SQLGEN_DUCKDB) + list(APPEND VCPKG_MANIFEST_FEATURES "duckdb") + endif() + if (SQLGEN_MYSQL) list(APPEND VCPKG_MANIFEST_FEATURES "mysql") endif() @@ -66,6 +72,16 @@ target_include_directories( $ $) + +if (SQLGEN_DUCKDB) + #list(APPEND SQLGEN_SOURCES src/sqlgen_duckdb.cpp) + if (NOT TARGET DuckDB) + find_package(DuckDB REQUIRED) + endif() + target_link_libraries(sqlgen PUBLIC $,duckdb,duckdb_static>) +endif() + + if(SQLGEN_MYSQL) list(APPEND SQLGEN_SOURCES src/sqlgen_mysql.cpp) if (SQLGEN_USE_VCPKG) diff --git a/vcpkg.json b/vcpkg.json index 9684759f..8fe0bd68 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -6,6 +6,15 @@ } ], "features": { + "duckdb": { + "description": "Enable DuckDB support", + "dependencies": [ + { + "name": "duckdb", + "version>=": "1.3.2" + } + ] + }, "mysql": { "description": "Enable MySQL/MariaDB support", "dependencies": [ From 41b941265a8013e8a5ff7746254132d6b604bdc0 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Tue, 16 Sep 2025 23:18:11 +0200 Subject: [PATCH 02/48] Added the DuckDBConnection --- CMakeLists.txt | 2 +- include/sqlgen/duckdb/Connection.hpp | 89 +++ include/sqlgen/duckdb/DuckDBConnection.hpp | 45 ++ src/sqlgen/duckdb/Connection.cpp | 155 ++++ src/sqlgen/duckdb/DuckDBConnection.cpp | 43 ++ src/sqlgen/duckdb/Iterator.cpp | 92 +++ src/sqlgen/duckdb/exec.cpp | 27 + src/sqlgen/duckdb/to_sql.cpp | 827 +++++++++++++++++++++ src/sqlgen_duckdb.cpp | 5 + 9 files changed, 1284 insertions(+), 1 deletion(-) create mode 100644 include/sqlgen/duckdb/Connection.hpp create mode 100644 include/sqlgen/duckdb/DuckDBConnection.hpp create mode 100644 src/sqlgen/duckdb/Connection.cpp create mode 100644 src/sqlgen/duckdb/DuckDBConnection.cpp create mode 100644 src/sqlgen/duckdb/Iterator.cpp create mode 100644 src/sqlgen/duckdb/exec.cpp create mode 100644 src/sqlgen/duckdb/to_sql.cpp create mode 100644 src/sqlgen_duckdb.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 87b58a40..92280825 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,7 +74,7 @@ target_include_directories( if (SQLGEN_DUCKDB) - #list(APPEND SQLGEN_SOURCES src/sqlgen_duckdb.cpp) + list(APPEND SQLGEN_SOURCES src/sqlgen_duckdb.cpp) if (NOT TARGET DuckDB) find_package(DuckDB REQUIRED) endif() diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp new file mode 100644 index 00000000..724b0fb4 --- /dev/null +++ b/include/sqlgen/duckdb/Connection.hpp @@ -0,0 +1,89 @@ +#ifndef SQLGEN_DUCKDB_CONNECTION_HPP_ +#define SQLGEN_DUCKDB_CONNECTION_HPP_ + +#include + +#include +#include +#include +#include +#include + +#include "../IteratorBase.hpp" +#include "../Ref.hpp" +#include "../Result.hpp" +#include "../Transaction.hpp" +#include "../dynamic/Write.hpp" +#include "../is_connection.hpp" +#include "to_sql.hpp" + +namespace sqlgen::duckdb { + +class Connection { + using ConnPtr = Ref; + using StmtPtr = std::shared_ptr; + + public: + Connection(const std::string& _fname) + : stmt_(nullptr), conn_(make_conn(_fname)) {} + + static rfl::Result> make(const std::string& _fname) noexcept; + + ~Connection() = default; + + Result begin_transaction() noexcept; + + Result commit() noexcept; + + Result execute(const std::string& _sql) noexcept; + + Result insert( + const dynamic::Insert& _stmt, + const std::vector>>& + _data) noexcept; + + Result> read(const dynamic::SelectFrom& _query); + + Result rollback() noexcept; + + std::string to_sql(const dynamic::Statement& _stmt) noexcept { + return duckdb::to_sql_impl(_stmt); + } + + Result start_write(const dynamic::Write& _stmt); + + Result end_write(); + + Result write( + const std::vector>>& _data); + + private: + /// Generates the underlying connection. + static ConnPtr make_conn(const std::string& _fname); + + /// Actually inserts data based on a prepared statement - + /// used by both .insert(...) and .write(...). + Result actual_insert( + const std::vector>>& _data, + duckdb3_stmt* _stmt) const noexcept; + + /// Generates a prepared statment, usually for inserts. + Result prepare_statement(const std::string& _sql) const noexcept; + + private: + /// A prepared statement - needed for the read and write operations. Note that + /// we have declared it before conn_, meaning it will be destroyed first. + StmtPtr stmt_; + + /// The underlying duckdb3 connection. + ConnPtr conn_; +}; + +static_assert(is_connection, + "Must fulfill the is_connection concept."); +static_assert(is_connection>, + "Must fulfill the is_connection concept."); + +} // namespace sqlgen::duckdb + +#endif diff --git a/include/sqlgen/duckdb/DuckDBConnection.hpp b/include/sqlgen/duckdb/DuckDBConnection.hpp new file mode 100644 index 00000000..e4ba79ad --- /dev/null +++ b/include/sqlgen/duckdb/DuckDBConnection.hpp @@ -0,0 +1,45 @@ +#ifndef SQLGEN_DUCKDB_CONNECTION_HPP_ +#define SQLGEN_DUCKDB_CONNECTION_HPP_ + +#include + +#include +#include + +#include "../Result.hpp" + +namespace sqlgen::duckdb { + +class DuckDBConnection { + public: + Result make(const std::optional& _fname); + + DuckDBConnection(duckdb_connection _conn, duckdb_database _db) + : conn_(_conn), db_(_db) {} + + ~DuckDBConnection() { + duckdb_disconnect(&conn_); + duckdb_close(&db_); + } + + DuckDBConnection(const DuckDBConnection& _other) = delete; + + DuckDBConnection(DuckDBConnection&& _other) + : conn_(_other.conn_), db_(_other.db_) { + _other.conn_ = NULL; + _other.db_ = NULL; + } + + DuckDBConnection& operator=(const DuckDBConnection& _other) = delete; + + DuckDBConnection& operator=(DuckDBConnection&& _other); + + private: + duckdb_connection conn_; + + duckdb_database db_; +}; + +} // namespace sqlgen::duckdb + +#endif diff --git a/src/sqlgen/duckdb/Connection.cpp b/src/sqlgen/duckdb/Connection.cpp new file mode 100644 index 00000000..b9a9f527 --- /dev/null +++ b/src/sqlgen/duckdb/Connection.cpp @@ -0,0 +1,155 @@ +#include "sqlgen/duckdb/Connection.hpp" + +#include +#include +#include +#include + +#include "sqlgen/duckdb/Iterator.hpp" +#include "sqlgen/internal/collect/vector.hpp" +#include "sqlgen/internal/strings/strings.hpp" + +namespace sqlgen::duckdb { + +Result Connection::begin_transaction() noexcept { + return execute("BEGIN TRANSACTION;"); +} + +Result Connection::commit() noexcept { return execute("COMMIT;"); } + +Result Connection::end_write() { + if (PQputCopyEnd(conn_.get(), NULL) == -1) { + return error(PQerrorMessage(conn_.get())); + } + const auto res = PQgetResult(conn_.get()); + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + return error(PQerrorMessage(conn_.get())); + } + return Nothing{}; +} + +Result Connection::insert( + const dynamic::Insert& _stmt, + const std::vector>>& + _data) noexcept { + if (_data.size() == 0) { + return Nothing{}; + } + + const auto sql = to_sql_impl(_stmt); + + const auto res = execute("PREPARE \"sqlgen_insert_into_table\" AS " + sql); + + if (!res) { + return res; + } + + std::vector current_row(_data[0].size()); + + const int n_params = static_cast(current_row.size()); + + for (size_t i = 0; i < _data.size(); ++i) { + const auto& d = _data[i]; + + if (d.size() != current_row.size()) { + execute("DEALLOCATE sqlgen_insert_into_table;"); + return error("Error in entry " + std::to_string(i) + ": Expected " + + std::to_string(current_row.size()) + " entries, got " + + std::to_string(d.size())); + } + + for (size_t j = 0; j < d.size(); ++j) { + current_row[j] = d[j] ? d[j]->c_str() : nullptr; + } + + const auto res = PQexecPrepared(conn_.get(), // conn + "sqlgen_insert_into_table", // stmtName + n_params, // nParams + current_row.data(), // paramValues + nullptr, // paramLengths + nullptr, // paramFormats + 0 // resultFormat + ); + + const auto status = PQresultStatus(res); + + if (status != PGRES_COMMAND_OK) { + const auto err = error(std::string("Executing INSERT failed: ") + + PQresultErrorMessage(res)); + execute("DEALLOCATE sqlgen_insert_into_table;"); + return err; + } + } + + return execute("DEALLOCATE sqlgen_insert_into_table;"); +} + +rfl::Result> Connection::make( + const Credentials& _credentials) noexcept { + try { + return Ref::make(_credentials); + } catch (std::exception& e) { + return error(e.what()); + } +} + +typename Connection::ConnPtr Connection::make_conn( + const std::string& _conn_str) { + const auto raw_ptr = PQconnectdb(_conn_str.c_str()); + + if (PQstatus(raw_ptr) != CONNECTION_OK) { + const auto msg = + std::string("Connection to duckdb failed: ") + PQerrorMessage(raw_ptr); + PQfinish(raw_ptr); + throw std::runtime_error(msg.c_str()); + } + + return ConnPtr::make(std::shared_ptr(raw_ptr, &PQfinish)).value(); +} + +Result> Connection::read(const dynamic::SelectFrom& _query) { + const auto sql = duckdb::to_sql_impl(_query); + try { + return Ref(Ref::make(sql, conn_)); + } catch (std::exception& e) { + return error(e.what()); + } +} + +Result Connection::rollback() noexcept { return execute("ROLLBACK;"); } + +std::string Connection::to_buffer( + const std::vector>& _line) const noexcept { + using namespace std::ranges::views; + + const auto edit_field = + [](const std::optional& _field) -> std::string { + if (!_field) { + return "\e"; + } + if (_field->find("\t") != std::string::npos) { + return "\a" + *_field + "\a"; + } + return *_field; + }; + + return internal::strings::join( + "\t", internal::collect::vector(_line | transform(edit_field))) + + "\n"; +} + +Result Connection::write( + const std::vector>>& _data) { + for (const auto& line : _data) { + const auto buffer = to_buffer(line); + const auto success = PQputCopyData(conn_.get(), buffer.c_str(), + static_cast(buffer.size())); + if (success != 1) { + PQputCopyEnd(conn_.get(), NULL); + return error("Error occurred while writing data to duckdb."); + } + } + return Nothing{}; +} + +} // namespace sqlgen::duckdb diff --git a/src/sqlgen/duckdb/DuckDBConnection.cpp b/src/sqlgen/duckdb/DuckDBConnection.cpp new file mode 100644 index 00000000..0918fd75 --- /dev/null +++ b/src/sqlgen/duckdb/DuckDBConnection.cpp @@ -0,0 +1,43 @@ +#include "sqlgen/duckdb/DuckDBConnection.hpp" + +namespace sqlgen::duckdb { + +Result DuckDBConnection::make( + const std::optional& _fname) { + duckdb_database db = NULL; + + const auto res_db = + _fname ? duckdb_open(_fname->c_str(), &db) : duckdb_open(NULL, &db); + + if (res_db == DuckDBError) { + duckdb_close(&db); + return error("Could not open database."); + } + + duckdb_connection conn = NULL; + + const auto res_conn = duckdb_connect(db, &conn); + + if (res_conn == DuckDBError) { + duckdb_disconnect(&conn); + duckdb_close(&db); + return error("Could not connect to database."); + } + + return DuckDBConnection(conn, db); +} + +DuckDBConnection& DuckDBConnection::operator=(DuckDBConnection&& _other) { + if (this == &_other) { + return *this; + } + duckdb_disconnect(&conn_); + duckdb_close(&db_); + conn_ = _other.conn_; + db_ = _other.db_; + _other.conn_ = NULL; + _other.db_ = NULL; + return *this; +} + +} // namespace sqlgen::duckdb diff --git a/src/sqlgen/duckdb/Iterator.cpp b/src/sqlgen/duckdb/Iterator.cpp new file mode 100644 index 00000000..0ecefbb9 --- /dev/null +++ b/src/sqlgen/duckdb/Iterator.cpp @@ -0,0 +1,92 @@ +#include "sqlgen/postgres/Iterator.hpp" + +#include +#include +#include + +#include "sqlgen/internal/collect/vector.hpp" +#include "sqlgen/internal/strings/strings.hpp" +#include "sqlgen/postgres/exec.hpp" + +namespace sqlgen::postgres { + +Iterator::Iterator(const std::string& _sql, const ConnPtr& _conn) + : cursor_name_(make_cursor_name()), conn_(_conn), end_(false) { + exec(conn_, "BEGIN").value(); + exec(conn_, "DECLARE " + cursor_name_ + " CURSOR FOR " + _sql).value(); +} + +Iterator::Iterator(Iterator&& _other) noexcept + : cursor_name_(std::move(_other.cursor_name_)), + conn_(std::move(_other.conn_)), + end_(_other.end_) { + _other.end_ = true; +} + +Iterator::~Iterator() { shutdown(); } + +bool Iterator::end() const { return end_; } + +Result>>> Iterator::next( + const size_t _batch_size) { + if (end()) { + return error("End is reached."); + } + + const auto to_vector = [](const Ref& _res) + -> std::vector>> { + const int num_rows = PQntuples(_res.get()); + const int num_cols = PQnfields(_res.get()); + + std::vector>> vec(num_rows); + + for (int i = 0; i < num_rows; ++i) { + std::vector> row(num_cols); + + for (int j = 0; j < num_cols; ++j) { + const bool is_null = PQgetisnull(_res.get(), i, j); + if (is_null) { + row[j] = std::nullopt; + } else { + row[j] = std::string(PQgetvalue(_res.get(), i, j)); + } + } + + vec[i] = std::move(row); + } + + return vec; + }; + + return exec(conn_, "FETCH FORWARD " + std::to_string(_batch_size) + " FROM " + + cursor_name_ + ";") + .transform(to_vector) + .transform([this](auto&& _vec) { + if (_vec.size() == 0) { + shutdown(); + } + return std::move(_vec); + }); +} + +Iterator& Iterator::operator=(Iterator&& _other) noexcept { + if (this == &_other) { + return *this; + } + shutdown(); + cursor_name_ = std::move(_other.cursor_name_); + conn_ = std::move(_other.conn_); + end_ = _other.end_; + _other.end_ = true; + return *this; +} + +void Iterator::shutdown() { + if (!end_) { + exec(conn_, "CLOSE " + cursor_name_); + exec(conn_, "END"); + end_ = true; + } +} + +} // namespace sqlgen::postgres diff --git a/src/sqlgen/duckdb/exec.cpp b/src/sqlgen/duckdb/exec.cpp new file mode 100644 index 00000000..19220f6f --- /dev/null +++ b/src/sqlgen/duckdb/exec.cpp @@ -0,0 +1,27 @@ +#include "sqlgen/postgres/exec.hpp" + +#include +#include +#include +#include + +namespace sqlgen::postgres { + +Result> exec(const Ref& _conn, + const std::string& _sql) noexcept { + const auto res = PQexec(_conn.get(), _sql.c_str()); + + const auto status = PQresultStatus(res); + + if (status != PGRES_COMMAND_OK && status != PGRES_TUPLES_OK && + status != PGRES_COPY_IN) { + const auto err = + error("Executing '" + _sql + "' failed: " + PQresultErrorMessage(res)); + PQclear(res); + return err; + } + + return Ref::make(std::shared_ptr(res, PQclear)); +} + +} // namespace sqlgen::postgres diff --git a/src/sqlgen/duckdb/to_sql.cpp b/src/sqlgen/duckdb/to_sql.cpp new file mode 100644 index 00000000..95cab03f --- /dev/null +++ b/src/sqlgen/duckdb/to_sql.cpp @@ -0,0 +1,827 @@ +#include "sqlgen/postgres/to_sql.hpp" + +#include +#include +#include +#include +#include + +#include "sqlgen/dynamic/Join.hpp" +#include "sqlgen/dynamic/Operation.hpp" +#include "sqlgen/internal/collect/vector.hpp" +#include "sqlgen/internal/strings/strings.hpp" + +namespace sqlgen::postgres { + +std::string aggregation_to_sql( + const dynamic::Aggregation& _aggregation) noexcept; + +std::string column_or_value_to_sql(const dynamic::ColumnOrValue& _col) noexcept; + +std::string condition_to_sql(const dynamic::Condition& _cond) noexcept; + +template +std::string condition_to_sql_impl(const ConditionType& _condition) noexcept; + +std::string column_to_sql_definition(const dynamic::Column& _col) noexcept; + +std::string create_index_to_sql(const dynamic::CreateIndex& _stmt) noexcept; + +std::string create_table_to_sql(const dynamic::CreateTable& _stmt) noexcept; + +std::string create_as_to_sql(const dynamic::CreateAs& _stmt) noexcept; + +std::string delete_from_to_sql(const dynamic::DeleteFrom& _stmt) noexcept; + +std::string drop_to_sql(const dynamic::Drop& _stmt) noexcept; + +std::string escape_single_quote(const std::string& _str) noexcept; + +std::string field_to_str(const dynamic::SelectFrom::Field& _field) noexcept; + +std::vector get_primary_keys( + const dynamic::CreateTable& _stmt) noexcept; + +std::string insert_to_sql(const dynamic::Insert& _stmt) noexcept; + +std::string join_to_sql(const dynamic::Join& _stmt) noexcept; + +std::string operation_to_sql(const dynamic::Operation& _stmt) noexcept; + +std::string properties_to_sql( + const dynamic::types::Properties& _properties) noexcept; + +std::string select_from_to_sql(const dynamic::SelectFrom& _stmt) noexcept; + +std::string table_or_query_to_sql( + const dynamic::SelectFrom::TableOrQueryType& _table_or_query) noexcept; + +std::string type_to_sql(const dynamic::Type& _type) noexcept; + +std::string update_to_sql(const dynamic::Update& _stmt) noexcept; + +std::string write_to_sql(const dynamic::Write& _stmt) noexcept; + +// ---------------------------------------------------------------------------- + +inline std::string get_name(const dynamic::Column& _col) { return _col.name; } + +inline std::string wrap_in_quotes(const std::string& _name) noexcept { + return "\"" + _name + "\""; +} + +// ---------------------------------------------------------------------------- + +std::string aggregation_to_sql( + const dynamic::Aggregation& _aggregation) noexcept { + return _aggregation.val.visit([](const auto& _agg) -> std::string { + using Type = std::remove_cvref_t; + std::stringstream stream; + if constexpr (std::is_same_v) { + stream << "AVG(" << operation_to_sql(*_agg.val) << ")"; + + } else if constexpr (std::is_same_v) { + const auto val = + std::string(_agg.val && _agg.distinct ? "DISTINCT " : "") + + (_agg.val ? column_or_value_to_sql(*_agg.val) : std::string("*")); + stream << "COUNT(" << val << ")"; + + } else if constexpr (std::is_same_v) { + stream << "MAX(" << operation_to_sql(*_agg.val) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "MIN(" << operation_to_sql(*_agg.val) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "SUM(" << operation_to_sql(*_agg.val) << ")"; + + } else { + static_assert(rfl::always_false_v, "Not all cases were covered."); + } + return stream.str(); + }); +} + +std::string column_or_value_to_sql( + const dynamic::ColumnOrValue& _col) noexcept { + const auto handle_value = [](const auto& _v) -> std::string { + using Type = std::remove_cvref_t; + if constexpr (std::is_same_v) { + return "'" + escape_single_quote(_v.val) + "'"; + + } else if constexpr (std::is_same_v) { + return "INTERVAL '" + std::to_string(_v.val) + " " + + rfl::enum_to_string(_v.unit) + "'"; + + } else if constexpr (std::is_same_v) { + return "to_timestamp(" + std::to_string(_v.seconds_since_unix) + ")"; + + } else { + return std::to_string(_v.val); + } + }; + + return _col.visit([&](const auto& _c) -> std::string { + using Type = std::remove_cvref_t; + if constexpr (std::is_same_v) { + if (_c.alias) { + return *_c.alias + "." + wrap_in_quotes(_c.name); + } else { + return wrap_in_quotes(_c.name); + } + } else { + return _c.val.visit(handle_value); + } + }); +} + +std::string condition_to_sql(const dynamic::Condition& _cond) noexcept { + return _cond.val.visit( + [&](const auto& _c) { return condition_to_sql_impl(_c); }); +} + +template +std::string condition_to_sql_impl(const ConditionType& _condition) noexcept { + using C = std::remove_cvref_t; + + std::stringstream stream; + + if constexpr (std::is_same_v) { + stream << "(" << condition_to_sql(*_condition.cond1) << ") AND (" + << condition_to_sql(*_condition.cond2) << ")"; + + } else if constexpr (std::is_same_v) { + stream << operation_to_sql(_condition.op1) << " = " + << operation_to_sql(_condition.op2); + + } else if constexpr (std::is_same_v) { + stream << operation_to_sql(_condition.op1) + << " >= " << operation_to_sql(_condition.op2); + + } else if constexpr (std::is_same_v) { + stream << operation_to_sql(_condition.op1) << " > " + << operation_to_sql(_condition.op2); + + } else if constexpr (std::is_same_v) { + stream << operation_to_sql(_condition.op) << " IS NULL"; + + } else if constexpr (std::is_same_v) { + stream << operation_to_sql(_condition.op) << " IS NOT NULL"; + + } else if constexpr (std::is_same_v) { + stream << operation_to_sql(_condition.op1) + << " <= " << operation_to_sql(_condition.op2); + + } else if constexpr (std::is_same_v) { + stream << operation_to_sql(_condition.op1) << " < " + << operation_to_sql(_condition.op2); + + } else if constexpr (std::is_same_v) { + stream << operation_to_sql(_condition.op) << " LIKE " + << column_or_value_to_sql(_condition.pattern); + + } else if constexpr (std::is_same_v) { + stream << "NOT (" << condition_to_sql(*_condition.cond) << ")"; + + } else if constexpr (std::is_same_v) { + stream << operation_to_sql(_condition.op1) + << " != " << operation_to_sql(_condition.op2); + + } else if constexpr (std::is_same_v) { + stream << operation_to_sql(_condition.op) << " NOT LIKE " + << column_or_value_to_sql(_condition.pattern); + + } else if constexpr (std::is_same_v) { + stream << "(" << condition_to_sql(*_condition.cond1) << ") OR (" + << condition_to_sql(*_condition.cond2) << ")"; + + } else { + static_assert(rfl::always_false_v, "Not all cases were covered."); + } + return stream.str(); +} + +std::string column_to_sql_definition(const dynamic::Column& _col) noexcept { + return wrap_in_quotes(_col.name) + " " + type_to_sql(_col.type) + + properties_to_sql( + _col.type.visit([](const auto& _t) { return _t.properties; })); +} + +std::string create_index_to_sql(const dynamic::CreateIndex& _stmt) noexcept { + using namespace std::ranges::views; + + std::stringstream stream; + + if (_stmt.unique) { + stream << "CREATE UNIQUE INDEX "; + } else { + stream << "CREATE INDEX "; + } + + if (_stmt.if_not_exists) { + stream << "IF NOT EXISTS "; + } + + stream << "\"" << _stmt.name << "\" "; + + stream << "ON "; + + if (_stmt.table.schema) { + stream << "\"" << *_stmt.table.schema << "\"."; + } + stream << "\"" << _stmt.table.name << "\""; + + stream << "("; + stream << internal::strings::join( + ", ", + internal::collect::vector(_stmt.columns | transform(wrap_in_quotes))); + stream << ")"; + + if (_stmt.where) { + stream << " WHERE " << condition_to_sql(*_stmt.where); + } + + stream << ";"; + + return stream.str(); +} + +std::string create_table_to_sql(const dynamic::CreateTable& _stmt) noexcept { + using namespace std::ranges::views; + + const auto col_to_sql = [&](const auto& _col) { + return column_to_sql_definition(_col); + }; + + std::stringstream stream; + stream << "CREATE TABLE "; + + if (_stmt.if_not_exists) { + stream << "IF NOT EXISTS "; + } + + if (_stmt.table.schema) { + stream << wrap_in_quotes(*_stmt.table.schema) << "."; + } + stream << wrap_in_quotes(_stmt.table.name) << " "; + + stream << "("; + stream << internal::strings::join( + ", ", internal::collect::vector(_stmt.columns | transform(col_to_sql))); + + const auto primary_keys = get_primary_keys(_stmt); + + if (primary_keys.size() != 0) { + stream << ", PRIMARY KEY (" << internal::strings::join(", ", primary_keys) + << ")"; + } + + stream << ");"; + + return stream.str(); +} + +std::string create_as_to_sql(const dynamic::CreateAs& _stmt) noexcept { + std::stringstream stream; + + stream << "CREATE "; + + if (_stmt.or_replace) { + stream << "OR REPLACE "; + } + + stream << internal::strings::replace_all( + internal::strings::to_upper(rfl::enum_to_string(_stmt.what)), + "_", " ") + << " "; + + if (_stmt.if_not_exists) { + stream << "IF NOT EXISTS "; + } + + if (_stmt.table_or_view.schema) { + stream << wrap_in_quotes(*_stmt.table_or_view.schema) << "."; + } + stream << wrap_in_quotes(_stmt.table_or_view.name) << " AS "; + + stream << select_from_to_sql(_stmt.query); + + return stream.str(); +} + +std::string delete_from_to_sql(const dynamic::DeleteFrom& _stmt) noexcept { + std::stringstream stream; + + stream << "DELETE FROM "; + + if (_stmt.table.schema) { + stream << wrap_in_quotes(*_stmt.table.schema) << "."; + } + stream << wrap_in_quotes(_stmt.table.name); + + if (_stmt.where) { + stream << " WHERE " << condition_to_sql(*_stmt.where); + } + + stream << ";"; + + return stream.str(); +} + +std::string drop_to_sql(const dynamic::Drop& _stmt) noexcept { + std::stringstream stream; + + stream << "DROP " + << internal::strings::replace_all( + internal::strings::to_upper(rfl::enum_to_string(_stmt.what)), + "_", " ") + << " "; + + if (_stmt.if_exists) { + stream << "IF EXISTS "; + } + + if (_stmt.table.schema) { + stream << wrap_in_quotes(*_stmt.table.schema) << "."; + } + stream << wrap_in_quotes(_stmt.table.name); + + if (_stmt.cascade) { + stream << " CASCADE"; + } + + stream << ";"; + + return stream.str(); +} + +std::string escape_single_quote(const std::string& _str) noexcept { + return internal::strings::replace_all(_str, "'", "''"); +} + +std::string field_to_str(const dynamic::SelectFrom::Field& _field) noexcept { + std::stringstream stream; + + stream << operation_to_sql(_field.val); + + if (_field.as) { + stream << " AS " << wrap_in_quotes(*_field.as); + } + + return stream.str(); +} + +std::vector get_primary_keys( + const dynamic::CreateTable& _stmt) noexcept { + using namespace std::ranges::views; + + const auto is_primary_key = [](const auto& _col) -> bool { + return _col.type.visit( + [](const auto& _t) -> bool { return _t.properties.primary; }); + }; + + return internal::collect::vector(_stmt.columns | filter(is_primary_key) | + transform(get_name) | + transform(wrap_in_quotes)); +} + +std::string insert_to_sql(const dynamic::Insert& _stmt) noexcept { + using namespace std::ranges::views; + + const auto to_placeholder = [](const size_t _i) -> std::string { + return "$" + std::to_string(_i + 1); + }; + + const auto as_excluded = [](const std::string& _str) -> std::string { + return _str + "=excluded." + _str; + }; + + std::stringstream stream; + stream << "INSERT INTO "; + if (_stmt.table.schema) { + stream << wrap_in_quotes(*_stmt.table.schema) << "."; + } + stream << wrap_in_quotes(_stmt.table.name); + + stream << " ("; + stream << internal::strings::join( + ", ", + internal::collect::vector(_stmt.columns | transform(wrap_in_quotes))); + stream << ")"; + + stream << " VALUES ("; + stream << internal::strings::join( + ", ", internal::collect::vector( + iota(static_cast(0), _stmt.columns.size()) | + transform(to_placeholder))); + stream << ")"; + + if (_stmt.or_replace) { + stream << " ON CONFLICT ("; + stream << internal::strings::join( + ", ", internal::collect::vector(_stmt.constraints)); + stream << ")"; + + stream << " DO UPDATE SET "; + stream << internal::strings::join( + ", ", + internal::collect::vector(_stmt.columns | transform(as_excluded))); + } + + stream << ";"; + return stream.str(); +} + +std::string join_to_sql(const dynamic::Join& _stmt) noexcept { + std::stringstream stream; + + stream << internal::strings::to_upper(internal::strings::replace_all( + rfl::enum_to_string(_stmt.how), "_", " ")) + << " " << table_or_query_to_sql(_stmt.table_or_query) << " " + << _stmt.alias << " "; + + if (_stmt.on) { + stream << "ON " << condition_to_sql(*_stmt.on); + } else { + stream << "ON 1 = 1"; + } + + return stream.str(); +} + +std::string operation_to_sql(const dynamic::Operation& _stmt) noexcept { + using namespace std::ranges::views; + return _stmt.val.visit([](const auto& _s) -> std::string { + using Type = std::remove_cvref_t; + + std::stringstream stream; + + if constexpr (std::is_same_v) { + stream << "abs(" << operation_to_sql(*_s.op1) << ")"; + + } else if constexpr (std::is_same_v) { + stream << aggregation_to_sql(_s); + + } else if constexpr (std::is_same_v) { + stream << "cast(" << operation_to_sql(*_s.op1) << " as " + << type_to_sql(_s.target_type) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "coalesce(" + << internal::strings::join( + ", ", internal::collect::vector( + _s.ops | transform([](const auto& _op) { + return operation_to_sql(*_op); + }))) + << ")"; + + } else if constexpr (std::is_same_v) { + stream << "ceil(" << operation_to_sql(*_s.op1) << ")"; + + } else if constexpr (std::is_same_v) { + stream << column_or_value_to_sql(_s); + + } else if constexpr (std::is_same_v) { + stream << "(" + << internal::strings::join( + " || ", internal::collect::vector( + _s.ops | transform([](const auto& _op) { + return operation_to_sql(*_op); + }))) + << ")"; + + } else if constexpr (std::is_same_v) { + stream << "cos(" << operation_to_sql(*_s.op1) << ")"; + + } else if constexpr (std::is_same_v) { + stream << operation_to_sql(*_s.date) << " + " + << internal::strings::join( + " + ", + internal::collect::vector( + _s.durations | transform([](const auto& _d) { + return column_or_value_to_sql(dynamic::Value{_d}); + }))); + + } else if constexpr (std::is_same_v) { + stream << "extract(DAY from " << operation_to_sql(*_s.op1) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "cast(" << operation_to_sql(*_s.op2) << " as DATE) - cast(" + << operation_to_sql(*_s.op1) << " as DATE)"; + + } else if constexpr (std::is_same_v) { + stream << "(" << operation_to_sql(*_s.op1) << ") / (" + << operation_to_sql(*_s.op2) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "exp(" << operation_to_sql(*_s.op1) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "floor(" << operation_to_sql(*_s.op1) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "extract(HOUR from " << operation_to_sql(*_s.op1) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "length(" << operation_to_sql(*_s.op1) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "ln(" << operation_to_sql(*_s.op1) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "log(2.0, " << operation_to_sql(*_s.op1) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "lower(" << operation_to_sql(*_s.op1) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "ltrim(" << operation_to_sql(*_s.op1) << ", " + << operation_to_sql(*_s.op2) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "(" << operation_to_sql(*_s.op1) << ") - (" + << operation_to_sql(*_s.op2) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "extract(MINUTE from " << operation_to_sql(*_s.op1) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "mod(" << operation_to_sql(*_s.op1) << ", " + << operation_to_sql(*_s.op2) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "extract(MONTH from " << operation_to_sql(*_s.op1) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "(" << operation_to_sql(*_s.op1) << ") * (" + << operation_to_sql(*_s.op2) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "(" << operation_to_sql(*_s.op1) << ") + (" + << operation_to_sql(*_s.op2) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "replace(" << operation_to_sql(*_s.op1) << ", " + << operation_to_sql(*_s.op2) << ", " << operation_to_sql(*_s.op3) + << ")"; + + } else if constexpr (std::is_same_v) { + stream << "round(" << operation_to_sql(*_s.op1) << ", " + << operation_to_sql(*_s.op2) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "rtrim(" << operation_to_sql(*_s.op1) << ", " + << operation_to_sql(*_s.op2) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "extract(SECOND from " << operation_to_sql(*_s.op1) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "sin(" << operation_to_sql(*_s.op1) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "sqrt(" << operation_to_sql(*_s.op1) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "tan(" << operation_to_sql(*_s.op1) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "trim(" << operation_to_sql(*_s.op1) << ", " + << operation_to_sql(*_s.op2) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "extract(EPOCH FROM " << operation_to_sql(*_s.op1) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "upper(" << operation_to_sql(*_s.op1) << ")"; + + } else if constexpr (std::is_same_v) { + stream << column_or_value_to_sql(_s); + + } else if constexpr (std::is_same_v) { + stream << "extract(DOW from " << operation_to_sql(*_s.op1) << ")"; + + } else if constexpr (std::is_same_v) { + stream << "extract(YEAR from " << operation_to_sql(*_s.op1) << ")"; + + } else { + static_assert(rfl::always_false_v, "Unsupported type."); + } + return stream.str(); + }); +} + +std::string properties_to_sql(const dynamic::types::Properties& _p) noexcept { + return [&]() -> std::string { + return std::string(_p.auto_incr ? " GENERATED ALWAYS AS IDENTITY" : "") + + std::string(_p.nullable ? "" : " NOT NULL") + + std::string(_p.unique ? " UNIQUE" : ""); + }() + [&]() -> std::string { + if (!_p.foreign_key_reference) { + return ""; + } + const auto& ref = *_p.foreign_key_reference; + return " REFERENCES " + wrap_in_quotes(ref.table) + "(" + + wrap_in_quotes(ref.column) + ")"; + }(); +} + +std::string select_from_to_sql(const dynamic::SelectFrom& _stmt) noexcept { + using namespace std::ranges::views; + + const auto order_by_to_str = [](const auto& _w) -> std::string { + return column_or_value_to_sql(_w.column) + (_w.desc ? " DESC" : ""); + }; + + std::stringstream stream; + + stream << "SELECT "; + stream << internal::strings::join( + ", ", internal::collect::vector(_stmt.fields | transform(field_to_str))); + + stream << " FROM " << table_or_query_to_sql(_stmt.table_or_query); + + if (_stmt.alias) { + stream << " " << *_stmt.alias; + } + + if (_stmt.joins) { + stream << " " + << internal::strings::join( + " ", internal::collect::vector(*_stmt.joins | + transform(join_to_sql))); + } + + if (_stmt.where) { + stream << " WHERE " << condition_to_sql(*_stmt.where); + } + + if (_stmt.group_by) { + stream << " GROUP BY " + << internal::strings::join( + ", ", + internal::collect::vector(_stmt.group_by->columns | + transform(column_or_value_to_sql))); + } + + if (_stmt.order_by) { + stream << " ORDER BY " + << internal::strings::join( + ", ", internal::collect::vector(_stmt.order_by->columns | + transform(order_by_to_str))); + } + + if (_stmt.limit) { + stream << " LIMIT " << _stmt.limit->val; + } + + return stream.str(); +} + +std::string table_or_query_to_sql( + const dynamic::SelectFrom::TableOrQueryType& _table_or_query) noexcept { + return _table_or_query.visit([](const auto& _t) -> std::string { + using Type = std::remove_cvref_t; + if constexpr (std::is_same_v) { + return wrap_in_quotes(_t.name); + } else { + return "(" + select_from_to_sql(*_t) + ")"; + } + }); +} + +std::string to_sql_impl(const dynamic::Statement& _stmt) noexcept { + return _stmt.visit([&](const auto& _s) -> std::string { + using S = std::remove_cvref_t; + + if constexpr (std::is_same_v) { + return create_index_to_sql(_s); + + } else if constexpr (std::is_same_v) { + return create_table_to_sql(_s); + + } else if constexpr (std::is_same_v) { + return create_as_to_sql(_s); + + } else if constexpr (std::is_same_v) { + return delete_from_to_sql(_s); + + } else if constexpr (std::is_same_v) { + return drop_to_sql(_s); + + } else if constexpr (std::is_same_v) { + return insert_to_sql(_s); + + } else if constexpr (std::is_same_v) { + return select_from_to_sql(_s); + + } else if constexpr (std::is_same_v) { + return update_to_sql(_s); + + } else if constexpr (std::is_same_v) { + return write_to_sql(_s); + + } else { + static_assert(rfl::always_false_v, "Unsupported type."); + } + }); +} + +std::string type_to_sql(const dynamic::Type& _type) noexcept { + return _type.visit([](const auto _t) -> std::string { + using T = std::remove_cvref_t; + if constexpr (std::is_same_v) { + return "BOOLEAN"; + + } else if constexpr (std::is_same_v) { + return _t.type_name; + + } else if constexpr (std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) { + return "SMALLINT"; + + } else if constexpr (std::is_same_v || + std::is_same_v) { + return "INTEGER"; + + } else if constexpr (std::is_same_v || + std::is_same_v) { + return "BIGINT"; + + } else if constexpr (std::is_same_v || + std::is_same_v) { + return "NUMERIC"; + + } else if constexpr (std::is_same_v) { + return "TEXT"; + + } else if constexpr (std::is_same_v) { + return "VARCHAR(" + std::to_string(_t.length) + ")"; + + } else if constexpr (std::is_same_v) { + return "JSONB"; + + } else if constexpr (std::is_same_v) { + return "DATE"; + + } else if constexpr (std::is_same_v) { + return "TIMESTAMP"; + + } else if constexpr (std::is_same_v) { + return "TIMESTAMP WITH TIME ZONE"; + + } else if constexpr (std::is_same_v) { + return "TEXT"; + } else { + static_assert(rfl::always_false_v, "Not all cases were covered."); + } + }); +} + +std::string update_to_sql(const dynamic::Update& _stmt) noexcept { + using namespace std::ranges::views; + + const auto to_str = [](const auto& _set) -> std::string { + return wrap_in_quotes(_set.col.name) + " = " + + column_or_value_to_sql(_set.to); + }; + + std::stringstream stream; + + stream << "UPDATE "; + + if (_stmt.table.schema) { + stream << wrap_in_quotes(*_stmt.table.schema) << "."; + } + stream << wrap_in_quotes(_stmt.table.name); + + stream << " SET "; + + stream << internal::strings::join( + ", ", internal::collect::vector(_stmt.sets | transform(to_str))); + + if (_stmt.where) { + stream << " WHERE " << condition_to_sql(*_stmt.where); + } + + stream << ";"; + + return stream.str(); +} + +std::string write_to_sql(const dynamic::Write& _stmt) noexcept { + using namespace std::ranges::views; + const auto schema = wrap_in_quotes(_stmt.table.schema.value_or("public")); + const auto table = wrap_in_quotes(_stmt.table.name); + const auto colnames = internal::strings::join( + ", ", + internal::collect::vector(_stmt.columns | transform(wrap_in_quotes))); + return "COPY " + schema + "." + table + "(" + colnames + + ") FROM STDIN WITH DELIMITER '\t' NULL '\e' CSV QUOTE '\a';"; +} + +} // namespace sqlgen::postgres diff --git a/src/sqlgen_duckdb.cpp b/src/sqlgen_duckdb.cpp new file mode 100644 index 00000000..6a84d56c --- /dev/null +++ b/src/sqlgen_duckdb.cpp @@ -0,0 +1,5 @@ +#include "sqlgen/duckdb/DuckDBConnection.cpp" +// #include "sqlgen/duckdb/Connection.cpp" +// #include "sqlgen/duckdb/Iterator.cpp" +// #include "sqlgen/duckdb/exec.cpp" +// #include "sqlgen/duckdb/to_sql.cpp" From ed38ddb21e985c2bfabf094fc98c63053675d8b7 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Wed, 17 Sep 2025 22:33:29 +0200 Subject: [PATCH 03/48] Started developing the Connection object --- include/sqlgen/duckdb/Connection.hpp | 31 ++--- include/sqlgen/duckdb/DuckDBConnection.hpp | 12 +- src/sqlgen/duckdb/Connection.cpp | 135 ++++----------------- src/sqlgen/duckdb/DuckDBConnection.cpp | 4 +- src/sqlgen_duckdb.cpp | 2 +- 5 files changed, 44 insertions(+), 140 deletions(-) diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index 724b0fb4..052fdb4f 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -15,19 +16,20 @@ #include "../Transaction.hpp" #include "../dynamic/Write.hpp" #include "../is_connection.hpp" -#include "to_sql.hpp" +#include "DuckDBConnection.hpp" +// #include "to_sql.hpp" namespace sqlgen::duckdb { class Connection { - using ConnPtr = Ref; - using StmtPtr = std::shared_ptr; + using ConnPtr = Ref; + // using StmtPtr = std::shared_ptr; public: - Connection(const std::string& _fname) - : stmt_(nullptr), conn_(make_conn(_fname)) {} + Connection(const ConnPtr& _conn) : /*stmt_(nullptr),*/ conn_(_conn) {} - static rfl::Result> make(const std::string& _fname) noexcept; + static rfl::Result> make( + const std::optional& _fname) noexcept; ~Connection() = default; @@ -47,7 +49,7 @@ class Connection { Result rollback() noexcept; std::string to_sql(const dynamic::Statement& _stmt) noexcept { - return duckdb::to_sql_impl(_stmt); + return "TODO"; // duckdb::to_sql_impl(_stmt); } Result start_write(const dynamic::Write& _stmt); @@ -57,23 +59,10 @@ class Connection { Result write( const std::vector>>& _data); - private: - /// Generates the underlying connection. - static ConnPtr make_conn(const std::string& _fname); - - /// Actually inserts data based on a prepared statement - - /// used by both .insert(...) and .write(...). - Result actual_insert( - const std::vector>>& _data, - duckdb3_stmt* _stmt) const noexcept; - - /// Generates a prepared statment, usually for inserts. - Result prepare_statement(const std::string& _sql) const noexcept; - private: /// A prepared statement - needed for the read and write operations. Note that /// we have declared it before conn_, meaning it will be destroyed first. - StmtPtr stmt_; + // StmtPtr stmt_; /// The underlying duckdb3 connection. ConnPtr conn_; diff --git a/include/sqlgen/duckdb/DuckDBConnection.hpp b/include/sqlgen/duckdb/DuckDBConnection.hpp index e4ba79ad..ab9ebe68 100644 --- a/include/sqlgen/duckdb/DuckDBConnection.hpp +++ b/include/sqlgen/duckdb/DuckDBConnection.hpp @@ -1,18 +1,20 @@ -#ifndef SQLGEN_DUCKDB_CONNECTION_HPP_ -#define SQLGEN_DUCKDB_CONNECTION_HPP_ +#ifndef SQLGEN_DUCKDB_DUCKDBCONNECTION_HPP_ +#define SQLGEN_DUCKDB_DUCKDBCONNECTION_HPP_ #include #include #include +#include "../Ref.hpp" #include "../Result.hpp" namespace sqlgen::duckdb { class DuckDBConnection { public: - Result make(const std::optional& _fname); + static Result> make( + const std::optional& _fname); DuckDBConnection(duckdb_connection _conn, duckdb_database _db) : conn_(_conn), db_(_db) {} @@ -34,6 +36,10 @@ class DuckDBConnection { DuckDBConnection& operator=(DuckDBConnection&& _other); + duckdb_connection conn() { return conn_; } + + duckdb_database db() { return db_; } + private: duckdb_connection conn_; diff --git a/src/sqlgen/duckdb/Connection.cpp b/src/sqlgen/duckdb/Connection.cpp index b9a9f527..eda9cee9 100644 --- a/src/sqlgen/duckdb/Connection.cpp +++ b/src/sqlgen/duckdb/Connection.cpp @@ -5,7 +5,7 @@ #include #include -#include "sqlgen/duckdb/Iterator.hpp" +// #include "sqlgen/duckdb/Iterator.hpp" #include "sqlgen/internal/collect/vector.hpp" #include "sqlgen/internal/strings/strings.hpp" @@ -18,13 +18,19 @@ Result Connection::begin_transaction() noexcept { Result Connection::commit() noexcept { return execute("COMMIT;"); } Result Connection::end_write() { - if (PQputCopyEnd(conn_.get(), NULL) == -1) { - return error(PQerrorMessage(conn_.get())); - } - const auto res = PQgetResult(conn_.get()); - if (PQresultStatus(res) != PGRES_COMMAND_OK) { - return error(PQerrorMessage(conn_.get())); + // TODO + return error("TODO"); +} + +Result Connection::execute(const std::string& _sql) noexcept { + duckdb_result res; + const auto state = duckdb_query(conn_->conn(), _sql.c_str(), &res); + if (state == DuckDBError) { + const auto err = error(duckdb_result_error(&res)); + duckdb_destroy_result(&res); + return err; } + duckdb_destroy_result(&res); return Nothing{}; } @@ -32,124 +38,27 @@ Result Connection::insert( const dynamic::Insert& _stmt, const std::vector>>& _data) noexcept { - if (_data.size() == 0) { - return Nothing{}; - } - - const auto sql = to_sql_impl(_stmt); - - const auto res = execute("PREPARE \"sqlgen_insert_into_table\" AS " + sql); - - if (!res) { - return res; - } - - std::vector current_row(_data[0].size()); - - const int n_params = static_cast(current_row.size()); - - for (size_t i = 0; i < _data.size(); ++i) { - const auto& d = _data[i]; - - if (d.size() != current_row.size()) { - execute("DEALLOCATE sqlgen_insert_into_table;"); - return error("Error in entry " + std::to_string(i) + ": Expected " + - std::to_string(current_row.size()) + " entries, got " + - std::to_string(d.size())); - } - - for (size_t j = 0; j < d.size(); ++j) { - current_row[j] = d[j] ? d[j]->c_str() : nullptr; - } - - const auto res = PQexecPrepared(conn_.get(), // conn - "sqlgen_insert_into_table", // stmtName - n_params, // nParams - current_row.data(), // paramValues - nullptr, // paramLengths - nullptr, // paramFormats - 0 // resultFormat - ); - - const auto status = PQresultStatus(res); - - if (status != PGRES_COMMAND_OK) { - const auto err = error(std::string("Executing INSERT failed: ") + - PQresultErrorMessage(res)); - execute("DEALLOCATE sqlgen_insert_into_table;"); - return err; - } - } - - return execute("DEALLOCATE sqlgen_insert_into_table;"); + // TODO + return error("TODO"); } rfl::Result> Connection::make( - const Credentials& _credentials) noexcept { - try { - return Ref::make(_credentials); - } catch (std::exception& e) { - return error(e.what()); - } -} - -typename Connection::ConnPtr Connection::make_conn( - const std::string& _conn_str) { - const auto raw_ptr = PQconnectdb(_conn_str.c_str()); - - if (PQstatus(raw_ptr) != CONNECTION_OK) { - const auto msg = - std::string("Connection to duckdb failed: ") + PQerrorMessage(raw_ptr); - PQfinish(raw_ptr); - throw std::runtime_error(msg.c_str()); - } - - return ConnPtr::make(std::shared_ptr(raw_ptr, &PQfinish)).value(); + const std::optional& _fname) noexcept { + return DuckDBConnection::make(_fname).transform( + [](auto&& _conn) { return Ref::make(std::move(_conn)); }); } Result> Connection::read(const dynamic::SelectFrom& _query) { - const auto sql = duckdb::to_sql_impl(_query); - try { - return Ref(Ref::make(sql, conn_)); - } catch (std::exception& e) { - return error(e.what()); - } + // TODO + return error("TODO"); } Result Connection::rollback() noexcept { return execute("ROLLBACK;"); } -std::string Connection::to_buffer( - const std::vector>& _line) const noexcept { - using namespace std::ranges::views; - - const auto edit_field = - [](const std::optional& _field) -> std::string { - if (!_field) { - return "\e"; - } - if (_field->find("\t") != std::string::npos) { - return "\a" + *_field + "\a"; - } - return *_field; - }; - - return internal::strings::join( - "\t", internal::collect::vector(_line | transform(edit_field))) + - "\n"; -} - Result Connection::write( const std::vector>>& _data) { - for (const auto& line : _data) { - const auto buffer = to_buffer(line); - const auto success = PQputCopyData(conn_.get(), buffer.c_str(), - static_cast(buffer.size())); - if (success != 1) { - PQputCopyEnd(conn_.get(), NULL); - return error("Error occurred while writing data to duckdb."); - } - } - return Nothing{}; + // TODO + return error("TODO"); } } // namespace sqlgen::duckdb diff --git a/src/sqlgen/duckdb/DuckDBConnection.cpp b/src/sqlgen/duckdb/DuckDBConnection.cpp index 0918fd75..55c0b462 100644 --- a/src/sqlgen/duckdb/DuckDBConnection.cpp +++ b/src/sqlgen/duckdb/DuckDBConnection.cpp @@ -2,7 +2,7 @@ namespace sqlgen::duckdb { -Result DuckDBConnection::make( +Result> DuckDBConnection::make( const std::optional& _fname) { duckdb_database db = NULL; @@ -24,7 +24,7 @@ Result DuckDBConnection::make( return error("Could not connect to database."); } - return DuckDBConnection(conn, db); + return Ref::make(conn, db); } DuckDBConnection& DuckDBConnection::operator=(DuckDBConnection&& _other) { diff --git a/src/sqlgen_duckdb.cpp b/src/sqlgen_duckdb.cpp index 6a84d56c..35afe55c 100644 --- a/src/sqlgen_duckdb.cpp +++ b/src/sqlgen_duckdb.cpp @@ -1,5 +1,5 @@ +#include "sqlgen/duckdb/Connection.cpp" #include "sqlgen/duckdb/DuckDBConnection.cpp" -// #include "sqlgen/duckdb/Connection.cpp" // #include "sqlgen/duckdb/Iterator.cpp" // #include "sqlgen/duckdb/exec.cpp" // #include "sqlgen/duckdb/to_sql.cpp" From 8987fcb6ba8d9afd833e72b400699a1815e779e1 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sun, 21 Sep 2025 09:01:56 +0200 Subject: [PATCH 04/48] Added the iterator --- include/sqlgen/duckdb/Connection.hpp | 1 - include/sqlgen/duckdb/Iterator.hpp | 60 ++++++++++++++++++ src/sqlgen/duckdb/Iterator.cpp | 93 +++++++--------------------- 3 files changed, 83 insertions(+), 71 deletions(-) create mode 100644 include/sqlgen/duckdb/Iterator.hpp diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index 052fdb4f..9e728089 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -23,7 +23,6 @@ namespace sqlgen::duckdb { class Connection { using ConnPtr = Ref; - // using StmtPtr = std::shared_ptr; public: Connection(const ConnPtr& _conn) : /*stmt_(nullptr),*/ conn_(_conn) {} diff --git a/include/sqlgen/duckdb/Iterator.hpp b/include/sqlgen/duckdb/Iterator.hpp new file mode 100644 index 00000000..05ff0d7b --- /dev/null +++ b/include/sqlgen/duckdb/Iterator.hpp @@ -0,0 +1,60 @@ +#ifndef SQLGEN_DUCKDB_ITERATOR_HPP_ +#define SQLGEN_DUCKDB_ITERATOR_HPP_ + +#include + +#include +#include +#include + +#include "../IteratorBase.hpp" +#include "../Ref.hpp" +#include "../Result.hpp" +#include "DuckDBConnection.hpp" + +namespace sqlgen::sqlite { + +class Iterator : public sqlgen::IteratorBase { + using ConnPtr = Ref; + using ResultPtr = Ref; + + public: + Iterator(const ResultPtr& _res, const ConnPtr& _conn); + + ~Iterator(); + + /// Whether the end of the available data has been reached. + bool end() const final; + + /// Returns the next batch of rows. + /// If _batch_size is greater than the number of rows left, returns all + /// of the rows left. + Result>>> next( + const size_t _batch_size) final; + + private: + static void destroy_result(duckdb_result* _r) { + duckdb_result(_r); + delete _r; + } + + private: + /// The underlying DuckDB result. + ResultPtr res_; + + /// The underlying connection. + ConnPtr conn_; + + /// The number of columns. + idx_t num_cols_; + + /// The number of rows. + idx_t num_rows_; + + /// The current rownumber. + idx_t rownum_; +}; + +} // namespace sqlgen::sqlite + +#endif diff --git a/src/sqlgen/duckdb/Iterator.cpp b/src/sqlgen/duckdb/Iterator.cpp index 0ecefbb9..aef6535b 100644 --- a/src/sqlgen/duckdb/Iterator.cpp +++ b/src/sqlgen/duckdb/Iterator.cpp @@ -1,92 +1,45 @@ -#include "sqlgen/postgres/Iterator.hpp" +#include "sqlgen/duckdb/Iterator.hpp" #include #include #include +#include "sqlgen/duckdb/exec.hpp" #include "sqlgen/internal/collect/vector.hpp" #include "sqlgen/internal/strings/strings.hpp" -#include "sqlgen/postgres/exec.hpp" -namespace sqlgen::postgres { +namespace sqlgen::duckdb { -Iterator::Iterator(const std::string& _sql, const ConnPtr& _conn) - : cursor_name_(make_cursor_name()), conn_(_conn), end_(false) { - exec(conn_, "BEGIN").value(); - exec(conn_, "DECLARE " + cursor_name_ + " CURSOR FOR " + _sql).value(); -} - -Iterator::Iterator(Iterator&& _other) noexcept - : cursor_name_(std::move(_other.cursor_name_)), - conn_(std::move(_other.conn_)), - end_(_other.end_) { - _other.end_ = true; -} +Iterator::Iterator(const ResultPtr& _res, const ConnPtr& _conn) + : res_(_res), + conn_(_conn), + num_cols_(duckdb_row_count(*_res)), + num_rows_(duckdb_column_count(*_res)), + rownum_(0) {} -Iterator::~Iterator() { shutdown(); } - -bool Iterator::end() const { return end_; } +bool Iterator::end() const { return rownum_ >= num_rows_; } Result>>> Iterator::next( const size_t _batch_size) { + auto vec = std::vector>>(); + if (end()) { - return error("End is reached."); + return vec; } - const auto to_vector = [](const Ref& _res) - -> std::vector>> { - const int num_rows = PQntuples(_res.get()); - const int num_cols = PQnfields(_res.get()); - - std::vector>> vec(num_rows); + const auto batch_size = + std::min(num_rows_ - rownum_, static_cast(_batch_size)); - for (int i = 0; i < num_rows; ++i) { - std::vector> row(num_cols); - - for (int j = 0; j < num_cols; ++j) { - const bool is_null = PQgetisnull(_res.get(), i, j); - if (is_null) { - row[j] = std::nullopt; - } else { - row[j] = std::string(PQgetvalue(_res.get(), i, j)); - } + for (idx_t i = 0; i < batch_size; ++i, ++rownum_) { + auto row = std::vector>(); + for (idx_t col = 0; col < num_cols_; ++col) { + auto str_val = duckdb_value_varchar(&result, col, rownum_); + if (str_val) { + duckdb_free(str_val); + } else { } - - vec[i] = std::move(row); } - - return vec; - }; - - return exec(conn_, "FETCH FORWARD " + std::to_string(_batch_size) + " FROM " + - cursor_name_ + ";") - .transform(to_vector) - .transform([this](auto&& _vec) { - if (_vec.size() == 0) { - shutdown(); - } - return std::move(_vec); - }); -} - -Iterator& Iterator::operator=(Iterator&& _other) noexcept { - if (this == &_other) { - return *this; - } - shutdown(); - cursor_name_ = std::move(_other.cursor_name_); - conn_ = std::move(_other.conn_); - end_ = _other.end_; - _other.end_ = true; - return *this; -} - -void Iterator::shutdown() { - if (!end_) { - exec(conn_, "CLOSE " + cursor_name_); - exec(conn_, "END"); - end_ = true; } } -} // namespace sqlgen::postgres +} // namespace sqlgen::duckdb From 26f5ea6633e64c44c91c73b2faf705ac9dc4d402 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sat, 11 Oct 2025 11:55:23 +0200 Subject: [PATCH 05/48] Adapted DuckDB to the new structure --- include/sqlgen/duckdb/Connection.hpp | 20 +++++++++++++------- src/sqlgen/duckdb/Connection.cpp | 19 ------------------- 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index 9e728089..6fa2e099 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -38,12 +38,16 @@ class Connection { Result execute(const std::string& _sql) noexcept; - Result insert( - const dynamic::Insert& _stmt, - const std::vector>>& - _data) noexcept; + template + Result insert(const dynamic::Insert& _stmt, ItBegin _begin, + ItEnd _end) noexcept { + return error("TODO"); + } - Result> read(const dynamic::SelectFrom& _query); + template + Result read(const dynamic::SelectFrom& _query) { + return error("TODO"); + } Result rollback() noexcept; @@ -55,8 +59,10 @@ class Connection { Result end_write(); - Result write( - const std::vector>>& _data); + template + Result write(ItBegin _begin, ItEnd _end) { + return error("TODO"); + } private: /// A prepared statement - needed for the read and write operations. Note that diff --git a/src/sqlgen/duckdb/Connection.cpp b/src/sqlgen/duckdb/Connection.cpp index eda9cee9..3975d92a 100644 --- a/src/sqlgen/duckdb/Connection.cpp +++ b/src/sqlgen/duckdb/Connection.cpp @@ -34,31 +34,12 @@ Result Connection::execute(const std::string& _sql) noexcept { return Nothing{}; } -Result Connection::insert( - const dynamic::Insert& _stmt, - const std::vector>>& - _data) noexcept { - // TODO - return error("TODO"); -} - rfl::Result> Connection::make( const std::optional& _fname) noexcept { return DuckDBConnection::make(_fname).transform( [](auto&& _conn) { return Ref::make(std::move(_conn)); }); } -Result> Connection::read(const dynamic::SelectFrom& _query) { - // TODO - return error("TODO"); -} - Result Connection::rollback() noexcept { return execute("ROLLBACK;"); } -Result Connection::write( - const std::vector>>& _data) { - // TODO - return error("TODO"); -} - } // namespace sqlgen::duckdb From 076324d6b4998835c39e4a417e01c1514c9db71d Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sat, 11 Oct 2025 12:31:37 +0200 Subject: [PATCH 06/48] Started writing the append_value function --- include/sqlgen/duckdb/Connection.hpp | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index 6fa2e099..0db93df4 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -40,9 +40,7 @@ class Connection { template Result insert(const dynamic::Insert& _stmt, ItBegin _begin, - ItEnd _end) noexcept { - return error("TODO"); - } + ItEnd _end) noexcept; template Result read(const dynamic::SelectFrom& _query) { @@ -64,6 +62,22 @@ class Connection { return error("TODO"); } + private: + template + Result append_value(const T& _t, duckdb_appender appender) noexcept { + using Type = std::remove_cvref_t; + if constexpr (std::is_same_v) { + } + } + + template + Result insert_row(const StructT& _struct, + duckdb_appender appender) noexcept { + Result res = Nothing{}; + rfl::to_view(_t).apply([&](const auto& _field) {}); + return res; + } + private: /// A prepared statement - needed for the read and write operations. Note that /// we have declared it before conn_, meaning it will be destroyed first. From 62bace82791ccd8e1213e47a3586074c6b59ce0e Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sat, 11 Oct 2025 12:35:35 +0200 Subject: [PATCH 07/48] Added error handling --- include/sqlgen/duckdb/Connection.hpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index 0db93df4..e4fdd5c8 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -64,15 +64,21 @@ class Connection { private: template - Result append_value(const T& _t, duckdb_appender appender) noexcept { + Result append_value(const T& _t, + duckdb_appender _appender) noexcept { using Type = std::remove_cvref_t; if constexpr (std::is_same_v) { + const auto state = duckdb_append_bool(_appender, _t); + if (state == DuckDBError) { + return error("Could not append boolean value."); + } } + return Nothing{}; } template Result insert_row(const StructT& _struct, - duckdb_appender appender) noexcept { + duckdb_appender _appender) noexcept { Result res = Nothing{}; rfl::to_view(_t).apply([&](const auto& _field) {}); return res; From d2e5d0a7eff29309744fbbe1e13f48185c3d4c97 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sat, 11 Oct 2025 16:12:04 +0200 Subject: [PATCH 08/48] Writing data into DuckDB appears to work --- include/sqlgen/duckdb.hpp | 8 ++ include/sqlgen/duckdb/Connection.hpp | 140 +++++++++++++++++++++++---- include/sqlgen/duckdb/connect.hpp | 16 +++ include/sqlgen/duckdb/to_sql.hpp | 26 +++++ src/sqlgen/duckdb/Connection.cpp | 5 - src/sqlgen/duckdb/to_sql.cpp | 75 +++++++++++++- src/sqlgen_duckdb.cpp | 2 +- tests/CMakeLists.txt | 4 + tests/duckdb/CMakeLists.txt | 19 ++++ tests/duckdb/test_write_and_read.cpp | 39 ++++++++ 10 files changed, 304 insertions(+), 30 deletions(-) create mode 100644 include/sqlgen/duckdb.hpp create mode 100644 include/sqlgen/duckdb/connect.hpp create mode 100644 include/sqlgen/duckdb/to_sql.hpp create mode 100644 tests/duckdb/CMakeLists.txt create mode 100644 tests/duckdb/test_write_and_read.cpp diff --git a/include/sqlgen/duckdb.hpp b/include/sqlgen/duckdb.hpp new file mode 100644 index 00000000..02b22f1b --- /dev/null +++ b/include/sqlgen/duckdb.hpp @@ -0,0 +1,8 @@ +#ifndef SQLGEN_DUCKDB_HPP_ +#define SQLGEN_DUCKDB_HPP_ + +#include "../sqlgen.hpp" +#include "duckdb/connect.hpp" +#include "duckdb/to_sql.hpp" + +#endif diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index e4fdd5c8..87df73fb 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -3,6 +3,7 @@ #include +#include #include #include #include @@ -16,8 +17,11 @@ #include "../Transaction.hpp" #include "../dynamic/Write.hpp" #include "../is_connection.hpp" +#include "../transpilation/get_tablename.hpp" +#include "../transpilation/has_reflection_method.hpp" +#include "../transpilation/is_nullable.hpp" #include "DuckDBConnection.hpp" -// #include "to_sql.hpp" +#include "to_sql.hpp" namespace sqlgen::duckdb { @@ -25,7 +29,7 @@ class Connection { using ConnPtr = Ref; public: - Connection(const ConnPtr& _conn) : /*stmt_(nullptr),*/ conn_(_conn) {} + Connection(const ConnPtr& _conn) : conn_(_conn) {} static rfl::Result> make( const std::optional& _fname) noexcept; @@ -39,8 +43,10 @@ class Connection { Result execute(const std::string& _sql) noexcept; template - Result insert(const dynamic::Insert& _stmt, ItBegin _begin, - ItEnd _end) noexcept; + Result insert(const dynamic::Insert&, ItBegin _begin, + ItEnd _end) noexcept { + return error("TODO"); + } template Result read(const dynamic::SelectFrom& _query) { @@ -50,16 +56,33 @@ class Connection { Result rollback() noexcept; std::string to_sql(const dynamic::Statement& _stmt) noexcept { - return "TODO"; // duckdb::to_sql_impl(_stmt); + return duckdb::to_sql_impl(_stmt); } - Result start_write(const dynamic::Write& _stmt); + Result start_write(const dynamic::Write&) { return Nothing{}; } - Result end_write(); + Result end_write() { return Nothing{}; } template Result write(ItBegin _begin, ItEnd _end) { - return error("TODO"); + using T = + std::remove_cvref_t::value_type>; + const auto tablename = transpilation::get_tablename(); + duckdb_appender appender{}; + if (duckdb_appender_create(conn_->conn(), nullptr, tablename.c_str(), + &appender) == DuckDBError) { + return error("Could not create appender."); + } + for (auto it = _begin; it < _end; ++it) { + const auto res = write_row(*it, appender); + if (!res) { + duckdb_appender_destroy(&appender); + return res; + } + duckdb_appender_end_row(appender); + } + duckdb_appender_destroy(&appender); + return Nothing{}; } private: @@ -67,28 +90,105 @@ class Connection { Result append_value(const T& _t, duckdb_appender _appender) noexcept { using Type = std::remove_cvref_t; - if constexpr (std::is_same_v) { - const auto state = duckdb_append_bool(_appender, _t); - if (state == DuckDBError) { - return error("Could not append boolean value."); + + if constexpr (transpilation::has_reflection_method) { + return append_value(_t.reflection(), _appender); + + } else if constexpr (std::is_same_v) { + return duckdb_append_bool(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append boolean value.")); + + } else if constexpr (std::is_same_v || + std::is_same_v) { + return duckdb_append_int8(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append int8 value.")); + + } else if constexpr (std::is_same_v) { + return duckdb_append_uint8(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append uint8 value.")); + + } else if constexpr (std::is_same_v) { + return duckdb_append_int16(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append int16 value.")); + + } else if constexpr (std::is_same_v) { + return duckdb_append_uint16(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append uint16 value.")); + + } else if constexpr (std::is_same_v) { + return duckdb_append_int32(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append int32 value.")); + + } else if constexpr (std::is_same_v) { + return duckdb_append_uint32(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append uint32 value.")); + + } else if constexpr (std::is_same_v) { + return duckdb_append_int64(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append int64 value.")); + + } else if constexpr (std::is_same_v) { + return duckdb_append_uint64(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append uint64 value.")); + + } else if constexpr (std::is_same_v) { + return duckdb_append_float(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append float value.")); + + } else if constexpr (std::is_same_v) { + return duckdb_append_double(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append double value.")); + + } else if constexpr (std::is_same_v) { + return duckdb_append_varchar(_appender, _t.c_str()) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append string value.")); + + } else if constexpr (std::is_same_v) { + return duckdb_append_varchar_length(_appender, _t.c_str(), _t.length()) != + DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append string value.")); + + } else if constexpr (transpilation::is_nullable_v) { + if (_t) { + return append_value(*_t, _appender); + } else { + return duckdb_append_null(_appender) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append null value.")); } + + } else { + static_assert(rfl::always_false_v, "Unsupported type."); + return error("Unsupported type."); } - return Nothing{}; } template - Result insert_row(const StructT& _struct, - duckdb_appender _appender) noexcept { + Result write_row(const StructT& _struct, + duckdb_appender _appender) noexcept { Result res = Nothing{}; - rfl::to_view(_t).apply([&](const auto& _field) {}); + rfl::to_view(_struct).apply([&](const auto& _field) { + if (res) { + res = append_value(*_field.value(), _appender); + } + }); return res; } private: - /// A prepared statement - needed for the read and write operations. Note that - /// we have declared it before conn_, meaning it will be destroyed first. - // StmtPtr stmt_; - /// The underlying duckdb3 connection. ConnPtr conn_; }; diff --git a/include/sqlgen/duckdb/connect.hpp b/include/sqlgen/duckdb/connect.hpp new file mode 100644 index 00000000..6cc6a0fc --- /dev/null +++ b/include/sqlgen/duckdb/connect.hpp @@ -0,0 +1,16 @@ +#ifndef SQLGEN_DUCKDB_CONNECT_HPP_ +#define SQLGEN_DUCKDB_CONNECT_HPP_ + +#include + +#include "Connection.hpp" + +namespace sqlgen::duckdb { + +inline auto connect(const std::string& _fname = ":memory:") { + return Connection::make(_fname); +} + +} // namespace sqlgen::duckdb + +#endif diff --git a/include/sqlgen/duckdb/to_sql.hpp b/include/sqlgen/duckdb/to_sql.hpp new file mode 100644 index 00000000..36d81945 --- /dev/null +++ b/include/sqlgen/duckdb/to_sql.hpp @@ -0,0 +1,26 @@ +#ifndef SQLGEN_DUCKDB_TO_SQL_HPP_ +#define SQLGEN_DUCKDB_TO_SQL_HPP_ + +#include + +#include "../dynamic/Statement.hpp" +#include "../transpilation/to_sql.hpp" + +namespace sqlgen::duckdb { + +/// Transpiles a dynamic general SQL statement to the duckdb dialect. +std::string to_sql_impl(const dynamic::Statement& _stmt) noexcept; + +/// Transpiles any SQL statement to the duckdb dialect. +template +std::string to_sql(const T& _t) noexcept { + if constexpr (std::is_same_v, dynamic::Statement>) { + return to_sql_impl(_t); + } else { + return to_sql_impl(transpilation::to_sql(_t)); + } +} + +} // namespace sqlgen::duckdb + +#endif diff --git a/src/sqlgen/duckdb/Connection.cpp b/src/sqlgen/duckdb/Connection.cpp index 3975d92a..a8aa8551 100644 --- a/src/sqlgen/duckdb/Connection.cpp +++ b/src/sqlgen/duckdb/Connection.cpp @@ -17,11 +17,6 @@ Result Connection::begin_transaction() noexcept { Result Connection::commit() noexcept { return execute("COMMIT;"); } -Result Connection::end_write() { - // TODO - return error("TODO"); -} - Result Connection::execute(const std::string& _sql) noexcept { duckdb_result res; const auto state = duckdb_query(conn_->conn(), _sql.c_str(), &res); diff --git a/src/sqlgen/duckdb/to_sql.cpp b/src/sqlgen/duckdb/to_sql.cpp index 95cab03f..55b13e6c 100644 --- a/src/sqlgen/duckdb/to_sql.cpp +++ b/src/sqlgen/duckdb/to_sql.cpp @@ -1,4 +1,4 @@ -#include "sqlgen/postgres/to_sql.hpp" +#include "sqlgen/duckdb/to_sql.hpp" #include #include @@ -11,7 +11,7 @@ #include "sqlgen/internal/collect/vector.hpp" #include "sqlgen/internal/strings/strings.hpp" -namespace sqlgen::postgres { +namespace sqlgen::duckdb { std::string aggregation_to_sql( const dynamic::Aggregation& _aggregation) noexcept; @@ -42,6 +42,9 @@ std::string field_to_str(const dynamic::SelectFrom::Field& _field) noexcept; std::vector get_primary_keys( const dynamic::CreateTable& _stmt) noexcept; +std::vector>> get_enum_types( + const dynamic::CreateTable& _stmt) noexcept; + std::string insert_to_sql(const dynamic::Insert& _stmt) noexcept; std::string join_to_sql(const dynamic::Join& _stmt) noexcept; @@ -66,10 +69,26 @@ std::string write_to_sql(const dynamic::Write& _stmt) noexcept; inline std::string get_name(const dynamic::Column& _col) { return _col.name; } +inline std::pair> get_enum_mapping( + const dynamic::Column& _col) { + return _col.type.visit( + [&](const auto& _t) -> std::pair> { + using T = std::remove_cvref_t; + if constexpr (std::is_same_v) { + return {type_to_sql(_t), _t.values}; + } + return {}; + }); +} + inline std::string wrap_in_quotes(const std::string& _name) noexcept { return "\"" + _name + "\""; } +inline std::string wrap_in_single_quotes(const std::string& _name) noexcept { + return "'" + _name + "'"; +} + // ---------------------------------------------------------------------------- std::string aggregation_to_sql( @@ -142,6 +161,8 @@ std::string condition_to_sql(const dynamic::Condition& _cond) noexcept { template std::string condition_to_sql_impl(const ConditionType& _condition) noexcept { + using namespace std::ranges::views; + using C = std::remove_cvref_t; std::stringstream stream; @@ -162,6 +183,14 @@ std::string condition_to_sql_impl(const ConditionType& _condition) noexcept { stream << operation_to_sql(_condition.op1) << " > " << operation_to_sql(_condition.op2); + } else if constexpr (std::is_same_v) { + stream << operation_to_sql(_condition.op) << " IN (" + << internal::strings::join( + ", ", + internal::collect::vector(_condition.patterns | + transform(column_or_value_to_sql))) + << ")"; + } else if constexpr (std::is_same_v) { stream << operation_to_sql(_condition.op) << " IS NULL"; @@ -191,6 +220,14 @@ std::string condition_to_sql_impl(const ConditionType& _condition) noexcept { stream << operation_to_sql(_condition.op) << " NOT LIKE " << column_or_value_to_sql(_condition.pattern); + } else if constexpr (std::is_same_v) { + stream << operation_to_sql(_condition.op) << " NOT IN (" + << internal::strings::join( + ", ", + internal::collect::vector(_condition.patterns | + transform(column_or_value_to_sql))) + << ")"; + } else if constexpr (std::is_same_v) { stream << "(" << condition_to_sql(*_condition.cond1) << ") OR (" << condition_to_sql(*_condition.cond2) << ")"; @@ -254,6 +291,21 @@ std::string create_table_to_sql(const dynamic::CreateTable& _stmt) noexcept { }; std::stringstream stream; + + for (const auto& [enum_name, enum_values] : get_enum_types(_stmt)) { + if (_stmt.if_not_exists) { + stream << "DO $$ BEGIN "; + } + stream << "CREATE TYPE " << enum_name << " AS ENUM (" + << internal::strings::join( + ", ", internal::collect::vector( + enum_values | transform(wrap_in_single_quotes))) + << "); "; + if (_stmt.if_not_exists) { + stream << "EXCEPTION WHEN duplicate_object THEN NULL; END $$;"; + } + } + stream << "CREATE TABLE "; if (_stmt.if_not_exists) { @@ -385,6 +437,20 @@ std::vector get_primary_keys( transform(wrap_in_quotes)); } +std::vector>> get_enum_types( + const dynamic::CreateTable& _stmt) noexcept { + using namespace std::ranges::views; + + const auto is_enum = [](const dynamic::Column& _col) -> bool { + return _col.type.visit([&](const auto& _t) -> bool { + using T = std::remove_cvref_t; + return std::is_same_v; + }); + }; + return internal::collect::vector(_stmt.columns | filter(is_enum) | + transform(get_enum_mapping)); +} + std::string insert_to_sql(const dynamic::Insert& _stmt) noexcept { using namespace std::ranges::views; @@ -751,7 +817,8 @@ std::string type_to_sql(const dynamic::Type& _type) noexcept { } else if constexpr (std::is_same_v || std::is_same_v) { return "BIGINT"; - + } else if constexpr (std::is_same_v) { + return _t.name; } else if constexpr (std::is_same_v || std::is_same_v) { return "NUMERIC"; @@ -824,4 +891,4 @@ std::string write_to_sql(const dynamic::Write& _stmt) noexcept { ") FROM STDIN WITH DELIMITER '\t' NULL '\e' CSV QUOTE '\a';"; } -} // namespace sqlgen::postgres +} // namespace sqlgen::duckdb diff --git a/src/sqlgen_duckdb.cpp b/src/sqlgen_duckdb.cpp index 35afe55c..1e254aa6 100644 --- a/src/sqlgen_duckdb.cpp +++ b/src/sqlgen_duckdb.cpp @@ -2,4 +2,4 @@ #include "sqlgen/duckdb/DuckDBConnection.cpp" // #include "sqlgen/duckdb/Iterator.cpp" // #include "sqlgen/duckdb/exec.cpp" -// #include "sqlgen/duckdb/to_sql.cpp" +#include "sqlgen/duckdb/to_sql.cpp" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f938e713..26b718a1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -8,6 +8,10 @@ if (SQLGEN_BUILD_DRY_TESTS_ONLY) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSQLGEN_BUILD_DRY_TESTS_ONLY") endif() +if(SQLGEN_DUCKDB) + add_subdirectory(duckdb) +endif() + if(SQLGEN_MYSQL) add_subdirectory(mysql) endif() diff --git a/tests/duckdb/CMakeLists.txt b/tests/duckdb/CMakeLists.txt new file mode 100644 index 00000000..fd827b82 --- /dev/null +++ b/tests/duckdb/CMakeLists.txt @@ -0,0 +1,19 @@ +project(sqlgen-duckdb-tests) + +file(GLOB_RECURSE SOURCES CONFIGURE_DEPENDS "*.cpp") + +add_executable( + sqlgen-duckdb-tests + ${SOURCES} +) +target_precompile_headers(sqlgen-duckdb-tests PRIVATE [["sqlgen.hpp"]] ) + + +target_link_libraries( + sqlgen-duckdb-tests + PRIVATE + "${SQLGEN_GTEST_LIB}" +) + +find_package(GTest) +gtest_discover_tests(sqlgen-duckdb-tests) diff --git a/tests/duckdb/test_write_and_read.cpp b/tests/duckdb/test_write_and_read.cpp new file mode 100644 index 00000000..fdac927f --- /dev/null +++ b/tests/duckdb/test_write_and_read.cpp @@ -0,0 +1,39 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_write_and_read { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_write_and_read) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + using namespace sqlgen; + + duckdb::connect().and_then(write(std::ref(people1))).value(); + + /*const auto people2 = sqlgen::read>(conn).value(); + + const auto json1 = rfl::json::write(people1); + const auto json2 = rfl::json::write(people2); + + EXPECT_EQ(json1, json2);*/ +} + +} // namespace test_write_and_read From 831855e9027f191a4ebfd79983aea9ed5c2d8cee Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sat, 11 Oct 2025 17:57:03 +0200 Subject: [PATCH 09/48] Began developing the Iterator --- include/sqlgen/duckdb/Iterator.hpp | 77 +++++++++++++++++++------- include/sqlgen/duckdb/chunk_ptrs_t.hpp | 36 ++++++++++++ src/sqlgen/duckdb/Iterator.cpp | 45 --------------- 3 files changed, 92 insertions(+), 66 deletions(-) create mode 100644 include/sqlgen/duckdb/chunk_ptrs_t.hpp delete mode 100644 src/sqlgen/duckdb/Iterator.cpp diff --git a/include/sqlgen/duckdb/Iterator.hpp b/include/sqlgen/duckdb/Iterator.hpp index 05ff0d7b..9c233278 100644 --- a/include/sqlgen/duckdb/Iterator.hpp +++ b/include/sqlgen/duckdb/Iterator.hpp @@ -4,38 +4,76 @@ #include #include +#include #include #include -#include "../IteratorBase.hpp" #include "../Ref.hpp" #include "../Result.hpp" +#include "DuckDBChunkPtrs.hpp" #include "DuckDBConnection.hpp" +#include "chunk_ptrs_t.hpp" -namespace sqlgen::sqlite { +namespace sqlgen::duckdb { -class Iterator : public sqlgen::IteratorBase { +template +class Iterator { using ConnPtr = Ref; using ResultPtr = Ref; + using ChunkPtrsT = chunk_ptrs_t; + + struct End { + bool operator==(const Iterator& _it) const noexcept { + return _it == *this; + } + + bool operator!=(const Iterator& _it) const noexcept { + return _it != *this; + } + }; public: + using difference_type = std::ptrdiff_t; + using value_type = Result; + Iterator(const ResultPtr& _res, const ConnPtr& _conn); ~Iterator(); - /// Whether the end of the available data has been reached. - bool end() const final; + Result& operator*() const noexcept { return (*current_batch_)[ix_]; } + + Result* operator->() const noexcept { return &(*current_batch_)[ix_]; } + + bool operator==(const End&) const noexcept { + return current_batch_->size() == 0; + } + + bool operator!=(const End& _end) const noexcept { return !(*this == _end); } - /// Returns the next batch of rows. - /// If _batch_size is greater than the number of rows left, returns all - /// of the rows left. - Result>>> next( - const size_t _batch_size) final; + Iterator& operator++() noexcept { + ++ix_; + if (ix_ >= current_batch_->size()) { + current_batch_ = get_next_batch(); + ix_ = 0; + } + return *this; + } + + void operator++(int) noexcept { ++*this; } private: - static void destroy_result(duckdb_result* _r) { - duckdb_result(_r); - delete _r; + static Ref>> get_next_batch( + const Ref& _it) noexcept { + duckdb_data_chunk chunk = duckdb_fetch_chunk(*res_); + auto batch = Ref>>::make(); + if (!chunk) { + return batch; + } + const idx_t row_count = duckdb_data_chunk_get_size(chunk); + const auto chunk_ptrs = make_chunk_ptrs(chunk); + for (idx_t i = 0; i < row_count; ++i) { + } + return batch; } private: @@ -45,16 +83,13 @@ class Iterator : public sqlgen::IteratorBase { /// The underlying connection. ConnPtr conn_; - /// The number of columns. - idx_t num_cols_; - - /// The number of rows. - idx_t num_rows_; + /// The current batch of results. + Ref>> current_batch_; - /// The current rownumber. - idx_t rownum_; + /// The index on the current_chunk + idx_t ix_; }; -} // namespace sqlgen::sqlite +} // namespace sqlgen::duckdb #endif diff --git a/include/sqlgen/duckdb/chunk_ptrs_t.hpp b/include/sqlgen/duckdb/chunk_ptrs_t.hpp new file mode 100644 index 00000000..629c6d03 --- /dev/null +++ b/include/sqlgen/duckdb/chunk_ptrs_t.hpp @@ -0,0 +1,36 @@ +#ifndef SQLGEN_DUCKDB_CHUNKPTRST_HPP_ +#define SQLGEN_DUCKDB_CHUNKPTRST_HPP_ + +#include + +#include +#include + +namespace sqlgen::duckdb { + +template +struct ColumnData { + duckdb_vector vec; + T *data; + uint64_t *validity; +}; + +template +struct ChunkPtrsType; + +template +struct ChunkPtrsType> { + using Type = rfl::Tuple...>; +}; + +template +struct ChunkPtrsType { + using Type = typename ChunkPtrsType>::Type; +}; + +template +using chunk_ptrs_t = typename ChunkPtrsType>::Type; + +} // namespace sqlgen::duckdb + +#endif diff --git a/src/sqlgen/duckdb/Iterator.cpp b/src/sqlgen/duckdb/Iterator.cpp deleted file mode 100644 index aef6535b..00000000 --- a/src/sqlgen/duckdb/Iterator.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include "sqlgen/duckdb/Iterator.hpp" - -#include -#include -#include - -#include "sqlgen/duckdb/exec.hpp" -#include "sqlgen/internal/collect/vector.hpp" -#include "sqlgen/internal/strings/strings.hpp" - -namespace sqlgen::duckdb { - -Iterator::Iterator(const ResultPtr& _res, const ConnPtr& _conn) - : res_(_res), - conn_(_conn), - num_cols_(duckdb_row_count(*_res)), - num_rows_(duckdb_column_count(*_res)), - rownum_(0) {} - -bool Iterator::end() const { return rownum_ >= num_rows_; } - -Result>>> Iterator::next( - const size_t _batch_size) { - auto vec = std::vector>>(); - - if (end()) { - return vec; - } - - const auto batch_size = - std::min(num_rows_ - rownum_, static_cast(_batch_size)); - - for (idx_t i = 0; i < batch_size; ++i, ++rownum_) { - auto row = std::vector>(); - for (idx_t col = 0; col < num_cols_; ++col) { - auto str_val = duckdb_value_varchar(&result, col, rownum_); - if (str_val) { - duckdb_free(str_val); - } else { - } - } - } -} - -} // namespace sqlgen::duckdb From 36b9b2e15dfb7430954e58cfa4a6ceb677ba8613 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sun, 12 Oct 2025 12:43:11 +0200 Subject: [PATCH 10/48] Continued writing the iterator --- include/sqlgen/duckdb/ColumnData.hpp | 17 ++++++++ include/sqlgen/duckdb/Iterator.hpp | 21 ++++++---- include/sqlgen/duckdb/chunk_ptrs_t.hpp | 11 +----- include/sqlgen/duckdb/from_chunk_ptrs.hpp | 35 +++++++++++++++++ include/sqlgen/duckdb/make_chunk_ptrs.hpp | 48 +++++++++++++++++++++++ 5 files changed, 115 insertions(+), 17 deletions(-) create mode 100644 include/sqlgen/duckdb/ColumnData.hpp create mode 100644 include/sqlgen/duckdb/from_chunk_ptrs.hpp create mode 100644 include/sqlgen/duckdb/make_chunk_ptrs.hpp diff --git a/include/sqlgen/duckdb/ColumnData.hpp b/include/sqlgen/duckdb/ColumnData.hpp new file mode 100644 index 00000000..ada64253 --- /dev/null +++ b/include/sqlgen/duckdb/ColumnData.hpp @@ -0,0 +1,17 @@ +#ifndef SQLGEN_DUCKDB_COLUMNDATA_HPP_ +#define SQLGEN_DUCKDB_COLUMNDATA_HPP_ + +#include + +namespace sqlgen::duckdb { + +template +struct ColumnData { + duckdb_vector vec; + T *data; + uint64_t *validity; +}; + +} // namespace sqlgen::duckdb + +#endif diff --git a/include/sqlgen/duckdb/Iterator.hpp b/include/sqlgen/duckdb/Iterator.hpp index 9c233278..97314de3 100644 --- a/include/sqlgen/duckdb/Iterator.hpp +++ b/include/sqlgen/duckdb/Iterator.hpp @@ -12,7 +12,7 @@ #include "../Result.hpp" #include "DuckDBChunkPtrs.hpp" #include "DuckDBConnection.hpp" -#include "chunk_ptrs_t.hpp" +#include "make_chunk_ptrs.hpp" namespace sqlgen::duckdb { @@ -20,7 +20,6 @@ template class Iterator { using ConnPtr = Ref; using ResultPtr = Ref; - using ChunkPtrsT = chunk_ptrs_t; struct End { bool operator==(const Iterator& _it) const noexcept { @@ -65,15 +64,21 @@ class Iterator { static Ref>> get_next_batch( const Ref& _it) noexcept { duckdb_data_chunk chunk = duckdb_fetch_chunk(*res_); - auto batch = Ref>>::make(); if (!chunk) { - return batch; + return Ref>>::make(); } const idx_t row_count = duckdb_data_chunk_get_size(chunk); - const auto chunk_ptrs = make_chunk_ptrs(chunk); - for (idx_t i = 0; i < row_count; ++i) { - } - return batch; + return make_chunk_ptrs(chunk) + .transform([&](auto&& _chunk_ptrs) { + auto batch = Ref>>::make(); + for (idx_t i = 0; i < row_count; ++i) { + batch->emplace_back(from_chunk_ptrs(_chunk_ptrs, i)); + } + return batch; + }) + .or_else( + [](auto _err) { return Ref>>::make({_err}); }) + .value(); } private: diff --git a/include/sqlgen/duckdb/chunk_ptrs_t.hpp b/include/sqlgen/duckdb/chunk_ptrs_t.hpp index 629c6d03..40e4c74c 100644 --- a/include/sqlgen/duckdb/chunk_ptrs_t.hpp +++ b/include/sqlgen/duckdb/chunk_ptrs_t.hpp @@ -1,19 +1,12 @@ #ifndef SQLGEN_DUCKDB_CHUNKPTRST_HPP_ #define SQLGEN_DUCKDB_CHUNKPTRST_HPP_ -#include - #include #include -namespace sqlgen::duckdb { +#include "ColumnData.hpp" -template -struct ColumnData { - duckdb_vector vec; - T *data; - uint64_t *validity; -}; +namespace sqlgen::duckdb { template struct ChunkPtrsType; diff --git a/include/sqlgen/duckdb/from_chunk_ptrs.hpp b/include/sqlgen/duckdb/from_chunk_ptrs.hpp new file mode 100644 index 00000000..a2cde98f --- /dev/null +++ b/include/sqlgen/duckdb/from_chunk_ptrs.hpp @@ -0,0 +1,35 @@ +#ifndef SQLGEN_DUCKDB_FROMCHUNKPTRS_HPP_ +#define SQLGEN_DUCKDB_FROMCHUNKPTRS_HPP_ + +#include + +#include +#include +#include + +#include "../Result.hpp" +#include "../Tuple.hpp" +#include "ColumnData.hpp" +#include "chunk_ptrs_t.hpp" + +namespace sqlgen::duckdb { + +template +struct FromChunkPtrs; + +template +struct FromChunkPtrs...>> { + Result operator(const Tuple...>& _chunk_ptrs, idx_t _i) { + return [&](std::integer_sequence) { + // TODO: Integrate this into the parser logic. + return T{*(rfl::get<_is>(_chunk_ptrs).data + _i)...}; + }(std::make_integer_sequence()); + } +}; + +template +auto from_chunk_ptrs = FromChunkPtrs, chunk_ptrs_t>{}; + +} // namespace sqlgen::duckdb + +#endif diff --git a/include/sqlgen/duckdb/make_chunk_ptrs.hpp b/include/sqlgen/duckdb/make_chunk_ptrs.hpp new file mode 100644 index 00000000..8cfc8f77 --- /dev/null +++ b/include/sqlgen/duckdb/make_chunk_ptrs.hpp @@ -0,0 +1,48 @@ +#ifndef SQLGEN_DUCKDB_MAKECHUNKPTRS_HPP_ +#define SQLGEN_DUCKDB_MAKECHUNKPTRS_HPP_ + +#include + +#include +#include +#include + +#include "../Tuple.hpp" +#include "ColumnData.hpp" +#include "chunk_ptrs_t.hpp" + +namespace sqlgen::duckdb { + +template +struct MakeChunkPtrs; + +template +struct MakeChunkPtrs...>> { + Result...>> operator(duckdb_data_chunk _chunk) { + // TODO: Runtime type checking + return [&](std::integer_sequence) { + return Tuple...>(make_column_data(_chunk)); + }(std::make_integer_sequence()); + } + + template + static auto make_column_data(duckdb_data_chunk _chunk) { + return ColumnData{.vec = duckdb_data_chunk_get_vector(_chunk, _i), + .data = static_cast(duckdb_vector_get_data(vec)), + .validity = duckdb_vector_get_validity(vec)}; + } +}; + +template +struct MakeChunkPtrs { + auto operator(duckdb_data_chunk _chunk) { + return MakeChunkPtrs>{}(_chunk); + } +}; + +template +auto make_chunk_ptrs = MakeChunkPtrs>{}; + +} // namespace sqlgen::duckdb + +#endif From 83b891d2e3808e8c84da6e4a7b24793a603e7456 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sat, 8 Nov 2025 15:34:52 +0100 Subject: [PATCH 11/48] Adapted DuckDB to recent changes --- include/sqlgen/duckdb/Connection.hpp | 12 +++++++++++- include/sqlgen/duckdb/Iterator.hpp | 7 +++---- include/sqlgen/duckdb/chunk_ptrs_t.hpp | 2 +- include/sqlgen/duckdb/make_chunk_ptrs.hpp | 15 ++++++++------- include/sqlgen/postgres/Connection.hpp | 1 + src/sqlgen/duckdb/to_sql.cpp | 7 +++++++ tests/duckdb/CMakeLists.txt | 8 ++++---- tests/duckdb/test_write_and_read.cpp | 1 - 8 files changed, 35 insertions(+), 18 deletions(-) diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index 87df73fb..2e41f37e 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -11,16 +11,17 @@ #include #include -#include "../IteratorBase.hpp" #include "../Ref.hpp" #include "../Result.hpp" #include "../Transaction.hpp" #include "../dynamic/Write.hpp" +#include "../internal/iterator_t.hpp" #include "../is_connection.hpp" #include "../transpilation/get_tablename.hpp" #include "../transpilation/has_reflection_method.hpp" #include "../transpilation/is_nullable.hpp" #include "DuckDBConnection.hpp" +#include "Iterator.hpp" #include "to_sql.hpp" namespace sqlgen::duckdb { @@ -200,4 +201,13 @@ static_assert(is_connection>, } // namespace sqlgen::duckdb +namespace sqlgen::internal { + +template +struct IteratorType { + using Type = duckdb::Iterator; +}; + +} // namespace sqlgen::internal + #endif diff --git a/include/sqlgen/duckdb/Iterator.hpp b/include/sqlgen/duckdb/Iterator.hpp index 97314de3..5364524e 100644 --- a/include/sqlgen/duckdb/Iterator.hpp +++ b/include/sqlgen/duckdb/Iterator.hpp @@ -10,7 +10,6 @@ #include "../Ref.hpp" #include "../Result.hpp" -#include "DuckDBChunkPtrs.hpp" #include "DuckDBConnection.hpp" #include "make_chunk_ptrs.hpp" @@ -52,7 +51,7 @@ class Iterator { Iterator& operator++() noexcept { ++ix_; if (ix_ >= current_batch_->size()) { - current_batch_ = get_next_batch(); + current_batch_ = get_next_batch(res_, conn_); ix_ = 0; } return *this; @@ -62,8 +61,8 @@ class Iterator { private: static Ref>> get_next_batch( - const Ref& _it) noexcept { - duckdb_data_chunk chunk = duckdb_fetch_chunk(*res_); + const ResultPtr& _res, const ConnPtr& _conn) noexcept { + duckdb_data_chunk chunk = duckdb_fetch_chunk(*_res); if (!chunk) { return Ref>>::make(); } diff --git a/include/sqlgen/duckdb/chunk_ptrs_t.hpp b/include/sqlgen/duckdb/chunk_ptrs_t.hpp index 40e4c74c..0e161de5 100644 --- a/include/sqlgen/duckdb/chunk_ptrs_t.hpp +++ b/include/sqlgen/duckdb/chunk_ptrs_t.hpp @@ -17,7 +17,7 @@ struct ChunkPtrsType> { }; template -struct ChunkPtrsType { +struct ChunkPtrsType { using Type = typename ChunkPtrsType>::Type; }; diff --git a/include/sqlgen/duckdb/make_chunk_ptrs.hpp b/include/sqlgen/duckdb/make_chunk_ptrs.hpp index 8cfc8f77..2892eb79 100644 --- a/include/sqlgen/duckdb/make_chunk_ptrs.hpp +++ b/include/sqlgen/duckdb/make_chunk_ptrs.hpp @@ -7,7 +7,6 @@ #include #include -#include "../Tuple.hpp" #include "ColumnData.hpp" #include "chunk_ptrs_t.hpp" @@ -17,25 +16,27 @@ template struct MakeChunkPtrs; template -struct MakeChunkPtrs...>> { - Result...>> operator(duckdb_data_chunk _chunk) { +struct MakeChunkPtrs...>> { + Result...>> operator()(duckdb_data_chunk _chunk) { // TODO: Runtime type checking return [&](std::integer_sequence) { - return Tuple...>(make_column_data(_chunk)); + return rfl::Tuple...>( + make_column_data(_chunk)...); }(std::make_integer_sequence()); } template static auto make_column_data(duckdb_data_chunk _chunk) { - return ColumnData{.vec = duckdb_data_chunk_get_vector(_chunk, _i), + auto vec = duckdb_data_chunk_get_vector(_chunk, _i); + return ColumnData{.vec = vec, .data = static_cast(duckdb_vector_get_data(vec)), .validity = duckdb_vector_get_validity(vec)}; } }; template -struct MakeChunkPtrs { - auto operator(duckdb_data_chunk _chunk) { +struct MakeChunkPtrs { + auto operator()(duckdb_data_chunk _chunk) { return MakeChunkPtrs>{}(_chunk); } }; diff --git a/include/sqlgen/postgres/Connection.hpp b/include/sqlgen/postgres/Connection.hpp index af50b109..94e743c9 100644 --- a/include/sqlgen/postgres/Connection.hpp +++ b/include/sqlgen/postgres/Connection.hpp @@ -15,6 +15,7 @@ #include "../dynamic/Column.hpp" #include "../dynamic/Statement.hpp" #include "../dynamic/Write.hpp" +#include "../internal/iterator_t.hpp" #include "../internal/to_container.hpp" #include "../internal/write_or_insert.hpp" #include "../is_connection.hpp" diff --git a/src/sqlgen/duckdb/to_sql.cpp b/src/sqlgen/duckdb/to_sql.cpp index 55b13e6c..59413d16 100644 --- a/src/sqlgen/duckdb/to_sql.cpp +++ b/src/sqlgen/duckdb/to_sql.cpp @@ -135,6 +135,9 @@ std::string column_or_value_to_sql( } else if constexpr (std::is_same_v) { return "to_timestamp(" + std::to_string(_v.seconds_since_unix) + ")"; + } else if constexpr (std::is_same_v) { + return _v.val ? "TRUE" : "FALSE"; + } else { return std::to_string(_v.val); } @@ -171,6 +174,10 @@ std::string condition_to_sql_impl(const ConditionType& _condition) noexcept { stream << "(" << condition_to_sql(*_condition.cond1) << ") AND (" << condition_to_sql(*_condition.cond2) << ")"; + } else if constexpr (std::is_same_v< + C, dynamic::Condition::BooleanColumnOrValue>) { + stream << column_or_value_to_sql(_condition.col_or_val); + } else if constexpr (std::is_same_v) { stream << operation_to_sql(_condition.op1) << " = " << operation_to_sql(_condition.op2); diff --git a/tests/duckdb/CMakeLists.txt b/tests/duckdb/CMakeLists.txt index fd827b82..956f0736 100644 --- a/tests/duckdb/CMakeLists.txt +++ b/tests/duckdb/CMakeLists.txt @@ -8,11 +8,11 @@ add_executable( ) target_precompile_headers(sqlgen-duckdb-tests PRIVATE [["sqlgen.hpp"]] ) +target_link_libraries(sqlgen-duckdb-tests PRIVATE sqlgen_tests_crt) -target_link_libraries( - sqlgen-duckdb-tests - PRIVATE - "${SQLGEN_GTEST_LIB}" +add_custom_command(TARGET sqlgen-duckdb-tests POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy -t $ $ + COMMAND_EXPAND_LISTS ) find_package(GTest) diff --git a/tests/duckdb/test_write_and_read.cpp b/tests/duckdb/test_write_and_read.cpp index fdac927f..5b118acf 100644 --- a/tests/duckdb/test_write_and_read.cpp +++ b/tests/duckdb/test_write_and_read.cpp @@ -2,7 +2,6 @@ #include #include -#include #include #include From fcade3dc412789b74721d84a8bf028708a64646c Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sat, 8 Nov 2025 17:02:11 +0100 Subject: [PATCH 12/48] Implemented the first reader --- include/sqlgen/duckdb/Connection.hpp | 8 +++++++- include/sqlgen/duckdb/Iterator.hpp | 16 ++++++++++++---- include/sqlgen/duckdb/from_chunk_ptrs.hpp | 6 +++--- tests/duckdb/test_write_and_read.cpp | 22 +++++++++++++--------- 4 files changed, 35 insertions(+), 17 deletions(-) diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index 2e41f37e..44f057b9 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -11,11 +11,13 @@ #include #include +#include "../Range.hpp" #include "../Ref.hpp" #include "../Result.hpp" #include "../Transaction.hpp" #include "../dynamic/Write.hpp" #include "../internal/iterator_t.hpp" +#include "../internal/to_container.hpp" #include "../is_connection.hpp" #include "../transpilation/get_tablename.hpp" #include "../transpilation/has_reflection_method.hpp" @@ -51,7 +53,11 @@ class Connection { template Result read(const dynamic::SelectFrom& _query) { - return error("TODO"); + using ValueType = transpilation::value_t; + auto res = Ref(); + duckdb_query(conn_->conn(), to_sql(_query).c_str(), res.get()); + return internal::to_container>( + Iterator(res, conn_)); } Result rollback() noexcept; diff --git a/include/sqlgen/duckdb/Iterator.hpp b/include/sqlgen/duckdb/Iterator.hpp index 5364524e..609c05c9 100644 --- a/include/sqlgen/duckdb/Iterator.hpp +++ b/include/sqlgen/duckdb/Iterator.hpp @@ -11,6 +11,7 @@ #include "../Ref.hpp" #include "../Result.hpp" #include "DuckDBConnection.hpp" +#include "from_chunk_ptrs.hpp" #include "make_chunk_ptrs.hpp" namespace sqlgen::duckdb { @@ -20,6 +21,7 @@ class Iterator { using ConnPtr = Ref; using ResultPtr = Ref; + public: struct End { bool operator==(const Iterator& _it) const noexcept { return _it == *this; @@ -34,9 +36,13 @@ class Iterator { using difference_type = std::ptrdiff_t; using value_type = Result; - Iterator(const ResultPtr& _res, const ConnPtr& _conn); + Iterator(const ResultPtr& _res, const ConnPtr& _conn) + : res_(_res), + conn_(_conn), + current_batch_(get_next_batch(_res, _conn)), + ix_(0) {} - ~Iterator(); + ~Iterator() = default; Result& operator*() const noexcept { return (*current_batch_)[ix_]; } @@ -75,8 +81,10 @@ class Iterator { } return batch; }) - .or_else( - [](auto _err) { return Ref>>::make({_err}); }) + .or_else([](auto _err) { + return Ref>>::make( + std::vector>({Result(_err)})); + }) .value(); } diff --git a/include/sqlgen/duckdb/from_chunk_ptrs.hpp b/include/sqlgen/duckdb/from_chunk_ptrs.hpp index a2cde98f..3ca6d05c 100644 --- a/include/sqlgen/duckdb/from_chunk_ptrs.hpp +++ b/include/sqlgen/duckdb/from_chunk_ptrs.hpp @@ -8,7 +8,6 @@ #include #include "../Result.hpp" -#include "../Tuple.hpp" #include "ColumnData.hpp" #include "chunk_ptrs_t.hpp" @@ -18,8 +17,9 @@ template struct FromChunkPtrs; template -struct FromChunkPtrs...>> { - Result operator(const Tuple...>& _chunk_ptrs, idx_t _i) { +struct FromChunkPtrs...>> { + Result operator()(const rfl::Tuple...>& _chunk_ptrs, + idx_t _i) { return [&](std::integer_sequence) { // TODO: Integrate this into the parser logic. return T{*(rfl::get<_is>(_chunk_ptrs).data + _i)...}; diff --git a/tests/duckdb/test_write_and_read.cpp b/tests/duckdb/test_write_and_read.cpp index 5b118acf..c84068bc 100644 --- a/tests/duckdb/test_write_and_read.cpp +++ b/tests/duckdb/test_write_and_read.cpp @@ -8,31 +8,35 @@ namespace test_write_and_read { struct Person { - sqlgen::PrimaryKey id; - std::string first_name; - std::string last_name; + // sqlgen::PrimaryKey id; + // std::string first_name; + // std::string last_name; int age; }; TEST(duckdb, test_write_and_read) { - const auto people1 = std::vector( + /*const auto people1 = std::vector( {Person{ .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, Person{ - .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = + 0}});*/ - using namespace sqlgen; + const auto people1 = std::vector({Person{.age = 45}}); - duckdb::connect().and_then(write(std::ref(people1))).value(); + using namespace sqlgen; - /*const auto people2 = sqlgen::read>(conn).value(); + const auto people2 = duckdb::connect() + .and_then(write(std::ref(people1))) + .and_then(sqlgen::read>) + .value(); const auto json1 = rfl::json::write(people1); const auto json2 = rfl::json::write(people2); - EXPECT_EQ(json1, json2);*/ + EXPECT_EQ(json1, json2); } } // namespace test_write_and_read From 4b8f58dbe937ff0f7c28b3319cd2133ad3f5e5d4 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sat, 8 Nov 2025 20:21:00 +0100 Subject: [PATCH 13/48] Added the raw type --- include/sqlgen/duckdb/Connection.hpp | 5 ----- include/sqlgen/internal/from_str_vec.hpp | 3 ++- include/sqlgen/internal/to_str.hpp | 3 ++- include/sqlgen/parsing/Parser_base.hpp | 4 +++- include/sqlgen/parsing/Parser_default.hpp | 15 ++++++++------- include/sqlgen/parsing/Parser_foreign_key.hpp | 16 +++++++++------- include/sqlgen/parsing/Parser_json.hpp | 5 +++-- include/sqlgen/parsing/Parser_optional.hpp | 11 ++++++----- include/sqlgen/parsing/Parser_primary_key.hpp | 11 ++++++----- include/sqlgen/parsing/Parser_shared_ptr.hpp | 11 ++++++----- include/sqlgen/parsing/Parser_string.hpp | 5 +++-- include/sqlgen/parsing/Parser_timestamp.hpp | 9 +++++---- include/sqlgen/parsing/Parser_unique.hpp | 11 ++++++----- include/sqlgen/parsing/Parser_unique_ptr.hpp | 11 ++++++----- include/sqlgen/parsing/Parser_varchar.hpp | 9 +++++---- include/sqlgen/parsing/RawType.hpp | 10 ++++++++++ include/sqlgen/transpilation/make_columns.hpp | 3 ++- include/sqlgen/transpilation/make_field.hpp | 4 +++- 18 files changed, 85 insertions(+), 61 deletions(-) create mode 100644 include/sqlgen/parsing/RawType.hpp diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index 44f057b9..24e056a5 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -157,11 +157,6 @@ class Connection { ? Result(Nothing{}) : Result(error("Could not append double value.")); - } else if constexpr (std::is_same_v) { - return duckdb_append_varchar(_appender, _t.c_str()) != DuckDBError - ? Result(Nothing{}) - : Result(error("Could not append string value.")); - } else if constexpr (std::is_same_v) { return duckdb_append_varchar_length(_appender, _t.c_str(), _t.length()) != DuckDBError diff --git a/include/sqlgen/internal/from_str_vec.hpp b/include/sqlgen/internal/from_str_vec.hpp index f919ef95..8a6eb961 100644 --- a/include/sqlgen/internal/from_str_vec.hpp +++ b/include/sqlgen/internal/from_str_vec.hpp @@ -13,6 +13,7 @@ #include "../Result.hpp" #include "../parsing/Parser.hpp" +#include "../parsing/RawType.hpp" #include "call_destructors_where_necessary.hpp" namespace sqlgen::internal { @@ -26,7 +27,7 @@ void assign_if_field_is_field_i( std::remove_cvref_t>; constexpr auto name = FieldType::name(); if (_i == i) { - auto res = parsing::Parser::read(_row[i]); + auto res = parsing::Parser::read(_row[i]); if (!res) { std::stringstream stream; stream << "Failed to parse field '" << std::string(name) diff --git a/include/sqlgen/internal/to_str.hpp b/include/sqlgen/internal/to_str.hpp index 228cf44c..36c7c7af 100644 --- a/include/sqlgen/internal/to_str.hpp +++ b/include/sqlgen/internal/to_str.hpp @@ -6,13 +6,14 @@ #include #include "../parsing/Parser.hpp" +#include "../parsing/RawType.hpp" namespace sqlgen::internal { template std::optional to_str(const T& _val) { using Type = std::remove_cvref_t; - return parsing::Parser::write(_val); + return parsing::Parser::write(_val); } } // namespace sqlgen::internal diff --git a/include/sqlgen/parsing/Parser_base.hpp b/include/sqlgen/parsing/Parser_base.hpp index d205549b..57fc7db2 100644 --- a/include/sqlgen/parsing/Parser_base.hpp +++ b/include/sqlgen/parsing/Parser_base.hpp @@ -1,9 +1,11 @@ #ifndef SQLGEN_PARSING_PARSER_BASE_HPP_ #define SQLGEN_PARSING_PARSER_BASE_HPP_ +#include "RawType.hpp" + namespace sqlgen::parsing { -template +template struct Parser; } diff --git a/include/sqlgen/parsing/Parser_default.hpp b/include/sqlgen/parsing/Parser_default.hpp index ec29ddf2..5a80758b 100644 --- a/include/sqlgen/parsing/Parser_default.hpp +++ b/include/sqlgen/parsing/Parser_default.hpp @@ -11,17 +11,18 @@ #include "../dynamic/types.hpp" #include "../transpilation/has_reflection_method.hpp" #include "Parser_base.hpp" +#include "RawType.hpp" namespace sqlgen::parsing { -template +template struct Parser { using Type = std::remove_cvref_t; static Result read(const std::optional& _str) noexcept { if constexpr (transpilation::has_reflection_method) { - return Parser>::read( - _str) + return Parser, + _raw_type>::read(_str) .transform([](auto&& _t) { return Type(std::move(_t)); }); } else { @@ -66,8 +67,8 @@ struct Parser { static std::optional write(const T& _t) noexcept { if constexpr (transpilation::has_reflection_method) { - return Parser>::write( - _t.reflection()); + return Parser, + _raw_type>::write(_t.reflection()); } else if constexpr (std::is_enum_v) { return rfl::enum_to_string(_t); } else { @@ -77,8 +78,8 @@ struct Parser { static dynamic::Type to_type() noexcept { if constexpr (transpilation::has_reflection_method) { - return Parser< - std::remove_cvref_t>::to_type(); + return Parser, + _raw_type>::to_type(); } else if constexpr (std::is_same_v) { return dynamic::types::Boolean{}; diff --git a/include/sqlgen/parsing/Parser_foreign_key.hpp b/include/sqlgen/parsing/Parser_foreign_key.hpp index 899044e8..0a5d47c2 100644 --- a/include/sqlgen/parsing/Parser_foreign_key.hpp +++ b/include/sqlgen/parsing/Parser_foreign_key.hpp @@ -9,26 +9,28 @@ #include "../dynamic/Type.hpp" #include "../transpilation/get_tablename.hpp" #include "Parser_base.hpp" +#include "RawType.hpp" namespace sqlgen::parsing { template -struct Parser> { + rfl::internal::StringLiteral _col_name, RawType _raw_type> +struct Parser, _raw_type> { static Result> read( const std::optional& _str) noexcept { - return Parser>::read(_str).transform([](auto&& _t) { - return ForeignKey(std::move(_t)); - }); + return Parser, _raw_type>::read(_str).transform( + [](auto&& _t) { + return ForeignKey(std::move(_t)); + }); } static std::optional write( const ForeignKey& _f) noexcept { - return Parser>::write(_f.value()); + return Parser, _raw_type>::write(_f.value()); } static dynamic::Type to_type() noexcept { - return Parser>::to_type().visit( + return Parser, _raw_type>::to_type().visit( [](auto _t) -> dynamic::Type { _t.properties.foreign_key_reference = dynamic::types::ForeignKeyReference{ diff --git a/include/sqlgen/parsing/Parser_json.hpp b/include/sqlgen/parsing/Parser_json.hpp index f846ee0e..b4031460 100644 --- a/include/sqlgen/parsing/Parser_json.hpp +++ b/include/sqlgen/parsing/Parser_json.hpp @@ -9,11 +9,12 @@ #include "../Result.hpp" #include "../dynamic/Type.hpp" #include "Parser_base.hpp" +#include "RawType.hpp" namespace sqlgen::parsing { -template -struct Parser> { +template +struct Parser, _raw_type> { static Result> read(const std::optional& _str) noexcept { if (!_str) { return error("NULL value encounted: JSON value cannot be NULL."); diff --git a/include/sqlgen/parsing/Parser_optional.hpp b/include/sqlgen/parsing/Parser_optional.hpp index 17a408b4..1ff7ed67 100644 --- a/include/sqlgen/parsing/Parser_optional.hpp +++ b/include/sqlgen/parsing/Parser_optional.hpp @@ -8,17 +8,18 @@ #include "../Result.hpp" #include "../dynamic/Type.hpp" #include "Parser_base.hpp" +#include "RawType.hpp" namespace sqlgen::parsing { -template -struct Parser> { +template +struct Parser, _raw_type> { static Result> read( const std::optional& _str) noexcept { if (!_str) { return std::optional(); } - return Parser>::read(_str).transform( + return Parser, _raw_type>::read(_str).transform( [](auto&& _t) -> std::optional { return std::make_optional(std::move(_t)); }); @@ -28,11 +29,11 @@ struct Parser> { if (!_o) { return std::nullopt; } - return Parser>::write(*_o); + return Parser, _raw_type>::write(*_o); } static dynamic::Type to_type() noexcept { - return Parser>::to_type().visit( + return Parser, _raw_type>::to_type().visit( [](auto _t) -> dynamic::Type { _t.properties.nullable = true; return _t; diff --git a/include/sqlgen/parsing/Parser_primary_key.hpp b/include/sqlgen/parsing/Parser_primary_key.hpp index 457a1692..4317f777 100644 --- a/include/sqlgen/parsing/Parser_primary_key.hpp +++ b/include/sqlgen/parsing/Parser_primary_key.hpp @@ -8,14 +8,15 @@ #include "../Result.hpp" #include "../dynamic/Type.hpp" #include "Parser_base.hpp" +#include "RawType.hpp" namespace sqlgen::parsing { -template -struct Parser> { +template +struct Parser, _raw_type> { static Result> read( const std::optional& _str) noexcept { - return Parser>::read(_str).transform( + return Parser, _raw_type>::read(_str).transform( [](auto&& _t) -> PrimaryKey { return PrimaryKey(std::move(_t)); }); @@ -26,12 +27,12 @@ struct Parser> { if constexpr (_auto_incr) { return std::nullopt; } else { - return Parser>::write(_p.value()); + return Parser, _raw_type>::write(_p.value()); } } static dynamic::Type to_type() noexcept { - return Parser>::to_type().visit( + return Parser, _raw_type>::to_type().visit( [](auto _t) -> dynamic::Type { _t.properties.auto_incr = _auto_incr; _t.properties.primary = true; diff --git a/include/sqlgen/parsing/Parser_shared_ptr.hpp b/include/sqlgen/parsing/Parser_shared_ptr.hpp index 0e8cb31b..da7de7b9 100644 --- a/include/sqlgen/parsing/Parser_shared_ptr.hpp +++ b/include/sqlgen/parsing/Parser_shared_ptr.hpp @@ -8,17 +8,18 @@ #include "../Result.hpp" #include "../dynamic/Type.hpp" #include "Parser_base.hpp" +#include "RawType.hpp" namespace sqlgen::parsing { -template -struct Parser> { +template +struct Parser, _raw_type> { static Result> read( const std::optional& _str) noexcept { if (!_str) { return std::shared_ptr(); } - return Parser>::read(_str).transform( + return Parser, _raw_type>::read(_str).transform( [](auto&& _t) -> std::shared_ptr { return std::make_shared(std::move(_t)); }); @@ -29,11 +30,11 @@ struct Parser> { if (!_ptr) { return std::nullopt; } - return Parser>::write(*_ptr); + return Parser, _raw_type>::write(*_ptr); } static dynamic::Type to_type() noexcept { - return Parser>::to_type().visit( + return Parser, _raw_type>::to_type().visit( [](auto _t) -> dynamic::Type { _t.properties.nullable = true; return _t; diff --git a/include/sqlgen/parsing/Parser_string.hpp b/include/sqlgen/parsing/Parser_string.hpp index 66ab60de..cec2191b 100644 --- a/include/sqlgen/parsing/Parser_string.hpp +++ b/include/sqlgen/parsing/Parser_string.hpp @@ -8,11 +8,12 @@ #include "../dynamic/Type.hpp" #include "../dynamic/types.hpp" #include "Parser_base.hpp" +#include "RawType.hpp" namespace sqlgen::parsing { -template <> -struct Parser { +template +struct Parser { static Result read( const std::optional& _str) noexcept { if (!_str) { diff --git a/include/sqlgen/parsing/Parser_timestamp.hpp b/include/sqlgen/parsing/Parser_timestamp.hpp index f3e8446a..5e990619 100644 --- a/include/sqlgen/parsing/Parser_timestamp.hpp +++ b/include/sqlgen/parsing/Parser_timestamp.hpp @@ -10,22 +10,23 @@ #include "../dynamic/types.hpp" #include "Parser_base.hpp" #include "Parser_default.hpp" +#include "RawType.hpp" namespace sqlgen::parsing { -template -struct Parser> { +template +struct Parser, _raw_type> { using TSType = Timestamp<_format>; static Result read(const std::optional& _str) noexcept { - return Parser::read(_str).and_then( + return Parser::read(_str).and_then( [](auto&& _s) -> Result { return TSType::from_string(std::move(_s)); }); } static std::optional write(const TSType& _t) noexcept { - return Parser::write(_t.str()); + return Parser::write(_t.str()); } static dynamic::Type to_type() noexcept { diff --git a/include/sqlgen/parsing/Parser_unique.hpp b/include/sqlgen/parsing/Parser_unique.hpp index dd4200c0..14e90443 100644 --- a/include/sqlgen/parsing/Parser_unique.hpp +++ b/include/sqlgen/parsing/Parser_unique.hpp @@ -8,23 +8,24 @@ #include "../Unique.hpp" #include "../dynamic/Type.hpp" #include "Parser_base.hpp" +#include "RawType.hpp" namespace sqlgen::parsing { -template -struct Parser> { +template +struct Parser, _raw_type> { static Result> read( const std::optional& _str) noexcept { - return Parser>::read(_str).transform( + return Parser, _raw_type>::read(_str).transform( [](auto&& _t) { return Unique(std::move(_t)); }); } static std::optional write(const Unique& _f) noexcept { - return Parser>::write(_f.value()); + return Parser, _raw_type>::write(_f.value()); } static dynamic::Type to_type() noexcept { - return Parser>::to_type().visit( + return Parser, _raw_type>::to_type().visit( [](auto _t) -> dynamic::Type { _t.properties.unique = true; return _t; diff --git a/include/sqlgen/parsing/Parser_unique_ptr.hpp b/include/sqlgen/parsing/Parser_unique_ptr.hpp index cb114d99..896f7a75 100644 --- a/include/sqlgen/parsing/Parser_unique_ptr.hpp +++ b/include/sqlgen/parsing/Parser_unique_ptr.hpp @@ -8,17 +8,18 @@ #include "../Result.hpp" #include "../dynamic/Type.hpp" #include "Parser_base.hpp" +#include "RawType.hpp" namespace sqlgen::parsing { -template -struct Parser> { +template +struct Parser, _raw_type> { static Result> read( const std::optional& _str) noexcept { if (!_str) { return std::unique_ptr(); } - return Parser>::read(_str).transform( + return Parser, _raw_type>::read(_str).transform( [](auto&& _t) -> std::unique_ptr { return std::make_unique(std::move(_t)); }); @@ -29,11 +30,11 @@ struct Parser> { if (!_ptr) { return std::nullopt; } - return Parser>::write(*_ptr); + return Parser, _raw_type>::write(*_ptr); } static dynamic::Type to_type() noexcept { - return Parser>::to_type().visit( + return Parser, _raw_type>::to_type().visit( [](auto _t) -> dynamic::Type { _t.properties.nullable = true; return _t; diff --git a/include/sqlgen/parsing/Parser_varchar.hpp b/include/sqlgen/parsing/Parser_varchar.hpp index 489b08bf..bd75ad60 100644 --- a/include/sqlgen/parsing/Parser_varchar.hpp +++ b/include/sqlgen/parsing/Parser_varchar.hpp @@ -10,21 +10,22 @@ #include "../dynamic/types.hpp" #include "Parser_base.hpp" #include "Parser_default.hpp" +#include "RawType.hpp" namespace sqlgen::parsing { -template -struct Parser> { +template +struct Parser, _raw_type> { static Result> read( const std::optional& _str) noexcept { - return Parser::read(_str).and_then( + return Parser::read(_str).and_then( [](auto&& _t) -> Result> { return Varchar<_size>::make(std::move(_t)); }); } static std::optional write(const Varchar<_size>& _v) noexcept { - return Parser::write(_v.value()); + return Parser::write(_v.value()); } static dynamic::Type to_type() noexcept { diff --git a/include/sqlgen/parsing/RawType.hpp b/include/sqlgen/parsing/RawType.hpp new file mode 100644 index 00000000..13e4719a --- /dev/null +++ b/include/sqlgen/parsing/RawType.hpp @@ -0,0 +1,10 @@ +#ifndef SQLGEN_PARSING_RAWTYPE_HPP_ +#define SQLGEN_PARSING_RAWTYPE_HPP_ + +namespace sqlgen::parsing { + +enum class RawType { original, string }; + +} + +#endif diff --git a/include/sqlgen/transpilation/make_columns.hpp b/include/sqlgen/transpilation/make_columns.hpp index 833882a3..39a09422 100644 --- a/include/sqlgen/transpilation/make_columns.hpp +++ b/include/sqlgen/transpilation/make_columns.hpp @@ -9,6 +9,7 @@ #include "../dynamic/Type.hpp" #include "../dynamic/types.hpp" #include "../parsing/Parser.hpp" +#include "../parsing/RawType.hpp" #include "has_reflection_method.hpp" #include "is_nullable.hpp" #include "is_primary_key.hpp" @@ -23,7 +24,7 @@ std::string to_colname() { template dynamic::Type to_type() { using T = std::remove_cvref_t; - return parsing::Parser::to_type(); + return parsing::Parser::to_type(); } template diff --git a/include/sqlgen/transpilation/make_field.hpp b/include/sqlgen/transpilation/make_field.hpp index 92d8cb08..76930718 100644 --- a/include/sqlgen/transpilation/make_field.hpp +++ b/include/sqlgen/transpilation/make_field.hpp @@ -24,6 +24,7 @@ #include "is_timestamp.hpp" #include "remove_as_t.hpp" #include "remove_nullable_t.hpp" +#include "sqlgen/parsing/RawType.hpp" #include "to_alias.hpp" #include "to_duration.hpp" #include "to_value.hpp" @@ -232,7 +233,8 @@ struct MakeField>::to_type()}}}; + parsing::Parser, + parsing::RawType::string>::to_type()}}}; } }; From 65f1bb722a7186dc1d90f9de0b1346c27f30b1e7 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sat, 8 Nov 2025 20:38:14 +0100 Subject: [PATCH 14/48] Revert "Added the raw type" This reverts commit 4b8f58dbe937ff0f7c28b3319cd2133ad3f5e5d4. --- include/sqlgen/duckdb/Connection.hpp | 5 +++++ include/sqlgen/internal/from_str_vec.hpp | 3 +-- include/sqlgen/internal/to_str.hpp | 3 +-- include/sqlgen/parsing/Parser_base.hpp | 4 +--- include/sqlgen/parsing/Parser_default.hpp | 15 +++++++-------- include/sqlgen/parsing/Parser_foreign_key.hpp | 16 +++++++--------- include/sqlgen/parsing/Parser_json.hpp | 5 ++--- include/sqlgen/parsing/Parser_optional.hpp | 11 +++++------ include/sqlgen/parsing/Parser_primary_key.hpp | 11 +++++------ include/sqlgen/parsing/Parser_shared_ptr.hpp | 11 +++++------ include/sqlgen/parsing/Parser_string.hpp | 5 ++--- include/sqlgen/parsing/Parser_timestamp.hpp | 9 ++++----- include/sqlgen/parsing/Parser_unique.hpp | 11 +++++------ include/sqlgen/parsing/Parser_unique_ptr.hpp | 11 +++++------ include/sqlgen/parsing/Parser_varchar.hpp | 9 ++++----- include/sqlgen/parsing/RawType.hpp | 10 ---------- include/sqlgen/transpilation/make_columns.hpp | 3 +-- include/sqlgen/transpilation/make_field.hpp | 4 +--- 18 files changed, 61 insertions(+), 85 deletions(-) delete mode 100644 include/sqlgen/parsing/RawType.hpp diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index 24e056a5..44f057b9 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -157,6 +157,11 @@ class Connection { ? Result(Nothing{}) : Result(error("Could not append double value.")); + } else if constexpr (std::is_same_v) { + return duckdb_append_varchar(_appender, _t.c_str()) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append string value.")); + } else if constexpr (std::is_same_v) { return duckdb_append_varchar_length(_appender, _t.c_str(), _t.length()) != DuckDBError diff --git a/include/sqlgen/internal/from_str_vec.hpp b/include/sqlgen/internal/from_str_vec.hpp index 8a6eb961..f919ef95 100644 --- a/include/sqlgen/internal/from_str_vec.hpp +++ b/include/sqlgen/internal/from_str_vec.hpp @@ -13,7 +13,6 @@ #include "../Result.hpp" #include "../parsing/Parser.hpp" -#include "../parsing/RawType.hpp" #include "call_destructors_where_necessary.hpp" namespace sqlgen::internal { @@ -27,7 +26,7 @@ void assign_if_field_is_field_i( std::remove_cvref_t>; constexpr auto name = FieldType::name(); if (_i == i) { - auto res = parsing::Parser::read(_row[i]); + auto res = parsing::Parser::read(_row[i]); if (!res) { std::stringstream stream; stream << "Failed to parse field '" << std::string(name) diff --git a/include/sqlgen/internal/to_str.hpp b/include/sqlgen/internal/to_str.hpp index 36c7c7af..228cf44c 100644 --- a/include/sqlgen/internal/to_str.hpp +++ b/include/sqlgen/internal/to_str.hpp @@ -6,14 +6,13 @@ #include #include "../parsing/Parser.hpp" -#include "../parsing/RawType.hpp" namespace sqlgen::internal { template std::optional to_str(const T& _val) { using Type = std::remove_cvref_t; - return parsing::Parser::write(_val); + return parsing::Parser::write(_val); } } // namespace sqlgen::internal diff --git a/include/sqlgen/parsing/Parser_base.hpp b/include/sqlgen/parsing/Parser_base.hpp index 57fc7db2..d205549b 100644 --- a/include/sqlgen/parsing/Parser_base.hpp +++ b/include/sqlgen/parsing/Parser_base.hpp @@ -1,11 +1,9 @@ #ifndef SQLGEN_PARSING_PARSER_BASE_HPP_ #define SQLGEN_PARSING_PARSER_BASE_HPP_ -#include "RawType.hpp" - namespace sqlgen::parsing { -template +template struct Parser; } diff --git a/include/sqlgen/parsing/Parser_default.hpp b/include/sqlgen/parsing/Parser_default.hpp index 5a80758b..ec29ddf2 100644 --- a/include/sqlgen/parsing/Parser_default.hpp +++ b/include/sqlgen/parsing/Parser_default.hpp @@ -11,18 +11,17 @@ #include "../dynamic/types.hpp" #include "../transpilation/has_reflection_method.hpp" #include "Parser_base.hpp" -#include "RawType.hpp" namespace sqlgen::parsing { -template +template struct Parser { using Type = std::remove_cvref_t; static Result read(const std::optional& _str) noexcept { if constexpr (transpilation::has_reflection_method) { - return Parser, - _raw_type>::read(_str) + return Parser>::read( + _str) .transform([](auto&& _t) { return Type(std::move(_t)); }); } else { @@ -67,8 +66,8 @@ struct Parser { static std::optional write(const T& _t) noexcept { if constexpr (transpilation::has_reflection_method) { - return Parser, - _raw_type>::write(_t.reflection()); + return Parser>::write( + _t.reflection()); } else if constexpr (std::is_enum_v) { return rfl::enum_to_string(_t); } else { @@ -78,8 +77,8 @@ struct Parser { static dynamic::Type to_type() noexcept { if constexpr (transpilation::has_reflection_method) { - return Parser, - _raw_type>::to_type(); + return Parser< + std::remove_cvref_t>::to_type(); } else if constexpr (std::is_same_v) { return dynamic::types::Boolean{}; diff --git a/include/sqlgen/parsing/Parser_foreign_key.hpp b/include/sqlgen/parsing/Parser_foreign_key.hpp index 0a5d47c2..899044e8 100644 --- a/include/sqlgen/parsing/Parser_foreign_key.hpp +++ b/include/sqlgen/parsing/Parser_foreign_key.hpp @@ -9,28 +9,26 @@ #include "../dynamic/Type.hpp" #include "../transpilation/get_tablename.hpp" #include "Parser_base.hpp" -#include "RawType.hpp" namespace sqlgen::parsing { template -struct Parser, _raw_type> { + rfl::internal::StringLiteral _col_name> +struct Parser> { static Result> read( const std::optional& _str) noexcept { - return Parser, _raw_type>::read(_str).transform( - [](auto&& _t) { - return ForeignKey(std::move(_t)); - }); + return Parser>::read(_str).transform([](auto&& _t) { + return ForeignKey(std::move(_t)); + }); } static std::optional write( const ForeignKey& _f) noexcept { - return Parser, _raw_type>::write(_f.value()); + return Parser>::write(_f.value()); } static dynamic::Type to_type() noexcept { - return Parser, _raw_type>::to_type().visit( + return Parser>::to_type().visit( [](auto _t) -> dynamic::Type { _t.properties.foreign_key_reference = dynamic::types::ForeignKeyReference{ diff --git a/include/sqlgen/parsing/Parser_json.hpp b/include/sqlgen/parsing/Parser_json.hpp index b4031460..f846ee0e 100644 --- a/include/sqlgen/parsing/Parser_json.hpp +++ b/include/sqlgen/parsing/Parser_json.hpp @@ -9,12 +9,11 @@ #include "../Result.hpp" #include "../dynamic/Type.hpp" #include "Parser_base.hpp" -#include "RawType.hpp" namespace sqlgen::parsing { -template -struct Parser, _raw_type> { +template +struct Parser> { static Result> read(const std::optional& _str) noexcept { if (!_str) { return error("NULL value encounted: JSON value cannot be NULL."); diff --git a/include/sqlgen/parsing/Parser_optional.hpp b/include/sqlgen/parsing/Parser_optional.hpp index 1ff7ed67..17a408b4 100644 --- a/include/sqlgen/parsing/Parser_optional.hpp +++ b/include/sqlgen/parsing/Parser_optional.hpp @@ -8,18 +8,17 @@ #include "../Result.hpp" #include "../dynamic/Type.hpp" #include "Parser_base.hpp" -#include "RawType.hpp" namespace sqlgen::parsing { -template -struct Parser, _raw_type> { +template +struct Parser> { static Result> read( const std::optional& _str) noexcept { if (!_str) { return std::optional(); } - return Parser, _raw_type>::read(_str).transform( + return Parser>::read(_str).transform( [](auto&& _t) -> std::optional { return std::make_optional(std::move(_t)); }); @@ -29,11 +28,11 @@ struct Parser, _raw_type> { if (!_o) { return std::nullopt; } - return Parser, _raw_type>::write(*_o); + return Parser>::write(*_o); } static dynamic::Type to_type() noexcept { - return Parser, _raw_type>::to_type().visit( + return Parser>::to_type().visit( [](auto _t) -> dynamic::Type { _t.properties.nullable = true; return _t; diff --git a/include/sqlgen/parsing/Parser_primary_key.hpp b/include/sqlgen/parsing/Parser_primary_key.hpp index 4317f777..457a1692 100644 --- a/include/sqlgen/parsing/Parser_primary_key.hpp +++ b/include/sqlgen/parsing/Parser_primary_key.hpp @@ -8,15 +8,14 @@ #include "../Result.hpp" #include "../dynamic/Type.hpp" #include "Parser_base.hpp" -#include "RawType.hpp" namespace sqlgen::parsing { -template -struct Parser, _raw_type> { +template +struct Parser> { static Result> read( const std::optional& _str) noexcept { - return Parser, _raw_type>::read(_str).transform( + return Parser>::read(_str).transform( [](auto&& _t) -> PrimaryKey { return PrimaryKey(std::move(_t)); }); @@ -27,12 +26,12 @@ struct Parser, _raw_type> { if constexpr (_auto_incr) { return std::nullopt; } else { - return Parser, _raw_type>::write(_p.value()); + return Parser>::write(_p.value()); } } static dynamic::Type to_type() noexcept { - return Parser, _raw_type>::to_type().visit( + return Parser>::to_type().visit( [](auto _t) -> dynamic::Type { _t.properties.auto_incr = _auto_incr; _t.properties.primary = true; diff --git a/include/sqlgen/parsing/Parser_shared_ptr.hpp b/include/sqlgen/parsing/Parser_shared_ptr.hpp index da7de7b9..0e8cb31b 100644 --- a/include/sqlgen/parsing/Parser_shared_ptr.hpp +++ b/include/sqlgen/parsing/Parser_shared_ptr.hpp @@ -8,18 +8,17 @@ #include "../Result.hpp" #include "../dynamic/Type.hpp" #include "Parser_base.hpp" -#include "RawType.hpp" namespace sqlgen::parsing { -template -struct Parser, _raw_type> { +template +struct Parser> { static Result> read( const std::optional& _str) noexcept { if (!_str) { return std::shared_ptr(); } - return Parser, _raw_type>::read(_str).transform( + return Parser>::read(_str).transform( [](auto&& _t) -> std::shared_ptr { return std::make_shared(std::move(_t)); }); @@ -30,11 +29,11 @@ struct Parser, _raw_type> { if (!_ptr) { return std::nullopt; } - return Parser, _raw_type>::write(*_ptr); + return Parser>::write(*_ptr); } static dynamic::Type to_type() noexcept { - return Parser, _raw_type>::to_type().visit( + return Parser>::to_type().visit( [](auto _t) -> dynamic::Type { _t.properties.nullable = true; return _t; diff --git a/include/sqlgen/parsing/Parser_string.hpp b/include/sqlgen/parsing/Parser_string.hpp index cec2191b..66ab60de 100644 --- a/include/sqlgen/parsing/Parser_string.hpp +++ b/include/sqlgen/parsing/Parser_string.hpp @@ -8,12 +8,11 @@ #include "../dynamic/Type.hpp" #include "../dynamic/types.hpp" #include "Parser_base.hpp" -#include "RawType.hpp" namespace sqlgen::parsing { -template -struct Parser { +template <> +struct Parser { static Result read( const std::optional& _str) noexcept { if (!_str) { diff --git a/include/sqlgen/parsing/Parser_timestamp.hpp b/include/sqlgen/parsing/Parser_timestamp.hpp index 5e990619..f3e8446a 100644 --- a/include/sqlgen/parsing/Parser_timestamp.hpp +++ b/include/sqlgen/parsing/Parser_timestamp.hpp @@ -10,23 +10,22 @@ #include "../dynamic/types.hpp" #include "Parser_base.hpp" #include "Parser_default.hpp" -#include "RawType.hpp" namespace sqlgen::parsing { -template -struct Parser, _raw_type> { +template +struct Parser> { using TSType = Timestamp<_format>; static Result read(const std::optional& _str) noexcept { - return Parser::read(_str).and_then( + return Parser::read(_str).and_then( [](auto&& _s) -> Result { return TSType::from_string(std::move(_s)); }); } static std::optional write(const TSType& _t) noexcept { - return Parser::write(_t.str()); + return Parser::write(_t.str()); } static dynamic::Type to_type() noexcept { diff --git a/include/sqlgen/parsing/Parser_unique.hpp b/include/sqlgen/parsing/Parser_unique.hpp index 14e90443..dd4200c0 100644 --- a/include/sqlgen/parsing/Parser_unique.hpp +++ b/include/sqlgen/parsing/Parser_unique.hpp @@ -8,24 +8,23 @@ #include "../Unique.hpp" #include "../dynamic/Type.hpp" #include "Parser_base.hpp" -#include "RawType.hpp" namespace sqlgen::parsing { -template -struct Parser, _raw_type> { +template +struct Parser> { static Result> read( const std::optional& _str) noexcept { - return Parser, _raw_type>::read(_str).transform( + return Parser>::read(_str).transform( [](auto&& _t) { return Unique(std::move(_t)); }); } static std::optional write(const Unique& _f) noexcept { - return Parser, _raw_type>::write(_f.value()); + return Parser>::write(_f.value()); } static dynamic::Type to_type() noexcept { - return Parser, _raw_type>::to_type().visit( + return Parser>::to_type().visit( [](auto _t) -> dynamic::Type { _t.properties.unique = true; return _t; diff --git a/include/sqlgen/parsing/Parser_unique_ptr.hpp b/include/sqlgen/parsing/Parser_unique_ptr.hpp index 896f7a75..cb114d99 100644 --- a/include/sqlgen/parsing/Parser_unique_ptr.hpp +++ b/include/sqlgen/parsing/Parser_unique_ptr.hpp @@ -8,18 +8,17 @@ #include "../Result.hpp" #include "../dynamic/Type.hpp" #include "Parser_base.hpp" -#include "RawType.hpp" namespace sqlgen::parsing { -template -struct Parser, _raw_type> { +template +struct Parser> { static Result> read( const std::optional& _str) noexcept { if (!_str) { return std::unique_ptr(); } - return Parser, _raw_type>::read(_str).transform( + return Parser>::read(_str).transform( [](auto&& _t) -> std::unique_ptr { return std::make_unique(std::move(_t)); }); @@ -30,11 +29,11 @@ struct Parser, _raw_type> { if (!_ptr) { return std::nullopt; } - return Parser, _raw_type>::write(*_ptr); + return Parser>::write(*_ptr); } static dynamic::Type to_type() noexcept { - return Parser, _raw_type>::to_type().visit( + return Parser>::to_type().visit( [](auto _t) -> dynamic::Type { _t.properties.nullable = true; return _t; diff --git a/include/sqlgen/parsing/Parser_varchar.hpp b/include/sqlgen/parsing/Parser_varchar.hpp index bd75ad60..489b08bf 100644 --- a/include/sqlgen/parsing/Parser_varchar.hpp +++ b/include/sqlgen/parsing/Parser_varchar.hpp @@ -10,22 +10,21 @@ #include "../dynamic/types.hpp" #include "Parser_base.hpp" #include "Parser_default.hpp" -#include "RawType.hpp" namespace sqlgen::parsing { -template -struct Parser, _raw_type> { +template +struct Parser> { static Result> read( const std::optional& _str) noexcept { - return Parser::read(_str).and_then( + return Parser::read(_str).and_then( [](auto&& _t) -> Result> { return Varchar<_size>::make(std::move(_t)); }); } static std::optional write(const Varchar<_size>& _v) noexcept { - return Parser::write(_v.value()); + return Parser::write(_v.value()); } static dynamic::Type to_type() noexcept { diff --git a/include/sqlgen/parsing/RawType.hpp b/include/sqlgen/parsing/RawType.hpp deleted file mode 100644 index 13e4719a..00000000 --- a/include/sqlgen/parsing/RawType.hpp +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef SQLGEN_PARSING_RAWTYPE_HPP_ -#define SQLGEN_PARSING_RAWTYPE_HPP_ - -namespace sqlgen::parsing { - -enum class RawType { original, string }; - -} - -#endif diff --git a/include/sqlgen/transpilation/make_columns.hpp b/include/sqlgen/transpilation/make_columns.hpp index 39a09422..833882a3 100644 --- a/include/sqlgen/transpilation/make_columns.hpp +++ b/include/sqlgen/transpilation/make_columns.hpp @@ -9,7 +9,6 @@ #include "../dynamic/Type.hpp" #include "../dynamic/types.hpp" #include "../parsing/Parser.hpp" -#include "../parsing/RawType.hpp" #include "has_reflection_method.hpp" #include "is_nullable.hpp" #include "is_primary_key.hpp" @@ -24,7 +23,7 @@ std::string to_colname() { template dynamic::Type to_type() { using T = std::remove_cvref_t; - return parsing::Parser::to_type(); + return parsing::Parser::to_type(); } template diff --git a/include/sqlgen/transpilation/make_field.hpp b/include/sqlgen/transpilation/make_field.hpp index 76930718..92d8cb08 100644 --- a/include/sqlgen/transpilation/make_field.hpp +++ b/include/sqlgen/transpilation/make_field.hpp @@ -24,7 +24,6 @@ #include "is_timestamp.hpp" #include "remove_as_t.hpp" #include "remove_nullable_t.hpp" -#include "sqlgen/parsing/RawType.hpp" #include "to_alias.hpp" #include "to_duration.hpp" #include "to_value.hpp" @@ -233,8 +232,7 @@ struct MakeField, - parsing::RawType::string>::to_type()}}}; + parsing::Parser>::to_type()}}}; } }; From a8ed2d1be1a8e90bbf60ba7449a19ca30775510b Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sun, 9 Nov 2025 00:04:51 +0100 Subject: [PATCH 15/48] Made sure we can check more complex types --- include/sqlgen/duckdb/Connection.hpp | 98 ++----------------- include/sqlgen/duckdb/chunk_ptrs_t.hpp | 4 +- include/sqlgen/duckdb/from_chunk_ptrs.hpp | 23 +++-- include/sqlgen/duckdb/parsing/Parser.hpp | 11 +++ include/sqlgen/duckdb/parsing/Parser_base.hpp | 11 +++ .../sqlgen/duckdb/parsing/Parser_default.hpp | 94 ++++++++++++++++++ .../sqlgen/duckdb/parsing/Parser_optional.hpp | 43 ++++++++ .../duckdb/parsing/Parser_reflection_type.hpp | 36 +++++++ .../duckdb/parsing/Parser_smart_ptr.hpp | 46 +++++++++ .../sqlgen/duckdb/parsing/Parser_string.hpp | 40 ++++++++ tests/duckdb/test_write_and_read.cpp | 13 +-- 11 files changed, 312 insertions(+), 107 deletions(-) create mode 100644 include/sqlgen/duckdb/parsing/Parser.hpp create mode 100644 include/sqlgen/duckdb/parsing/Parser_base.hpp create mode 100644 include/sqlgen/duckdb/parsing/Parser_default.hpp create mode 100644 include/sqlgen/duckdb/parsing/Parser_optional.hpp create mode 100644 include/sqlgen/duckdb/parsing/Parser_reflection_type.hpp create mode 100644 include/sqlgen/duckdb/parsing/Parser_smart_ptr.hpp create mode 100644 include/sqlgen/duckdb/parsing/Parser_string.hpp diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index 44f057b9..7092bb01 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include "../Range.hpp" #include "../Ref.hpp" @@ -22,8 +23,10 @@ #include "../transpilation/get_tablename.hpp" #include "../transpilation/has_reflection_method.hpp" #include "../transpilation/is_nullable.hpp" +#include "./parsing/Parser_default.hpp" #include "DuckDBConnection.hpp" #include "Iterator.hpp" +#include "parsing/Parser.hpp" #include "to_sql.hpp" namespace sqlgen::duckdb { @@ -93,103 +96,16 @@ class Connection { } private: - template - Result append_value(const T& _t, - duckdb_appender _appender) noexcept { - using Type = std::remove_cvref_t; - - if constexpr (transpilation::has_reflection_method) { - return append_value(_t.reflection(), _appender); - - } else if constexpr (std::is_same_v) { - return duckdb_append_bool(_appender, _t) != DuckDBError - ? Result(Nothing{}) - : Result(error("Could not append boolean value.")); - - } else if constexpr (std::is_same_v || - std::is_same_v) { - return duckdb_append_int8(_appender, _t) != DuckDBError - ? Result(Nothing{}) - : Result(error("Could not append int8 value.")); - - } else if constexpr (std::is_same_v) { - return duckdb_append_uint8(_appender, _t) != DuckDBError - ? Result(Nothing{}) - : Result(error("Could not append uint8 value.")); - - } else if constexpr (std::is_same_v) { - return duckdb_append_int16(_appender, _t) != DuckDBError - ? Result(Nothing{}) - : Result(error("Could not append int16 value.")); - - } else if constexpr (std::is_same_v) { - return duckdb_append_uint16(_appender, _t) != DuckDBError - ? Result(Nothing{}) - : Result(error("Could not append uint16 value.")); - - } else if constexpr (std::is_same_v) { - return duckdb_append_int32(_appender, _t) != DuckDBError - ? Result(Nothing{}) - : Result(error("Could not append int32 value.")); - - } else if constexpr (std::is_same_v) { - return duckdb_append_uint32(_appender, _t) != DuckDBError - ? Result(Nothing{}) - : Result(error("Could not append uint32 value.")); - - } else if constexpr (std::is_same_v) { - return duckdb_append_int64(_appender, _t) != DuckDBError - ? Result(Nothing{}) - : Result(error("Could not append int64 value.")); - - } else if constexpr (std::is_same_v) { - return duckdb_append_uint64(_appender, _t) != DuckDBError - ? Result(Nothing{}) - : Result(error("Could not append uint64 value.")); - - } else if constexpr (std::is_same_v) { - return duckdb_append_float(_appender, _t) != DuckDBError - ? Result(Nothing{}) - : Result(error("Could not append float value.")); - - } else if constexpr (std::is_same_v) { - return duckdb_append_double(_appender, _t) != DuckDBError - ? Result(Nothing{}) - : Result(error("Could not append double value.")); - - } else if constexpr (std::is_same_v) { - return duckdb_append_varchar(_appender, _t.c_str()) != DuckDBError - ? Result(Nothing{}) - : Result(error("Could not append string value.")); - - } else if constexpr (std::is_same_v) { - return duckdb_append_varchar_length(_appender, _t.c_str(), _t.length()) != - DuckDBError - ? Result(Nothing{}) - : Result(error("Could not append string value.")); - - } else if constexpr (transpilation::is_nullable_v) { - if (_t) { - return append_value(*_t, _appender); - } else { - return duckdb_append_null(_appender) != DuckDBError - ? Result(Nothing{}) - : Result(error("Could not append null value.")); - } - - } else { - static_assert(rfl::always_false_v, "Unsupported type."); - return error("Unsupported type."); - } - } - template Result write_row(const StructT& _struct, duckdb_appender _appender) noexcept { Result res = Nothing{}; rfl::to_view(_struct).apply([&](const auto& _field) { + using ValueType = std::remove_cvref_t::Type>>; if (res) { - res = append_value(*_field.value(), _appender); + res = duckdb::parsing::Parser::write(*_field.value(), + _appender); } }); return res; diff --git a/include/sqlgen/duckdb/chunk_ptrs_t.hpp b/include/sqlgen/duckdb/chunk_ptrs_t.hpp index 0e161de5..7969bad1 100644 --- a/include/sqlgen/duckdb/chunk_ptrs_t.hpp +++ b/include/sqlgen/duckdb/chunk_ptrs_t.hpp @@ -4,6 +4,7 @@ #include #include +#include "./parsing/Parser.hpp" #include "ColumnData.hpp" namespace sqlgen::duckdb { @@ -13,7 +14,8 @@ struct ChunkPtrsType; template struct ChunkPtrsType> { - using Type = rfl::Tuple...>; + using Type = rfl::Tuple::ResultingType>...>; }; template diff --git a/include/sqlgen/duckdb/from_chunk_ptrs.hpp b/include/sqlgen/duckdb/from_chunk_ptrs.hpp index 3ca6d05c..2305992b 100644 --- a/include/sqlgen/duckdb/from_chunk_ptrs.hpp +++ b/include/sqlgen/duckdb/from_chunk_ptrs.hpp @@ -4,31 +4,40 @@ #include #include +#include #include #include #include "../Result.hpp" +#include "./parsing/Parser_default.hpp" #include "ColumnData.hpp" #include "chunk_ptrs_t.hpp" namespace sqlgen::duckdb { -template +template struct FromChunkPtrs; -template -struct FromChunkPtrs...>> { +template +struct FromChunkPtrs, + rfl::Tuple...>> { Result operator()(const rfl::Tuple...>& _chunk_ptrs, idx_t _i) { - return [&](std::integer_sequence) { - // TODO: Integrate this into the parser logic. - return T{*(rfl::get<_is>(_chunk_ptrs).data + _i)...}; + return [&](std::integer_sequence) -> Result { + try { + return T{duckdb::parsing::Parser::read( + rfl::get<_is>(_chunk_ptrs).data + _i) + .value()...}; + } catch (const std::exception& e) { + return error(e.what()); + } }(std::make_integer_sequence()); } }; template -auto from_chunk_ptrs = FromChunkPtrs, chunk_ptrs_t>{}; +auto from_chunk_ptrs = FromChunkPtrs, + rfl::named_tuple_t, chunk_ptrs_t>{}; } // namespace sqlgen::duckdb diff --git a/include/sqlgen/duckdb/parsing/Parser.hpp b/include/sqlgen/duckdb/parsing/Parser.hpp new file mode 100644 index 00000000..58985f87 --- /dev/null +++ b/include/sqlgen/duckdb/parsing/Parser.hpp @@ -0,0 +1,11 @@ +#ifndef SQLGEN_DUCKDB_PARSING_PARSER_HPP_ +#define SQLGEN_DUCKDB_PARSING_PARSER_HPP_ + +#include "Parser_base.hpp" +#include "Parser_default.hpp" +#include "Parser_optional.hpp" +#include "Parser_reflection_type.hpp" +#include "Parser_smart_ptr.hpp" +#include "Parser_string.hpp" + +#endif diff --git a/include/sqlgen/duckdb/parsing/Parser_base.hpp b/include/sqlgen/duckdb/parsing/Parser_base.hpp new file mode 100644 index 00000000..dd39974b --- /dev/null +++ b/include/sqlgen/duckdb/parsing/Parser_base.hpp @@ -0,0 +1,11 @@ +#ifndef SQLGEN_DUCKDB_PARSING_PARSER_BASE_HPP_ +#define SQLGEN_DUCKDB_PARSING_PARSER_BASE_HPP_ + +namespace sqlgen::duckdb::parsing { + +template +struct Parser; + +} + +#endif diff --git a/include/sqlgen/duckdb/parsing/Parser_default.hpp b/include/sqlgen/duckdb/parsing/Parser_default.hpp new file mode 100644 index 00000000..400cf0b6 --- /dev/null +++ b/include/sqlgen/duckdb/parsing/Parser_default.hpp @@ -0,0 +1,94 @@ +#ifndef SQLGEN_DUCKDB_PARSING_PARSER_DEFAULT_HPP_ +#define SQLGEN_DUCKDB_PARSING_PARSER_DEFAULT_HPP_ + +#include + +#include +#include +#include + +#include "../../Result.hpp" +#include "Parser_base.hpp" + +namespace sqlgen::duckdb::parsing { + +template +struct Parser { + using Type = std::remove_cvref_t; + using ResultingType = Type; + + static Result read(const ResultingType* _r) noexcept { + if (!_r) { + return error("Numeric or boolean value cannot be NULL."); + } + return Type(*_r); + } + + static Result write(const T& _t, + duckdb_appender _appender) noexcept { + if constexpr (std::is_same_v) { + return duckdb_append_bool(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append boolean value.")); + + } else if constexpr (std::is_same_v || + std::is_same_v) { + return duckdb_append_int8(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append int8 value.")); + + } else if constexpr (std::is_same_v) { + return duckdb_append_uint8(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append uint8 value.")); + + } else if constexpr (std::is_same_v) { + return duckdb_append_int16(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append int16 value.")); + + } else if constexpr (std::is_same_v) { + return duckdb_append_uint16(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append uint16 value.")); + + } else if constexpr (std::is_same_v) { + return duckdb_append_int32(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append int32 value.")); + + } else if constexpr (std::is_same_v) { + return duckdb_append_uint32(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append uint32 value.")); + + } else if constexpr (std::is_same_v) { + return duckdb_append_int64(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append int64 value.")); + + } else if constexpr (std::is_same_v) { + return duckdb_append_uint64(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append uint64 value.")); + + } else if constexpr (std::is_same_v) { + return duckdb_append_float(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append float value.")); + + } else if constexpr (std::is_same_v) { + return duckdb_append_double(_appender, _t) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append double value.")); + + } else { + static_assert(rfl::always_false_v, "Unsupported type."); + return error("Unsupported type."); + } + } +}; + +} // namespace sqlgen::duckdb::parsing + +#endif diff --git a/include/sqlgen/duckdb/parsing/Parser_optional.hpp b/include/sqlgen/duckdb/parsing/Parser_optional.hpp new file mode 100644 index 00000000..edb0211b --- /dev/null +++ b/include/sqlgen/duckdb/parsing/Parser_optional.hpp @@ -0,0 +1,43 @@ +#ifndef SQLGEN_DUCKDB_PARSING_PARSER_OPTIONAL_HPP_ +#define SQLGEN_DUCKDB_PARSING_PARSER_OPTIONAL_HPP_ + +#include + +#include +#include +#include + +#include "../../Result.hpp" +#include "Parser_base.hpp" + +namespace sqlgen::duckdb::parsing { + +template +struct Parser> { + using Type = std::remove_cvref_t; + using ResultingType = typename Parser::ResultingType; + + static Result> read(const ResultingType* _r) noexcept { + if (!_r) { + return std::optional(); + } + return Parser>::read(_r).transform( + [](auto&& _t) -> std::optional { + return std::make_optional(std::move(_t)); + }); + } + + static Result write(const std::optional& _o, + duckdb_appender _appender) noexcept { + if (!_o) { + return duckdb_append_null(_appender) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append null value.")); + } + return Parser>::write(*_o); + } +}; + +} // namespace sqlgen::duckdb::parsing + +#endif diff --git a/include/sqlgen/duckdb/parsing/Parser_reflection_type.hpp b/include/sqlgen/duckdb/parsing/Parser_reflection_type.hpp new file mode 100644 index 00000000..7a347a61 --- /dev/null +++ b/include/sqlgen/duckdb/parsing/Parser_reflection_type.hpp @@ -0,0 +1,36 @@ +#ifndef SQLGEN_DUCKDB_PARSING_PARSER_FOREIGN_KEY_HPP_ +#define SQLGEN_DUCKDB_PARSING_PARSER_FOREIGN_KEY_HPP_ + +#include + +#include +#include + +#include "../../Result.hpp" +#include "../../transpilation/has_reflection_method.hpp" +#include "Parser_base.hpp" + +namespace sqlgen::duckdb::parsing { + +template + requires transpilation::has_reflection_method> +struct Parser { + using Type = std::remove_cvref_t; + using ResultingType = + typename Parser::ResultingType; + + static Result read(const ResultingType* _r) noexcept { + return Parser>::read(_r) + .transform([](auto&& _t) { return T(std::move(_t)); }); + } + + static Result write(const T& _t, + duckdb_appender _appender) noexcept { + return Parser>::write( + _t.reflection(), _appender); + } +}; + +} // namespace sqlgen::duckdb::parsing + +#endif diff --git a/include/sqlgen/duckdb/parsing/Parser_smart_ptr.hpp b/include/sqlgen/duckdb/parsing/Parser_smart_ptr.hpp new file mode 100644 index 00000000..6863f5a6 --- /dev/null +++ b/include/sqlgen/duckdb/parsing/Parser_smart_ptr.hpp @@ -0,0 +1,46 @@ +#ifndef SQLGEN_DUCKDB_PARSING_PARSER_SMART_PTR_HPP_ +#define SQLGEN_DUCKDB_PARSING_PARSER_SMART_PTR_HPP_ + +#include + +#include +#include + +#include "../../Result.hpp" +#include "../../transpilation/is_nullable.hpp" +#include "Parser_base.hpp" + +namespace sqlgen::duckdb::parsing { + +template + requires transpilation::is_nullable_v> +struct Parser { + using Type = std::remove_cvref_t; + using ResultingType = + typename Parser::ResultingType; + + static Result read(const ResultingType* _r) noexcept { + if (!_r) { + return T(); + } + return Parser::read(_r).transform( + [](auto&& _u) -> T { + using U = std::remove_cvref; + return T(new U(std::move(_u))); + }); + } + + static Result write(const T& _ptr, + duckdb_appender _appender) noexcept { + if (!_ptr) { + return duckdb_append_null(_appender) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append null value.")); + } + return Parser>::write(*_ptr); + } +}; + +} // namespace sqlgen::duckdb::parsing + +#endif diff --git a/include/sqlgen/duckdb/parsing/Parser_string.hpp b/include/sqlgen/duckdb/parsing/Parser_string.hpp new file mode 100644 index 00000000..27cb341b --- /dev/null +++ b/include/sqlgen/duckdb/parsing/Parser_string.hpp @@ -0,0 +1,40 @@ +#ifndef SQLGEN_DUCKDB_PARSING_PARSER_STRING_HPP_ +#define SQLGEN_DUCKDB_PARSING_PARSER_STRING_HPP_ + +#include + +#include +#include + +#include "../../Result.hpp" +#include "Parser_base.hpp" + +namespace sqlgen::duckdb::parsing { + +template <> +struct Parser { + using ResultingType = duckdb_string_t; + + static Result read(const ResultingType* _r) noexcept { + if (!_r) { + return error("String value cannot be NULL."); + } + if (duckdb_string_is_inlined(*_r)) { + return std::string(_r->value.inlined.inlined, _r->value.inlined.length); + } else { + return std::string(_r->value.pointer.ptr, _r->value.pointer.length); + } + } + + static Result write(const std::string& _t, + duckdb_appender _appender) noexcept { + return duckdb_append_varchar_length(_appender, _t.c_str(), _t.length()) != + DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append string value.")); + } +}; + +} // namespace sqlgen::duckdb::parsing + +#endif diff --git a/tests/duckdb/test_write_and_read.cpp b/tests/duckdb/test_write_and_read.cpp index c84068bc..e4c7e596 100644 --- a/tests/duckdb/test_write_and_read.cpp +++ b/tests/duckdb/test_write_and_read.cpp @@ -8,23 +8,20 @@ namespace test_write_and_read { struct Person { - // sqlgen::PrimaryKey id; - // std::string first_name; - // std::string last_name; + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; int age; }; TEST(duckdb, test_write_and_read) { - /*const auto people1 = std::vector( + const auto people1 = std::vector( {Person{ .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, Person{ - .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = - 0}});*/ - - const auto people1 = std::vector({Person{.age = 45}}); + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); using namespace sqlgen; From 9d2d17b4391a3cf284c512ca770b4b836ec2bdc5 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sun, 9 Nov 2025 01:10:09 +0100 Subject: [PATCH 16/48] Added type checks --- include/sqlgen/duckdb/Iterator.hpp | 2 +- include/sqlgen/duckdb/get_duckdb_type.hpp | 61 +++++++++++++++++++ include/sqlgen/duckdb/make_chunk_ptrs.hpp | 32 +++++++--- .../duckdb/parsing/Parser_reflection_type.hpp | 4 +- src/sqlgen/duckdb/to_sql.cpp | 36 +++++++---- 5 files changed, 112 insertions(+), 23 deletions(-) create mode 100644 include/sqlgen/duckdb/get_duckdb_type.hpp diff --git a/include/sqlgen/duckdb/Iterator.hpp b/include/sqlgen/duckdb/Iterator.hpp index 609c05c9..767a7b01 100644 --- a/include/sqlgen/duckdb/Iterator.hpp +++ b/include/sqlgen/duckdb/Iterator.hpp @@ -73,7 +73,7 @@ class Iterator { return Ref>>::make(); } const idx_t row_count = duckdb_data_chunk_get_size(chunk); - return make_chunk_ptrs(chunk) + return make_chunk_ptrs(_res, chunk) .transform([&](auto&& _chunk_ptrs) { auto batch = Ref>>::make(); for (idx_t i = 0; i < row_count; ++i) { diff --git a/include/sqlgen/duckdb/get_duckdb_type.hpp b/include/sqlgen/duckdb/get_duckdb_type.hpp new file mode 100644 index 00000000..20583998 --- /dev/null +++ b/include/sqlgen/duckdb/get_duckdb_type.hpp @@ -0,0 +1,61 @@ +#ifndef SQLGEN_DUCKDB_GETDUCKDBTYPE_HPP_ +#define SQLGEN_DUCKDB_GETDUCKDBTYPE_HPP_ + +#include + +#include +#include + +#include "../Result.hpp" + +namespace sqlgen::duckdb { + +template +duckdb_type get_duckdb_type() { + using Type = std::remove_cvref_t; + + if constexpr (std::is_same_v) { + return DUCKDB_TYPE_BOOLEAN; + + } else if constexpr (std::is_same_v || + std::is_same_v) { + return DUCKDB_TYPE_TINYINT; + + } else if constexpr (std::is_same_v) { + return DUCKDB_TYPE_UTINYINT; + + } else if constexpr (std::is_same_v) { + return DUCKDB_TYPE_SMALLINT; + + } else if constexpr (std::is_same_v) { + return DUCKDB_TYPE_USMALLINT; + + } else if constexpr (std::is_same_v) { + return DUCKDB_TYPE_INTEGER; + + } else if constexpr (std::is_same_v) { + return DUCKDB_TYPE_UINTEGER; + + } else if constexpr (std::is_same_v) { + return DUCKDB_TYPE_BIGINT; + + } else if constexpr (std::is_same_v) { + return DUCKDB_TYPE_UBIGINT; + + } else if constexpr (std::is_same_v) { + return DUCKDB_TYPE_FLOAT; + + } else if constexpr (std::is_same_v) { + return DUCKDB_TYPE_DOUBLE; + + } else if constexpr (std::is_same_v) { + return DUCKDB_TYPE_VARCHAR; + + } else { + static_assert(rfl::always_false_v, "Unsupported type."); + } +} + +} // namespace sqlgen::duckdb + +#endif diff --git a/include/sqlgen/duckdb/make_chunk_ptrs.hpp b/include/sqlgen/duckdb/make_chunk_ptrs.hpp index 2892eb79..c0c7347e 100644 --- a/include/sqlgen/duckdb/make_chunk_ptrs.hpp +++ b/include/sqlgen/duckdb/make_chunk_ptrs.hpp @@ -4,11 +4,14 @@ #include #include +#include #include #include +#include "./parsing/Parser.hpp" #include "ColumnData.hpp" #include "chunk_ptrs_t.hpp" +#include "get_duckdb_type.hpp" namespace sqlgen::duckdb { @@ -17,16 +20,27 @@ struct MakeChunkPtrs; template struct MakeChunkPtrs...>> { - Result...>> operator()(duckdb_data_chunk _chunk) { - // TODO: Runtime type checking - return [&](std::integer_sequence) { - return rfl::Tuple...>( - make_column_data(_chunk)...); - }(std::make_integer_sequence()); + Result...>> operator()( + const Ref& _res, duckdb_data_chunk _chunk) { + try { + return [&](std::integer_sequence) { + return rfl::Tuple...>( + make_column_data(_res, _chunk)...); + }(std::make_integer_sequence()); + } catch (const std::exception& e) { + return error(e.what()); + } } template - static auto make_column_data(duckdb_data_chunk _chunk) { + static auto make_column_data(const Ref& _res, + duckdb_data_chunk _chunk) { + if (duckdb_column_type(_res.get(), _i) != get_duckdb_type()) { + throw std::runtime_error( + "Wrong type in field " + std::to_string(_i) + ". Expected " + + rfl::enum_to_string(get_duckdb_type()) + ", got " + + rfl::enum_to_string(duckdb_column_type(_res.get(), _i)) + "."); + } auto vec = duckdb_data_chunk_get_vector(_chunk, _i); return ColumnData{.vec = vec, .data = static_cast(duckdb_vector_get_data(vec)), @@ -36,8 +50,8 @@ struct MakeChunkPtrs...>> { template struct MakeChunkPtrs { - auto operator()(duckdb_data_chunk _chunk) { - return MakeChunkPtrs>{}(_chunk); + auto operator()(const Ref& _res, duckdb_data_chunk _chunk) { + return MakeChunkPtrs>{}(_res, _chunk); } }; diff --git a/include/sqlgen/duckdb/parsing/Parser_reflection_type.hpp b/include/sqlgen/duckdb/parsing/Parser_reflection_type.hpp index 7a347a61..8f3a0f10 100644 --- a/include/sqlgen/duckdb/parsing/Parser_reflection_type.hpp +++ b/include/sqlgen/duckdb/parsing/Parser_reflection_type.hpp @@ -16,8 +16,8 @@ template requires transpilation::has_reflection_method> struct Parser { using Type = std::remove_cvref_t; - using ResultingType = - typename Parser::ResultingType; + using ResultingType = typename Parser< + std::remove_cvref_t>::ResultingType; static Result read(const ResultingType* _r) noexcept { return Parser>::read(_r) diff --git a/src/sqlgen/duckdb/to_sql.cpp b/src/sqlgen/duckdb/to_sql.cpp index 59413d16..421e4693 100644 --- a/src/sqlgen/duckdb/to_sql.cpp +++ b/src/sqlgen/duckdb/to_sql.cpp @@ -811,24 +811,38 @@ std::string type_to_sql(const dynamic::Type& _type) noexcept { } else if constexpr (std::is_same_v) { return _t.type_name; - } else if constexpr (std::is_same_v || - std::is_same_v || - std::is_same_v || - std::is_same_v) { + } else if constexpr (std::is_same_v) { + return "TINYINT"; + + } else if constexpr (std::is_same_v) { + return "UTINYINT"; + + } else if constexpr (std::is_same_v) { return "SMALLINT"; - } else if constexpr (std::is_same_v || - std::is_same_v) { + } else if constexpr (std::is_same_v) { + return "USMALLINT"; + + } else if constexpr (std::is_same_v) { return "INTEGER"; - } else if constexpr (std::is_same_v || - std::is_same_v) { + } else if constexpr (std::is_same_v) { + return "UINTEGER"; + + } else if constexpr (std::is_same_v) { return "BIGINT"; + + } else if constexpr (std::is_same_v) { + return "UBIGINT"; + } else if constexpr (std::is_same_v) { return _t.name; - } else if constexpr (std::is_same_v || - std::is_same_v) { - return "NUMERIC"; + + } else if constexpr (std::is_same_v) { + return "FLOAT"; + + } else if constexpr (std::is_same_v) { + return "DOUBLE"; } else if constexpr (std::is_same_v) { return "TEXT"; From b20f211874860cf505d20562ede9a96986fda146 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sun, 9 Nov 2025 10:48:41 +0100 Subject: [PATCH 17/48] Added NULL checks --- include/sqlgen/duckdb/ColumnData.hpp | 4 ++++ include/sqlgen/duckdb/from_chunk_ptrs.hpp | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/include/sqlgen/duckdb/ColumnData.hpp b/include/sqlgen/duckdb/ColumnData.hpp index ada64253..2bbc8c7c 100644 --- a/include/sqlgen/duckdb/ColumnData.hpp +++ b/include/sqlgen/duckdb/ColumnData.hpp @@ -10,6 +10,10 @@ struct ColumnData { duckdb_vector vec; T *data; uint64_t *validity; + + bool is_not_null(idx_t _i) const { + return (validity == nullptr) || duckdb_validity_row_is_valid(validity, _i); + } }; } // namespace sqlgen::duckdb diff --git a/include/sqlgen/duckdb/from_chunk_ptrs.hpp b/include/sqlgen/duckdb/from_chunk_ptrs.hpp index 2305992b..0517bb4b 100644 --- a/include/sqlgen/duckdb/from_chunk_ptrs.hpp +++ b/include/sqlgen/duckdb/from_chunk_ptrs.hpp @@ -26,7 +26,9 @@ struct FromChunkPtrs, return [&](std::integer_sequence) -> Result { try { return T{duckdb::parsing::Parser::read( - rfl::get<_is>(_chunk_ptrs).data + _i) + rfl::get<_is>(_chunk_ptrs).is_not_null(_i) + ? rfl::get<_is>(_chunk_ptrs).data + _i + : nullptr) .value()...}; } catch (const std::exception& e) { return error(e.what()); From b833423cf41553fe6b419402f660819a279b4f7e Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sun, 9 Nov 2025 12:01:32 +0100 Subject: [PATCH 18/48] More sophisticated approach for appender_create --- include/sqlgen/duckdb/Connection.hpp | 87 ++++++++++++++++++++++------ 1 file changed, 68 insertions(+), 19 deletions(-) diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index 7092bb01..1f39b146 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -27,6 +27,10 @@ #include "DuckDBConnection.hpp" #include "Iterator.hpp" #include "parsing/Parser.hpp" +#include "sqlgen/dynamic/Operation.hpp" +#include "sqlgen/dynamic/SelectFrom.hpp" +#include "sqlgen/transpilation/get_schema.hpp" +#include "sqlgen/transpilation/to_table_or_query.hpp" #include "to_sql.hpp" namespace sqlgen::duckdb { @@ -35,10 +39,10 @@ class Connection { using ConnPtr = Ref; public: - Connection(const ConnPtr& _conn) : conn_(_conn) {} + Connection(const ConnPtr &_conn) : conn_(_conn) {} static rfl::Result> make( - const std::optional& _fname) noexcept; + const std::optional &_fname) noexcept; ~Connection() = default; @@ -46,16 +50,30 @@ class Connection { Result commit() noexcept; - Result execute(const std::string& _sql) noexcept; + Result execute(const std::string &_sql) noexcept; template - Result insert(const dynamic::Insert&, ItBegin _begin, + Result insert(const dynamic::Insert &_insert_stmt, ItBegin _begin, ItEnd _end) noexcept { - return error("TODO"); + const auto sql = to_sql(_insert_stmt); + + return get_duckdb_logical_types(_insert_stmt.table, _insert_stmt.columns) + .and_then([&](const auto &_types) -> Result { + duckdb_appender appender{}; + if (duckdb_appender_create_query( + conn_->conn(), sql.c_str(), + static_cast(_insert_stmt.columns.size()), + _types.data(), nullptr, nullptr, &appender) == DuckDBError) { + return error("Could not create appender."); + } + const auto res = write_to_appender(_begin, _end, appender); + duckdb_appender_destroy(&appender); + return res; + }); } template - Result read(const dynamic::SelectFrom& _query) { + Result read(const dynamic::SelectFrom &_query) { using ValueType = transpilation::value_t; auto res = Ref(); duckdb_query(conn_->conn(), to_sql(_query).c_str(), res.get()); @@ -65,11 +83,11 @@ class Connection { Result rollback() noexcept; - std::string to_sql(const dynamic::Statement& _stmt) noexcept { + std::string to_sql(const dynamic::Statement &_stmt) noexcept { return duckdb::to_sql_impl(_stmt); } - Result start_write(const dynamic::Write&) { return Nothing{}; } + Result start_write(const dynamic::Write &) { return Nothing{}; } Result end_write() { return Nothing{}; } @@ -77,30 +95,62 @@ class Connection { Result write(ItBegin _begin, ItEnd _end) { using T = std::remove_cvref_t::value_type>; - const auto tablename = transpilation::get_tablename(); + const auto schema = transpilation::get_schema(); + const auto table = transpilation::get_tablename(); duckdb_appender appender{}; - if (duckdb_appender_create(conn_->conn(), nullptr, tablename.c_str(), - &appender) == DuckDBError) { + if (duckdb_appender_create(conn_->conn(), + schema ? schema->c_str() : nullptr, + table.c_str(), &appender) == DuckDBError) { return error("Could not create appender."); } + const auto res = write_to_appender(_begin, _end, appender); + duckdb_appender_destroy(&appender); + return res; + } + + private: + Result> get_duckdb_logical_types( + const dynamic::Table &_table, const std::vector &_columns) { + using namespace std::ranges::views; + + const auto fields = internal::collect::vector( + _columns | transform([](const auto &_name) { + return dynamic::SelectFrom::Field{ + .val = dynamic::Operation{dynamic::Column{.alias = std::nullopt, + .name = _name}}, + .as = std::nullopt}; + })); + + const auto select_from = dynamic::SelectFrom{ + .table_or_query = _table, .fields = fields, .limit = dynamic::Limit{0}}; + + auto res = Ref(); + + duckdb_query(conn_->conn(), to_sql(select_from).c_str(), res.get()); + + return internal::collect::vector( + iota(static_cast(fields.size())) | + transform(std::bind_front(duckdb_column_logical_type, res.get()))); + } + + template + Result write_to_appender(ItBegin _begin, ItEnd _end, + duckdb_appender _appender) { for (auto it = _begin; it < _end; ++it) { - const auto res = write_row(*it, appender); + const auto res = write_row(*it, _appender); if (!res) { - duckdb_appender_destroy(&appender); return res; } - duckdb_appender_end_row(appender); + duckdb_appender_end_row(_appender); } - duckdb_appender_destroy(&appender); return Nothing{}; } - private: template - Result write_row(const StructT& _struct, + Result write_row(const StructT &_struct, duckdb_appender _appender) noexcept { Result res = Nothing{}; - rfl::to_view(_struct).apply([&](const auto& _field) { + rfl::to_view(_struct).apply([&](const auto &_field) { using ValueType = std::remove_cvref_t::Type>>; if (res) { @@ -124,7 +174,6 @@ static_assert(is_connection>, } // namespace sqlgen::duckdb namespace sqlgen::internal { - template struct IteratorType { using Type = duckdb::Iterator; From a56edd6015ecfd8be65696df02fba3824ca5a9da Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sun, 9 Nov 2025 12:26:14 +0100 Subject: [PATCH 19/48] More efficient (and more idiomatic) approach to write --- include/sqlgen/duckdb/Connection.hpp | 52 +++++++++++++++++++--------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index 1f39b146..d68d2277 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -39,12 +39,16 @@ class Connection { using ConnPtr = Ref; public: - Connection(const ConnPtr &_conn) : conn_(_conn) {} + Connection(const ConnPtr &_conn) : appender_(nullptr), conn_(_conn) {} static rfl::Result> make( const std::optional &_fname) noexcept; - ~Connection() = default; + ~Connection() { + if (appender_) { + duckdb_appender_destroy(appender_.get()); + } + } Result begin_transaction() noexcept; @@ -87,25 +91,38 @@ class Connection { return duckdb::to_sql_impl(_stmt); } - Result start_write(const dynamic::Write &) { return Nothing{}; } + Result start_write(const dynamic::Write &_write_stmt) { + if (appender_) { + return error( + "Write operation already in progress - you cannot start another."); + } + appender_ = std::make_unique(); + if (duckdb_appender_create( + conn_->conn(), + _write_stmt.table.schema ? _write_stmt.table.schema->c_str() + : nullptr, + _write_stmt.table.name.c_str(), appender_.get()) == DuckDBError) { + appender_ = nullptr; + return error("Could not create appender."); + } + return Nothing{}; + } - Result end_write() { return Nothing{}; } + Result end_write() { + if (!appender_) { + return error("No write operation in progress - nothing to end."); + } + duckdb_appender_destroy(appender_.get()); + appender_ = nullptr; + return Nothing{}; + } template Result write(ItBegin _begin, ItEnd _end) { - using T = - std::remove_cvref_t::value_type>; - const auto schema = transpilation::get_schema(); - const auto table = transpilation::get_tablename(); - duckdb_appender appender{}; - if (duckdb_appender_create(conn_->conn(), - schema ? schema->c_str() : nullptr, - table.c_str(), &appender) == DuckDBError) { - return error("Could not create appender."); + if (!appender_) { + return error("No write operation in progress - nothing to write."); } - const auto res = write_to_appender(_begin, _end, appender); - duckdb_appender_destroy(&appender); - return res; + return write_to_appender(_begin, _end, *appender_); } private: @@ -162,6 +179,9 @@ class Connection { } private: + /// The appender to be used for the write statements + std::unique_ptr appender_; + /// The underlying duckdb3 connection. ConnPtr conn_; }; From 1db68cd9f44b78eb1e662b6a4b9977a4928dc6cf Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sun, 9 Nov 2025 14:08:12 +0100 Subject: [PATCH 20/48] Made sure the INSERT statement works as expected --- include/sqlgen/duckdb/Connection.hpp | 71 ++++++++++++++++++------- src/sqlgen/duckdb/Connection.cpp | 2 +- src/sqlgen/duckdb/to_sql.cpp | 71 ++++++++++++------------- tests/duckdb/test_insert_and_read.cpp | 44 +++++++++++++++ tests/duckdb/test_insert_or_replace.cpp | 70 ++++++++++++++++++++++++ vcpkg.json | 2 +- 6 files changed, 202 insertions(+), 58 deletions(-) create mode 100644 tests/duckdb/test_insert_and_read.cpp create mode 100644 tests/duckdb/test_insert_or_replace.cpp diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index d68d2277..bf3b0651 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -59,15 +59,22 @@ class Connection { template Result insert(const dynamic::Insert &_insert_stmt, ItBegin _begin, ItEnd _end) noexcept { + using namespace std::ranges::views; + const auto sql = to_sql(_insert_stmt); + auto columns = internal::collect::vector( + _insert_stmt.columns | + transform([](const auto &_str) { return _str.c_str(); })); + return get_duckdb_logical_types(_insert_stmt.table, _insert_stmt.columns) - .and_then([&](const auto &_types) -> Result { + .and_then([&](auto _types) -> Result { duckdb_appender appender{}; if (duckdb_appender_create_query( conn_->conn(), sql.c_str(), static_cast(_insert_stmt.columns.size()), - _types.data(), nullptr, nullptr, &appender) == DuckDBError) { + _types.data(), "sqlgen_appended_data", columns.data(), + &appender) == DuckDBError) { return error("Could not create appender."); } const auto res = write_to_appender(_begin, _end, appender); @@ -81,8 +88,12 @@ class Connection { using ValueType = transpilation::value_t; auto res = Ref(); duckdb_query(conn_->conn(), to_sql(_query).c_str(), res.get()); - return internal::to_container>( - Iterator(res, conn_)); + const auto result = + internal::to_container>( + Iterator(res, conn_)); + // TODO: Destroy result inside of iterator. + duckdb_destroy_result(res.get()); + return result; } Result rollback() noexcept; @@ -96,16 +107,27 @@ class Connection { return error( "Write operation already in progress - you cannot start another."); } - appender_ = std::make_unique(); - if (duckdb_appender_create( - conn_->conn(), - _write_stmt.table.schema ? _write_stmt.table.schema->c_str() - : nullptr, - _write_stmt.table.name.c_str(), appender_.get()) == DuckDBError) { - appender_ = nullptr; - return error("Could not create appender."); - } - return Nothing{}; + + using namespace std::ranges::views; + + auto columns = internal::collect::vector( + _write_stmt.columns | + transform([](const auto &_str) { return _str.c_str(); })); + + const auto sql = to_sql(_write_stmt); + + return get_duckdb_logical_types(_write_stmt.table, _write_stmt.columns) + .and_then([&](auto _types) -> Result { + appender_ = std::make_unique(); + if (duckdb_appender_create_query( + conn_->conn(), sql.c_str(), + static_cast(_write_stmt.columns.size()), _types.data(), + "sqlgen_appended_data", columns.data(), + appender_.get()) == DuckDBError) { + return error("Could not create appender."); + } + return Nothing{}; + }); } Result end_write() { @@ -141,13 +163,24 @@ class Connection { const auto select_from = dynamic::SelectFrom{ .table_or_query = _table, .fields = fields, .limit = dynamic::Limit{0}}; - auto res = Ref(); + duckdb_result res{}; + + const auto state = + duckdb_query(conn_->conn(), to_sql(select_from).c_str(), &res); + + if (state == DuckDBError) { + const auto err = error(duckdb_result_error(&res)); + duckdb_destroy_result(&res); + return err; + } + + const auto types = internal::collect::vector( + iota(static_cast(0), static_cast(fields.size())) | + transform(std::bind_front(duckdb_column_logical_type, &res))); - duckdb_query(conn_->conn(), to_sql(select_from).c_str(), res.get()); + duckdb_destroy_result(&res); - return internal::collect::vector( - iota(static_cast(fields.size())) | - transform(std::bind_front(duckdb_column_logical_type, res.get()))); + return types; } template diff --git a/src/sqlgen/duckdb/Connection.cpp b/src/sqlgen/duckdb/Connection.cpp index a8aa8551..b78222e5 100644 --- a/src/sqlgen/duckdb/Connection.cpp +++ b/src/sqlgen/duckdb/Connection.cpp @@ -18,7 +18,7 @@ Result Connection::begin_transaction() noexcept { Result Connection::commit() noexcept { return execute("COMMIT;"); } Result Connection::execute(const std::string& _sql) noexcept { - duckdb_result res; + duckdb_result res{}; const auto state = duckdb_query(conn_->conn(), _sql.c_str(), &res); if (state == DuckDBError) { const auto err = error(duckdb_result_error(&res)); diff --git a/src/sqlgen/duckdb/to_sql.cpp b/src/sqlgen/duckdb/to_sql.cpp index 421e4693..2fb6e79d 100644 --- a/src/sqlgen/duckdb/to_sql.cpp +++ b/src/sqlgen/duckdb/to_sql.cpp @@ -461,47 +461,32 @@ std::vector>> get_enum_types( std::string insert_to_sql(const dynamic::Insert& _stmt) noexcept { using namespace std::ranges::views; - const auto to_placeholder = [](const size_t _i) -> std::string { - return "$" + std::to_string(_i + 1); - }; + std::stringstream stream; - const auto as_excluded = [](const std::string& _str) -> std::string { - return _str + "=excluded." + _str; - }; + stream << "INSERT "; + + if (_stmt.or_replace) { + stream << "OR REPLACE "; + } + + stream << "INTO "; - std::stringstream stream; - stream << "INSERT INTO "; if (_stmt.table.schema) { stream << wrap_in_quotes(*_stmt.table.schema) << "."; } - stream << wrap_in_quotes(_stmt.table.name); - stream << " ("; - stream << internal::strings::join( - ", ", - internal::collect::vector(_stmt.columns | transform(wrap_in_quotes))); - stream << ")"; + stream << wrap_in_quotes(_stmt.table.name); - stream << " VALUES ("; + stream << " BY NAME ( SELECT "; stream << internal::strings::join( ", ", internal::collect::vector( - iota(static_cast(0), _stmt.columns.size()) | - transform(to_placeholder))); - stream << ")"; - - if (_stmt.or_replace) { - stream << " ON CONFLICT ("; - stream << internal::strings::join( - ", ", internal::collect::vector(_stmt.constraints)); - stream << ")"; - - stream << " DO UPDATE SET "; - stream << internal::strings::join( - ", ", - internal::collect::vector(_stmt.columns | transform(as_excluded))); - } + _stmt.columns | transform([&](const auto _name) { + return wrap_in_quotes(_name) + " AS " + wrap_in_quotes(_name); + }))); + stream << " FROM sqlgen_appended_data)"; stream << ";"; + return stream.str(); } @@ -903,13 +888,25 @@ std::string update_to_sql(const dynamic::Update& _stmt) noexcept { std::string write_to_sql(const dynamic::Write& _stmt) noexcept { using namespace std::ranges::views; - const auto schema = wrap_in_quotes(_stmt.table.schema.value_or("public")); - const auto table = wrap_in_quotes(_stmt.table.name); - const auto colnames = internal::strings::join( - ", ", - internal::collect::vector(_stmt.columns | transform(wrap_in_quotes))); - return "COPY " + schema + "." + table + "(" + colnames + - ") FROM STDIN WITH DELIMITER '\t' NULL '\e' CSV QUOTE '\a';"; + + std::stringstream stream; + stream << "INSERT INTO "; + if (_stmt.table.schema) { + stream << wrap_in_quotes(*_stmt.table.schema) << "."; + } + stream << wrap_in_quotes(_stmt.table.name); + + stream << " BY NAME ( SELECT "; + stream << internal::strings::join( + ", ", internal::collect::vector( + _stmt.columns | transform([&](const auto _name) { + return wrap_in_quotes(_name) + " AS " + wrap_in_quotes(_name); + }))); + stream << " FROM sqlgen_appended_data)"; + + stream << ";"; + + return stream.str(); } } // namespace sqlgen::duckdb diff --git a/tests/duckdb/test_insert_and_read.cpp b/tests/duckdb/test_insert_and_read.cpp new file mode 100644 index 00000000..8efb523a --- /dev/null +++ b/tests/duckdb/test_insert_and_read.cpp @@ -0,0 +1,44 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_insert_and_read { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_insert_and_read) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto people2 = duckdb::connect() + .and_then(begin_transaction) + .and_then(create_table | if_not_exists) + .and_then(insert(people1)) + .and_then(commit) + .and_then(sqlgen::read>) + .value(); + + const auto json1 = rfl::json::write(people1); + const auto json2 = rfl::json::write(people2); + + EXPECT_EQ(json1, json2); +} + +} // namespace test_insert_and_read diff --git a/tests/duckdb/test_insert_or_replace.cpp b/tests/duckdb/test_insert_or_replace.cpp new file mode 100644 index 00000000..e2f1c28f --- /dev/null +++ b/tests/duckdb/test_insert_or_replace.cpp @@ -0,0 +1,70 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_insert_or_replace { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_insert_or_replace) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + const auto people2 = std::vector({Person{.id = 1, + .first_name = "Bartholomew", + .last_name = "Simpson", + .age = 10}, + Person{.id = 3, + .first_name = "Margaret", + .last_name = "Simpson", + .age = 1}}); + + const auto people3 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, + .first_name = "Bartholomew", + .last_name = "Simpson", + .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{.id = 3, + .first_name = "Margaret", + .last_name = "Simpson", + .age = 1}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto people4 = + duckdb::connect() + .and_then(begin_transaction) + .and_then(create_table | if_not_exists) + .and_then(insert(std::ref(people1))) + .and_then(commit) + .and_then(begin_transaction) + .and_then(insert_or_replace(std::ref(people2))) + .and_then(commit) + .and_then(sqlgen::read> | order_by("id"_c)) + .value(); + + const auto json3 = rfl::json::write(people3); + const auto json4 = rfl::json::write(people4); + + EXPECT_EQ(json3, json4); +} + +} // namespace test_insert_or_replace diff --git a/vcpkg.json b/vcpkg.json index 85a0665d..42171303 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -11,7 +11,7 @@ "dependencies": [ { "name": "duckdb", - "version>=": "1.3.2" + "version>=": "1.4.1" } ] }, From 8a69cad2838b66c7b4200671c95d7d2428bb98f3 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sun, 9 Nov 2025 14:28:38 +0100 Subject: [PATCH 21/48] Began developing the autoincr primary key --- include/sqlgen/duckdb/Connection.hpp | 11 ++--- src/sqlgen/duckdb/to_sql.cpp | 4 +- tests/duckdb/test_auto_incr_primary_key.cpp | 45 +++++++++++++++++++++ 3 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 tests/duckdb/test_auto_incr_primary_key.cpp diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index bf3b0651..7031e8f4 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -18,19 +18,14 @@ #include "../Transaction.hpp" #include "../dynamic/Write.hpp" #include "../internal/iterator_t.hpp" +#include "../internal/remove_auto_incr_primary_t.hpp" #include "../internal/to_container.hpp" #include "../is_connection.hpp" -#include "../transpilation/get_tablename.hpp" -#include "../transpilation/has_reflection_method.hpp" -#include "../transpilation/is_nullable.hpp" #include "./parsing/Parser_default.hpp" #include "DuckDBConnection.hpp" #include "Iterator.hpp" -#include "parsing/Parser.hpp" #include "sqlgen/dynamic/Operation.hpp" #include "sqlgen/dynamic/SelectFrom.hpp" -#include "sqlgen/transpilation/get_schema.hpp" -#include "sqlgen/transpilation/to_table_or_query.hpp" #include "to_sql.hpp" namespace sqlgen::duckdb { @@ -199,8 +194,10 @@ class Connection { template Result write_row(const StructT &_struct, duckdb_appender _appender) noexcept { + using ViewType = + internal::remove_auto_incr_primary_t>; Result res = Nothing{}; - rfl::to_view(_struct).apply([&](const auto &_field) { + ViewType(rfl::to_view(_struct)).apply([&](const auto &_field) { using ValueType = std::remove_cvref_t::Type>>; if (res) { diff --git a/src/sqlgen/duckdb/to_sql.cpp b/src/sqlgen/duckdb/to_sql.cpp index 2fb6e79d..c95c7a37 100644 --- a/src/sqlgen/duckdb/to_sql.cpp +++ b/src/sqlgen/duckdb/to_sql.cpp @@ -673,8 +673,8 @@ std::string operation_to_sql(const dynamic::Operation& _stmt) noexcept { std::string properties_to_sql(const dynamic::types::Properties& _p) noexcept { return [&]() -> std::string { - return std::string(_p.auto_incr ? " GENERATED ALWAYS AS IDENTITY" : "") + - std::string(_p.nullable ? "" : " NOT NULL") + + return std::string(_p.nullable ? "" : " NOT NULL") + + std::string(_p.auto_incr ? " GENERATED ALWAYS" : "") + std::string(_p.unique ? " UNIQUE" : ""); }() + [&]() -> std::string { if (!_p.foreign_key_reference) { diff --git a/tests/duckdb/test_auto_incr_primary_key.cpp b/tests/duckdb/test_auto_incr_primary_key.cpp new file mode 100644 index 00000000..2d3b5f0a --- /dev/null +++ b/tests/duckdb/test_auto_incr_primary_key.cpp @@ -0,0 +1,45 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_auto_incr_primary_key { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_auto_incr_primary_key) { + auto people1 = std::vector( + {Person{.first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{.first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto people2 = duckdb::connect() + .and_then(write(std::ref(people1))) + .and_then(sqlgen::read> | + order_by("age"_c.desc())) + .value(); + + people1.at(0).id = 1; + people1.at(1).id = 2; + people1.at(2).id = 3; + people1.at(3).id = 4; + + const auto json1 = rfl::json::write(people1); + const auto json2 = rfl::json::write(people2); + + EXPECT_EQ(json1, json2); +} + +} // namespace test_auto_incr_primary_key From a818c6fbf49b26e6ea3ebeeca277de709487167f Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sun, 9 Nov 2025 22:20:37 +0100 Subject: [PATCH 22/48] Use RAII for the appender and results --- include/sqlgen/duckdb/Connection.hpp | 75 +++++++---------------- include/sqlgen/duckdb/DuckDBAppender.hpp | 75 +++++++++++++++++++++++ include/sqlgen/duckdb/DuckDBResult.hpp | 68 ++++++++++++++++++++ include/sqlgen/duckdb/Iterator.hpp | 39 ++++++------ include/sqlgen/duckdb/make_chunk_ptrs.hpp | 13 ++-- include/sqlgen/internal/to_container.hpp | 5 +- 6 files changed, 197 insertions(+), 78 deletions(-) create mode 100644 include/sqlgen/duckdb/DuckDBAppender.hpp create mode 100644 include/sqlgen/duckdb/DuckDBResult.hpp diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index 7031e8f4..52a4c4d2 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -22,7 +22,9 @@ #include "../internal/to_container.hpp" #include "../is_connection.hpp" #include "./parsing/Parser_default.hpp" +#include "DuckDBAppender.hpp" #include "DuckDBConnection.hpp" +#include "DuckDBResult.hpp" #include "Iterator.hpp" #include "sqlgen/dynamic/Operation.hpp" #include "sqlgen/dynamic/SelectFrom.hpp" @@ -39,11 +41,7 @@ class Connection { static rfl::Result> make( const std::optional &_fname) noexcept; - ~Connection() { - if (appender_) { - duckdb_appender_destroy(appender_.get()); - } - } + ~Connection() = default; Result begin_transaction() noexcept; @@ -63,31 +61,20 @@ class Connection { transform([](const auto &_str) { return _str.c_str(); })); return get_duckdb_logical_types(_insert_stmt.table, _insert_stmt.columns) - .and_then([&](auto _types) -> Result { - duckdb_appender appender{}; - if (duckdb_appender_create_query( - conn_->conn(), sql.c_str(), - static_cast(_insert_stmt.columns.size()), - _types.data(), "sqlgen_appended_data", columns.data(), - &appender) == DuckDBError) { - return error("Could not create appender."); - } - const auto res = write_to_appender(_begin, _end, appender); - duckdb_appender_destroy(&appender); - return res; + .and_then([&](const auto &_types) { + return DuckDBAppender::make(sql, conn_, columns, _types); + }) + .and_then([&](const auto &_appender) { + return write_to_appender(_begin, _end, _appender->appender()); }); } template Result read(const dynamic::SelectFrom &_query) { using ValueType = transpilation::value_t; - auto res = Ref(); - duckdb_query(conn_->conn(), to_sql(_query).c_str(), res.get()); const auto result = internal::to_container>( - Iterator(res, conn_)); - // TODO: Destroy result inside of iterator. - duckdb_destroy_result(res.get()); + Iterator(to_sql(_query), conn_)); return result; } @@ -112,15 +99,11 @@ class Connection { const auto sql = to_sql(_write_stmt); return get_duckdb_logical_types(_write_stmt.table, _write_stmt.columns) - .and_then([&](auto _types) -> Result { - appender_ = std::make_unique(); - if (duckdb_appender_create_query( - conn_->conn(), sql.c_str(), - static_cast(_write_stmt.columns.size()), _types.data(), - "sqlgen_appended_data", columns.data(), - appender_.get()) == DuckDBError) { - return error("Could not create appender."); - } + .and_then([&](auto _types) { + return DuckDBAppender::make(sql, conn_, columns, _types); + }) + .transform([&](auto &&_appender) { + appender_ = _appender.ptr(); return Nothing{}; }); } @@ -129,7 +112,6 @@ class Connection { if (!appender_) { return error("No write operation in progress - nothing to end."); } - duckdb_appender_destroy(appender_.get()); appender_ = nullptr; return Nothing{}; } @@ -139,7 +121,7 @@ class Connection { if (!appender_) { return error("No write operation in progress - nothing to write."); } - return write_to_appender(_begin, _end, *appender_); + return write_to_appender(_begin, _end, appender_->appender()); } private: @@ -158,24 +140,13 @@ class Connection { const auto select_from = dynamic::SelectFrom{ .table_or_query = _table, .fields = fields, .limit = dynamic::Limit{0}}; - duckdb_result res{}; - - const auto state = - duckdb_query(conn_->conn(), to_sql(select_from).c_str(), &res); - - if (state == DuckDBError) { - const auto err = error(duckdb_result_error(&res)); - duckdb_destroy_result(&res); - return err; - } - - const auto types = internal::collect::vector( - iota(static_cast(0), static_cast(fields.size())) | - transform(std::bind_front(duckdb_column_logical_type, &res))); - - duckdb_destroy_result(&res); - - return types; + return DuckDBResult::make(to_sql(select_from), conn_) + .transform([&](const auto &_res) { + return internal::collect::vector( + iota(static_cast(0), static_cast(fields.size())) | + transform( + std::bind_front(duckdb_column_logical_type, &_res->res()))); + }); } template @@ -210,7 +181,7 @@ class Connection { private: /// The appender to be used for the write statements - std::unique_ptr appender_; + std::shared_ptr appender_; /// The underlying duckdb3 connection. ConnPtr conn_; diff --git a/include/sqlgen/duckdb/DuckDBAppender.hpp b/include/sqlgen/duckdb/DuckDBAppender.hpp new file mode 100644 index 00000000..add9d034 --- /dev/null +++ b/include/sqlgen/duckdb/DuckDBAppender.hpp @@ -0,0 +1,75 @@ +#ifndef SQLGEN_DUCKDB_DUCKDBAPPENDER_HPP_ +#define SQLGEN_DUCKDB_DUCKDBAPPENDER_HPP_ + +#include + +#include + +#include "DuckDBConnection.hpp" + +namespace sqlgen::duckdb { + +class DuckDBAppender { + using ConnPtr = Ref; + + public: + static Result> make( + const std::string& _sql, const ConnPtr& _conn, + const std::vector& _columns, + const std::vector& _types) { + try { + return Ref::make(_sql, _conn, _columns, _types); + } catch (const std::exception& e) { + return error(e.what()); + } + } + + DuckDBAppender(const std::string& _sql, const ConnPtr& _conn, + std::vector _columns, + std::vector _types) + : destroy_(false) { + if (duckdb_appender_create_query( + _conn->conn(), _sql.c_str(), static_cast(_columns.size()), + _types.data(), "sqlgen_appended_data", _columns.data(), + &appender_) == DuckDBError) { + throw std::runtime_error("Could not create appender."); + } + destroy_ = true; + } + + ~DuckDBAppender() { + if (destroy_) { + duckdb_appender_destroy(&appender_); + } + } + + DuckDBAppender(const DuckDBAppender& _other) = delete; + + DuckDBAppender(DuckDBAppender&& _other) + : destroy_(_other.destroy_), appender_(_other.appender_) { + _other.destroy_ = false; + } + + DuckDBAppender& operator=(const DuckDBAppender& _other) = delete; + + DuckDBAppender& operator=(DuckDBAppender&& _other) { + if (this == &_other) { + return *this; + } + destroy_ = _other.destroy_; + appender_ = _other.appender_; + _other.destroy_ = false; + return *this; + } + + duckdb_appender& appender() { return appender_; } + + private: + bool destroy_; + + duckdb_appender appender_; +}; + +} // namespace sqlgen::duckdb + +#endif diff --git a/include/sqlgen/duckdb/DuckDBResult.hpp b/include/sqlgen/duckdb/DuckDBResult.hpp new file mode 100644 index 00000000..14c0b396 --- /dev/null +++ b/include/sqlgen/duckdb/DuckDBResult.hpp @@ -0,0 +1,68 @@ +#ifndef SQLGEN_DUCKDB_DUCKDBRESULT_HPP_ +#define SQLGEN_DUCKDB_DUCKDBRESULT_HPP_ + +#include + +#include + +#include "DuckDBConnection.hpp" + +namespace sqlgen::duckdb { + +class DuckDBResult { + using ConnPtr = Ref; + + public: + static Result> make(const std::string& _query, + const ConnPtr& _conn) { + try { + return Ref::make(_query, _conn); + } catch (const std::exception& e) { + return error(e.what()); + } + } + + DuckDBResult(const std::string& _query, const ConnPtr& _conn) + : destroy_(false) { + if (duckdb_query(_conn->conn(), _query.c_str(), &res_) == DuckDBError) { + throw std::runtime_error(duckdb_result_error(&res_)); + } + destroy_ = true; + } + + ~DuckDBResult() { + if (destroy_) { + duckdb_destroy_result(&res_); + } + } + + DuckDBResult(const DuckDBResult& _other) = delete; + + DuckDBResult(DuckDBResult&& _other) + : destroy_(_other.destroy_), res_(_other.res_) { + _other.destroy_ = false; + } + + DuckDBResult& operator=(const DuckDBResult& _other) = delete; + + DuckDBResult& operator=(DuckDBResult&& _other) { + if (this == &_other) { + return *this; + } + destroy_ = _other.destroy_; + res_ = _other.res_; + _other.destroy_ = false; + return *this; + } + + duckdb_result& res() { return res_; } + + private: + bool destroy_; + + duckdb_result res_; +}; + +} // namespace sqlgen::duckdb + +#endif diff --git a/include/sqlgen/duckdb/Iterator.hpp b/include/sqlgen/duckdb/Iterator.hpp index 767a7b01..940d7d95 100644 --- a/include/sqlgen/duckdb/Iterator.hpp +++ b/include/sqlgen/duckdb/Iterator.hpp @@ -3,7 +3,6 @@ #include -#include #include #include #include @@ -11,6 +10,7 @@ #include "../Ref.hpp" #include "../Result.hpp" #include "DuckDBConnection.hpp" +#include "DuckDBResult.hpp" #include "from_chunk_ptrs.hpp" #include "make_chunk_ptrs.hpp" @@ -19,7 +19,7 @@ namespace sqlgen::duckdb { template class Iterator { using ConnPtr = Ref; - using ResultPtr = Ref; + using ResultPtr = Ref; public: struct End { @@ -36,10 +36,10 @@ class Iterator { using difference_type = std::ptrdiff_t; using value_type = Result; - Iterator(const ResultPtr& _res, const ConnPtr& _conn) - : res_(_res), + Iterator(const std::string& _query, const ConnPtr& _conn) + : res_(DuckDBResult::make(_query, _conn)), conn_(_conn), - current_batch_(get_next_batch(_res, _conn)), + current_batch_(get_next_batch(res_, _conn)), ix_(0) {} ~Iterator() = default; @@ -67,19 +67,22 @@ class Iterator { private: static Ref>> get_next_batch( - const ResultPtr& _res, const ConnPtr& _conn) noexcept { - duckdb_data_chunk chunk = duckdb_fetch_chunk(*_res); - if (!chunk) { - return Ref>>::make(); - } - const idx_t row_count = duckdb_data_chunk_get_size(chunk); - return make_chunk_ptrs(_res, chunk) - .transform([&](auto&& _chunk_ptrs) { - auto batch = Ref>>::make(); - for (idx_t i = 0; i < row_count; ++i) { - batch->emplace_back(from_chunk_ptrs(_chunk_ptrs, i)); + const Result& _result_ptr, const ConnPtr& _conn) noexcept { + return _result_ptr + .and_then([&](const auto& _res) -> Result>>> { + duckdb_data_chunk chunk = duckdb_fetch_chunk(_res->res()); + if (!chunk) { + return Ref>>::make(); } - return batch; + const idx_t row_count = duckdb_data_chunk_get_size(chunk); + return make_chunk_ptrs(_res, chunk) + .transform([&](auto&& _chunk_ptrs) { + auto batch = Ref>>::make(); + for (idx_t i = 0; i < row_count; ++i) { + batch->emplace_back(from_chunk_ptrs(_chunk_ptrs, i)); + } + return batch; + }); }) .or_else([](auto _err) { return Ref>>::make( @@ -90,7 +93,7 @@ class Iterator { private: /// The underlying DuckDB result. - ResultPtr res_; + Result res_; /// The underlying connection. ConnPtr conn_; diff --git a/include/sqlgen/duckdb/make_chunk_ptrs.hpp b/include/sqlgen/duckdb/make_chunk_ptrs.hpp index c0c7347e..1f41068d 100644 --- a/include/sqlgen/duckdb/make_chunk_ptrs.hpp +++ b/include/sqlgen/duckdb/make_chunk_ptrs.hpp @@ -8,8 +8,9 @@ #include #include -#include "./parsing/Parser.hpp" +#include "../Ref.hpp" #include "ColumnData.hpp" +#include "DuckDBResult.hpp" #include "chunk_ptrs_t.hpp" #include "get_duckdb_type.hpp" @@ -21,7 +22,7 @@ struct MakeChunkPtrs; template struct MakeChunkPtrs...>> { Result...>> operator()( - const Ref& _res, duckdb_data_chunk _chunk) { + const Ref& _res, duckdb_data_chunk _chunk) { try { return [&](std::integer_sequence) { return rfl::Tuple...>( @@ -33,13 +34,13 @@ struct MakeChunkPtrs...>> { } template - static auto make_column_data(const Ref& _res, + static auto make_column_data(const Ref& _res, duckdb_data_chunk _chunk) { - if (duckdb_column_type(_res.get(), _i) != get_duckdb_type()) { + if (duckdb_column_type(&_res->res(), _i) != get_duckdb_type()) { throw std::runtime_error( "Wrong type in field " + std::to_string(_i) + ". Expected " + rfl::enum_to_string(get_duckdb_type()) + ", got " + - rfl::enum_to_string(duckdb_column_type(_res.get(), _i)) + "."); + rfl::enum_to_string(duckdb_column_type(&_res->res(), _i)) + "."); } auto vec = duckdb_data_chunk_get_vector(_chunk, _i); return ColumnData{.vec = vec, @@ -50,7 +51,7 @@ struct MakeChunkPtrs...>> { template struct MakeChunkPtrs { - auto operator()(const Ref& _res, duckdb_data_chunk _chunk) { + auto operator()(const Ref& _res, duckdb_data_chunk _chunk) { return MakeChunkPtrs>{}(_res, _chunk); } }; diff --git a/include/sqlgen/internal/to_container.hpp b/include/sqlgen/internal/to_container.hpp index 852ef6f4..ceaacbc1 100644 --- a/include/sqlgen/internal/to_container.hpp +++ b/include/sqlgen/internal/to_container.hpp @@ -12,11 +12,12 @@ namespace sqlgen::internal { template auto to_container(const Result& _res) { if constexpr (internal::is_range_v) { - return _res.transform([](auto&& _it) { return Range(_it); }); + return _res.transform( + [](auto&& _it) { return Range(std::move(_it)); }); } else { return to_container>(_res).and_then( - [](auto range) -> Result { + [](const auto& range) -> Result { ContainerType container; for (auto& res : range) { if (res) { From be595a7135ef820eeb708b626aaa390c25b1767f Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Tue, 11 Nov 2025 22:22:49 +0100 Subject: [PATCH 23/48] Reworked the appender --- include/sqlgen/duckdb/Connection.hpp | 75 +++++++---------------- include/sqlgen/duckdb/DuckDBAppender.hpp | 75 +++++++++++++++++++++++ include/sqlgen/duckdb/DuckDBResult.hpp | 68 ++++++++++++++++++++ include/sqlgen/duckdb/Iterator.hpp | 39 ++++++------ include/sqlgen/duckdb/make_chunk_ptrs.hpp | 13 ++-- include/sqlgen/internal/to_container.hpp | 5 +- 6 files changed, 197 insertions(+), 78 deletions(-) create mode 100644 include/sqlgen/duckdb/DuckDBAppender.hpp create mode 100644 include/sqlgen/duckdb/DuckDBResult.hpp diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index 7031e8f4..52a4c4d2 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -22,7 +22,9 @@ #include "../internal/to_container.hpp" #include "../is_connection.hpp" #include "./parsing/Parser_default.hpp" +#include "DuckDBAppender.hpp" #include "DuckDBConnection.hpp" +#include "DuckDBResult.hpp" #include "Iterator.hpp" #include "sqlgen/dynamic/Operation.hpp" #include "sqlgen/dynamic/SelectFrom.hpp" @@ -39,11 +41,7 @@ class Connection { static rfl::Result> make( const std::optional &_fname) noexcept; - ~Connection() { - if (appender_) { - duckdb_appender_destroy(appender_.get()); - } - } + ~Connection() = default; Result begin_transaction() noexcept; @@ -63,31 +61,20 @@ class Connection { transform([](const auto &_str) { return _str.c_str(); })); return get_duckdb_logical_types(_insert_stmt.table, _insert_stmt.columns) - .and_then([&](auto _types) -> Result { - duckdb_appender appender{}; - if (duckdb_appender_create_query( - conn_->conn(), sql.c_str(), - static_cast(_insert_stmt.columns.size()), - _types.data(), "sqlgen_appended_data", columns.data(), - &appender) == DuckDBError) { - return error("Could not create appender."); - } - const auto res = write_to_appender(_begin, _end, appender); - duckdb_appender_destroy(&appender); - return res; + .and_then([&](const auto &_types) { + return DuckDBAppender::make(sql, conn_, columns, _types); + }) + .and_then([&](const auto &_appender) { + return write_to_appender(_begin, _end, _appender->appender()); }); } template Result read(const dynamic::SelectFrom &_query) { using ValueType = transpilation::value_t; - auto res = Ref(); - duckdb_query(conn_->conn(), to_sql(_query).c_str(), res.get()); const auto result = internal::to_container>( - Iterator(res, conn_)); - // TODO: Destroy result inside of iterator. - duckdb_destroy_result(res.get()); + Iterator(to_sql(_query), conn_)); return result; } @@ -112,15 +99,11 @@ class Connection { const auto sql = to_sql(_write_stmt); return get_duckdb_logical_types(_write_stmt.table, _write_stmt.columns) - .and_then([&](auto _types) -> Result { - appender_ = std::make_unique(); - if (duckdb_appender_create_query( - conn_->conn(), sql.c_str(), - static_cast(_write_stmt.columns.size()), _types.data(), - "sqlgen_appended_data", columns.data(), - appender_.get()) == DuckDBError) { - return error("Could not create appender."); - } + .and_then([&](auto _types) { + return DuckDBAppender::make(sql, conn_, columns, _types); + }) + .transform([&](auto &&_appender) { + appender_ = _appender.ptr(); return Nothing{}; }); } @@ -129,7 +112,6 @@ class Connection { if (!appender_) { return error("No write operation in progress - nothing to end."); } - duckdb_appender_destroy(appender_.get()); appender_ = nullptr; return Nothing{}; } @@ -139,7 +121,7 @@ class Connection { if (!appender_) { return error("No write operation in progress - nothing to write."); } - return write_to_appender(_begin, _end, *appender_); + return write_to_appender(_begin, _end, appender_->appender()); } private: @@ -158,24 +140,13 @@ class Connection { const auto select_from = dynamic::SelectFrom{ .table_or_query = _table, .fields = fields, .limit = dynamic::Limit{0}}; - duckdb_result res{}; - - const auto state = - duckdb_query(conn_->conn(), to_sql(select_from).c_str(), &res); - - if (state == DuckDBError) { - const auto err = error(duckdb_result_error(&res)); - duckdb_destroy_result(&res); - return err; - } - - const auto types = internal::collect::vector( - iota(static_cast(0), static_cast(fields.size())) | - transform(std::bind_front(duckdb_column_logical_type, &res))); - - duckdb_destroy_result(&res); - - return types; + return DuckDBResult::make(to_sql(select_from), conn_) + .transform([&](const auto &_res) { + return internal::collect::vector( + iota(static_cast(0), static_cast(fields.size())) | + transform( + std::bind_front(duckdb_column_logical_type, &_res->res()))); + }); } template @@ -210,7 +181,7 @@ class Connection { private: /// The appender to be used for the write statements - std::unique_ptr appender_; + std::shared_ptr appender_; /// The underlying duckdb3 connection. ConnPtr conn_; diff --git a/include/sqlgen/duckdb/DuckDBAppender.hpp b/include/sqlgen/duckdb/DuckDBAppender.hpp new file mode 100644 index 00000000..add9d034 --- /dev/null +++ b/include/sqlgen/duckdb/DuckDBAppender.hpp @@ -0,0 +1,75 @@ +#ifndef SQLGEN_DUCKDB_DUCKDBAPPENDER_HPP_ +#define SQLGEN_DUCKDB_DUCKDBAPPENDER_HPP_ + +#include + +#include + +#include "DuckDBConnection.hpp" + +namespace sqlgen::duckdb { + +class DuckDBAppender { + using ConnPtr = Ref; + + public: + static Result> make( + const std::string& _sql, const ConnPtr& _conn, + const std::vector& _columns, + const std::vector& _types) { + try { + return Ref::make(_sql, _conn, _columns, _types); + } catch (const std::exception& e) { + return error(e.what()); + } + } + + DuckDBAppender(const std::string& _sql, const ConnPtr& _conn, + std::vector _columns, + std::vector _types) + : destroy_(false) { + if (duckdb_appender_create_query( + _conn->conn(), _sql.c_str(), static_cast(_columns.size()), + _types.data(), "sqlgen_appended_data", _columns.data(), + &appender_) == DuckDBError) { + throw std::runtime_error("Could not create appender."); + } + destroy_ = true; + } + + ~DuckDBAppender() { + if (destroy_) { + duckdb_appender_destroy(&appender_); + } + } + + DuckDBAppender(const DuckDBAppender& _other) = delete; + + DuckDBAppender(DuckDBAppender&& _other) + : destroy_(_other.destroy_), appender_(_other.appender_) { + _other.destroy_ = false; + } + + DuckDBAppender& operator=(const DuckDBAppender& _other) = delete; + + DuckDBAppender& operator=(DuckDBAppender&& _other) { + if (this == &_other) { + return *this; + } + destroy_ = _other.destroy_; + appender_ = _other.appender_; + _other.destroy_ = false; + return *this; + } + + duckdb_appender& appender() { return appender_; } + + private: + bool destroy_; + + duckdb_appender appender_; +}; + +} // namespace sqlgen::duckdb + +#endif diff --git a/include/sqlgen/duckdb/DuckDBResult.hpp b/include/sqlgen/duckdb/DuckDBResult.hpp new file mode 100644 index 00000000..14c0b396 --- /dev/null +++ b/include/sqlgen/duckdb/DuckDBResult.hpp @@ -0,0 +1,68 @@ +#ifndef SQLGEN_DUCKDB_DUCKDBRESULT_HPP_ +#define SQLGEN_DUCKDB_DUCKDBRESULT_HPP_ + +#include + +#include + +#include "DuckDBConnection.hpp" + +namespace sqlgen::duckdb { + +class DuckDBResult { + using ConnPtr = Ref; + + public: + static Result> make(const std::string& _query, + const ConnPtr& _conn) { + try { + return Ref::make(_query, _conn); + } catch (const std::exception& e) { + return error(e.what()); + } + } + + DuckDBResult(const std::string& _query, const ConnPtr& _conn) + : destroy_(false) { + if (duckdb_query(_conn->conn(), _query.c_str(), &res_) == DuckDBError) { + throw std::runtime_error(duckdb_result_error(&res_)); + } + destroy_ = true; + } + + ~DuckDBResult() { + if (destroy_) { + duckdb_destroy_result(&res_); + } + } + + DuckDBResult(const DuckDBResult& _other) = delete; + + DuckDBResult(DuckDBResult&& _other) + : destroy_(_other.destroy_), res_(_other.res_) { + _other.destroy_ = false; + } + + DuckDBResult& operator=(const DuckDBResult& _other) = delete; + + DuckDBResult& operator=(DuckDBResult&& _other) { + if (this == &_other) { + return *this; + } + destroy_ = _other.destroy_; + res_ = _other.res_; + _other.destroy_ = false; + return *this; + } + + duckdb_result& res() { return res_; } + + private: + bool destroy_; + + duckdb_result res_; +}; + +} // namespace sqlgen::duckdb + +#endif diff --git a/include/sqlgen/duckdb/Iterator.hpp b/include/sqlgen/duckdb/Iterator.hpp index 767a7b01..940d7d95 100644 --- a/include/sqlgen/duckdb/Iterator.hpp +++ b/include/sqlgen/duckdb/Iterator.hpp @@ -3,7 +3,6 @@ #include -#include #include #include #include @@ -11,6 +10,7 @@ #include "../Ref.hpp" #include "../Result.hpp" #include "DuckDBConnection.hpp" +#include "DuckDBResult.hpp" #include "from_chunk_ptrs.hpp" #include "make_chunk_ptrs.hpp" @@ -19,7 +19,7 @@ namespace sqlgen::duckdb { template class Iterator { using ConnPtr = Ref; - using ResultPtr = Ref; + using ResultPtr = Ref; public: struct End { @@ -36,10 +36,10 @@ class Iterator { using difference_type = std::ptrdiff_t; using value_type = Result; - Iterator(const ResultPtr& _res, const ConnPtr& _conn) - : res_(_res), + Iterator(const std::string& _query, const ConnPtr& _conn) + : res_(DuckDBResult::make(_query, _conn)), conn_(_conn), - current_batch_(get_next_batch(_res, _conn)), + current_batch_(get_next_batch(res_, _conn)), ix_(0) {} ~Iterator() = default; @@ -67,19 +67,22 @@ class Iterator { private: static Ref>> get_next_batch( - const ResultPtr& _res, const ConnPtr& _conn) noexcept { - duckdb_data_chunk chunk = duckdb_fetch_chunk(*_res); - if (!chunk) { - return Ref>>::make(); - } - const idx_t row_count = duckdb_data_chunk_get_size(chunk); - return make_chunk_ptrs(_res, chunk) - .transform([&](auto&& _chunk_ptrs) { - auto batch = Ref>>::make(); - for (idx_t i = 0; i < row_count; ++i) { - batch->emplace_back(from_chunk_ptrs(_chunk_ptrs, i)); + const Result& _result_ptr, const ConnPtr& _conn) noexcept { + return _result_ptr + .and_then([&](const auto& _res) -> Result>>> { + duckdb_data_chunk chunk = duckdb_fetch_chunk(_res->res()); + if (!chunk) { + return Ref>>::make(); } - return batch; + const idx_t row_count = duckdb_data_chunk_get_size(chunk); + return make_chunk_ptrs(_res, chunk) + .transform([&](auto&& _chunk_ptrs) { + auto batch = Ref>>::make(); + for (idx_t i = 0; i < row_count; ++i) { + batch->emplace_back(from_chunk_ptrs(_chunk_ptrs, i)); + } + return batch; + }); }) .or_else([](auto _err) { return Ref>>::make( @@ -90,7 +93,7 @@ class Iterator { private: /// The underlying DuckDB result. - ResultPtr res_; + Result res_; /// The underlying connection. ConnPtr conn_; diff --git a/include/sqlgen/duckdb/make_chunk_ptrs.hpp b/include/sqlgen/duckdb/make_chunk_ptrs.hpp index c0c7347e..1f41068d 100644 --- a/include/sqlgen/duckdb/make_chunk_ptrs.hpp +++ b/include/sqlgen/duckdb/make_chunk_ptrs.hpp @@ -8,8 +8,9 @@ #include #include -#include "./parsing/Parser.hpp" +#include "../Ref.hpp" #include "ColumnData.hpp" +#include "DuckDBResult.hpp" #include "chunk_ptrs_t.hpp" #include "get_duckdb_type.hpp" @@ -21,7 +22,7 @@ struct MakeChunkPtrs; template struct MakeChunkPtrs...>> { Result...>> operator()( - const Ref& _res, duckdb_data_chunk _chunk) { + const Ref& _res, duckdb_data_chunk _chunk) { try { return [&](std::integer_sequence) { return rfl::Tuple...>( @@ -33,13 +34,13 @@ struct MakeChunkPtrs...>> { } template - static auto make_column_data(const Ref& _res, + static auto make_column_data(const Ref& _res, duckdb_data_chunk _chunk) { - if (duckdb_column_type(_res.get(), _i) != get_duckdb_type()) { + if (duckdb_column_type(&_res->res(), _i) != get_duckdb_type()) { throw std::runtime_error( "Wrong type in field " + std::to_string(_i) + ". Expected " + rfl::enum_to_string(get_duckdb_type()) + ", got " + - rfl::enum_to_string(duckdb_column_type(_res.get(), _i)) + "."); + rfl::enum_to_string(duckdb_column_type(&_res->res(), _i)) + "."); } auto vec = duckdb_data_chunk_get_vector(_chunk, _i); return ColumnData{.vec = vec, @@ -50,7 +51,7 @@ struct MakeChunkPtrs...>> { template struct MakeChunkPtrs { - auto operator()(const Ref& _res, duckdb_data_chunk _chunk) { + auto operator()(const Ref& _res, duckdb_data_chunk _chunk) { return MakeChunkPtrs>{}(_res, _chunk); } }; diff --git a/include/sqlgen/internal/to_container.hpp b/include/sqlgen/internal/to_container.hpp index 852ef6f4..ceaacbc1 100644 --- a/include/sqlgen/internal/to_container.hpp +++ b/include/sqlgen/internal/to_container.hpp @@ -12,11 +12,12 @@ namespace sqlgen::internal { template auto to_container(const Result& _res) { if constexpr (internal::is_range_v) { - return _res.transform([](auto&& _it) { return Range(_it); }); + return _res.transform( + [](auto&& _it) { return Range(std::move(_it)); }); } else { return to_container>(_res).and_then( - [](auto range) -> Result { + [](const auto& range) -> Result { ContainerType container; for (auto& res : range) { if (res) { From a8038d2535a7bacc46142a4911e7ed42b75f8f6f Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Tue, 11 Nov 2025 22:22:58 +0100 Subject: [PATCH 24/48] Added auto incr primary keys --- src/sqlgen/duckdb/to_sql.cpp | 92 +++++++++++++++++++++++++++++------- 1 file changed, 76 insertions(+), 16 deletions(-) diff --git a/src/sqlgen/duckdb/to_sql.cpp b/src/sqlgen/duckdb/to_sql.cpp index c95c7a37..65c2fd86 100644 --- a/src/sqlgen/duckdb/to_sql.cpp +++ b/src/sqlgen/duckdb/to_sql.cpp @@ -1,11 +1,13 @@ #include "sqlgen/duckdb/to_sql.hpp" +#include #include #include #include #include #include +#include "sqlgen/dynamic/Column.hpp" #include "sqlgen/dynamic/Join.hpp" #include "sqlgen/dynamic/Operation.hpp" #include "sqlgen/internal/collect/vector.hpp" @@ -23,10 +25,16 @@ std::string condition_to_sql(const dynamic::Condition& _cond) noexcept; template std::string condition_to_sql_impl(const ConditionType& _condition) noexcept; -std::string column_to_sql_definition(const dynamic::Column& _col) noexcept; +std::string column_to_sql_definition(const dynamic::Table& _table, + const dynamic::Column& _col) noexcept; + +std::string create_enums(const dynamic::CreateTable& _stmt) noexcept; std::string create_index_to_sql(const dynamic::CreateIndex& _stmt) noexcept; +std::string create_sequences_for_auto_incr( + const dynamic::CreateTable& _stmt) noexcept; + std::string create_table_to_sql(const dynamic::CreateTable& _stmt) noexcept; std::string create_as_to_sql(const dynamic::CreateAs& _stmt) noexcept; @@ -51,8 +59,11 @@ std::string join_to_sql(const dynamic::Join& _stmt) noexcept; std::string operation_to_sql(const dynamic::Operation& _stmt) noexcept; -std::string properties_to_sql( - const dynamic::types::Properties& _properties) noexcept; +std::string make_sequence_name(const dynamic::Table& _table, + const dynamic::Column& _col) noexcept; + +std::string properties_to_sql(const dynamic::Table& _table, + const dynamic::Column& _col) noexcept; std::string select_from_to_sql(const dynamic::SelectFrom& _stmt) noexcept; @@ -245,10 +256,10 @@ std::string condition_to_sql_impl(const ConditionType& _condition) noexcept { return stream.str(); } -std::string column_to_sql_definition(const dynamic::Column& _col) noexcept { +std::string column_to_sql_definition(const dynamic::Table& _table, + const dynamic::Column& _col) noexcept { return wrap_in_quotes(_col.name) + " " + type_to_sql(_col.type) + - properties_to_sql( - _col.type.visit([](const auto& _t) { return _t.properties; })); + properties_to_sql(_table, _col); } std::string create_index_to_sql(const dynamic::CreateIndex& _stmt) noexcept { @@ -290,13 +301,41 @@ std::string create_index_to_sql(const dynamic::CreateIndex& _stmt) noexcept { return stream.str(); } -std::string create_table_to_sql(const dynamic::CreateTable& _stmt) noexcept { +std::string make_sequence_name(const dynamic::Table& _table, + const dynamic::Column& _col) noexcept { + return "sqlgen_seq_" + (_table.alias ? *_table.alias + "_" : std::string()) + + _table.name + "_" + _col.name; +} + +std::string create_sequences_for_auto_incr( + const dynamic::CreateTable& _stmt) noexcept { using namespace std::ranges::views; - const auto col_to_sql = [&](const auto& _col) { - return column_to_sql_definition(_col); + const auto is_auto_incr = [](const auto& _col) { + return _col.type.visit( + [](const auto& _t) { return _t.properties.auto_incr; }); + }; + + const auto create_one_sequence = + [&](const dynamic::Column& _col) -> std::string { + std::stringstream stream; + stream << "CREATE SEQUENCE "; + if (_stmt.if_not_exists) { + stream << "IF NOT EXISTS "; + } + stream << wrap_in_quotes(make_sequence_name(_stmt.table, _col)) << ";"; + return stream.str(); }; + return internal::strings::join(" ", internal::collect::vector( + _stmt.columns | filter(is_auto_incr) | + transform(create_one_sequence))) + + " "; +} + +std::string create_enums(const dynamic::CreateTable& _stmt) noexcept { + using namespace std::ranges::views; + std::stringstream stream; for (const auto& [enum_name, enum_values] : get_enum_types(_stmt)) { @@ -313,6 +352,18 @@ std::string create_table_to_sql(const dynamic::CreateTable& _stmt) noexcept { } } + return stream.str(); +} + +std::string create_table_to_sql(const dynamic::CreateTable& _stmt) noexcept { + using namespace std::ranges::views; + + std::stringstream stream; + + stream << create_enums(_stmt); + + stream << create_sequences_for_auto_incr(_stmt); + stream << "CREATE TABLE "; if (_stmt.if_not_exists) { @@ -326,7 +377,9 @@ std::string create_table_to_sql(const dynamic::CreateTable& _stmt) noexcept { stream << "("; stream << internal::strings::join( - ", ", internal::collect::vector(_stmt.columns | transform(col_to_sql))); + ", ", internal::collect::vector( + _stmt.columns | transform(std::bind_front( + column_to_sql_definition, _stmt.table)))); const auto primary_keys = get_primary_keys(_stmt); @@ -671,16 +724,23 @@ std::string operation_to_sql(const dynamic::Operation& _stmt) noexcept { }); } -std::string properties_to_sql(const dynamic::types::Properties& _p) noexcept { +std::string properties_to_sql(const dynamic::Table& _table, + const dynamic::Column& _col) noexcept { + const auto properties = + _col.type.visit([](const auto& _t) { return _t.properties; }); + return [&]() -> std::string { - return std::string(_p.nullable ? "" : " NOT NULL") + - std::string(_p.auto_incr ? " GENERATED ALWAYS" : "") + - std::string(_p.unique ? " UNIQUE" : ""); + return std::string(properties.nullable ? "" : " NOT NULL") + + std::string(properties.auto_incr + ? " DEFAULT nextval('" + + make_sequence_name(_table, _col) + "')" + : "") + + std::string(properties.unique ? " UNIQUE" : ""); }() + [&]() -> std::string { - if (!_p.foreign_key_reference) { + if (!properties.foreign_key_reference) { return ""; } - const auto& ref = *_p.foreign_key_reference; + const auto& ref = *properties.foreign_key_reference; return " REFERENCES " + wrap_in_quotes(ref.table) + "(" + wrap_in_quotes(ref.column) + ")"; }(); From d4e7d258e433ed9ba93937a598419d18f8935348 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Tue, 11 Nov 2025 22:56:07 +0100 Subject: [PATCH 25/48] Fixed typos --- include/sqlgen/duckdb/parsing/Parser_optional.hpp | 2 +- include/sqlgen/duckdb/parsing/Parser_smart_ptr.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/sqlgen/duckdb/parsing/Parser_optional.hpp b/include/sqlgen/duckdb/parsing/Parser_optional.hpp index edb0211b..e5b373d9 100644 --- a/include/sqlgen/duckdb/parsing/Parser_optional.hpp +++ b/include/sqlgen/duckdb/parsing/Parser_optional.hpp @@ -34,7 +34,7 @@ struct Parser> { ? Result(Nothing{}) : Result(error("Could not append null value.")); } - return Parser>::write(*_o); + return Parser>::write(*_o, _appender); } }; diff --git a/include/sqlgen/duckdb/parsing/Parser_smart_ptr.hpp b/include/sqlgen/duckdb/parsing/Parser_smart_ptr.hpp index 6863f5a6..dcea1519 100644 --- a/include/sqlgen/duckdb/parsing/Parser_smart_ptr.hpp +++ b/include/sqlgen/duckdb/parsing/Parser_smart_ptr.hpp @@ -37,7 +37,7 @@ struct Parser { ? Result(Nothing{}) : Result(error("Could not append null value.")); } - return Parser>::write(*_ptr); + return Parser>::write(*_ptr, _appender); } }; From 777e7fa422d36412428586f29a2adc01dd1c7080 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Tue, 11 Nov 2025 22:56:28 +0100 Subject: [PATCH 26/48] Add tests --- include/sqlgen/duckdb/Connection.hpp | 8 +- tests/duckdb/test_aggregations.cpp | 64 ++++++++++++++ .../test_aggregations_with_nullable.cpp | 63 ++++++++++++++ tests/duckdb/test_alpha_numeric.cpp | 39 +++++++++ tests/duckdb/test_alpha_numeric_query.cpp | 47 ++++++++++ tests/duckdb/test_boolean.cpp | 52 +++++++++++ tests/duckdb/test_boolean_conditions.cpp | 54 ++++++++++++ tests/duckdb/test_boolean_update.cpp | 59 +++++++++++++ tests/duckdb/test_cache.cpp | 45 ++++++++++ tests/duckdb/test_create_index.cpp | 34 ++++++++ tests/duckdb/test_create_table.cpp | 31 +++++++ tests/duckdb/test_create_table_as.cpp | 60 +++++++++++++ tests/duckdb/test_create_view_as.cpp | 65 ++++++++++++++ tests/duckdb/test_delete_from.cpp | 48 +++++++++++ tests/duckdb/test_drop.cpp | 41 +++++++++ tests/duckdb/test_foreign_key.cpp | 58 +++++++++++++ tests/duckdb/test_full_join.cpp | 69 +++++++++++++++ tests/duckdb/test_group_by.cpp | 67 +++++++++++++++ .../duckdb/test_group_by_with_operations.cpp | 63 ++++++++++++++ tests/duckdb/test_hello_world.cpp | 32 +++++++ tests/duckdb/test_in.cpp | 48 +++++++++++ tests/duckdb/test_in_vec.cpp | 49 +++++++++++ tests/duckdb/test_insert_by_ref_and_read.cpp | 44 ++++++++++ tests/duckdb/test_insert_fail.cpp | 59 +++++++++++++ tests/duckdb/test_is_null.cpp | 55 ++++++++++++ tests/duckdb/test_join.cpp | 51 +++++++++++ tests/duckdb/test_joins_from.cpp | 86 +++++++++++++++++++ tests/duckdb/test_joins_nested.cpp | 82 ++++++++++++++++++ tests/duckdb/test_joins_nested_grouped.cpp | 81 +++++++++++++++++ tests/duckdb/test_joins_two_tables.cpp | 77 +++++++++++++++++ .../duckdb/test_joins_two_tables_grouped.cpp | 78 +++++++++++++++++ tests/duckdb/test_left_join.cpp | 69 +++++++++++++++ tests/duckdb/test_like.cpp | 65 ++++++++++++++ tests/duckdb/test_limit.cpp | 47 ++++++++++ tests/duckdb/test_not_in.cpp | 48 +++++++++++ tests/duckdb/test_not_in_vec.cpp | 49 +++++++++++ tests/duckdb/test_operations.cpp | 76 ++++++++++++++++ .../duckdb/test_operations_with_nullable.cpp | 63 ++++++++++++++ tests/duckdb/test_order_by.cpp | 47 ++++++++++ tests/duckdb/test_range.cpp | 47 ++++++++++ tests/duckdb/test_range_select_from.cpp | 52 +++++++++++ .../duckdb/test_range_select_from_with_to.cpp | 51 +++++++++++ tests/duckdb/test_right_join.cpp | 69 +++++++++++++++ .../test_select_from_with_timestamps.cpp | 79 +++++++++++++++++ tests/duckdb/test_single_read.cpp | 43 ++++++++++ tests/duckdb/test_timestamp.cpp | 49 +++++++++++ tests/duckdb/test_to_create_table.cpp | 25 ++++++ tests/duckdb/test_to_insert.cpp | 27 ++++++ tests/duckdb/test_to_insert_or_replace.cpp | 32 +++++++ tests/duckdb/test_to_select_from.cpp | 24 ++++++ .../test_to_select_from_with_schema.cpp | 27 ++++++ tests/duckdb/test_transaction.cpp | 55 ++++++++++++ tests/duckdb/test_unique.cpp | 46 ++++++++++ tests/duckdb/test_update.cpp | 50 +++++++++++ tests/duckdb/test_varchar.cpp | 39 +++++++++ tests/duckdb/test_where.cpp | 48 +++++++++++ tests/duckdb/test_where_with_nullable.cpp | 48 +++++++++++ .../test_where_with_nullable_operations.cpp | 49 +++++++++++ tests/duckdb/test_where_with_operations.cpp | 48 +++++++++++ tests/duckdb/test_where_with_timestamps.cpp | 60 +++++++++++++ tests/duckdb/test_write_and_read_curried.cpp | 42 +++++++++ tests/duckdb/test_write_and_read_to_file.cpp | 44 ++++++++++ 62 files changed, 3222 insertions(+), 5 deletions(-) create mode 100644 tests/duckdb/test_aggregations.cpp create mode 100644 tests/duckdb/test_aggregations_with_nullable.cpp create mode 100644 tests/duckdb/test_alpha_numeric.cpp create mode 100644 tests/duckdb/test_alpha_numeric_query.cpp create mode 100644 tests/duckdb/test_boolean.cpp create mode 100644 tests/duckdb/test_boolean_conditions.cpp create mode 100644 tests/duckdb/test_boolean_update.cpp create mode 100644 tests/duckdb/test_cache.cpp create mode 100644 tests/duckdb/test_create_index.cpp create mode 100644 tests/duckdb/test_create_table.cpp create mode 100644 tests/duckdb/test_create_table_as.cpp create mode 100644 tests/duckdb/test_create_view_as.cpp create mode 100644 tests/duckdb/test_delete_from.cpp create mode 100644 tests/duckdb/test_drop.cpp create mode 100644 tests/duckdb/test_foreign_key.cpp create mode 100644 tests/duckdb/test_full_join.cpp create mode 100644 tests/duckdb/test_group_by.cpp create mode 100644 tests/duckdb/test_group_by_with_operations.cpp create mode 100644 tests/duckdb/test_hello_world.cpp create mode 100644 tests/duckdb/test_in.cpp create mode 100644 tests/duckdb/test_in_vec.cpp create mode 100644 tests/duckdb/test_insert_by_ref_and_read.cpp create mode 100644 tests/duckdb/test_insert_fail.cpp create mode 100644 tests/duckdb/test_is_null.cpp create mode 100644 tests/duckdb/test_join.cpp create mode 100644 tests/duckdb/test_joins_from.cpp create mode 100644 tests/duckdb/test_joins_nested.cpp create mode 100644 tests/duckdb/test_joins_nested_grouped.cpp create mode 100644 tests/duckdb/test_joins_two_tables.cpp create mode 100644 tests/duckdb/test_joins_two_tables_grouped.cpp create mode 100644 tests/duckdb/test_left_join.cpp create mode 100644 tests/duckdb/test_like.cpp create mode 100644 tests/duckdb/test_limit.cpp create mode 100644 tests/duckdb/test_not_in.cpp create mode 100644 tests/duckdb/test_not_in_vec.cpp create mode 100644 tests/duckdb/test_operations.cpp create mode 100644 tests/duckdb/test_operations_with_nullable.cpp create mode 100644 tests/duckdb/test_order_by.cpp create mode 100644 tests/duckdb/test_range.cpp create mode 100644 tests/duckdb/test_range_select_from.cpp create mode 100644 tests/duckdb/test_range_select_from_with_to.cpp create mode 100644 tests/duckdb/test_right_join.cpp create mode 100644 tests/duckdb/test_select_from_with_timestamps.cpp create mode 100644 tests/duckdb/test_single_read.cpp create mode 100644 tests/duckdb/test_timestamp.cpp create mode 100644 tests/duckdb/test_to_create_table.cpp create mode 100644 tests/duckdb/test_to_insert.cpp create mode 100644 tests/duckdb/test_to_insert_or_replace.cpp create mode 100644 tests/duckdb/test_to_select_from.cpp create mode 100644 tests/duckdb/test_to_select_from_with_schema.cpp create mode 100644 tests/duckdb/test_transaction.cpp create mode 100644 tests/duckdb/test_unique.cpp create mode 100644 tests/duckdb/test_update.cpp create mode 100644 tests/duckdb/test_varchar.cpp create mode 100644 tests/duckdb/test_where.cpp create mode 100644 tests/duckdb/test_where_with_nullable.cpp create mode 100644 tests/duckdb/test_where_with_nullable_operations.cpp create mode 100644 tests/duckdb/test_where_with_operations.cpp create mode 100644 tests/duckdb/test_where_with_timestamps.cpp create mode 100644 tests/duckdb/test_write_and_read_curried.cpp create mode 100644 tests/duckdb/test_write_and_read_to_file.cpp diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index 52a4c4d2..b36934f5 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -70,12 +70,10 @@ class Connection { } template - Result read(const dynamic::SelectFrom &_query) { + auto read(const dynamic::SelectFrom &_query) { using ValueType = transpilation::value_t; - const auto result = - internal::to_container>( - Iterator(to_sql(_query), conn_)); - return result; + return internal::to_container>( + Iterator(to_sql(_query), conn_)); } Result rollback() noexcept; diff --git a/tests/duckdb/test_aggregations.cpp b/tests/duckdb/test_aggregations.cpp new file mode 100644 index 00000000..a1beb0d6 --- /dev/null +++ b/tests/duckdb/test_aggregations.cpp @@ -0,0 +1,64 @@ + +#include + +#include +#include +#include +#include +#include +#include + +namespace test_aggregations { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_aggregations) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + struct Children { + int num_children; + int num_last_names; + double avg_age; + double max_age; + double min_age; + double sum_age; + }; + + const auto get_children = + select_from( + avg("age"_c).as<"avg_age">(), count().as<"num_children">(), + max("age"_c).as<"max_age">(), min("age"_c).as<"min_age">(), + sum("age"_c).as<"sum_age">(), + count_distinct("last_name"_c).as<"num_last_names">()) | + where("age"_c < 18) | to; + + const auto children = duckdb::connect() + .and_then(drop | if_exists) + .and_then(write(std::ref(people1))) + .and_then(get_children) + .value(); + + EXPECT_EQ(children.num_children, 3); + EXPECT_EQ(children.num_last_names, 1); + EXPECT_EQ(children.avg_age, 6.0); + EXPECT_EQ(children.max_age, 10.0); + EXPECT_EQ(children.min_age, 0.0); + EXPECT_EQ(children.sum_age, 18.0); +} + +} // namespace test_aggregations + diff --git a/tests/duckdb/test_aggregations_with_nullable.cpp b/tests/duckdb/test_aggregations_with_nullable.cpp new file mode 100644 index 00000000..37afa1da --- /dev/null +++ b/tests/duckdb/test_aggregations_with_nullable.cpp @@ -0,0 +1,63 @@ + +#include + +#include +#include +#include +#include +#include +#include + +namespace test_aggregations_with_nullable { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + std::optional age; +}; + +TEST(duckdb, test_aggregations_with_nullable) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{.id = 3, .first_name = "Maggie", .last_name = "Simpson"}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + struct Children { + int num_children; + int num_last_names; + std::optional avg_age; + std::optional max_age; + std::optional min_age; + std::optional sum_age; + }; + + const auto get_children = + select_from( + avg("age"_c).as<"avg_age">(), count().as<"num_children">(), + max("age"_c).as<"max_age">(), min("age"_c).as<"min_age">(), + sum("age"_c).as<"sum_age">(), + count_distinct("last_name"_c).as<"num_last_names">()) | + where("age"_c < 18 or "age"_c.is_null()) | to; + + const auto children = duckdb::connect() + .and_then(drop | if_exists) + .and_then(write(std::ref(people1))) + .and_then(get_children) + .value(); + + EXPECT_EQ(children.num_children, 3); + EXPECT_EQ(children.num_last_names, 1); + EXPECT_EQ(children.avg_age, 9.0); + EXPECT_EQ(children.max_age, 10.0); + EXPECT_EQ(children.min_age, 8.0); + EXPECT_EQ(children.sum_age, 18.0); +} + +} // namespace test_aggregations_with_nullable + diff --git a/tests/duckdb/test_alpha_numeric.cpp b/tests/duckdb/test_alpha_numeric.cpp new file mode 100644 index 00000000..2e81d7e3 --- /dev/null +++ b/tests/duckdb/test_alpha_numeric.cpp @@ -0,0 +1,39 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_alpha_numeric { + +struct Person { + sqlgen::PrimaryKey id; + sqlgen::AlphaNumeric first_name; + sqlgen::AlphaNumeric last_name; + int age; +}; + +TEST(duckdb, test_alpha_numeric) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + const auto conn = sqlgen::duckdb::connect(); + + sqlgen::write(conn, people1); + + const auto people2 = sqlgen::read>(conn).value(); + + const auto json1 = rfl::json::write(people1); + const auto json2 = rfl::json::write(people2); + + EXPECT_EQ(json1, json2); +} + +} // namespace test_alpha_numeric diff --git a/tests/duckdb/test_alpha_numeric_query.cpp b/tests/duckdb/test_alpha_numeric_query.cpp new file mode 100644 index 00000000..39104413 --- /dev/null +++ b/tests/duckdb/test_alpha_numeric_query.cpp @@ -0,0 +1,47 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_alpha_numeric_query { + +struct Person { + sqlgen::PrimaryKey id; + sqlgen::AlphaNumeric first_name; + sqlgen::AlphaNumeric last_name; + int age; +}; + +sqlgen::Result> get_people( + const auto& _conn, const sqlgen::AlphaNumeric& _first_name) { + using namespace sqlgen; + using namespace sqlgen::literals; + const auto query = + sqlgen::read> | where("first_name"_c == _first_name); + return query(_conn); +} + +TEST(duckdb, test_alpha_numeric_query) { + const auto people1 = std::vector({Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}}); + + const auto conn = sqlgen::duckdb::connect(); + + sqlgen::write(conn, people1); + + const auto people2 = sqlgen::AlphaNumeric::from_value("Homer") + .and_then([&](const auto& _first_name) { + return get_people(conn, _first_name); + }) + .value(); + + const auto json1 = rfl::json::write(people1); + const auto json2 = rfl::json::write(people2); + + EXPECT_EQ(json1, json2); +} + +} // namespace test_alpha_numeric_query diff --git a/tests/duckdb/test_boolean.cpp b/tests/duckdb/test_boolean.cpp new file mode 100644 index 00000000..ff754994 --- /dev/null +++ b/tests/duckdb/test_boolean.cpp @@ -0,0 +1,52 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_boolean { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + bool has_children; +}; + +TEST(duckdb, test_boolean) { + const auto people1 = std::vector({Person{.id = 0, + .first_name = "Homer", + .last_name = "Simpson", + .has_children = true}, + Person{.id = 1, + .first_name = "Bart", + .last_name = "Simpson", + .has_children = false}, + Person{.id = 2, + .first_name = "Lisa", + .last_name = "Simpson", + .has_children = false}, + Person{.id = 3, + .first_name = "Maggie", + .last_name = "Simpson", + .has_children = false}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto conn = duckdb::connect(); + + const auto people2 = sqlgen::write(conn, people1) + .and_then(sqlgen::read>) + .value(); + + const auto json1 = rfl::json::write(people1); + const auto json2 = rfl::json::write(people2); + + EXPECT_EQ(json1, json2); +} + +} // namespace test_boolean + diff --git a/tests/duckdb/test_boolean_conditions.cpp b/tests/duckdb/test_boolean_conditions.cpp new file mode 100644 index 00000000..8fe076aa --- /dev/null +++ b/tests/duckdb/test_boolean_conditions.cpp @@ -0,0 +1,54 @@ + +#include + +#include +#include +#include +#include +#include + +namespace test_boolean_conditions { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + bool has_children; +}; + +TEST(duckdb, test_boolean_conditions) { + const auto people1 = std::vector({Person{.id = 0, + .first_name = "Homer", + .last_name = "Simpson", + .has_children = true}, + Person{.id = 1, + .first_name = "Bart", + .last_name = "Simpson", + .has_children = false}, + Person{.id = 2, + .first_name = "Lisa", + .last_name = "Simpson", + .has_children = false}, + Person{.id = 3, + .first_name = "Maggie", + .last_name = "Simpson", + .has_children = false}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto homer = + duckdb::connect() + .and_then(sqlgen::write(people1)) + .and_then(sqlgen::read | where("has_children"_c == true) | + order_by("id"_c)) + .value(); + + const auto json1 = rfl::json::write(people1.at(0)); + const auto json2 = rfl::json::write(homer); + + EXPECT_EQ(json1, json2); +} + +} // namespace test_boolean_conditions + diff --git a/tests/duckdb/test_boolean_update.cpp b/tests/duckdb/test_boolean_update.cpp new file mode 100644 index 00000000..a202cffa --- /dev/null +++ b/tests/duckdb/test_boolean_update.cpp @@ -0,0 +1,59 @@ + +#include + +#include +#include +#include +#include +#include + +namespace test_boolean_update { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + bool has_children; +}; + +TEST(duckdb, test_boolean_update) { + auto people1 = std::vector({Person{.id = 0, + .first_name = "Homer", + .last_name = "Simpson", + .has_children = true}, + Person{.id = 1, + .first_name = "Bart", + .last_name = "Simpson", + .has_children = false}, + Person{.id = 2, + .first_name = "Lisa", + .last_name = "Simpson", + .has_children = false}, + Person{.id = 3, + .first_name = "Maggie", + .last_name = "Simpson", + .has_children = false}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto conn = duckdb::connect(); + + const auto people2 = + sqlgen::write(conn, people1) + .and_then(update("has_children"_c.set(false)) | + where("has_children"_c == true)) + .and_then(sqlgen::read> | + where("has_children"_c == false) | order_by("id"_c)) + .value(); + + people1.at(0).has_children = false; + + const auto json1 = rfl::json::write(people1); + const auto json2 = rfl::json::write(people2); + + EXPECT_EQ(json1, json2); +} + +} // namespace test_boolean_update + diff --git a/tests/duckdb/test_cache.cpp b/tests/duckdb/test_cache.cpp new file mode 100644 index 00000000..322442b7 --- /dev/null +++ b/tests/duckdb/test_cache.cpp @@ -0,0 +1,45 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_cache { + +struct User { + std::string name; + int age; +}; + +TEST(duckdb, test_cache) { + const auto conn = sqlgen::duckdb::connect(); + + const auto user = User{.name = "John", .age = 30}; + sqlgen::write(conn, user); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto query = sqlgen::read | where("name"_c == "John"); + + const auto cached_query = sqlgen::cache<100>(query); + + const auto user1 = conn.and_then(cache<100>(query)).value(); + + EXPECT_EQ(cached_query.cache(conn).size(), 1); + + const auto user2 = cached_query(conn).value(); + const auto user3 = cached_query(conn).value(); + + EXPECT_EQ(user1.name, "John"); + EXPECT_EQ(user1.age, 30); + EXPECT_EQ(user2.name, "John"); + EXPECT_EQ(user2.age, 30); + EXPECT_EQ(cached_query.cache(conn).size(), 1); + EXPECT_EQ(user3.name, "John"); + EXPECT_EQ(user3.age, 30); +} + +} // namespace test_cache diff --git a/tests/duckdb/test_create_index.cpp b/tests/duckdb/test_create_index.cpp new file mode 100644 index 00000000..cbce32d0 --- /dev/null +++ b/tests/duckdb/test_create_index.cpp @@ -0,0 +1,34 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_create_index { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_create_index) { + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto people = sqlgen::duckdb::connect() + .and_then(create_table | if_not_exists) + .and_then(create_index<"person_ix", Person>( + "first_name"_c, "last_name"_c) | + if_not_exists) + .and_then(sqlgen::read>); + + const std::string expected = R"([])"; + + EXPECT_EQ(rfl::json::write(people), expected); +} + +} // namespace test_create_index diff --git a/tests/duckdb/test_create_table.cpp b/tests/duckdb/test_create_table.cpp new file mode 100644 index 00000000..cc2174ce --- /dev/null +++ b/tests/duckdb/test_create_table.cpp @@ -0,0 +1,31 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_create_table { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_create_table) { + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto people = sqlgen::duckdb::connect() + .and_then(create_table | if_not_exists) + .and_then(sqlgen::read>); + + const std::string expected = R"([])"; + + EXPECT_EQ(rfl::json::write(people), expected); +} + +} // namespace test_create_table diff --git a/tests/duckdb/test_create_table_as.cpp b/tests/duckdb/test_create_table_as.cpp new file mode 100644 index 00000000..f10e11ef --- /dev/null +++ b/tests/duckdb/test_create_table_as.cpp @@ -0,0 +1,60 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_create_table_as { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +struct Name { + std::string first_name; + std::string last_name; +}; + +TEST(duckdb, test_create_table_as) { + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto people = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{ + .id = 1, .first_name = "Marge", .last_name = "Simpson", .age = 40}, + Person{.id = 2, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 3, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 4, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + const auto names_query = select_from("first_name"_c, "last_name"_c); + + const auto get_names = create_as(names_query) | if_not_exists; + + const auto names = sqlgen::duckdb::connect() + .and_then(drop | if_exists) + .and_then(drop | if_exists) + .and_then(write(std::ref(people))) + .and_then(get_names) + .and_then(sqlgen::read>) + .value(); + + const std::string expected_query = + R"(CREATE TABLE IF NOT EXISTS "Name" AS SELECT "first_name", "last_name" FROM "Person")"; + + const std::string expected = + R"([{"first_name":"Homer","last_name":"Simpson"},{"first_name":"Marge","last_name":"Simpson"},{"first_name":"Bart","last_name":"Simpson"},{"first_name":"Lisa","last_name":"Simpson"},{"first_name":"Maggie","last_name":"Simpson"}])"; + + EXPECT_EQ(duckdb::to_sql(get_names), expected_query); + EXPECT_EQ(rfl::json::write(names), expected); +} + +} // namespace test_create_table_as + diff --git a/tests/duckdb/test_create_view_as.cpp b/tests/duckdb/test_create_view_as.cpp new file mode 100644 index 00000000..bfb2b5ad --- /dev/null +++ b/tests/duckdb/test_create_view_as.cpp @@ -0,0 +1,65 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_create_view_as { + +struct Person { + static constexpr const char* tablename = "PEOPLE"; + + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +struct Name { + static constexpr const char* viewname = "NAMES"; + static constexpr bool is_view = true; + + std::string first_name; + std::string last_name; +}; + +TEST(duckdb, test_create_view_as) { + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto people = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{ + .id = 1, .first_name = "Marge", .last_name = "Simpson", .age = 40}, + Person{.id = 2, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 3, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 4, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + const auto names_query = select_from("first_name"_c, "last_name"_c); + + const auto get_names = create_as(names_query) | if_not_exists; + + const auto names = duckdb::connect() + .and_then(drop | if_exists) + .and_then(drop | if_exists) + .and_then(write(std::ref(people))) + .and_then(get_names) + .and_then(sqlgen::read>) + .value(); + + const std::string expected_query = + R"(CREATE VIEW IF NOT EXISTS "NAMES" AS SELECT "first_name", "last_name" FROM "PEOPLE")"; + + const std::string expected = + R"([{"first_name":"Homer","last_name":"Simpson"},{"first_name":"Marge","last_name":"Simpson"},{"first_name":"Bart","last_name":"Simpson"},{"first_name":"Lisa","last_name":"Simpson"},{"first_name":"Maggie","last_name":"Simpson"}])"; + + EXPECT_EQ(duckdb::to_sql(get_names), expected_query); + EXPECT_EQ(rfl::json::write(names), expected); +} + +} // namespace test_create_view_as + diff --git a/tests/duckdb/test_delete_from.cpp b/tests/duckdb/test_delete_from.cpp new file mode 100644 index 00000000..642345ea --- /dev/null +++ b/tests/duckdb/test_delete_from.cpp @@ -0,0 +1,48 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_delete_from { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_delete_from) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}, + Person{ + .id = 4, .first_name = "Hugo", .last_name = "Simpson", .age = 10}}); + + const auto conn = sqlgen::duckdb::connect(); + + sqlgen::write(conn, people1); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto query = delete_from | where("first_name"_c == "Hugo"); + + query(conn).value(); + + const auto people2 = sqlgen::read>(conn).value(); + + const std::string expected = + R"([{"id":0,"first_name":"Homer","last_name":"Simpson","age":45},{"id":1,"first_name":"Bart","last_name":"Simpson","age":10},{"id":2,"first_name":"Lisa","last_name":"Simpson","age":8},{"id":3,"first_name":"Maggie","last_name":"Simpson","age":0}])"; + + EXPECT_EQ(rfl::json::write(people2), expected); +} + +} // namespace test_delete_from diff --git a/tests/duckdb/test_drop.cpp b/tests/duckdb/test_drop.cpp new file mode 100644 index 00000000..3d0e4fa3 --- /dev/null +++ b/tests/duckdb/test_drop.cpp @@ -0,0 +1,41 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_drop { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_drop) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}, + Person{ + .id = 4, .first_name = "Hugo", .last_name = "Simpson", .age = 10}}); + + const auto conn = sqlgen::duckdb::connect(); + + sqlgen::write(conn, people1); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto query = drop | if_exists; + + query(conn).value(); +} + +} // namespace test_drop diff --git a/tests/duckdb/test_foreign_key.cpp b/tests/duckdb/test_foreign_key.cpp new file mode 100644 index 00000000..322136c0 --- /dev/null +++ b/tests/duckdb/test_foreign_key.cpp @@ -0,0 +1,58 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_foreign_key { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +struct Relationship { + sqlgen::ForeignKey parent_id; + uint32_t child_id; +}; + +TEST(duckdb, test_foreign_key) { + auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + const auto relationships = + std::vector({Relationship{.parent_id = 0, .child_id = 2}, + Relationship{.parent_id = 0, .child_id = 3}, + Relationship{.parent_id = 0, .child_id = 4}, + Relationship{.parent_id = 1, .child_id = 2}, + Relationship{.parent_id = 1, .child_id = 3}, + Relationship{.parent_id = 1, .child_id = 4}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto people = duckdb::connect() + .and_then(drop | if_exists) + .and_then(drop | if_exists) + .and_then(begin_transaction) + .and_then(create_table) + .and_then(create_table) + .and_then(insert(std::ref(people1))) + .and_then(insert(std::ref(relationships))) + .and_then(drop | if_exists) + .and_then(drop | if_exists) + .and_then(commit) + .value(); +} + +} // namespace test_foreign_key + diff --git a/tests/duckdb/test_full_join.cpp b/tests/duckdb/test_full_join.cpp new file mode 100644 index 00000000..b9e4aedf --- /dev/null +++ b/tests/duckdb/test_full_join.cpp @@ -0,0 +1,69 @@ +#include + +#include +#include +#include +#include +#include +#include + +namespace test_full_join { + +TEST(duckdb, test_full_join) { + struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + }; + + struct Pet { + sqlgen::PrimaryKey id; + std::string name; + uint32_t owner_id; + }; + + const auto people = std::vector({ + Person{.id = 1, .first_name = "Homer", .last_name = "Simpson"}, + Person{.id = 2, .first_name = "Marge", .last_name = "Simpson"}, + Person{.id = 3, .first_name = "Bart", .last_name = "Simpson"}, + Person{.id = 4, .first_name = "Lisa", .last_name = "Simpson"}, + }); + + const auto pets = std::vector({ + Pet{.id = 1, .name = "Santa's Little Helper", .owner_id = 1}, + Pet{.id = 2, .name = "Snowball", .owner_id = 4}, + Pet{.id = 3, .name = "Mr. Teeny", .owner_id = 99}, + }); + + using namespace sqlgen; + using namespace sqlgen::literals; + + struct PersonAndPet { + std::optional first_name; + std::optional last_name; + std::optional pet_name; + }; + + const auto get_all = + select_from("first_name"_t1 | as<"first_name">, + "last_name"_t1 | as<"last_name">, + "name"_t2 | as<"pet_name">) | + full_join("id"_t1 == "owner_id"_t2) | + order_by("id"_t1, "id"_t2) | to>; + + const auto result = duckdb::connect() + .and_then(write(std::ref(people))) + .and_then(write(std::ref(pets))) + .and_then(get_all) + .value(); + + const std::string expected_query = + R"(SELECT t1."first_name" AS "first_name", t1."last_name" AS "last_name", t2."name" AS "pet_name" FROM "Person" t1 FULL JOIN "Pet" t2 ON t1."id" = t2."owner_id" ORDER BY t1."id", t2."id")"; + const std::string expected_json = + R"([{"pet_name":"Mr. Teeny"},{"first_name":"Homer","last_name":"Simpson","pet_name":"Santa's Little Helper"},{"first_name":"Marge","last_name":"Simpson"},{"first_name":"Bart","last_name":"Simpson"},{"first_name":"Lisa","last_name":"Simpson","pet_name":"Snowball"}])"; + + EXPECT_EQ(duckdb::to_sql(get_all), expected_query); + EXPECT_EQ(rfl::json::write(result), expected_json); +} + +} // namespace test_full_join diff --git a/tests/duckdb/test_group_by.cpp b/tests/duckdb/test_group_by.cpp new file mode 100644 index 00000000..a04e841e --- /dev/null +++ b/tests/duckdb/test_group_by.cpp @@ -0,0 +1,67 @@ + +#include + +#include +#include +#include +#include +#include +#include + +namespace test_group_by { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_group_by) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + struct Children { + std::string last_name; + int num_children; + int num_last_names; + double avg_age; + double max_age; + double min_age; + double sum_age; + }; + + const auto get_children = + select_from( + "last_name"_c, avg("age"_c).as<"avg_age">(), + count().as<"num_children">(), max("age"_c).as<"max_age">(), + min("age"_c).as<"min_age">(), sum("age"_c).as<"sum_age">(), + count_distinct("last_name"_c).as<"num_last_names">()) | + where("age"_c < 18) | group_by("last_name"_c) | to>; + + const auto children = duckdb::connect() + .and_then(drop | if_exists) + .and_then(write(std::ref(people1))) + .and_then(get_children) + .value(); + + EXPECT_EQ(children.size(), 1); + EXPECT_EQ(children.at(0).last_name, "Simpson"); + EXPECT_EQ(children.at(0).num_children, 3); + EXPECT_EQ(children.at(0).num_last_names, 1); + EXPECT_EQ(children.at(0).avg_age, 6.0); + EXPECT_EQ(children.at(0).max_age, 10.0); + EXPECT_EQ(children.at(0).min_age, 0.0); + EXPECT_EQ(children.at(0).sum_age, 18.0); +} + +} // namespace test_group_by + diff --git a/tests/duckdb/test_group_by_with_operations.cpp b/tests/duckdb/test_group_by_with_operations.cpp new file mode 100644 index 00000000..c3da4567 --- /dev/null +++ b/tests/duckdb/test_group_by_with_operations.cpp @@ -0,0 +1,63 @@ + +#include + +#include +#include +#include +#include +#include +#include + +namespace test_group_by_with_operations { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_group_by_with_operations) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + struct Children { + std::string last_name; + std::string last_name_trimmed; + double avg_age; + double max_age_plus_one; + double min_age_plus_one; + }; + + const auto get_children = + select_from( + "last_name"_c, trim("last_name"_c).as<"last_name_trimmed">(), + max(cast("age"_c) + 1.0).as<"max_age_plus_one">(), + (min(cast("age"_c)) + 1.0).as<"min_age_plus_one">(), + round(avg(cast("age"_c))).as<"avg_age">()) | + where("age"_c < 18) | group_by("last_name"_c) | to>; + + const auto conn = duckdb::connect() + .and_then(drop | if_exists) + .and_then(write(std::ref(people1))); + + const auto children = get_children(conn).value(); + + EXPECT_EQ(children.size(), 1); + EXPECT_EQ(children.at(0).last_name, "Simpson"); + EXPECT_EQ(children.at(0).last_name_trimmed, "Simpson"); + EXPECT_EQ(children.at(0).avg_age, 6.0); + EXPECT_EQ(children.at(0).max_age_plus_one, 11.0); + EXPECT_EQ(children.at(0).min_age_plus_one, 1.0); +} + +} // namespace test_group_by_with_operations + diff --git a/tests/duckdb/test_hello_world.cpp b/tests/duckdb/test_hello_world.cpp new file mode 100644 index 00000000..cf69eb0d --- /dev/null +++ b/tests/duckdb/test_hello_world.cpp @@ -0,0 +1,32 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_hello_world { + +struct User { + std::string name; + int age; +}; + +TEST(duckdb, test_hello_world) { + // Connect to duckdb database + const auto conn = sqlgen::duckdb::connect("test.db"); + + // Create and insert a user + const auto user = User{.name = "John", .age = 30}; + sqlgen::write(conn, user); + + // Read all users + const auto users = sqlgen::read>(conn).value(); + + EXPECT_EQ(users.size(), 1); + EXPECT_EQ(users.at(0).name, "John"); + EXPECT_EQ(users.at(0).age, 30); +} + +} // namespace test_hello_world diff --git a/tests/duckdb/test_in.cpp b/tests/duckdb/test_in.cpp new file mode 100644 index 00000000..dc1923e5 --- /dev/null +++ b/tests/duckdb/test_in.cpp @@ -0,0 +1,48 @@ + +#include + +#include +#include +#include +#include +#include + +namespace test_in { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_in) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}, + Person{ + .id = 4, .first_name = "Hugo", .last_name = "Simpson", .age = 10}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto people2 = + duckdb::connect() + .and_then(write(std::ref(people1))) + .and_then(sqlgen::read> | + where("first_name"_c.in("Bart", "Lisa", "Maggie")) | + order_by("age"_c)) + .value(); + + const std::string expected1 = + R"([{"id":3,"first_name":"Maggie","last_name":"Simpson","age":0},{"id":2,"first_name":"Lisa","last_name":"Simpson","age":8},{"id":1,"first_name":"Bart","last_name":"Simpson","age":10}])"; + + EXPECT_EQ(rfl::json::write(people2), expected1); +} + +} // namespace test_in + diff --git a/tests/duckdb/test_in_vec.cpp b/tests/duckdb/test_in_vec.cpp new file mode 100644 index 00000000..2160aba7 --- /dev/null +++ b/tests/duckdb/test_in_vec.cpp @@ -0,0 +1,49 @@ + +#include + +#include +#include +#include +#include +#include + +namespace test_in_vec { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_in_vec) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}, + Person{ + .id = 4, .first_name = "Hugo", .last_name = "Simpson", .age = 10}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto people2 = + duckdb::connect() + .and_then(write(std::ref(people1))) + .and_then(sqlgen::read> | + where("first_name"_c.in( + std::vector({"Bart", "Lisa", "Maggie"}))) | + order_by("age"_c)) + .value(); + + const std::string expected1 = + R"([{"id":3,"first_name":"Maggie","last_name":"Simpson","age":0},{"id":2,"first_name":"Lisa","last_name":"Simpson","age":8},{"id":1,"first_name":"Bart","last_name":"Simpson","age":10}])"; + + EXPECT_EQ(rfl::json::write(people2), expected1); +} + +} // namespace test_in_vec + diff --git a/tests/duckdb/test_insert_by_ref_and_read.cpp b/tests/duckdb/test_insert_by_ref_and_read.cpp new file mode 100644 index 00000000..ffa866e5 --- /dev/null +++ b/tests/duckdb/test_insert_by_ref_and_read.cpp @@ -0,0 +1,44 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_insert_by_ref_and_read { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_insert_by_ref_and_read) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto people2 = duckdb::connect() + .and_then(begin_transaction) + .and_then(create_table | if_not_exists) + .and_then(insert(std::ref(people1))) + .and_then(commit) + .and_then(sqlgen::read>) + .value(); + + const auto json1 = rfl::json::write(people1); + const auto json2 = rfl::json::write(people2); + + EXPECT_EQ(json1, json2); +} + +} // namespace test_insert_by_ref_and_read diff --git a/tests/duckdb/test_insert_fail.cpp b/tests/duckdb/test_insert_fail.cpp new file mode 100644 index 00000000..d614baf8 --- /dev/null +++ b/tests/duckdb/test_insert_fail.cpp @@ -0,0 +1,59 @@ +#ifndef SQLGEN_BUILD_DRY_TESTS_ONLY + +#include + +#include +#include +#include +#include +#include +#include + +namespace test_insert_fail { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_insert_fail) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto conn = duckdb::connect() + .and_then(drop | if_exists) + .and_then(write(people1.at(0))); + + const auto res = conn.and_then(begin_transaction) + .and_then(insert(people1.at(0))) + .and_then(commit); + + // Should fail - duplicate key violation. + EXPECT_FALSE(res && true); + + const auto people2 = + conn.and_then(begin_transaction) + .and_then(insert(people1 | std::ranges::views::drop(1))) + .and_then(commit) + .and_then(sqlgen::read>) + .value(); + + const auto json1 = rfl::json::write(people1); + const auto json2 = rfl::json::write(people2); + + EXPECT_EQ(json1, json2); +} + +} // namespace test_insert_fail + +#endif diff --git a/tests/duckdb/test_is_null.cpp b/tests/duckdb/test_is_null.cpp new file mode 100644 index 00000000..3910f484 --- /dev/null +++ b/tests/duckdb/test_is_null.cpp @@ -0,0 +1,55 @@ + +#include + +#include +#include +#include +#include +#include + +namespace test_is_null { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + std::optional age; +}; + +TEST(duckdb, test_is_null) { + const auto people1 = std::vector( + {Person{.id = 0, .first_name = "Homer", .last_name = "Simpson"}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}, + Person{.id = 4, .first_name = "Hugo", .last_name = "Simpson"}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto conn = duckdb::connect(); + + const auto people2 = + conn.and_then(write(std::ref(people1))) + .and_then(sqlgen::read> | + where("age"_c.is_null()) | order_by("first_name"_c.desc())) + .value(); + + const auto people3 = + conn.and_then(sqlgen::read> | + where("age"_c.is_not_null()) | order_by("age"_c)) + .value(); + + const std::string expected1 = + R"([{"id":4,"first_name":"Hugo","last_name":"Simpson"},{"id":0,"first_name":"Homer","last_name":"Simpson"}])"; + + const std::string expected2 = + R"([{"id":3,"first_name":"Maggie","last_name":"Simpson","age":0},{"id":2,"first_name":"Lisa","last_name":"Simpson","age":8},{"id":1,"first_name":"Bart","last_name":"Simpson","age":10}])"; + + EXPECT_EQ(rfl::json::write(people2), expected1); + EXPECT_EQ(rfl::json::write(people3), expected2); +} + +} // namespace test_is_null + diff --git a/tests/duckdb/test_join.cpp b/tests/duckdb/test_join.cpp new file mode 100644 index 00000000..7e69ee14 --- /dev/null +++ b/tests/duckdb/test_join.cpp @@ -0,0 +1,51 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_join { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + double age; +}; + +TEST(duckdb, test_join) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto get_people = + select_from( + "id"_t1 | as<"id">, "first_name"_t1 | as<"first_name">, + "last_name"_t2 | as<"last_name">, "age"_t2 | as<"age">) | + inner_join("id"_t1 == "id"_t2) | order_by("id"_t1) | + to>; + + const auto people = duckdb::connect() + .and_then(write(std::ref(people1))) + .and_then(get_people) + .value(); + + const std::string expected_query = + R"(SELECT t1."id" AS "id", t1."first_name" AS "first_name", t2."last_name" AS "last_name", t2."age" AS "age" FROM "Person" t1 INNER JOIN "Person" t2 ON t1."id" = t2."id" ORDER BY t1."id")"; + const std::string expected = + R"([{"id":0,"first_name":"Homer","last_name":"Simpson","age":45.0},{"id":1,"first_name":"Bart","last_name":"Simpson","age":10.0},{"id":2,"first_name":"Lisa","last_name":"Simpson","age":8.0},{"id":3,"first_name":"Maggie","last_name":"Simpson","age":0.0}])"; + + EXPECT_EQ(duckdb::to_sql(get_people), expected_query); + EXPECT_EQ(rfl::json::write(people), expected); +} + +} // namespace test_join diff --git a/tests/duckdb/test_joins_from.cpp b/tests/duckdb/test_joins_from.cpp new file mode 100644 index 00000000..30aba805 --- /dev/null +++ b/tests/duckdb/test_joins_from.cpp @@ -0,0 +1,86 @@ +#include + +#include +#include +#include +#include +#include +#include + +namespace test_joins_from { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + double age; +}; + +struct Relationship { + uint32_t parent_id; + uint32_t child_id; +}; + +TEST(duckdb, test_joins_from) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{ + .id = 1, .first_name = "Marge", .last_name = "Simpson", .age = 40}, + Person{.id = 2, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 3, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 4, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + const auto relationships = + std::vector({Relationship{.parent_id = 0, .child_id = 2}, + Relationship{.parent_id = 0, .child_id = 3}, + Relationship{.parent_id = 0, .child_id = 4}, + Relationship{.parent_id = 1, .child_id = 2}, + Relationship{.parent_id = 1, .child_id = 3}, + Relationship{.parent_id = 1, .child_id = 4}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + struct ParentAndChild { + std::string last_name; + std::string first_name_parent; + std::string first_name_child; + double parent_age_at_birth; + }; + + const auto get_parents = + select_from( + "child_id"_t2 | as<"id">, "first_name"_t1 | as<"first_name">, + "last_name"_t1 | as<"last_name">, "age"_t1 | as<"age">) | + inner_join("id"_t1 == "parent_id"_t2); + + const auto get_people = + select_from<"t1">(get_parents, "last_name"_t1 | as<"last_name">, + "first_name"_t1 | as<"first_name_parent">, + "first_name"_t2 | as<"first_name_child">, + ("age"_t1 - "age"_t2) | as<"parent_age_at_birth">) | + inner_join("id"_t1 == "id"_t2) | + order_by("id"_t2, "id"_t1) | to>; + + const auto people = duckdb::connect() + .and_then(drop | if_exists) + .and_then(drop | if_exists) + .and_then(write(std::ref(people1))) + .and_then(write(std::ref(relationships))) + .and_then(get_people) + .value(); + + const std::string expected_query = + R"(SELECT t1."last_name" AS "last_name", t1."first_name" AS "first_name_parent", t2."first_name" AS "first_name_child", (t1."age") - (t2."age") AS "parent_age_at_birth" FROM (SELECT t2."child_id" AS "id", t1."first_name" AS "first_name", t1."last_name" AS "last_name", t1."age" AS "age" FROM "Person" t1 INNER JOIN "Relationship" t2 ON t1."id" = t2."parent_id") t1 INNER JOIN "Person" t2 ON t1."id" = t2."id" ORDER BY t2."id", t1."id")"; + + const std::string expected = + R"([{"last_name":"Simpson","first_name_parent":"Homer","first_name_child":"Bart","parent_age_at_birth":35.0},{"last_name":"Simpson","first_name_parent":"Marge","first_name_child":"Bart","parent_age_at_birth":30.0},{"last_name":"Simpson","first_name_parent":"Homer","first_name_child":"Lisa","parent_age_at_birth":37.0},{"last_name":"Simpson","first_name_parent":"Marge","first_name_child":"Lisa","parent_age_at_birth":32.0},{"last_name":"Simpson","first_name_parent":"Homer","first_name_child":"Maggie","parent_age_at_birth":45.0},{"last_name":"Simpson","first_name_parent":"Marge","first_name_child":"Maggie","parent_age_at_birth":40.0}])"; + + EXPECT_EQ(duckdb::to_sql(get_people), expected_query); + EXPECT_EQ(rfl::json::write(people), expected); +} + +} // namespace test_joins_from + diff --git a/tests/duckdb/test_joins_nested.cpp b/tests/duckdb/test_joins_nested.cpp new file mode 100644 index 00000000..b0292e31 --- /dev/null +++ b/tests/duckdb/test_joins_nested.cpp @@ -0,0 +1,82 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_joins_nested { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + double age; +}; + +struct Relationship { + uint32_t parent_id; + uint32_t child_id; +}; + +TEST(duckdb, test_joins_nested) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{ + .id = 1, .first_name = "Marge", .last_name = "Simpson", .age = 40}, + Person{.id = 2, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 3, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 4, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + const auto relationships = + std::vector({Relationship{.parent_id = 0, .child_id = 2}, + Relationship{.parent_id = 0, .child_id = 3}, + Relationship{.parent_id = 0, .child_id = 4}, + Relationship{.parent_id = 1, .child_id = 2}, + Relationship{.parent_id = 1, .child_id = 3}, + Relationship{.parent_id = 1, .child_id = 4}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + struct ParentAndChild { + std::string last_name; + std::string first_name_parent; + std::string first_name_child; + double parent_age_at_birth; + }; + + const auto get_children = + select_from("parent_id"_t1 | as<"id">, + "first_name"_t2 | as<"first_name">, + "age"_t2 | as<"age">) | + inner_join("id"_t2 == "child_id"_t1); + + const auto get_people = + select_from( + "last_name"_t1 | as<"last_name">, + "first_name"_t1 | as<"first_name_parent">, + "first_name"_t2 | as<"first_name_child">, + ("age"_t1 - "age"_t2) | as<"parent_age_at_birth">) | + inner_join<"t2">(get_children, "id"_t1 == "id"_t2) | + order_by("id"_t1, "id"_t2) | to>; + + const auto people = duckdb::connect() + .and_then(write(std::ref(people1))) + .and_then(write(std::ref(relationships))) + .and_then(get_people) + .value(); + + const std::string expected_query = + R"(SELECT t1."last_name" AS "last_name", t1."first_name" AS "first_name_parent", t2."first_name" AS "first_name_child", (t1."age") - (t2."age") AS "parent_age_at_birth" FROM "Person" t1 INNER JOIN (SELECT t1."parent_id" AS "id", t2."first_name" AS "first_name", t2."age" AS "age" FROM "Relationship" t1 INNER JOIN "Person" t2 ON t2."id" = t1."child_id") t2 ON t1."id" = t2."id" ORDER BY t1."id", t2."id")"; + const std::string expected = + R"([{"last_name":"Simpson","first_name_parent":"Homer","first_name_child":"Bart","parent_age_at_birth":35.0},{"last_name":"Simpson","first_name_parent":"Homer","first_name_child":"Lisa","parent_age_at_birth":37.0},{"last_name":"Simpson","first_name_parent":"Homer","first_name_child":"Maggie","parent_age_at_birth":45.0},{"last_name":"Simpson","first_name_parent":"Marge","first_name_child":"Bart","parent_age_at_birth":30.0},{"last_name":"Simpson","first_name_parent":"Marge","first_name_child":"Lisa","parent_age_at_birth":32.0},{"last_name":"Simpson","first_name_parent":"Marge","first_name_child":"Maggie","parent_age_at_birth":40.0}])"; + + EXPECT_EQ(duckdb::to_sql(get_people), expected_query); + EXPECT_EQ(rfl::json::write(people), expected); +} + +} // namespace test_joins_nested diff --git a/tests/duckdb/test_joins_nested_grouped.cpp b/tests/duckdb/test_joins_nested_grouped.cpp new file mode 100644 index 00000000..990e4d59 --- /dev/null +++ b/tests/duckdb/test_joins_nested_grouped.cpp @@ -0,0 +1,81 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_joins_nested_grouped { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + double age; +}; + +struct Relationship { + uint32_t parent_id; + uint32_t child_id; +}; + +TEST(duckdb, test_joins_nested_grouped) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{ + .id = 1, .first_name = "Marge", .last_name = "Simpson", .age = 40}, + Person{.id = 2, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 3, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 4, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + const auto relationships = + std::vector({Relationship{.parent_id = 0, .child_id = 2}, + Relationship{.parent_id = 0, .child_id = 3}, + Relationship{.parent_id = 0, .child_id = 4}, + Relationship{.parent_id = 1, .child_id = 2}, + Relationship{.parent_id = 1, .child_id = 3}, + Relationship{.parent_id = 1, .child_id = 4}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + struct ParentAndChild { + std::string last_name; + std::string first_name_child; + double avg_parent_age_at_birth; + }; + + const auto get_children = + select_from("parent_id"_t1 | as<"id">, + "first_name"_t2 | as<"first_name">, + "age"_t2 | as<"age">) | + inner_join("id"_t2 == "child_id"_t1); + + const auto get_people = + select_from( + "last_name"_t1 | as<"last_name">, + "first_name"_t2 | as<"first_name_child">, + avg("age"_t1 - "age"_t2) | as<"avg_parent_age_at_birth">) | + inner_join<"t2">(get_children, "id"_t1 == "id"_t2) | + group_by("last_name"_t1, "first_name"_t2) | order_by("first_name"_t2) | + to>; + + const auto people = duckdb::connect() + .and_then(write(std::ref(people1))) + .and_then(write(std::ref(relationships))) + .and_then(get_people) + .value(); + + const std::string expected_query = + R"(SELECT t1."last_name" AS "last_name", t2."first_name" AS "first_name_child", AVG((t1."age") - (t2."age")) AS "avg_parent_age_at_birth" FROM "Person" t1 INNER JOIN (SELECT t1."parent_id" AS "id", t2."first_name" AS "first_name", t2."age" AS "age" FROM "Relationship" t1 INNER JOIN "Person" t2 ON t2."id" = t1."child_id") t2 ON t1."id" = t2."id" GROUP BY t1."last_name", t2."first_name" ORDER BY t2."first_name")"; + + const std::string expected = + R"([{"last_name":"Simpson","first_name_child":"Bart","avg_parent_age_at_birth":32.5},{"last_name":"Simpson","first_name_child":"Lisa","avg_parent_age_at_birth":34.5},{"last_name":"Simpson","first_name_child":"Maggie","avg_parent_age_at_birth":42.5}])"; + EXPECT_EQ(duckdb::to_sql(get_people), expected_query); + EXPECT_EQ(rfl::json::write(people), expected); +} + +} // namespace test_joins_nested_grouped diff --git a/tests/duckdb/test_joins_two_tables.cpp b/tests/duckdb/test_joins_two_tables.cpp new file mode 100644 index 00000000..2caa0279 --- /dev/null +++ b/tests/duckdb/test_joins_two_tables.cpp @@ -0,0 +1,77 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_joins_two_tables { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + double age; +}; + +struct Relationship { + uint32_t parent_id; + uint32_t child_id; +}; + +TEST(duckdb, test_joins_two_tables) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{ + .id = 1, .first_name = "Marge", .last_name = "Simpson", .age = 40}, + Person{.id = 2, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 3, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 4, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + const auto relationships = + std::vector({Relationship{.parent_id = 0, .child_id = 2}, + Relationship{.parent_id = 0, .child_id = 3}, + Relationship{.parent_id = 0, .child_id = 4}, + Relationship{.parent_id = 1, .child_id = 2}, + Relationship{.parent_id = 1, .child_id = 3}, + Relationship{.parent_id = 1, .child_id = 4}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + struct ParentAndChild { + std::string last_name; + std::string first_name_parent; + std::string first_name_child; + double parent_age_at_birth; + }; + + const auto get_people = + select_from( + "last_name"_t1 | as<"last_name">, + "first_name"_t1 | as<"first_name_parent">, + "first_name"_t3 | as<"first_name_child">, + ("age"_t1 - "age"_t3) | as<"parent_age_at_birth">) | + inner_join("id"_t1 == "parent_id"_t2) | + inner_join("id"_t3 == "child_id"_t2) | + order_by("id"_t1, "id"_t3) | to>; + + const auto people = duckdb::connect() + .and_then(write(std::ref(people1))) + .and_then(write(std::ref(relationships))) + .and_then(get_people) + .value(); + + const std::string expected_query = + R"(SELECT t1."last_name" AS "last_name", t1."first_name" AS "first_name_parent", t3."first_name" AS "first_name_child", (t1."age") - (t3."age") AS "parent_age_at_birth" FROM "Person" t1 INNER JOIN "Relationship" t2 ON t1."id" = t2."parent_id" INNER JOIN "Person" t3 ON t3."id" = t2."child_id" ORDER BY t1."id", t3."id")"; + const std::string expected = + R"([{"last_name":"Simpson","first_name_parent":"Homer","first_name_child":"Bart","parent_age_at_birth":35.0},{"last_name":"Simpson","first_name_parent":"Homer","first_name_child":"Lisa","parent_age_at_birth":37.0},{"last_name":"Simpson","first_name_parent":"Homer","first_name_child":"Maggie","parent_age_at_birth":45.0},{"last_name":"Simpson","first_name_parent":"Marge","first_name_child":"Bart","parent_age_at_birth":30.0},{"last_name":"Simpson","first_name_parent":"Marge","first_name_child":"Lisa","parent_age_at_birth":32.0},{"last_name":"Simpson","first_name_parent":"Marge","first_name_child":"Maggie","parent_age_at_birth":40.0}])"; + + EXPECT_EQ(duckdb::to_sql(get_people), expected_query); + EXPECT_EQ(rfl::json::write(people), expected); +} + +} // namespace test_joins_two_tables diff --git a/tests/duckdb/test_joins_two_tables_grouped.cpp b/tests/duckdb/test_joins_two_tables_grouped.cpp new file mode 100644 index 00000000..9ec6167c --- /dev/null +++ b/tests/duckdb/test_joins_two_tables_grouped.cpp @@ -0,0 +1,78 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_joins_two_tables_grouped { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + double age; +}; + +struct Relationship { + uint32_t parent_id; + uint32_t child_id; +}; + +TEST(duckdb, test_joins_two_tables_grouped) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{ + .id = 1, .first_name = "Marge", .last_name = "Simpson", .age = 40}, + Person{.id = 2, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 3, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 4, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + const auto relationships = + std::vector({Relationship{.parent_id = 0, .child_id = 2}, + Relationship{.parent_id = 0, .child_id = 3}, + Relationship{.parent_id = 0, .child_id = 4}, + Relationship{.parent_id = 1, .child_id = 2}, + Relationship{.parent_id = 1, .child_id = 3}, + Relationship{.parent_id = 1, .child_id = 4}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + struct ParentAndChild { + std::string last_name; + std::string first_name_child; + double avg_parent_age_at_birth; + }; + + const auto get_people = + select_from( + "last_name"_t1 | as<"last_name">, + "first_name"_t3 | as<"first_name_child">, + avg("age"_t1 - "age"_t3) | as<"avg_parent_age_at_birth">) | + inner_join("id"_t1 == "parent_id"_t2) | + inner_join("id"_t3 == "child_id"_t2) | + group_by("last_name"_t1, "first_name"_t3) | + order_by("last_name"_t1, "first_name"_t3) | + to>; + + const auto people = duckdb::connect() + .and_then(write(std::ref(people1))) + .and_then(write(std::ref(relationships))) + .and_then(get_people) + .value(); + + const std::string expected_query = + R"(SELECT t1."last_name" AS "last_name", t3."first_name" AS "first_name_child", AVG((t1."age") - (t3."age")) AS "avg_parent_age_at_birth" FROM "Person" t1 INNER JOIN "Relationship" t2 ON t1."id" = t2."parent_id" INNER JOIN "Person" t3 ON t3."id" = t2."child_id" GROUP BY t1."last_name", t3."first_name" ORDER BY t1."last_name", t3."first_name")"; + + const std::string expected = + R"([{"last_name":"Simpson","first_name_child":"Bart","avg_parent_age_at_birth":32.5},{"last_name":"Simpson","first_name_child":"Lisa","avg_parent_age_at_birth":34.5},{"last_name":"Simpson","first_name_child":"Maggie","avg_parent_age_at_birth":42.5}])"; + + EXPECT_EQ(duckdb::to_sql(get_people), expected_query); + EXPECT_EQ(rfl::json::write(people), expected); +} + +} // namespace test_joins_two_tables_grouped diff --git a/tests/duckdb/test_left_join.cpp b/tests/duckdb/test_left_join.cpp new file mode 100644 index 00000000..4e7cb37d --- /dev/null +++ b/tests/duckdb/test_left_join.cpp @@ -0,0 +1,69 @@ +#include + +#include +#include +#include +#include +#include +#include + +namespace test_left_join { + +TEST(duckdb, test_left_join) { + struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + }; + + struct Pet { + sqlgen::PrimaryKey id; + std::string name; + uint32_t owner_id; + }; + + const auto people = std::vector({ + Person{.id = 1, .first_name = "Homer", .last_name = "Simpson"}, + Person{.id = 2, .first_name = "Marge", .last_name = "Simpson"}, + Person{.id = 3, .first_name = "Bart", .last_name = "Simpson"}, + Person{.id = 4, .first_name = "Lisa", .last_name = "Simpson"}, + }); + + const auto pets = std::vector({ + Pet{.id = 1, .name = "Santa's Little Helper", .owner_id = 1}, + Pet{.id = 2, .name = "Snowball", .owner_id = 4}, + Pet{.id = 3, .name = "Mr. Teeny", .owner_id = 99}, + }); + + using namespace sqlgen; + using namespace sqlgen::literals; + + struct PersonWithPet { + std::string first_name; + std::string last_name; + std::optional pet_name; + }; + + const auto get_people_with_pets = + select_from("first_name"_t1 | as<"first_name">, + "last_name"_t1 | as<"last_name">, + "name"_t2 | as<"pet_name">) | + left_join("id"_t1 == "owner_id"_t2) | order_by("id"_t1) | + to>; + + const auto result = duckdb::connect() + .and_then(write(std::ref(people))) + .and_then(write(std::ref(pets))) + .and_then(get_people_with_pets) + .value(); + + const std::string expected_query = + R"(SELECT t1."first_name" AS "first_name", t1."last_name" AS "last_name", t2."name" AS "pet_name" FROM "Person" t1 LEFT JOIN "Pet" t2 ON t1."id" = t2."owner_id" ORDER BY t1."id")"; + const std::string expected_json = + R"([{"first_name":"Homer","last_name":"Simpson","pet_name":"Santa's Little Helper"},{"first_name":"Marge","last_name":"Simpson"},{"first_name":"Bart","last_name":"Simpson"},{"first_name":"Lisa","last_name":"Simpson","pet_name":"Snowball"}])"; + + EXPECT_EQ(duckdb::to_sql(get_people_with_pets), expected_query); + EXPECT_EQ(rfl::json::write(result), expected_json); +} + +} // namespace test_left_join diff --git a/tests/duckdb/test_like.cpp b/tests/duckdb/test_like.cpp new file mode 100644 index 00000000..a968121c --- /dev/null +++ b/tests/duckdb/test_like.cpp @@ -0,0 +1,65 @@ + +#include + +#include +#include +#include +#include +#include + +namespace test_like { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_like) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}, + Person{ + .id = 4, .first_name = "Hugo", .last_name = "Simpson", .age = 10}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto conn = duckdb::connect(); + + const auto people2 = + conn.and_then(write(std::ref(people1))) + .and_then(sqlgen::read> | + where("first_name"_c.like("H%")) | order_by("age"_c)) + .value(); + + const auto people3 = + conn.and_then(sqlgen::read> | + where("first_name"_c.not_like("H%")) | order_by("age"_c)) + .value(); + + const auto people4 = + conn.and_then(sqlgen::read> | + where("first_name"_c.like("O'Reilly")) | order_by("age"_c)) + .value(); + + const std::string expected1 = + R"([{"id":4,"first_name":"Hugo","last_name":"Simpson","age":10},{"id":0,"first_name":"Homer","last_name":"Simpson","age":45}])"; + + const std::string expected2 = + R"([{"id":3,"first_name":"Maggie","last_name":"Simpson","age":0},{"id":2,"first_name":"Lisa","last_name":"Simpson","age":8},{"id":1,"first_name":"Bart","last_name":"Simpson","age":10}])"; + + const std::string expected3 = R"([])"; + + EXPECT_EQ(rfl::json::write(people2), expected1); + EXPECT_EQ(rfl::json::write(people3), expected2); + EXPECT_EQ(rfl::json::write(people4), expected3); +} + +} // namespace test_like + diff --git a/tests/duckdb/test_limit.cpp b/tests/duckdb/test_limit.cpp new file mode 100644 index 00000000..e3cf2d4b --- /dev/null +++ b/tests/duckdb/test_limit.cpp @@ -0,0 +1,47 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_limit { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_limit) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}, + Person{ + .id = 4, .first_name = "Hugo", .last_name = "Simpson", .age = 10}}); + + const auto conn = sqlgen::duckdb::connect(); + + sqlgen::write(conn, people1); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto query = + sqlgen::read> | order_by("age"_c) | limit(2); + + const auto people2 = query(conn).value(); + + const std::string expected = + R"([{"id":3,"first_name":"Maggie","last_name":"Simpson","age":0},{"id":2,"first_name":"Lisa","last_name":"Simpson","age":8}])"; + + EXPECT_EQ(rfl::json::write(people2), expected); +} + +} // namespace test_limit diff --git a/tests/duckdb/test_not_in.cpp b/tests/duckdb/test_not_in.cpp new file mode 100644 index 00000000..44dcca79 --- /dev/null +++ b/tests/duckdb/test_not_in.cpp @@ -0,0 +1,48 @@ + +#include + +#include +#include +#include +#include +#include + +namespace test_not_in { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_not_in) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}, + Person{ + .id = 4, .first_name = "Hugo", .last_name = "Simpson", .age = 10}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto people2 = + duckdb::connect() + .and_then(write(std::ref(people1))) + .and_then(sqlgen::read> | + where("first_name"_c.not_in("Homer", "Hugo")) | + order_by("age"_c)) + .value(); + + const std::string expected1 = + R"([{"id":3,"first_name":"Maggie","last_name":"Simpson","age":0},{"id":2,"first_name":"Lisa","last_name":"Simpson","age":8},{"id":1,"first_name":"Bart","last_name":"Simpson","age":10}])"; + + EXPECT_EQ(rfl::json::write(people2), expected1); +} + +} // namespace test_not_in + diff --git a/tests/duckdb/test_not_in_vec.cpp b/tests/duckdb/test_not_in_vec.cpp new file mode 100644 index 00000000..3996b032 --- /dev/null +++ b/tests/duckdb/test_not_in_vec.cpp @@ -0,0 +1,49 @@ + +#include + +#include +#include +#include +#include +#include + +namespace test_not_in_vec { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_not_in_vec) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}, + Person{ + .id = 4, .first_name = "Hugo", .last_name = "Simpson", .age = 10}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto people2 = + duckdb::connect() + .and_then(write(std::ref(people1))) + .and_then(sqlgen::read> | + where("first_name"_c.not_in( + std::vector({"Homer", "Hugo"}))) | + order_by("age"_c)) + .value(); + + const std::string expected1 = + R"([{"id":3,"first_name":"Maggie","last_name":"Simpson","age":0},{"id":2,"first_name":"Lisa","last_name":"Simpson","age":8},{"id":1,"first_name":"Bart","last_name":"Simpson","age":10}])"; + + EXPECT_EQ(rfl::json::write(people2), expected1); +} + +} // namespace test_not_in_vec + diff --git a/tests/duckdb/test_operations.cpp b/tests/duckdb/test_operations.cpp new file mode 100644 index 00000000..42bc6c45 --- /dev/null +++ b/tests/duckdb/test_operations.cpp @@ -0,0 +1,76 @@ + +#include + +#include +#include +#include +#include +#include +#include + +namespace test_operations { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_operations) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + struct Children { + int id_plus_age; + int age_times_2; + int id_plus_2_minus_age; + int age_mod_3; + int abs_age; + double exp_age; + double sqrt_age; + size_t length_first_name; + std::string full_name; + std::string first_name_lower; + std::string first_name_upper; + std::string first_name_replaced; + }; + + const auto get_children = + select_from( + ("id"_c + "age"_c) | as<"id_plus_age">, + ("age"_c * 2) | as<"age_times_2">, ("age"_c % 3) | as<"age_mod_3">, + abs("age"_c * (-1)) | as<"abs_age">, + round(exp(cast("age"_c)), 2) | as<"exp_age">, + round(sqrt(cast("age"_c)), 2) | as<"sqrt_age">, + ("id"_c + 2 - "age"_c) | as<"id_plus_2_minus_age">, + length(trim("first_name"_c)) | as<"length_first_name">, + concat(ltrim("first_name"_c), " ", rtrim("last_name"_c)) | + as<"full_name">, + upper(rtrim(concat("first_name"_c, " "))) | as<"first_name_upper">, + lower(ltrim(concat(" ", "first_name"_c))) | as<"first_name_lower">, + replace("first_name"_c, "Bart", "Hugo") | as<"first_name_replaced">) | + where("age"_c < 18) | order_by("age"_c.desc()) | + to>; + + const auto children = duckdb::connect() + .and_then(write(std::ref(people1))) + .and_then(get_children) + .value(); + + const std::string expected = + R"([{"id_plus_age":11,"age_times_2":20,"id_plus_2_minus_age":-7,"age_mod_3":1,"abs_age":10,"exp_age":22026.47,"sqrt_age":3.16,"length_first_name":4,"full_name":"Bart Simpson","first_name_lower":"bart","first_name_upper":"BART","first_name_replaced":"Hugo"},{"id_plus_age":10,"age_times_2":16,"id_plus_2_minus_age":-4,"age_mod_3":2,"abs_age":8,"exp_age":2980.96,"sqrt_age":2.83,"length_first_name":4,"full_name":"Lisa Simpson","first_name_lower":"lisa","first_name_upper":"LISA","first_name_replaced":"Lisa"},{"id_plus_age":3,"age_times_2":0,"id_plus_2_minus_age":5,"age_mod_3":0,"abs_age":0,"exp_age":1.0,"sqrt_age":0.0,"length_first_name":6,"full_name":"Maggie Simpson","first_name_lower":"maggie","first_name_upper":"MAGGIE","first_name_replaced":"Maggie"}])"; + + EXPECT_EQ(rfl::json::write(children), expected); +} + +} // namespace test_operations + diff --git a/tests/duckdb/test_operations_with_nullable.cpp b/tests/duckdb/test_operations_with_nullable.cpp new file mode 100644 index 00000000..c59d969d --- /dev/null +++ b/tests/duckdb/test_operations_with_nullable.cpp @@ -0,0 +1,63 @@ + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace test_operations_with_nullable { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::optional last_name; + std::optional age; +}; + +TEST(duckdb, test_operations_with_nullable) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Hugo", .age = 10}, + Person{.id = 3, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 4, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + struct Children { + std::optional id_plus_age; + std::optional age_times_2; + std::optional id_plus_2_minus_age; + std::optional full_name; + std::string last_name_or_none; + }; + + const auto get_children = + select_from( + ("id"_c + "age"_c) | as<"id_plus_age">, + ("age"_c * 2) | as<"age_times_2">, + ("id"_c + 2 - "age"_c) | as<"id_plus_2_minus_age">, + concat(upper("last_name"_c), ", ", "first_name"_c) | as<"full_name">, + coalesce(upper("last_name"_c), "none") | as<"last_name_or_none">) | + where("age"_c < 18) | to>; + + const auto children = duckdb::connect() + .and_then(write(std::ref(people1))) + .and_then(get_children) + .value(); + + const std::string expected = + R"([{"id_plus_age":11,"age_times_2":20,"id_plus_2_minus_age":-7,"full_name":"SIMPSON, Bart","last_name_or_none":"SIMPSON"},{"id_plus_age":12,"age_times_2":20,"id_plus_2_minus_age":-6,"last_name_or_none":"none"},{"id_plus_age":11,"age_times_2":16,"id_plus_2_minus_age":-3,"full_name":"SIMPSON, Lisa","last_name_or_none":"SIMPSON"},{"id_plus_age":4,"age_times_2":0,"id_plus_2_minus_age":6,"full_name":"SIMPSON, Maggie","last_name_or_none":"SIMPSON"}])"; + + EXPECT_EQ(rfl::json::write(children), expected); +} + +} // namespace test_operations_with_nullable + diff --git a/tests/duckdb/test_order_by.cpp b/tests/duckdb/test_order_by.cpp new file mode 100644 index 00000000..ea6615ec --- /dev/null +++ b/tests/duckdb/test_order_by.cpp @@ -0,0 +1,47 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_order_by { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_order_by) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}, + Person{ + .id = 4, .first_name = "Hugo", .last_name = "Simpson", .age = 10}}); + + const auto conn = sqlgen::duckdb::connect(); + + sqlgen::write(conn, people1); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto query = sqlgen::read> | + order_by("age"_c, "first_name"_c.desc()); + + const auto people2 = query(conn).value(); + + const std::string expected = + R"([{"id":3,"first_name":"Maggie","last_name":"Simpson","age":0},{"id":2,"first_name":"Lisa","last_name":"Simpson","age":8},{"id":4,"first_name":"Hugo","last_name":"Simpson","age":10},{"id":1,"first_name":"Bart","last_name":"Simpson","age":10},{"id":0,"first_name":"Homer","last_name":"Simpson","age":45}])"; + + EXPECT_EQ(rfl::json::write(people2), expected); +} + +} // namespace test_order_by diff --git a/tests/duckdb/test_range.cpp b/tests/duckdb/test_range.cpp new file mode 100644 index 00000000..dfd32bdb --- /dev/null +++ b/tests/duckdb/test_range.cpp @@ -0,0 +1,47 @@ +#include + +#include +#include +#include +#include +#include +#include + +namespace test_range { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_range) { + using namespace std::ranges::views; + + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + const auto conn = sqlgen::duckdb::connect(); + + sqlgen::write(conn, people1); + + const auto people2 = sqlgen::read>(conn).value(); + + const auto first_names = + sqlgen::internal::collect::vector(people2 | transform([](const auto& _r) { + return _r.value().first_name; + })); + + EXPECT_EQ(first_names.at(0), "Homer"); + EXPECT_EQ(first_names.at(1), "Bart"); + EXPECT_EQ(first_names.at(2), "Lisa"); + EXPECT_EQ(first_names.at(3), "Maggie"); +} + +} // namespace test_range diff --git a/tests/duckdb/test_range_select_from.cpp b/tests/duckdb/test_range_select_from.cpp new file mode 100644 index 00000000..2905c09b --- /dev/null +++ b/tests/duckdb/test_range_select_from.cpp @@ -0,0 +1,52 @@ + +#include + +#include +#include +#include +#include +#include +#include + +namespace test_range_select_from { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_range_select_from) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto people2 = + sqlgen::duckdb::connect() + .and_then(sqlgen::write(std::ref(people1))) + .and_then(select_from("first_name"_c) | order_by("id"_c)) + .value(); + + using namespace std::ranges::views; + + const auto first_names = + internal::collect::vector(people2 | transform([](const auto& _r) { + return rfl::get<"first_name">(_r.value()); + })); + + EXPECT_EQ(first_names.at(0), "Homer"); + EXPECT_EQ(first_names.at(1), "Bart"); + EXPECT_EQ(first_names.at(2), "Lisa"); + EXPECT_EQ(first_names.at(3), "Maggie"); +} + +} // namespace test_range_select_from + diff --git a/tests/duckdb/test_range_select_from_with_to.cpp b/tests/duckdb/test_range_select_from_with_to.cpp new file mode 100644 index 00000000..fce1cebe --- /dev/null +++ b/tests/duckdb/test_range_select_from_with_to.cpp @@ -0,0 +1,51 @@ + +#include + +#include +#include +#include +#include +#include +#include + +namespace test_range_select_from_with_to { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_range_select_from_with_to) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + struct FirstName { + std::string first_name; + }; + + const auto people2 = + duckdb::connect() + .and_then(drop | if_exists) + .and_then(write(std::ref(people1))) + .and_then(select_from("first_name"_c) | order_by("id"_c) | + to>) + .value(); + + EXPECT_EQ(people2.at(0).first_name, "Homer"); + EXPECT_EQ(people2.at(1).first_name, "Bart"); + EXPECT_EQ(people2.at(2).first_name, "Lisa"); + EXPECT_EQ(people2.at(3).first_name, "Maggie"); +} + +} // namespace test_range_select_from_with_to + diff --git a/tests/duckdb/test_right_join.cpp b/tests/duckdb/test_right_join.cpp new file mode 100644 index 00000000..dea2b4f0 --- /dev/null +++ b/tests/duckdb/test_right_join.cpp @@ -0,0 +1,69 @@ +#include + +#include +#include +#include +#include +#include +#include + +namespace test_right_join { + +TEST(duckdb, test_right_join) { + struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + }; + + struct Pet { + sqlgen::PrimaryKey id; + std::string name; + uint32_t owner_id; + }; + + const auto people = std::vector({ + Person{.id = 1, .first_name = "Homer", .last_name = "Simpson"}, + Person{.id = 2, .first_name = "Marge", .last_name = "Simpson"}, + Person{.id = 3, .first_name = "Bart", .last_name = "Simpson"}, + Person{.id = 4, .first_name = "Lisa", .last_name = "Simpson"}, + }); + + const auto pets = std::vector({ + Pet{.id = 1, .name = "Santa's Little Helper", .owner_id = 1}, + Pet{.id = 2, .name = "Snowball", .owner_id = 4}, + Pet{.id = 3, .name = "Mr. Teeny", .owner_id = 99}, + }); + + using namespace sqlgen; + using namespace sqlgen::literals; + + struct PetWithOwner { + std::string pet_name; + std::optional owner_first_name; + std::optional owner_last_name; + }; + + const auto get_pets_with_owners = + select_from("name"_t2 | as<"pet_name">, + "first_name"_t1 | as<"owner_first_name">, + "last_name"_t1 | as<"owner_last_name">) | + right_join("id"_t1 == "owner_id"_t2) | order_by("id"_t2) | + to>; + + const auto result = duckdb::connect() + .and_then(write(std::ref(people))) + .and_then(write(std::ref(pets))) + .and_then(get_pets_with_owners) + .value(); + + const std::string expected_query = + R"(SELECT t2."name" AS "pet_name", t1."first_name" AS "owner_first_name", t1."last_name" AS "owner_last_name" FROM "Person" t1 RIGHT JOIN "Pet" t2 ON t1."id" = t2."owner_id" ORDER BY t2."id")"; + const std::string expected_json = + R"([{"pet_name":"Santa's Little Helper","owner_first_name":"Homer","owner_last_name":"Simpson"},{"pet_name":"Snowball","owner_first_name":"Lisa","owner_last_name":"Simpson"},{"pet_name":"Mr. Teeny"}])"; + + EXPECT_EQ(duckdb::to_sql(get_pets_with_owners), expected_query); + EXPECT_EQ(rfl::json::write(result), expected_json); +} + +} // namespace test_right_join diff --git a/tests/duckdb/test_select_from_with_timestamps.cpp b/tests/duckdb/test_select_from_with_timestamps.cpp new file mode 100644 index 00000000..2ccda80a --- /dev/null +++ b/tests/duckdb/test_select_from_with_timestamps.cpp @@ -0,0 +1,79 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace test_range_select_from_with_timestamps { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + sqlgen::Date birthday; +}; + +TEST(duckdb, test_range_select_from_with_timestamps) { + const auto people1 = + std::vector({Person{.first_name = "Homer", + .last_name = "Simpson", + .birthday = sqlgen::Date("1970-01-01")}, + Person{.first_name = "Bart", + .last_name = "Simpson", + .birthday = sqlgen::Date("2000-01-01")}, + Person{.first_name = "Lisa", + .last_name = "Simpson", + .birthday = sqlgen::Date("2002-01-01")}, + Person{.first_name = "Maggie", + .last_name = "Simpson", + .birthday = sqlgen::Date("2010-01-01")}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + struct Birthday { + Date birthday; + Date birthday_recreated; + time_t birthday_unixepoch; + double age_in_days; + int hour; + int minute; + int second; + int weekday; + }; + + const auto get_birthdays = + select_from( + ("birthday"_c + std::chrono::days(10)) | as<"birthday">, + ((cast(concat(cast(year("birthday"_c)), "-", + cast(month("birthday"_c)), "-", + cast(day("birthday"_c)))))) | + as<"birthday_recreated">, + days_between("birthday"_c, Date("2011-01-01")) | as<"age_in_days">, + unixepoch("birthday"_c + std::chrono::days(10)) | + as<"birthday_unixepoch">, + hour("birthday"_c) | as<"hour">, minute("birthday"_c) | as<"minute">, + second("birthday"_c) | as<"second">, + weekday("birthday"_c) | as<"weekday">) | + order_by("id"_c) | to>; + + const auto birthdays = duckdb::connect() + .and_then(write(std::ref(people1))) + .and_then(get_birthdays) + .value(); + + const std::string expected_query = + R"(SELECT datetime("birthday", '+10 days') AS "birthday", cast((cast(cast(strftime('%Y', "birthday") as INT) as TEXT) || '-' || cast(cast(strftime('%m', "birthday") as INT) as TEXT) || '-' || cast(cast(strftime('%d', "birthday") as INT) as TEXT)) as TEXT) AS "birthday_recreated", julianday('2011-01-01') - julianday("birthday") AS "age_in_days", unixepoch(datetime("birthday", '+10 days'), 'subsec') AS "birthday_unixepoch", cast(strftime('%H', "birthday") as INT) AS "hour", cast(strftime('%M', "birthday") as INT) AS "minute", cast(strftime('%S', "birthday") as INT) AS "second", cast(strftime('%w', "birthday") as INT) AS "weekday" FROM "Person" ORDER BY "id")"; + const std::string expected = + R"([{"birthday":"1970-01-11","birthday_recreated":"1970-01-01","birthday_unixepoch":864000,"age_in_days":14975.0,"hour":0,"minute":0,"second":0,"weekday":4},{"birthday":"2000-01-11","birthday_recreated":"2000-01-01","birthday_unixepoch":947548800,"age_in_days":4018.0,"hour":0,"minute":0,"second":0,"weekday":6},{"birthday":"2002-01-11","birthday_recreated":"2002-01-01","birthday_unixepoch":1010707200,"age_in_days":3287.0,"hour":0,"minute":0,"second":0,"weekday":2},{"birthday":"2010-01-11","birthday_recreated":"2010-01-01","birthday_unixepoch":1263168000,"age_in_days":365.0,"hour":0,"minute":0,"second":0,"weekday":5}])"; + + EXPECT_EQ(duckdb::to_sql(get_birthdays), expected_query); + EXPECT_EQ(rfl::json::write(birthdays), expected); +} + +} // namespace test_range_select_from_with_timestamps + diff --git a/tests/duckdb/test_single_read.cpp b/tests/duckdb/test_single_read.cpp new file mode 100644 index 00000000..ce3ecdd9 --- /dev/null +++ b/tests/duckdb/test_single_read.cpp @@ -0,0 +1,43 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_single_read { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_single_read) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + const auto conn = sqlgen::duckdb::connect(); + + sqlgen::write(conn, people1); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto people2 = + (sqlgen::read | where("id"_c == 0))(conn).value(); + + const auto json1 = rfl::json::write(people1.at(0)); + const auto json2 = rfl::json::write(people2); + + EXPECT_EQ(json1, json2); +} + +} // namespace test_single_read diff --git a/tests/duckdb/test_timestamp.cpp b/tests/duckdb/test_timestamp.cpp new file mode 100644 index 00000000..0e5fc1c8 --- /dev/null +++ b/tests/duckdb/test_timestamp.cpp @@ -0,0 +1,49 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_timestamp { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + sqlgen::Timestamp<"%Y-%m-%d %H:%M:%S"> birthdate; +}; + +TEST(duckdb, test_timestamp) { + const auto people1 = + std::vector({Person{.id = 0, + .first_name = "Homer", + .last_name = "Simpson", + .birthdate = "1989-12-17 12:00:00"}, + Person{.id = 1, + .first_name = "Bart", + .last_name = "Simpson", + .birthdate = "1989-12-17 12:00:00"}, + Person{.id = 2, + .first_name = "Lisa", + .last_name = "Simpson", + .birthdate = "1989-12-17 12:00:00"}, + Person{.id = 3, + .first_name = "Maggie", + .last_name = "Simpson", + .birthdate = "1989-12-17 12:00:00"}}); + + const auto conn = sqlgen::duckdb::connect(); + + sqlgen::write(conn, people1); + + const auto people2 = sqlgen::read>(conn).value(); + + const auto json1 = rfl::json::write(people1); + const auto json2 = rfl::json::write(people2); + + EXPECT_EQ(json1, json2); +} + +} // namespace test_timestamp diff --git a/tests/duckdb/test_to_create_table.cpp b/tests/duckdb/test_to_create_table.cpp new file mode 100644 index 00000000..b928211b --- /dev/null +++ b/tests/duckdb/test_to_create_table.cpp @@ -0,0 +1,25 @@ +#include + +#include +#include +#include + +namespace test_to_create_table { + +struct TestTable { + std::string field1; + int32_t field2; + sqlgen::PrimaryKey id; + std::optional nullable; +}; + +TEST(duckdb, test_to_create_table) { + const auto create_table_stmt = + sqlgen::transpilation::to_create_table(); + const auto conn = sqlgen::duckdb::connect().value(); + const auto expected = + R"(CREATE TABLE IF NOT EXISTS "TestTable" ("field1" TEXT NOT NULL, "field2" INTEGER NOT NULL, "id" INTEGER PRIMARY KEY NOT NULL, "nullable" TEXT);)"; + + EXPECT_EQ(conn->to_sql(create_table_stmt), expected); +} +} // namespace test_to_create_table diff --git a/tests/duckdb/test_to_insert.cpp b/tests/duckdb/test_to_insert.cpp new file mode 100644 index 00000000..545b9a0c --- /dev/null +++ b/tests/duckdb/test_to_insert.cpp @@ -0,0 +1,27 @@ +#include + +#include +#include +#include +#include + +namespace test_to_insert { + +struct TestTable { + std::string field1; + int32_t field2; + sqlgen::PrimaryKey id; + std::optional nullable; +}; + +TEST(duckdb, test_to_insert) { + const auto insert_stmt = + sqlgen::transpilation::to_insert_or_write(); + const auto conn = sqlgen::duckdb::connect().value(); + const auto expected = + R"(INSERT INTO "TestTable" ("field1", "field2", "id", "nullable") VALUES (?, ?, ?, ?);)"; + + EXPECT_EQ(conn->to_sql(insert_stmt), expected); +} +} // namespace test_to_insert diff --git a/tests/duckdb/test_to_insert_or_replace.cpp b/tests/duckdb/test_to_insert_or_replace.cpp new file mode 100644 index 00000000..96e6a8b2 --- /dev/null +++ b/tests/duckdb/test_to_insert_or_replace.cpp @@ -0,0 +1,32 @@ +#include + +#include +#include +#include +#include + +namespace test_to_insert_or_replace { + +struct TestTable { + std::string field1; + int32_t field2; + sqlgen::Unique field3; + sqlgen::PrimaryKey id; + std::optional nullable; +}; + +TEST(duckdb, test_to_insert_or_replace) { + static_assert(sqlgen::internal::has_constraint_v, + "The table must have a primary key or unique column for " + "insert_or_replace(...) to work."); + + const auto insert_stmt = + sqlgen::transpilation::to_insert_or_write(true); + const auto conn = sqlgen::duckdb::connect().value(); + const auto expected = + R"(INSERT INTO "TestTable" ("field1", "field2", "field3", "id", "nullable") VALUES (?, ?, ?, ?, ?) ON CONFLICT (field3, id) DO UPDATE SET field1=excluded.field1, field2=excluded.field2, field3=excluded.field3, id=excluded.id, nullable=excluded.nullable;)"; + + EXPECT_EQ(conn->to_sql(insert_stmt), expected); +} +} // namespace test_to_insert_or_replace diff --git a/tests/duckdb/test_to_select_from.cpp b/tests/duckdb/test_to_select_from.cpp new file mode 100644 index 00000000..ff64e1e2 --- /dev/null +++ b/tests/duckdb/test_to_select_from.cpp @@ -0,0 +1,24 @@ +#include + +#include +#include + +namespace test_to_select_from { + +struct TestTable { + std::string field1; + int32_t field2; + sqlgen::PrimaryKey id; + std::optional nullable; +}; + +TEST(duckdb, test_to_select_from) { + const auto select_from_stmt = + sqlgen::transpilation::read_to_select_from(); + const auto conn = sqlgen::duckdb::connect().value(); + const auto expected = + R"(SELECT "field1", "field2", "id", "nullable" FROM "TestTable")"; + + EXPECT_EQ(conn->to_sql(select_from_stmt), expected); +} +} // namespace test_to_select_from diff --git a/tests/duckdb/test_to_select_from_with_schema.cpp b/tests/duckdb/test_to_select_from_with_schema.cpp new file mode 100644 index 00000000..0711a487 --- /dev/null +++ b/tests/duckdb/test_to_select_from_with_schema.cpp @@ -0,0 +1,27 @@ +#include + +#include +#include + +namespace test_to_select_from_with_schema { + +struct TestTable { + constexpr static const char* tablename = "test_table"; + constexpr static const char* schema = "my_schema"; + + std::string field1; + int32_t field2; + sqlgen::PrimaryKey id; + std::optional nullable; +}; + +TEST(duckdb, test_to_select_from_with_schema) { + const auto select_from_stmt = + sqlgen::transpilation::read_to_select_from(); + const auto conn = sqlgen::duckdb::connect().value(); + const auto expected = + R"(SELECT "field1", "field2", "id", "nullable" FROM "my_schema"."test_table")"; + + EXPECT_EQ(conn->to_sql(select_from_stmt), expected); +} +} // namespace test_to_select_from_with_schema diff --git a/tests/duckdb/test_transaction.cpp b/tests/duckdb/test_transaction.cpp new file mode 100644 index 00000000..fbf7f611 --- /dev/null +++ b/tests/duckdb/test_transaction.cpp @@ -0,0 +1,55 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_update { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_transaction) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}, + Person{ + .id = 4, .first_name = "Hugo", .last_name = "Simpson", .age = 10}}); + + auto conn = sqlgen::duckdb::connect(); + + sqlgen::write(conn, people1); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto delete_hugo = + delete_from | where("first_name"_c == "Hugo"); + + const auto update_homers_age = + update("age"_c.set(46)) | where("first_name"_c == "Homer"); + + conn = sqlgen::begin_transaction(conn) + .and_then(delete_hugo) + .and_then(update_homers_age) + .and_then(sqlgen::commit); + + const auto people2 = sqlgen::read>(conn).value(); + + const std::string expected = + R"([{"id":0,"first_name":"Homer","last_name":"Simpson","age":46},{"id":1,"first_name":"Bart","last_name":"Simpson","age":10},{"id":2,"first_name":"Lisa","last_name":"Simpson","age":8},{"id":3,"first_name":"Maggie","last_name":"Simpson","age":0}])"; + + EXPECT_EQ(rfl::json::write(people2), expected); +} + +} // namespace test_update diff --git a/tests/duckdb/test_unique.cpp b/tests/duckdb/test_unique.cpp new file mode 100644 index 00000000..43452f1d --- /dev/null +++ b/tests/duckdb/test_unique.cpp @@ -0,0 +1,46 @@ +#include + +#include +#include +#include +#include +#include +#include + +namespace test_unique { + +struct Person { + sqlgen::PrimaryKey id; + sqlgen::Unique first_name; + std::string last_name; + double age; +}; + +TEST(duckdb, test_unique) { + const auto people = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{ + .id = 1, .first_name = "Marge", .last_name = "Simpson", .age = 40}, + Person{.id = 2, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 3, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 4, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + duckdb::connect() + .and_then(drop | if_exists) + .and_then(create_table) + .and_then(insert(std::ref(people))) + .value(); + + const std::string expected_query = + R"(CREATE TABLE IF NOT EXISTS "Person" ("id" INTEGER PRIMARY KEY NOT NULL, "first_name" TEXT NOT NULL UNIQUE, "last_name" TEXT NOT NULL, "age" REAL NOT NULL);)"; + + EXPECT_EQ(duckdb::to_sql(create_table), expected_query); +} + +} // namespace test_unique + diff --git a/tests/duckdb/test_update.cpp b/tests/duckdb/test_update.cpp new file mode 100644 index 00000000..aef5b9fc --- /dev/null +++ b/tests/duckdb/test_update.cpp @@ -0,0 +1,50 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_update { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_update) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}, + Person{ + .id = 4, .first_name = "Hugo", .last_name = "Simpson", .age = 10}}); + + const auto conn = sqlgen::duckdb::connect(); + + sqlgen::write(conn, people1); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto query = + update("first_name"_c.set("last_name"_c), "age"_c.set(100)) | + where("first_name"_c == "Hugo"); + + query(conn).value(); + + const auto people2 = sqlgen::read>(conn).value(); + + const std::string expected = + R"([{"id":0,"first_name":"Homer","last_name":"Simpson","age":45},{"id":1,"first_name":"Bart","last_name":"Simpson","age":10},{"id":2,"first_name":"Lisa","last_name":"Simpson","age":8},{"id":3,"first_name":"Maggie","last_name":"Simpson","age":0},{"id":4,"first_name":"Simpson","last_name":"Simpson","age":100}])"; + + EXPECT_EQ(rfl::json::write(people2), expected); +} + +} // namespace test_update diff --git a/tests/duckdb/test_varchar.cpp b/tests/duckdb/test_varchar.cpp new file mode 100644 index 00000000..db6ec28a --- /dev/null +++ b/tests/duckdb/test_varchar.cpp @@ -0,0 +1,39 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_varchar { + +struct Person { + sqlgen::PrimaryKey id; + sqlgen::Varchar<6> first_name; + sqlgen::Varchar<7> last_name; + int age; +}; + +TEST(duckdb, test_varchar) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + const auto conn = sqlgen::duckdb::connect(); + + sqlgen::write(conn, people1); + + const auto people2 = sqlgen::read>(conn).value(); + + const auto json1 = rfl::json::write(people1); + const auto json2 = rfl::json::write(people2); + + EXPECT_EQ(json1, json2); +} + +} // namespace test_varchar diff --git a/tests/duckdb/test_where.cpp b/tests/duckdb/test_where.cpp new file mode 100644 index 00000000..6c69afa3 --- /dev/null +++ b/tests/duckdb/test_where.cpp @@ -0,0 +1,48 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_where { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_where) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}, + Person{ + .id = 4, .first_name = "Hugo", .last_name = "Simpson", .age = 10}}); + + const auto conn = sqlgen::duckdb::connect(); + + sqlgen::write(conn, people1); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto query = sqlgen::read> | + where("age"_c < 18 and not("first_name"_c == "Hugo")) | + order_by("age"_c); + + const auto people2 = query(conn).value(); + + const std::string expected = + R"([{"id":3,"first_name":"Maggie","last_name":"Simpson","age":0},{"id":2,"first_name":"Lisa","last_name":"Simpson","age":8},{"id":1,"first_name":"Bart","last_name":"Simpson","age":10}])"; + + EXPECT_EQ(rfl::json::write(people2), expected); +} + +} // namespace test_where diff --git a/tests/duckdb/test_where_with_nullable.cpp b/tests/duckdb/test_where_with_nullable.cpp new file mode 100644 index 00000000..307a2e04 --- /dev/null +++ b/tests/duckdb/test_where_with_nullable.cpp @@ -0,0 +1,48 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_where_with_nullable { + +struct Person { + sqlgen::PrimaryKey id; + std::optional first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_where_with_nullable) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}, + Person{ + .id = 4, .first_name = "Hugo", .last_name = "Simpson", .age = 10}}); + + const auto conn = sqlgen::duckdb::connect(); + + sqlgen::write(conn, people1); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto query = sqlgen::read> | + where("age"_c < 18 and "first_name"_c != "Hugo") | + order_by("age"_c); + + const auto people2 = query(conn).value(); + + const std::string expected = + R"([{"id":3,"first_name":"Maggie","last_name":"Simpson","age":0},{"id":1,"first_name":"Bart","last_name":"Simpson","age":10}])"; + + EXPECT_EQ(rfl::json::write(people2), expected); +} + +} // namespace test_where_with_nullable diff --git a/tests/duckdb/test_where_with_nullable_operations.cpp b/tests/duckdb/test_where_with_nullable_operations.cpp new file mode 100644 index 00000000..3f73bf7d --- /dev/null +++ b/tests/duckdb/test_where_with_nullable_operations.cpp @@ -0,0 +1,49 @@ +#include + +#include +#include +#include +#include +#include +#include + +namespace test_where_with_nullable_operations { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + std::optional age; +}; + +TEST(duckdb, test_where_with_nullable_operations) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}, + Person{ + .id = 4, .first_name = "Hugo", .last_name = "Simpson", .age = 10}}); + + const auto conn = sqlgen::duckdb::connect(); + + sqlgen::write(conn, people1); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto query = sqlgen::read> | + where("age"_c * 2 + 4 < 40 and "first_name"_c != "Hugo") | + order_by("age"_c); + + const auto people2 = query(conn).value(); + + const std::string expected = + R"([{"id":3,"first_name":"Maggie","last_name":"Simpson","age":0},{"id":2,"first_name":"Lisa","last_name":"Simpson","age":8},{"id":1,"first_name":"Bart","last_name":"Simpson","age":10}])"; + + EXPECT_EQ(rfl::json::write(people2), expected); +} + +} // namespace test_where_with_nullable_operations diff --git a/tests/duckdb/test_where_with_operations.cpp b/tests/duckdb/test_where_with_operations.cpp new file mode 100644 index 00000000..7480b8de --- /dev/null +++ b/tests/duckdb/test_where_with_operations.cpp @@ -0,0 +1,48 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_where_with_operations { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_where_with_operations) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}, + Person{ + .id = 4, .first_name = "Hugo", .last_name = "Simpson", .age = 10}}); + + const auto conn = sqlgen::duckdb::connect(); + + sqlgen::write(conn, people1); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto query = sqlgen::read> | + where("age"_c * 2 + 4 < 40 and "first_name"_c != "Hugo") | + order_by("age"_c); + + const auto people2 = query(conn).value(); + + const std::string expected = + R"([{"id":3,"first_name":"Maggie","last_name":"Simpson","age":0},{"id":2,"first_name":"Lisa","last_name":"Simpson","age":8},{"id":1,"first_name":"Bart","last_name":"Simpson","age":10}])"; + + EXPECT_EQ(rfl::json::write(people2), expected); +} + +} // namespace test_where_with_operations diff --git a/tests/duckdb/test_where_with_timestamps.cpp b/tests/duckdb/test_where_with_timestamps.cpp new file mode 100644 index 00000000..2e077d9c --- /dev/null +++ b/tests/duckdb/test_where_with_timestamps.cpp @@ -0,0 +1,60 @@ + +#include + +#include +#include +#include +#include +#include +#include + +namespace test_where_with_timestamps { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + sqlgen::Date birthday; +}; + +TEST(duckdb, test_where_with_timestamps) { + const auto people1 = + std::vector({Person{.first_name = "Homer", + .last_name = "Simpson", + .birthday = sqlgen::Date("1970-01-01")}, + Person{.first_name = "Bart", + .last_name = "Simpson", + .birthday = sqlgen::Date("2000-01-01")}, + Person{.first_name = "Lisa", + .last_name = "Simpson", + .birthday = sqlgen::Date("2002-01-01")}, + Person{.first_name = "Maggie", + .last_name = "Simpson", + .birthday = sqlgen::Date("2010-01-01")}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto conn = sqlgen::duckdb::connect(); + + sqlgen::write(conn, people1).value(); + + const auto query = + sqlgen::read> | + where("birthday"_c + std::chrono::years(11) - std::chrono::weeks(10) + + std::chrono::milliseconds(4000005) > + Date("2010-01-01")) | + order_by("id"_c); + + const auto people2 = query(conn).value(); + + const std::string expected_query = + R"(SELECT "id", "first_name", "last_name", "birthday" FROM "Person" WHERE datetime("birthday", '+11 years', '-70 days', '+01:06:40.005') > '2010-01-01' ORDER BY "id";)"; + const std::string expected = + R"([{"id":2,"first_name":"Bart","last_name":"Simpson","birthday":"2000-01-01"},{"id":3,"first_name":"Lisa","last_name":"Simpson","birthday":"2002-01-01"},{"id":4,"first_name":"Maggie","last_name":"Simpson","birthday":"2010-01-01"}])"; + + EXPECT_EQ(rfl::json::write(people2), expected); +} + +} // namespace test_where_with_timestamps + diff --git a/tests/duckdb/test_write_and_read_curried.cpp b/tests/duckdb/test_write_and_read_curried.cpp new file mode 100644 index 00000000..4b5c1c6c --- /dev/null +++ b/tests/duckdb/test_write_and_read_curried.cpp @@ -0,0 +1,42 @@ +#include + +#include +#include +#include +#include +#include +#include + +namespace test_write_and_read_curried { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_write_and_read_curried) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto people2 = duckdb::connect() + .and_then(write(std::ref(people1))) + .and_then(sqlgen::read>) + .value(); + + const auto json1 = rfl::json::write(people1); + const auto json2 = rfl::json::write(people2); + + EXPECT_EQ(json1, json2); +} + +} // namespace test_write_and_read_curried diff --git a/tests/duckdb/test_write_and_read_to_file.cpp b/tests/duckdb/test_write_and_read_to_file.cpp new file mode 100644 index 00000000..47e358e1 --- /dev/null +++ b/tests/duckdb/test_write_and_read_to_file.cpp @@ -0,0 +1,44 @@ +#include + +#include +#include +#include +#include +#include +#include + +namespace test_write_and_read_to_file { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_write_and_read_to_file) { + const auto people1 = std::vector( + {Person{ + .id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{ + .id = 3, .first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + sqlgen::duckdb::connect("test.db") + .and_then([&](auto&& _conn) { return sqlgen::write(_conn, people1); }) + .value(); + + const auto people2 = sqlgen::duckdb::connect("test.db") + .and_then(sqlgen::read>) + .value(); + + std::remove("test.db"); + + const auto json1 = rfl::json::write(people1); + const auto json2 = rfl::json::write(people2); + + EXPECT_EQ(json1, json2); +} + +} // namespace test_write_and_read_to_file From 7318b03bc0d617953ddfaffb253d90bfeead25e2 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sat, 15 Nov 2025 10:09:34 +0100 Subject: [PATCH 27/48] Some fixes --- include/sqlgen/duckdb/ColumnData.hpp | 14 +++++++++++++- include/sqlgen/duckdb/chunk_ptrs_t.hpp | 5 +++-- include/sqlgen/duckdb/from_chunk_ptrs.hpp | 8 ++++---- tests/duckdb/test_aggregations_with_nullable.cpp | 4 ++-- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/include/sqlgen/duckdb/ColumnData.hpp b/include/sqlgen/duckdb/ColumnData.hpp index 2bbc8c7c..b7ee2038 100644 --- a/include/sqlgen/duckdb/ColumnData.hpp +++ b/include/sqlgen/duckdb/ColumnData.hpp @@ -3,14 +3,26 @@ #include +#include +#include +#include +#include + namespace sqlgen::duckdb { -template +template struct ColumnData { + using ColName = _ColName; + duckdb_vector vec; T *data; uint64_t *validity; + // This is only needed if the data returned by DuckDB is not of the + // same type as T, but can be converted to T. In this case, + // data actually points to ptr->data(). Otherwise, ptr is a nullptr. + std::shared_ptr> ptr; + bool is_not_null(idx_t _i) const { return (validity == nullptr) || duckdb_validity_row_is_valid(validity, _i); } diff --git a/include/sqlgen/duckdb/chunk_ptrs_t.hpp b/include/sqlgen/duckdb/chunk_ptrs_t.hpp index 7969bad1..afe732a6 100644 --- a/include/sqlgen/duckdb/chunk_ptrs_t.hpp +++ b/include/sqlgen/duckdb/chunk_ptrs_t.hpp @@ -14,8 +14,9 @@ struct ChunkPtrsType; template struct ChunkPtrsType> { - using Type = rfl::Tuple::ResultingType>...>; + using Type = rfl::Tuple::ResultingType, + typename FieldTs::Name>...>; }; template diff --git a/include/sqlgen/duckdb/from_chunk_ptrs.hpp b/include/sqlgen/duckdb/from_chunk_ptrs.hpp index 0517bb4b..7b35ece5 100644 --- a/include/sqlgen/duckdb/from_chunk_ptrs.hpp +++ b/include/sqlgen/duckdb/from_chunk_ptrs.hpp @@ -18,11 +18,11 @@ namespace sqlgen::duckdb { template struct FromChunkPtrs; -template +template struct FromChunkPtrs, - rfl::Tuple...>> { - Result operator()(const rfl::Tuple...>& _chunk_ptrs, - idx_t _i) { + rfl::Tuple...>> { + Result operator()( + const rfl::Tuple...>& _chunk_ptrs, idx_t _i) { return [&](std::integer_sequence) -> Result { try { return T{duckdb::parsing::Parser::read( diff --git a/tests/duckdb/test_aggregations_with_nullable.cpp b/tests/duckdb/test_aggregations_with_nullable.cpp index 37afa1da..dd7b81aa 100644 --- a/tests/duckdb/test_aggregations_with_nullable.cpp +++ b/tests/duckdb/test_aggregations_with_nullable.cpp @@ -29,8 +29,8 @@ TEST(duckdb, test_aggregations_with_nullable) { using namespace sqlgen::literals; struct Children { - int num_children; - int num_last_names; + int64_t num_children; + uint64_t num_last_names; std::optional avg_age; std::optional max_age; std::optional min_age; From c78bdaff750bd725bf4a7a8b6b802673be143b7e Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sat, 15 Nov 2025 10:24:43 +0100 Subject: [PATCH 28/48] Added ColName --- include/sqlgen/duckdb/make_chunk_ptrs.hpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/include/sqlgen/duckdb/make_chunk_ptrs.hpp b/include/sqlgen/duckdb/make_chunk_ptrs.hpp index 1f41068d..e5e500d3 100644 --- a/include/sqlgen/duckdb/make_chunk_ptrs.hpp +++ b/include/sqlgen/duckdb/make_chunk_ptrs.hpp @@ -19,21 +19,21 @@ namespace sqlgen::duckdb { template struct MakeChunkPtrs; -template -struct MakeChunkPtrs...>> { - Result...>> operator()( +template +struct MakeChunkPtrs...>> { + Result...>> operator()( const Ref& _res, duckdb_data_chunk _chunk) { try { return [&](std::integer_sequence) { - return rfl::Tuple...>( - make_column_data(_res, _chunk)...); + return rfl::Tuple...>( + make_column_data(_res, _chunk)...); }(std::make_integer_sequence()); } catch (const std::exception& e) { return error(e.what()); } } - template + template static auto make_column_data(const Ref& _res, duckdb_data_chunk _chunk) { if (duckdb_column_type(&_res->res(), _i) != get_duckdb_type()) { @@ -43,9 +43,10 @@ struct MakeChunkPtrs...>> { rfl::enum_to_string(duckdb_column_type(&_res->res(), _i)) + "."); } auto vec = duckdb_data_chunk_get_vector(_chunk, _i); - return ColumnData{.vec = vec, - .data = static_cast(duckdb_vector_get_data(vec)), - .validity = duckdb_vector_get_validity(vec)}; + return ColumnData{ + .vec = vec, + .data = static_cast(duckdb_vector_get_data(vec)), + .validity = duckdb_vector_get_validity(vec)}; } }; From c8b5f5a2063dff9f55cea459457bb6db289390da Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sat, 15 Nov 2025 11:08:39 +0100 Subject: [PATCH 29/48] Added numeric casting --- include/sqlgen/duckdb/cast_duckdb_type.hpp | 69 ++++++++++++++++++++++ include/sqlgen/duckdb/make_chunk_ptrs.hpp | 40 ++++++++++--- 2 files changed, 101 insertions(+), 8 deletions(-) create mode 100644 include/sqlgen/duckdb/cast_duckdb_type.hpp diff --git a/include/sqlgen/duckdb/cast_duckdb_type.hpp b/include/sqlgen/duckdb/cast_duckdb_type.hpp new file mode 100644 index 00000000..62eb64a3 --- /dev/null +++ b/include/sqlgen/duckdb/cast_duckdb_type.hpp @@ -0,0 +1,69 @@ +#ifndef SQLGEN_DUCKDB_CASTDUCKDBTYPE_HPP_ +#define SQLGEN_DUCKDB_CASTDUCKDBTYPE_HPP_ + +#include + +#include +#include + +#include "../Ref.hpp" +#include "../Result.hpp" + +namespace sqlgen::duckdb { + +template +Ref> cast_as_vector(const size_t _size, U* _ptr) { + auto vec = Ref>::make(_size); + for (size_t i = 0; i < _size; ++i) { + (*vec)[i] = static_cast(_ptr[i]); + } + return vec; +} + +template +Result>> cast_duckdb_type(const duckdb_type _type, + const size_t _size, + void* _raw_ptr) { + if constexpr (!std::is_floating_point_v && !std::is_integral_v) { + return error("Could not cast"); + + } else { + if (_type == DUCKDB_TYPE_TINYINT) { + return cast_as_vector(_size, static_cast(_raw_ptr)); + + } else if (_type == DUCKDB_TYPE_UTINYINT) { + return cast_as_vector(_size, static_cast(_raw_ptr)); + + } else if (_type == DUCKDB_TYPE_SMALLINT) { + return cast_as_vector(_size, static_cast(_raw_ptr)); + + } else if (_type == DUCKDB_TYPE_USMALLINT) { + return cast_as_vector(_size, static_cast(_raw_ptr)); + + } else if (_type == DUCKDB_TYPE_INTEGER) { + return cast_as_vector(_size, static_cast(_raw_ptr)); + + } else if (_type == DUCKDB_TYPE_UINTEGER) { + return cast_as_vector(_size, static_cast(_raw_ptr)); + + } else if (_type == DUCKDB_TYPE_BIGINT) { + return cast_as_vector(_size, static_cast(_raw_ptr)); + + } else if (_type == DUCKDB_TYPE_UBIGINT) { + return cast_as_vector(_size, static_cast(_raw_ptr)); + + } else if (_type == DUCKDB_TYPE_FLOAT) { + return cast_as_vector(_size, static_cast(_raw_ptr)); + + } else if (_type == DUCKDB_TYPE_DOUBLE) { + return cast_as_vector(_size, static_cast(_raw_ptr)); + + } else { + return error("Could not cast"); + } + } +} + +} // namespace sqlgen::duckdb + +#endif diff --git a/include/sqlgen/duckdb/make_chunk_ptrs.hpp b/include/sqlgen/duckdb/make_chunk_ptrs.hpp index e5e500d3..d76737fa 100644 --- a/include/sqlgen/duckdb/make_chunk_ptrs.hpp +++ b/include/sqlgen/duckdb/make_chunk_ptrs.hpp @@ -11,6 +11,7 @@ #include "../Ref.hpp" #include "ColumnData.hpp" #include "DuckDBResult.hpp" +#include "cast_duckdb_type.hpp" #include "chunk_ptrs_t.hpp" #include "get_duckdb_type.hpp" @@ -36,17 +37,40 @@ struct MakeChunkPtrs...>> { template static auto make_column_data(const Ref& _res, duckdb_data_chunk _chunk) { - if (duckdb_column_type(&_res->res(), _i) != get_duckdb_type()) { + const auto actual_duckdb_type = duckdb_column_type(&_res->res(), _i); + + auto vec = duckdb_data_chunk_get_vector(_chunk, _i); + + if (actual_duckdb_type == get_duckdb_type()) { + return ColumnData{ + .vec = vec, + .data = static_cast(duckdb_vector_get_data(vec)), + .validity = duckdb_vector_get_validity(vec)}; + } + + if constexpr (std::is_same_v) { throw std::runtime_error( - "Wrong type in field " + std::to_string(_i) + ". Expected " + + "Wrong type in field '" + ColName().str() + "'. Expected " + rfl::enum_to_string(get_duckdb_type()) + ", got " + - rfl::enum_to_string(duckdb_column_type(&_res->res(), _i)) + "."); + rfl::enum_to_string(actual_duckdb_type) + "."); + + } else { + const auto ptr_res = cast_duckdb_type( + actual_duckdb_type, duckdb_data_chunk_get_size(_chunk), + duckdb_vector_get_data(vec)); + + if (!ptr_res) { + throw std::runtime_error( + "Wrong type in field '" + ColName().str() + "'. Expected " + + rfl::enum_to_string(get_duckdb_type()) + ", got " + + rfl::enum_to_string(actual_duckdb_type) + "."); + } + + return ColumnData{.vec = vec, + .data = (*ptr_res)->data(), + .validity = duckdb_vector_get_validity(vec), + .ptr = ptr_res->ptr()}; } - auto vec = duckdb_data_chunk_get_vector(_chunk, _i); - return ColumnData{ - .vec = vec, - .data = static_cast(duckdb_vector_get_data(vec)), - .validity = duckdb_vector_get_validity(vec)}; } }; From 46d7f8d21864f5f9bc27a76c4dac0d48d877499e Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sat, 15 Nov 2025 11:23:27 +0100 Subject: [PATCH 30/48] Destroy chunk pointers --- include/sqlgen/duckdb/Iterator.hpp | 19 +++++++++++-------- include/sqlgen/duckdb/from_chunk_ptrs.hpp | 3 ++- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/include/sqlgen/duckdb/Iterator.hpp b/include/sqlgen/duckdb/Iterator.hpp index 940d7d95..3b3e733e 100644 --- a/include/sqlgen/duckdb/Iterator.hpp +++ b/include/sqlgen/duckdb/Iterator.hpp @@ -75,14 +75,17 @@ class Iterator { return Ref>>::make(); } const idx_t row_count = duckdb_data_chunk_get_size(chunk); - return make_chunk_ptrs(_res, chunk) - .transform([&](auto&& _chunk_ptrs) { - auto batch = Ref>>::make(); - for (idx_t i = 0; i < row_count; ++i) { - batch->emplace_back(from_chunk_ptrs(_chunk_ptrs, i)); - } - return batch; - }); + auto res = + make_chunk_ptrs(_res, chunk) + .transform([&](auto&& _chunk_ptrs) { + auto batch = Ref>>::make(); + for (idx_t i = 0; i < row_count; ++i) { + batch->emplace_back(from_chunk_ptrs(_chunk_ptrs, i)); + } + return batch; + }); + duckdb_destroy_data_chunk(&chunk); + return res; }) .or_else([](auto _err) { return Ref>>::make( diff --git a/include/sqlgen/duckdb/from_chunk_ptrs.hpp b/include/sqlgen/duckdb/from_chunk_ptrs.hpp index 7b35ece5..1586713c 100644 --- a/include/sqlgen/duckdb/from_chunk_ptrs.hpp +++ b/include/sqlgen/duckdb/from_chunk_ptrs.hpp @@ -22,7 +22,8 @@ template struct FromChunkPtrs, rfl::Tuple...>> { Result operator()( - const rfl::Tuple...>& _chunk_ptrs, idx_t _i) { + const rfl::Tuple...>& _chunk_ptrs, + idx_t _i) noexcept { return [&](std::integer_sequence) -> Result { try { return T{duckdb::parsing::Parser::read( From b122e3ce7ee5c0c010d3ef109a93d5aa310c8917 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sat, 15 Nov 2025 17:22:36 +0100 Subject: [PATCH 31/48] Handle duckdb_hugeint --- include/sqlgen/duckdb/cast_duckdb_type.hpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/include/sqlgen/duckdb/cast_duckdb_type.hpp b/include/sqlgen/duckdb/cast_duckdb_type.hpp index 62eb64a3..672d4b0b 100644 --- a/include/sqlgen/duckdb/cast_duckdb_type.hpp +++ b/include/sqlgen/duckdb/cast_duckdb_type.hpp @@ -15,7 +15,11 @@ template Ref> cast_as_vector(const size_t _size, U* _ptr) { auto vec = Ref>::make(_size); for (size_t i = 0; i < _size; ++i) { - (*vec)[i] = static_cast(_ptr[i]); + if constexpr (std::is_same_v) { + (*vec)[i] = static_cast(duckdb_hugeint_to_double(_ptr[i])); + } else { + (*vec)[i] = static_cast(_ptr[i]); + } } return vec; } @@ -58,6 +62,9 @@ Result>> cast_duckdb_type(const duckdb_type _type, } else if (_type == DUCKDB_TYPE_DOUBLE) { return cast_as_vector(_size, static_cast(_raw_ptr)); + } else if (_type == DUCKDB_TYPE_HUGEINT) { + return cast_as_vector(_size, static_cast(_raw_ptr)); + } else { return error("Could not cast"); } From 0434d0fc339587384cde51b92b87094f5e5d75dc Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sat, 15 Nov 2025 17:35:58 +0100 Subject: [PATCH 32/48] Fixed some tests --- src/sqlgen/duckdb/to_sql.cpp | 7 +++---- tests/duckdb/test_full_join.cpp | 2 +- tests/duckdb/test_joins_nested.cpp | 2 +- tests/duckdb/test_to_create_table.cpp | 2 +- tests/duckdb/test_unique.cpp | 2 +- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/sqlgen/duckdb/to_sql.cpp b/src/sqlgen/duckdb/to_sql.cpp index 65c2fd86..71dc0294 100644 --- a/src/sqlgen/duckdb/to_sql.cpp +++ b/src/sqlgen/duckdb/to_sql.cpp @@ -327,10 +327,9 @@ std::string create_sequences_for_auto_incr( return stream.str(); }; - return internal::strings::join(" ", internal::collect::vector( - _stmt.columns | filter(is_auto_incr) | - transform(create_one_sequence))) + - " "; + return internal::strings::join( + " ", internal::collect::vector(_stmt.columns | filter(is_auto_incr) | + transform(create_one_sequence))); } std::string create_enums(const dynamic::CreateTable& _stmt) noexcept { diff --git a/tests/duckdb/test_full_join.cpp b/tests/duckdb/test_full_join.cpp index b9e4aedf..27a0111d 100644 --- a/tests/duckdb/test_full_join.cpp +++ b/tests/duckdb/test_full_join.cpp @@ -60,7 +60,7 @@ TEST(duckdb, test_full_join) { const std::string expected_query = R"(SELECT t1."first_name" AS "first_name", t1."last_name" AS "last_name", t2."name" AS "pet_name" FROM "Person" t1 FULL JOIN "Pet" t2 ON t1."id" = t2."owner_id" ORDER BY t1."id", t2."id")"; const std::string expected_json = - R"([{"pet_name":"Mr. Teeny"},{"first_name":"Homer","last_name":"Simpson","pet_name":"Santa's Little Helper"},{"first_name":"Marge","last_name":"Simpson"},{"first_name":"Bart","last_name":"Simpson"},{"first_name":"Lisa","last_name":"Simpson","pet_name":"Snowball"}])"; + R"([{"first_name":"Homer","last_name":"Simpson","pet_name":"Santa's Little Helper"},{"first_name":"Marge","last_name":"Simpson"},{"first_name":"Bart","last_name":"Simpson"},{"first_name":"Lisa","last_name":"Simpson","pet_name":"Snowball"},{"pet_name":"Mr. Teeny"}])"; EXPECT_EQ(duckdb::to_sql(get_all), expected_query); EXPECT_EQ(rfl::json::write(result), expected_json); diff --git a/tests/duckdb/test_joins_nested.cpp b/tests/duckdb/test_joins_nested.cpp index b0292e31..abdfc4bc 100644 --- a/tests/duckdb/test_joins_nested.cpp +++ b/tests/duckdb/test_joins_nested.cpp @@ -73,7 +73,7 @@ TEST(duckdb, test_joins_nested) { const std::string expected_query = R"(SELECT t1."last_name" AS "last_name", t1."first_name" AS "first_name_parent", t2."first_name" AS "first_name_child", (t1."age") - (t2."age") AS "parent_age_at_birth" FROM "Person" t1 INNER JOIN (SELECT t1."parent_id" AS "id", t2."first_name" AS "first_name", t2."age" AS "age" FROM "Relationship" t1 INNER JOIN "Person" t2 ON t2."id" = t1."child_id") t2 ON t1."id" = t2."id" ORDER BY t1."id", t2."id")"; const std::string expected = - R"([{"last_name":"Simpson","first_name_parent":"Homer","first_name_child":"Bart","parent_age_at_birth":35.0},{"last_name":"Simpson","first_name_parent":"Homer","first_name_child":"Lisa","parent_age_at_birth":37.0},{"last_name":"Simpson","first_name_parent":"Homer","first_name_child":"Maggie","parent_age_at_birth":45.0},{"last_name":"Simpson","first_name_parent":"Marge","first_name_child":"Bart","parent_age_at_birth":30.0},{"last_name":"Simpson","first_name_parent":"Marge","first_name_child":"Lisa","parent_age_at_birth":32.0},{"last_name":"Simpson","first_name_parent":"Marge","first_name_child":"Maggie","parent_age_at_birth":40.0}])"; + R"([{"last_name":"Simpson","first_name_parent":"Homer","first_name_child":"Maggie","parent_age_at_birth":45.0},{"last_name":"Simpson","first_name_parent":"Homer","first_name_child":"Lisa","parent_age_at_birth":37.0},{"last_name":"Simpson","first_name_parent":"Homer","first_name_child":"Bart","parent_age_at_birth":35.0},{"last_name":"Simpson","first_name_parent":"Marge","first_name_child":"Maggie","parent_age_at_birth":40.0},{"last_name":"Simpson","first_name_parent":"Marge","first_name_child":"Lisa","parent_age_at_birth":32.0},{"last_name":"Simpson","first_name_parent":"Marge","first_name_child":"Bart","parent_age_at_birth":30.0}])"; EXPECT_EQ(duckdb::to_sql(get_people), expected_query); EXPECT_EQ(rfl::json::write(people), expected); diff --git a/tests/duckdb/test_to_create_table.cpp b/tests/duckdb/test_to_create_table.cpp index b928211b..7c74a209 100644 --- a/tests/duckdb/test_to_create_table.cpp +++ b/tests/duckdb/test_to_create_table.cpp @@ -18,7 +18,7 @@ TEST(duckdb, test_to_create_table) { sqlgen::transpilation::to_create_table(); const auto conn = sqlgen::duckdb::connect().value(); const auto expected = - R"(CREATE TABLE IF NOT EXISTS "TestTable" ("field1" TEXT NOT NULL, "field2" INTEGER NOT NULL, "id" INTEGER PRIMARY KEY NOT NULL, "nullable" TEXT);)"; + R"(CREATE TABLE IF NOT EXISTS "TestTable" ("field1" TEXT NOT NULL, "field2" INTEGER NOT NULL, "id" UINTEGER NOT NULL, "nullable" TEXT, PRIMARY KEY ("id"));)"; EXPECT_EQ(conn->to_sql(create_table_stmt), expected); } diff --git a/tests/duckdb/test_unique.cpp b/tests/duckdb/test_unique.cpp index 43452f1d..828fbeda 100644 --- a/tests/duckdb/test_unique.cpp +++ b/tests/duckdb/test_unique.cpp @@ -37,7 +37,7 @@ TEST(duckdb, test_unique) { .value(); const std::string expected_query = - R"(CREATE TABLE IF NOT EXISTS "Person" ("id" INTEGER PRIMARY KEY NOT NULL, "first_name" TEXT NOT NULL UNIQUE, "last_name" TEXT NOT NULL, "age" REAL NOT NULL);)"; + R"(CREATE TABLE IF NOT EXISTS "Person" ("id" UINTEGER NOT NULL, "first_name" TEXT NOT NULL UNIQUE, "last_name" TEXT NOT NULL, "age" DOUBLE NOT NULL, PRIMARY KEY ("id"));)"; EXPECT_EQ(duckdb::to_sql(create_table), expected_query); } From d67d4c3b916889de98e7c322e5992245214a8b1b Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sat, 15 Nov 2025 21:28:48 +0100 Subject: [PATCH 33/48] Fixed table schema --- src/sqlgen/duckdb/to_sql.cpp | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/src/sqlgen/duckdb/to_sql.cpp b/src/sqlgen/duckdb/to_sql.cpp index 71dc0294..7ccc0b88 100644 --- a/src/sqlgen/duckdb/to_sql.cpp +++ b/src/sqlgen/duckdb/to_sql.cpp @@ -281,10 +281,7 @@ std::string create_index_to_sql(const dynamic::CreateIndex& _stmt) noexcept { stream << "ON "; - if (_stmt.table.schema) { - stream << "\"" << *_stmt.table.schema << "\"."; - } - stream << "\"" << _stmt.table.name << "\""; + stream << table_or_query_to_sql(_stmt.table); stream << "("; stream << internal::strings::join( @@ -369,10 +366,7 @@ std::string create_table_to_sql(const dynamic::CreateTable& _stmt) noexcept { stream << "IF NOT EXISTS "; } - if (_stmt.table.schema) { - stream << wrap_in_quotes(*_stmt.table.schema) << "."; - } - stream << wrap_in_quotes(_stmt.table.name) << " "; + stream << table_or_query_to_sql(_stmt.table); stream << "("; stream << internal::strings::join( @@ -802,6 +796,9 @@ std::string table_or_query_to_sql( return _table_or_query.visit([](const auto& _t) -> std::string { using Type = std::remove_cvref_t; if constexpr (std::is_same_v) { + if (_t.schema) { + return wrap_in_quotes(*_t.schema) + "." + wrap_in_quotes(_t.name); + } return wrap_in_quotes(_t.name); } else { return "(" + select_from_to_sql(*_t) + ")"; @@ -926,10 +923,7 @@ std::string update_to_sql(const dynamic::Update& _stmt) noexcept { stream << "UPDATE "; - if (_stmt.table.schema) { - stream << wrap_in_quotes(*_stmt.table.schema) << "."; - } - stream << wrap_in_quotes(_stmt.table.name); + stream << table_or_query_to_sql(_stmt.table); stream << " SET "; @@ -950,10 +944,7 @@ std::string write_to_sql(const dynamic::Write& _stmt) noexcept { std::stringstream stream; stream << "INSERT INTO "; - if (_stmt.table.schema) { - stream << wrap_in_quotes(*_stmt.table.schema) << "."; - } - stream << wrap_in_quotes(_stmt.table.name); + stream << table_or_query_to_sql(_stmt.table); stream << " BY NAME ( SELECT "; stream << internal::strings::join( From 930e28bea2c81f27968f78f34a49a3b6770f87bd Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sat, 15 Nov 2025 21:35:07 +0100 Subject: [PATCH 34/48] Fixed more tests --- tests/duckdb/test_to_create_table.cpp | 2 +- tests/duckdb/test_to_insert.cpp | 2 +- tests/duckdb/test_to_insert_or_replace.cpp | 3 ++- tests/duckdb/test_unique.cpp | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/duckdb/test_to_create_table.cpp b/tests/duckdb/test_to_create_table.cpp index 7c74a209..326343ac 100644 --- a/tests/duckdb/test_to_create_table.cpp +++ b/tests/duckdb/test_to_create_table.cpp @@ -18,7 +18,7 @@ TEST(duckdb, test_to_create_table) { sqlgen::transpilation::to_create_table(); const auto conn = sqlgen::duckdb::connect().value(); const auto expected = - R"(CREATE TABLE IF NOT EXISTS "TestTable" ("field1" TEXT NOT NULL, "field2" INTEGER NOT NULL, "id" UINTEGER NOT NULL, "nullable" TEXT, PRIMARY KEY ("id"));)"; + R"(CREATE TABLE IF NOT EXISTS "TestTable"("field1" TEXT NOT NULL, "field2" INTEGER NOT NULL, "id" UINTEGER NOT NULL, "nullable" TEXT, PRIMARY KEY ("id"));)"; EXPECT_EQ(conn->to_sql(create_table_stmt), expected); } diff --git a/tests/duckdb/test_to_insert.cpp b/tests/duckdb/test_to_insert.cpp index 545b9a0c..90fd641b 100644 --- a/tests/duckdb/test_to_insert.cpp +++ b/tests/duckdb/test_to_insert.cpp @@ -20,7 +20,7 @@ TEST(duckdb, test_to_insert) { sqlgen::dynamic::Insert>(); const auto conn = sqlgen::duckdb::connect().value(); const auto expected = - R"(INSERT INTO "TestTable" ("field1", "field2", "id", "nullable") VALUES (?, ?, ?, ?);)"; + R"(INSERT INTO "TestTable" BY NAME ( SELECT "field1" AS "field1", "field2" AS "field2", "id" AS "id", "nullable" AS "nullable" FROM sqlgen_appended_data);)"; EXPECT_EQ(conn->to_sql(insert_stmt), expected); } diff --git a/tests/duckdb/test_to_insert_or_replace.cpp b/tests/duckdb/test_to_insert_or_replace.cpp index 96e6a8b2..81e68be3 100644 --- a/tests/duckdb/test_to_insert_or_replace.cpp +++ b/tests/duckdb/test_to_insert_or_replace.cpp @@ -24,8 +24,9 @@ TEST(duckdb, test_to_insert_or_replace) { sqlgen::transpilation::to_insert_or_write(true); const auto conn = sqlgen::duckdb::connect().value(); + const auto expected = - R"(INSERT INTO "TestTable" ("field1", "field2", "field3", "id", "nullable") VALUES (?, ?, ?, ?, ?) ON CONFLICT (field3, id) DO UPDATE SET field1=excluded.field1, field2=excluded.field2, field3=excluded.field3, id=excluded.id, nullable=excluded.nullable;)"; + R"(INSERT OR REPLACE INTO "TestTable" BY NAME ( SELECT "field1" AS "field1", "field2" AS "field2", "field3" AS "field3", "id" AS "id", "nullable" AS "nullable" FROM sqlgen_appended_data);)"; EXPECT_EQ(conn->to_sql(insert_stmt), expected); } diff --git a/tests/duckdb/test_unique.cpp b/tests/duckdb/test_unique.cpp index 828fbeda..a9a7c304 100644 --- a/tests/duckdb/test_unique.cpp +++ b/tests/duckdb/test_unique.cpp @@ -37,7 +37,7 @@ TEST(duckdb, test_unique) { .value(); const std::string expected_query = - R"(CREATE TABLE IF NOT EXISTS "Person" ("id" UINTEGER NOT NULL, "first_name" TEXT NOT NULL UNIQUE, "last_name" TEXT NOT NULL, "age" DOUBLE NOT NULL, PRIMARY KEY ("id"));)"; + R"(CREATE TABLE IF NOT EXISTS "Person"("id" UINTEGER NOT NULL, "first_name" TEXT NOT NULL UNIQUE, "last_name" TEXT NOT NULL, "age" DOUBLE NOT NULL, PRIMARY KEY ("id"));)"; EXPECT_EQ(duckdb::to_sql(create_table), expected_query); } From 8ad759117a4223cdfd34a80569059ac7848f1c3f Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sat, 15 Nov 2025 22:26:21 +0100 Subject: [PATCH 35/48] Handling timestamps and dates --- include/sqlgen/duckdb/get_duckdb_type.hpp | 6 +++ include/sqlgen/duckdb/parsing/Parser.hpp | 2 + include/sqlgen/duckdb/parsing/Parser_date.hpp | 42 +++++++++++++++++++ .../duckdb/parsing/Parser_timestamp.hpp | 40 ++++++++++++++++++ 4 files changed, 90 insertions(+) create mode 100644 include/sqlgen/duckdb/parsing/Parser_date.hpp create mode 100644 include/sqlgen/duckdb/parsing/Parser_timestamp.hpp diff --git a/include/sqlgen/duckdb/get_duckdb_type.hpp b/include/sqlgen/duckdb/get_duckdb_type.hpp index 20583998..919ec7ed 100644 --- a/include/sqlgen/duckdb/get_duckdb_type.hpp +++ b/include/sqlgen/duckdb/get_duckdb_type.hpp @@ -48,9 +48,15 @@ duckdb_type get_duckdb_type() { } else if constexpr (std::is_same_v) { return DUCKDB_TYPE_DOUBLE; + } else if constexpr (std::is_same_v) { + return DUCKDB_TYPE_DATE; + } else if constexpr (std::is_same_v) { return DUCKDB_TYPE_VARCHAR; + } else if constexpr (std::is_same_v) { + return DUCKDB_TYPE_TIMESTAMP; + } else { static_assert(rfl::always_false_v, "Unsupported type."); } diff --git a/include/sqlgen/duckdb/parsing/Parser.hpp b/include/sqlgen/duckdb/parsing/Parser.hpp index 58985f87..575f3100 100644 --- a/include/sqlgen/duckdb/parsing/Parser.hpp +++ b/include/sqlgen/duckdb/parsing/Parser.hpp @@ -2,10 +2,12 @@ #define SQLGEN_DUCKDB_PARSING_PARSER_HPP_ #include "Parser_base.hpp" +#include "Parser_date.hpp" #include "Parser_default.hpp" #include "Parser_optional.hpp" #include "Parser_reflection_type.hpp" #include "Parser_smart_ptr.hpp" #include "Parser_string.hpp" +#include "Parser_timestamp.hpp" #endif diff --git a/include/sqlgen/duckdb/parsing/Parser_date.hpp b/include/sqlgen/duckdb/parsing/Parser_date.hpp new file mode 100644 index 00000000..615c32ed --- /dev/null +++ b/include/sqlgen/duckdb/parsing/Parser_date.hpp @@ -0,0 +1,42 @@ +#ifndef SQLGEN_DUCKDB_PARSING_PARSER_DATE_HPP_ +#define SQLGEN_DUCKDB_PARSING_PARSER_DATE_HPP_ + +#include + +#include +#include +#include + +#include "../../Result.hpp" +#include "../../Timestamp.hpp" +#include "Parser_base.hpp" + +namespace sqlgen::duckdb::parsing { + +template <> +struct Parser { + using ResultingType = duckdb_date; + + static constexpr time_t seconds_per_day = 24 * 60 * 60; + + static Result read(const ResultingType* _r) noexcept { + if (!_r) { + return error("Date value cannot be NULL."); + } + return Date(static_cast(_r->days) * seconds_per_day); + } + + static Result write(const Date& _t, + duckdb_appender _appender) noexcept { + return duckdb_append_date( + _appender, duckdb_date{.days = static_cast( + _t.to_time_t() / seconds_per_day)}) != + DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append date value.")); + } +}; + +} // namespace sqlgen::duckdb::parsing + +#endif diff --git a/include/sqlgen/duckdb/parsing/Parser_timestamp.hpp b/include/sqlgen/duckdb/parsing/Parser_timestamp.hpp new file mode 100644 index 00000000..2d9f12a0 --- /dev/null +++ b/include/sqlgen/duckdb/parsing/Parser_timestamp.hpp @@ -0,0 +1,40 @@ +#ifndef SQLGEN_DUCKDB_PARSING_PARSER_TIMESTAMP_HPP_ +#define SQLGEN_DUCKDB_PARSING_PARSER_TIMESTAMP_HPP_ + +#include + +#include +#include +#include + +#include "../../Result.hpp" +#include "Parser_base.hpp" + +namespace sqlgen::duckdb::parsing { + +template +struct Parser> { + using ResultingType = duckdb_timestamp; + + static Result> read( + const ResultingType* _r) noexcept { + if (!_r) { + return error("Timestamp value cannot be NULL."); + } + return rfl::Timestamp<_format>(static_cast(_r->micros / 1000000)); + } + + static Result write(const rfl::Timestamp<_format>& _t, + duckdb_appender _appender) noexcept { + return duckdb_append_timestamp( + _appender, + duckdb_timestamp{.micros = static_cast(_t.to_time_t()) * + 1000000}) != DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append timestamp value.")); + } +}; + +} // namespace sqlgen::duckdb::parsing + +#endif From 7b5758624117708c75f18fa56083e90a4a5ed75c Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sat, 15 Nov 2025 23:13:41 +0100 Subject: [PATCH 36/48] Cast date to timestamp and vice-versa --- include/sqlgen/duckdb/cast_duckdb_type.hpp | 28 +++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/include/sqlgen/duckdb/cast_duckdb_type.hpp b/include/sqlgen/duckdb/cast_duckdb_type.hpp index 672d4b0b..a74711d8 100644 --- a/include/sqlgen/duckdb/cast_duckdb_type.hpp +++ b/include/sqlgen/duckdb/cast_duckdb_type.hpp @@ -13,10 +13,24 @@ namespace sqlgen::duckdb { template Ref> cast_as_vector(const size_t _size, U* _ptr) { + constexpr int64_t microseconds_per_day = + static_cast(24 * 60 * 60) * static_cast(1000000); + auto vec = Ref>::make(_size); for (size_t i = 0; i < _size; ++i) { if constexpr (std::is_same_v) { (*vec)[i] = static_cast(duckdb_hugeint_to_double(_ptr[i])); + + } else if constexpr (std::is_same_v && + std::is_same_v) { + (*vec)[i] = duckdb_timestamp{ + .micros = static_cast(_ptr[i].days) * microseconds_per_day}; + + } else if constexpr (std::is_same_v && + std::is_same_v) { + (*vec)[i] = duckdb_date{ + .days = static_cast(_ptr[i].micros / microseconds_per_day)}; + } else { (*vec)[i] = static_cast(_ptr[i]); } @@ -28,7 +42,19 @@ template Result>> cast_duckdb_type(const duckdb_type _type, const size_t _size, void* _raw_ptr) { - if constexpr (!std::is_floating_point_v && !std::is_integral_v) { + if constexpr (std::is_same_v) { + if (_type == DUCKDB_TYPE_DATE) { + return cast_as_vector(_size, static_cast(_raw_ptr)); + } + return error("Could not cast"); + + } else if constexpr (std::is_same_v) { + if (_type == DUCKDB_TYPE_TIMESTAMP) { + return cast_as_vector(_size, static_cast(_raw_ptr)); + } + return error("Could not cast"); + + } else if constexpr (!std::is_floating_point_v && !std::is_integral_v) { return error("Could not cast"); } else { From 82d0c5f2e894784d3172df7cd01ab6f5c683b9c0 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sat, 15 Nov 2025 23:28:42 +0100 Subject: [PATCH 37/48] Fixed yet more tests --- tests/duckdb/test_foreign_key.cpp | 2 +- tests/duckdb/test_insert_fail.cpp | 4 +--- tests/duckdb/test_select_from_with_timestamps.cpp | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/duckdb/test_foreign_key.cpp b/tests/duckdb/test_foreign_key.cpp index 322136c0..2224fa06 100644 --- a/tests/duckdb/test_foreign_key.cpp +++ b/tests/duckdb/test_foreign_key.cpp @@ -48,8 +48,8 @@ TEST(duckdb, test_foreign_key) { .and_then(create_table) .and_then(insert(std::ref(people1))) .and_then(insert(std::ref(relationships))) - .and_then(drop | if_exists) .and_then(drop | if_exists) + .and_then(drop | if_exists) .and_then(commit) .value(); } diff --git a/tests/duckdb/test_insert_fail.cpp b/tests/duckdb/test_insert_fail.cpp index d614baf8..cfb62d1d 100644 --- a/tests/duckdb/test_insert_fail.cpp +++ b/tests/duckdb/test_insert_fail.cpp @@ -34,9 +34,7 @@ TEST(duckdb, test_insert_fail) { .and_then(drop | if_exists) .and_then(write(people1.at(0))); - const auto res = conn.and_then(begin_transaction) - .and_then(insert(people1.at(0))) - .and_then(commit); + const auto res = conn.and_then(insert(people1.at(0))); // Should fail - duplicate key violation. EXPECT_FALSE(res && true); diff --git a/tests/duckdb/test_select_from_with_timestamps.cpp b/tests/duckdb/test_select_from_with_timestamps.cpp index 2ccda80a..f11d7b2b 100644 --- a/tests/duckdb/test_select_from_with_timestamps.cpp +++ b/tests/duckdb/test_select_from_with_timestamps.cpp @@ -67,7 +67,7 @@ TEST(duckdb, test_range_select_from_with_timestamps) { .value(); const std::string expected_query = - R"(SELECT datetime("birthday", '+10 days') AS "birthday", cast((cast(cast(strftime('%Y', "birthday") as INT) as TEXT) || '-' || cast(cast(strftime('%m', "birthday") as INT) as TEXT) || '-' || cast(cast(strftime('%d', "birthday") as INT) as TEXT)) as TEXT) AS "birthday_recreated", julianday('2011-01-01') - julianday("birthday") AS "age_in_days", unixepoch(datetime("birthday", '+10 days'), 'subsec') AS "birthday_unixepoch", cast(strftime('%H', "birthday") as INT) AS "hour", cast(strftime('%M', "birthday") as INT) AS "minute", cast(strftime('%S', "birthday") as INT) AS "second", cast(strftime('%w', "birthday") as INT) AS "weekday" FROM "Person" ORDER BY "id")"; + R"(SELECT "birthday" + INTERVAL '10 days' AS "birthday", cast((cast(extract(YEAR from "birthday") as TEXT) || '-' || cast(extract(MONTH from "birthday") as TEXT) || '-' || cast(extract(DAY from "birthday") as TEXT)) as DATE) AS "birthday_recreated", cast('2011-01-01' as DATE) - cast("birthday" as DATE) AS "age_in_days", extract(EPOCH FROM "birthday" + INTERVAL '10 days') AS "birthday_unixepoch", extract(HOUR from "birthday") AS "hour", extract(MINUTE from "birthday") AS "minute", extract(SECOND from "birthday") AS "second", extract(DOW from "birthday") AS "weekday" FROM "Person" ORDER BY "id")"; const std::string expected = R"([{"birthday":"1970-01-11","birthday_recreated":"1970-01-01","birthday_unixepoch":864000,"age_in_days":14975.0,"hour":0,"minute":0,"second":0,"weekday":4},{"birthday":"2000-01-11","birthday_recreated":"2000-01-01","birthday_unixepoch":947548800,"age_in_days":4018.0,"hour":0,"minute":0,"second":0,"weekday":6},{"birthday":"2002-01-11","birthday_recreated":"2002-01-01","birthday_unixepoch":1010707200,"age_in_days":3287.0,"hour":0,"minute":0,"second":0,"weekday":2},{"birthday":"2010-01-11","birthday_recreated":"2010-01-01","birthday_unixepoch":1263168000,"age_in_days":365.0,"hour":0,"minute":0,"second":0,"weekday":5}])"; From f291067ab36d29dcc8a45a51c01030c6ce4f0de2 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sun, 16 Nov 2025 00:04:27 +0100 Subject: [PATCH 38/48] Make sure we properly handle contraint errors --- include/sqlgen/duckdb/Connection.hpp | 31 ++++++++++++++---------- include/sqlgen/duckdb/DuckDBAppender.hpp | 8 ++++++ 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index b36934f5..3894f9ec 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -64,8 +64,9 @@ class Connection { .and_then([&](const auto &_types) { return DuckDBAppender::make(sql, conn_, columns, _types); }) - .and_then([&](const auto &_appender) { - return write_to_appender(_begin, _end, _appender->appender()); + .and_then([&](auto _appender) { + return write_to_appender(_begin, _end, _appender->appender()) + .and_then([&](const auto &) { return _appender->close(); }); }); } @@ -155,7 +156,10 @@ class Connection { if (!res) { return res; } - duckdb_appender_end_row(_appender); + const auto state = duckdb_appender_end_row(_appender); + if (state == DuckDBError) { + return error(duckdb_appender_error(_appender)); + } } return Nothing{}; } @@ -165,16 +169,17 @@ class Connection { duckdb_appender _appender) noexcept { using ViewType = internal::remove_auto_incr_primary_t>; - Result res = Nothing{}; - ViewType(rfl::to_view(_struct)).apply([&](const auto &_field) { - using ValueType = std::remove_cvref_t::Type>>; - if (res) { - res = duckdb::parsing::Parser::write(*_field.value(), - _appender); - } - }); - return res; + try { + ViewType(rfl::to_view(_struct)).apply([&](const auto &_field) { + using ValueType = std::remove_cvref_t::Type>>; + duckdb::parsing::Parser::write(*_field.value(), _appender) + .value(); + }); + } catch (const std::exception &e) { + return error(e.what()); + } + return Nothing{}; } private: diff --git a/include/sqlgen/duckdb/DuckDBAppender.hpp b/include/sqlgen/duckdb/DuckDBAppender.hpp index add9d034..4a42dead 100644 --- a/include/sqlgen/duckdb/DuckDBAppender.hpp +++ b/include/sqlgen/duckdb/DuckDBAppender.hpp @@ -64,6 +64,14 @@ class DuckDBAppender { duckdb_appender& appender() { return appender_; } + Result close() { + const auto state = duckdb_appender_close(appender_); + if (state == DuckDBError) { + return error(duckdb_appender_error(appender_)); + } + return Nothing{}; + } + private: bool destroy_; From a4399a59412b48447c61e2931c9a0b7558922a7a Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Sun, 16 Nov 2025 19:01:46 +0100 Subject: [PATCH 39/48] Started adding enums --- include/sqlgen/duckdb/cast_duckdb_type.hpp | 50 +++++----- include/sqlgen/duckdb/parsing/Parser.hpp | 1 + include/sqlgen/duckdb/parsing/Parser_enum.hpp | 43 ++++++++ src/sqlgen/duckdb/to_sql.cpp | 8 +- tests/duckdb/test_enum_crosstable.cpp | 99 +++++++++++++++++++ 5 files changed, 172 insertions(+), 29 deletions(-) create mode 100644 include/sqlgen/duckdb/parsing/Parser_enum.hpp create mode 100644 tests/duckdb/test_enum_crosstable.cpp diff --git a/include/sqlgen/duckdb/cast_duckdb_type.hpp b/include/sqlgen/duckdb/cast_duckdb_type.hpp index a74711d8..0284d110 100644 --- a/include/sqlgen/duckdb/cast_duckdb_type.hpp +++ b/include/sqlgen/duckdb/cast_duckdb_type.hpp @@ -58,41 +58,43 @@ Result>> cast_duckdb_type(const duckdb_type _type, return error("Could not cast"); } else { - if (_type == DUCKDB_TYPE_TINYINT) { - return cast_as_vector(_size, static_cast(_raw_ptr)); + switch (_type) { + case DUCKDB_TYPE_TINYINT: + return cast_as_vector(_size, static_cast(_raw_ptr)); - } else if (_type == DUCKDB_TYPE_UTINYINT) { - return cast_as_vector(_size, static_cast(_raw_ptr)); + case DUCKDB_TYPE_UTINYINT: + case DUCKDB_TYPE_ENUM: + return cast_as_vector(_size, static_cast(_raw_ptr)); - } else if (_type == DUCKDB_TYPE_SMALLINT) { - return cast_as_vector(_size, static_cast(_raw_ptr)); + case DUCKDB_TYPE_SMALLINT: + return cast_as_vector(_size, static_cast(_raw_ptr)); - } else if (_type == DUCKDB_TYPE_USMALLINT) { - return cast_as_vector(_size, static_cast(_raw_ptr)); + case DUCKDB_TYPE_USMALLINT: + return cast_as_vector(_size, static_cast(_raw_ptr)); - } else if (_type == DUCKDB_TYPE_INTEGER) { - return cast_as_vector(_size, static_cast(_raw_ptr)); + case DUCKDB_TYPE_INTEGER: + return cast_as_vector(_size, static_cast(_raw_ptr)); - } else if (_type == DUCKDB_TYPE_UINTEGER) { - return cast_as_vector(_size, static_cast(_raw_ptr)); + case DUCKDB_TYPE_UINTEGER: + return cast_as_vector(_size, static_cast(_raw_ptr)); - } else if (_type == DUCKDB_TYPE_BIGINT) { - return cast_as_vector(_size, static_cast(_raw_ptr)); + case DUCKDB_TYPE_BIGINT: + return cast_as_vector(_size, static_cast(_raw_ptr)); - } else if (_type == DUCKDB_TYPE_UBIGINT) { - return cast_as_vector(_size, static_cast(_raw_ptr)); + case DUCKDB_TYPE_UBIGINT: + return cast_as_vector(_size, static_cast(_raw_ptr)); - } else if (_type == DUCKDB_TYPE_FLOAT) { - return cast_as_vector(_size, static_cast(_raw_ptr)); + case DUCKDB_TYPE_FLOAT: + return cast_as_vector(_size, static_cast(_raw_ptr)); - } else if (_type == DUCKDB_TYPE_DOUBLE) { - return cast_as_vector(_size, static_cast(_raw_ptr)); + case DUCKDB_TYPE_DOUBLE: + return cast_as_vector(_size, static_cast(_raw_ptr)); - } else if (_type == DUCKDB_TYPE_HUGEINT) { - return cast_as_vector(_size, static_cast(_raw_ptr)); + case DUCKDB_TYPE_HUGEINT: + return cast_as_vector(_size, static_cast(_raw_ptr)); - } else { - return error("Could not cast"); + default: + return error("Could not cast"); } } } diff --git a/include/sqlgen/duckdb/parsing/Parser.hpp b/include/sqlgen/duckdb/parsing/Parser.hpp index 575f3100..9c76e9fa 100644 --- a/include/sqlgen/duckdb/parsing/Parser.hpp +++ b/include/sqlgen/duckdb/parsing/Parser.hpp @@ -4,6 +4,7 @@ #include "Parser_base.hpp" #include "Parser_date.hpp" #include "Parser_default.hpp" +#include "Parser_enum.hpp" #include "Parser_optional.hpp" #include "Parser_reflection_type.hpp" #include "Parser_smart_ptr.hpp" diff --git a/include/sqlgen/duckdb/parsing/Parser_enum.hpp b/include/sqlgen/duckdb/parsing/Parser_enum.hpp new file mode 100644 index 00000000..0428b1d8 --- /dev/null +++ b/include/sqlgen/duckdb/parsing/Parser_enum.hpp @@ -0,0 +1,43 @@ +#ifndef SQLGEN_DUCKDB_PARSING_PARSER_ENUM_HPP_ +#define SQLGEN_DUCKDB_PARSING_PARSER_ENUM_HPP_ + +#include + +#include +#include +#include +#include + +#include "../../Result.hpp" +#include "Parser_base.hpp" + +namespace sqlgen::duckdb::parsing { + +template + requires std::is_enum_v +struct Parser { + using ResultingType = uint8_t; + + static Result read(const ResultingType* _r) noexcept { + static_assert(enchantum::ScopedEnum, "The enum must be scoped."); + constexpr auto arr = rfl::get_enumerator_array(); + static_assert(arr.size() < 255, "Enum size cannot exceed 255."); + if (!_r) { + return error("Enum value cannot be NULL."); + } + return static_cast(*_r); + } + + static Result write(const EnumT& _t, + duckdb_appender _appender) noexcept { + const auto str = rfl::enum_to_string(_t); + return duckdb_append_varchar_length(_appender, str.c_str(), str.length()) != + DuckDBError + ? Result(Nothing{}) + : Result(error("Could not append string value.")); + } +}; + +} // namespace sqlgen::duckdb::parsing + +#endif diff --git a/src/sqlgen/duckdb/to_sql.cpp b/src/sqlgen/duckdb/to_sql.cpp index 7ccc0b88..bc4430fb 100644 --- a/src/sqlgen/duckdb/to_sql.cpp +++ b/src/sqlgen/duckdb/to_sql.cpp @@ -335,17 +335,15 @@ std::string create_enums(const dynamic::CreateTable& _stmt) noexcept { std::stringstream stream; for (const auto& [enum_name, enum_values] : get_enum_types(_stmt)) { + stream << "CREATE TYPE "; if (_stmt.if_not_exists) { - stream << "DO $$ BEGIN "; + stream << "IF NOT EXISTS "; } - stream << "CREATE TYPE " << enum_name << " AS ENUM (" + stream << enum_name << " AS ENUM (" << internal::strings::join( ", ", internal::collect::vector( enum_values | transform(wrap_in_single_quotes))) << "); "; - if (_stmt.if_not_exists) { - stream << "EXCEPTION WHEN duplicate_object THEN NULL; END $$;"; - } } return stream.str(); diff --git a/tests/duckdb/test_enum_crosstable.cpp b/tests/duckdb/test_enum_crosstable.cpp new file mode 100644 index 00000000..b3b76409 --- /dev/null +++ b/tests/duckdb/test_enum_crosstable.cpp @@ -0,0 +1,99 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_enum_cross_table { +enum class AccessRestriction { PUBLIC = 1, INTERNAL = 2, CONFIDENTIAL = 3 }; +struct Employee { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + AccessRestriction access_level; +}; + +struct Document { + sqlgen::PrimaryKey id; + AccessRestriction min_access_level; + std::string name; + std::string path; +}; + +TEST(duckdb, test_enum_cross_table) { + using namespace sqlgen; + using namespace sqlgen::literals; + + auto employees = std::vector({ + Employee{.first_name = "Homer", + .last_name = "Simpson", + .access_level = AccessRestriction::PUBLIC}, + Employee{.first_name = "Waylon", + .last_name = "Smithers", + .access_level = AccessRestriction::INTERNAL}, + Employee{.first_name = "Montgomery", + .last_name = "Burns", + .access_level = AccessRestriction::CONFIDENTIAL}, + }); + auto documents = std::vector({ + Document{.min_access_level = AccessRestriction::PUBLIC, + .name = "Power Plant Safety Manual", + .path = "/documents/powerplant/safety_manual.txt"}, + Document{.min_access_level = AccessRestriction::INTERNAL, + .name = "Staff Memo", + .path = "/documents/powerplant/staff_memo.txt"}, + Document{.min_access_level = AccessRestriction::CONFIDENTIAL, + .name = "Operations Report", + .path = "/documents/powerplant/operations_report.pdf"}, + Document{.min_access_level = AccessRestriction::PUBLIC, + .name = "Project Plan", + .path = "/documents/powerplant/project_plan.md"}, + Document{.min_access_level = AccessRestriction::INTERNAL, + .name = "Budget Q1", + .path = "/documents/powerplant/budget_q1.pdf"}, + Document{.min_access_level = AccessRestriction::CONFIDENTIAL, + .name = "HR Policies", + .path = "/documents/powerplant/hr_policies.pdf"}, + Document{.min_access_level = AccessRestriction::PUBLIC, + .name = "Team Photo", + .path = "/documents/powerplant/team.jpg"}, + Document{.min_access_level = AccessRestriction::CONFIDENTIAL, + .name = "Executive Summary", + .path = "/documents/powerplant/executive_summary.docx"}, + Document{.min_access_level = AccessRestriction::INTERNAL, + .name = "Release Notes", + .path = "/documents/powerplant/release_notes.txt"}, + }); + + const auto conn = duckdb::connect() + .and_then(drop | if_exists) + .and_then(drop | if_exists) + .and_then(write(employees)) + .and_then(write(documents)); + + const auto smithers = conn.and_then(sqlgen::read | + where("last_name"_c == "Smithers" and + "first_name"_c == "Waylon")) + .value(); + + const auto smithers_level = smithers.access_level; + const auto smithers_documents = + conn.and_then(sqlgen::read> | + where("min_access_level"_c == smithers_level || + "min_access_level"_c == AccessRestriction::PUBLIC) | + order_by("name"_c)) + .value(); + + const auto expected_ids = std::set{1, 2, 4, 5, 7, 9}; + std::set actual_ids; + for (const auto &d : smithers_documents) { + actual_ids.emplace(d.id()); + } + + EXPECT_EQ(expected_ids, actual_ids); +} + +} // namespace test_enum_cross_table + From 065507680b5e5619c16f58ba64bece1fb2369058 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Mon, 17 Nov 2025 22:01:13 +0100 Subject: [PATCH 40/48] Added more tests for enums --- include/sqlgen/duckdb/cast_duckdb_type.hpp | 1 - include/sqlgen/duckdb/check_duckdb_type.hpp | 68 +++++++++++++++ include/sqlgen/duckdb/get_duckdb_type.hpp | 67 --------------- include/sqlgen/duckdb/make_chunk_ptrs.hpp | 19 +++-- include/sqlgen/duckdb/parsing/Parser_enum.hpp | 7 +- tests/duckdb/test_enum_crosstable.cpp | 3 +- tests/duckdb/test_enum_lookup.cpp | 84 +++++++++++++++++++ tests/duckdb/test_enum_namespace.cpp | 59 +++++++++++++ 8 files changed, 227 insertions(+), 81 deletions(-) create mode 100644 include/sqlgen/duckdb/check_duckdb_type.hpp delete mode 100644 include/sqlgen/duckdb/get_duckdb_type.hpp create mode 100644 tests/duckdb/test_enum_lookup.cpp create mode 100644 tests/duckdb/test_enum_namespace.cpp diff --git a/include/sqlgen/duckdb/cast_duckdb_type.hpp b/include/sqlgen/duckdb/cast_duckdb_type.hpp index 0284d110..53dd8632 100644 --- a/include/sqlgen/duckdb/cast_duckdb_type.hpp +++ b/include/sqlgen/duckdb/cast_duckdb_type.hpp @@ -63,7 +63,6 @@ Result>> cast_duckdb_type(const duckdb_type _type, return cast_as_vector(_size, static_cast(_raw_ptr)); case DUCKDB_TYPE_UTINYINT: - case DUCKDB_TYPE_ENUM: return cast_as_vector(_size, static_cast(_raw_ptr)); case DUCKDB_TYPE_SMALLINT: diff --git a/include/sqlgen/duckdb/check_duckdb_type.hpp b/include/sqlgen/duckdb/check_duckdb_type.hpp new file mode 100644 index 00000000..4f7a86f7 --- /dev/null +++ b/include/sqlgen/duckdb/check_duckdb_type.hpp @@ -0,0 +1,68 @@ +#ifndef SQLGEN_DUCKDB_CHECKDUCKDBTYPE_HPP_ +#define SQLGEN_DUCKDB_CHECKDUCKDBTYPE_HPP_ + +#include + +#include +#include + +#include "../Result.hpp" + +namespace sqlgen::duckdb { + +template +bool check_duckdb_type(duckdb_type _t) { + using Type = std::remove_cvref_t; + + switch (_t) { + case DUCKDB_TYPE_BOOLEAN: + return std::is_same_v; + + case DUCKDB_TYPE_TINYINT: + return std::is_same_v || std::is_same_v; + + case DUCKDB_TYPE_ENUM: + case DUCKDB_TYPE_UTINYINT: + return std::is_same_v; + + case DUCKDB_TYPE_SMALLINT: + return std::is_same_v; + + case DUCKDB_TYPE_USMALLINT: + return std::is_same_v; + + case DUCKDB_TYPE_INTEGER: + return std::is_same_v; + + case DUCKDB_TYPE_UINTEGER: + return std::is_same_v; + + case DUCKDB_TYPE_BIGINT: + return std::is_same_v; + + case DUCKDB_TYPE_UBIGINT: + return std::is_same_v; + + case DUCKDB_TYPE_FLOAT: + return std::is_same_v; + + case DUCKDB_TYPE_DOUBLE: + return std::is_same_v; + + case DUCKDB_TYPE_DATE: + return std::is_same_v; + + case DUCKDB_TYPE_VARCHAR: + return std::is_same_v; + + case DUCKDB_TYPE_TIMESTAMP: + return std::is_same_v; + + default: + return false; + } +} + +} // namespace sqlgen::duckdb + +#endif diff --git a/include/sqlgen/duckdb/get_duckdb_type.hpp b/include/sqlgen/duckdb/get_duckdb_type.hpp deleted file mode 100644 index 919ec7ed..00000000 --- a/include/sqlgen/duckdb/get_duckdb_type.hpp +++ /dev/null @@ -1,67 +0,0 @@ -#ifndef SQLGEN_DUCKDB_GETDUCKDBTYPE_HPP_ -#define SQLGEN_DUCKDB_GETDUCKDBTYPE_HPP_ - -#include - -#include -#include - -#include "../Result.hpp" - -namespace sqlgen::duckdb { - -template -duckdb_type get_duckdb_type() { - using Type = std::remove_cvref_t; - - if constexpr (std::is_same_v) { - return DUCKDB_TYPE_BOOLEAN; - - } else if constexpr (std::is_same_v || - std::is_same_v) { - return DUCKDB_TYPE_TINYINT; - - } else if constexpr (std::is_same_v) { - return DUCKDB_TYPE_UTINYINT; - - } else if constexpr (std::is_same_v) { - return DUCKDB_TYPE_SMALLINT; - - } else if constexpr (std::is_same_v) { - return DUCKDB_TYPE_USMALLINT; - - } else if constexpr (std::is_same_v) { - return DUCKDB_TYPE_INTEGER; - - } else if constexpr (std::is_same_v) { - return DUCKDB_TYPE_UINTEGER; - - } else if constexpr (std::is_same_v) { - return DUCKDB_TYPE_BIGINT; - - } else if constexpr (std::is_same_v) { - return DUCKDB_TYPE_UBIGINT; - - } else if constexpr (std::is_same_v) { - return DUCKDB_TYPE_FLOAT; - - } else if constexpr (std::is_same_v) { - return DUCKDB_TYPE_DOUBLE; - - } else if constexpr (std::is_same_v) { - return DUCKDB_TYPE_DATE; - - } else if constexpr (std::is_same_v) { - return DUCKDB_TYPE_VARCHAR; - - } else if constexpr (std::is_same_v) { - return DUCKDB_TYPE_TIMESTAMP; - - } else { - static_assert(rfl::always_false_v, "Unsupported type."); - } -} - -} // namespace sqlgen::duckdb - -#endif diff --git a/include/sqlgen/duckdb/make_chunk_ptrs.hpp b/include/sqlgen/duckdb/make_chunk_ptrs.hpp index d76737fa..6a9c6b1a 100644 --- a/include/sqlgen/duckdb/make_chunk_ptrs.hpp +++ b/include/sqlgen/duckdb/make_chunk_ptrs.hpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -12,8 +13,8 @@ #include "ColumnData.hpp" #include "DuckDBResult.hpp" #include "cast_duckdb_type.hpp" +#include "check_duckdb_type.hpp" #include "chunk_ptrs_t.hpp" -#include "get_duckdb_type.hpp" namespace sqlgen::duckdb { @@ -41,7 +42,7 @@ struct MakeChunkPtrs...>> { auto vec = duckdb_data_chunk_get_vector(_chunk, _i); - if (actual_duckdb_type == get_duckdb_type()) { + if (check_duckdb_type(actual_duckdb_type)) { return ColumnData{ .vec = vec, .data = static_cast(duckdb_vector_get_data(vec)), @@ -49,10 +50,10 @@ struct MakeChunkPtrs...>> { } if constexpr (std::is_same_v) { - throw std::runtime_error( - "Wrong type in field '" + ColName().str() + "'. Expected " + - rfl::enum_to_string(get_duckdb_type()) + ", got " + - rfl::enum_to_string(actual_duckdb_type) + "."); + throw std::runtime_error("Wrong type in field '" + ColName().str() + + "'. " + rfl::enum_to_string(actual_duckdb_type) + + " could not be cast to " + + rfl::type_name_t().str() + "."); } else { const auto ptr_res = cast_duckdb_type( @@ -61,9 +62,9 @@ struct MakeChunkPtrs...>> { if (!ptr_res) { throw std::runtime_error( - "Wrong type in field '" + ColName().str() + "'. Expected " + - rfl::enum_to_string(get_duckdb_type()) + ", got " + - rfl::enum_to_string(actual_duckdb_type) + "."); + "Wrong type in field '" + ColName().str() + "'. " + + rfl::enum_to_string(actual_duckdb_type) + " could not be cast to " + + rfl::type_name_t().str() + "."); } return ColumnData{.vec = vec, diff --git a/include/sqlgen/duckdb/parsing/Parser_enum.hpp b/include/sqlgen/duckdb/parsing/Parser_enum.hpp index 0428b1d8..566251e0 100644 --- a/include/sqlgen/duckdb/parsing/Parser_enum.hpp +++ b/include/sqlgen/duckdb/parsing/Parser_enum.hpp @@ -18,10 +18,11 @@ template struct Parser { using ResultingType = uint8_t; + static_assert(enchantum::ScopedEnum, "The enum must be scoped."); + static constexpr auto arr = rfl::get_enumerator_array(); + static_assert(arr.size() < 255, "Enum size cannot exceed 255."); + static Result read(const ResultingType* _r) noexcept { - static_assert(enchantum::ScopedEnum, "The enum must be scoped."); - constexpr auto arr = rfl::get_enumerator_array(); - static_assert(arr.size() < 255, "Enum size cannot exceed 255."); if (!_r) { return error("Enum value cannot be NULL."); } diff --git a/tests/duckdb/test_enum_crosstable.cpp b/tests/duckdb/test_enum_crosstable.cpp index b3b76409..c11f1d29 100644 --- a/tests/duckdb/test_enum_crosstable.cpp +++ b/tests/duckdb/test_enum_crosstable.cpp @@ -7,7 +7,7 @@ #include namespace test_enum_cross_table { -enum class AccessRestriction { PUBLIC = 1, INTERNAL = 2, CONFIDENTIAL = 3 }; +enum class AccessRestriction { PUBLIC, INTERNAL, CONFIDENTIAL }; struct Employee { sqlgen::PrimaryKey id; std::string first_name; @@ -79,6 +79,7 @@ TEST(duckdb, test_enum_cross_table) { .value(); const auto smithers_level = smithers.access_level; + const auto smithers_documents = conn.and_then(sqlgen::read> | where("min_access_level"_c == smithers_level || diff --git a/tests/duckdb/test_enum_lookup.cpp b/tests/duckdb/test_enum_lookup.cpp new file mode 100644 index 00000000..b4f8895a --- /dev/null +++ b/tests/duckdb/test_enum_lookup.cpp @@ -0,0 +1,84 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_enum_lookup { + +enum class AccessRestriction { PUBLIC, INTERNAL, CONFIDENTIAL }; + +struct Document { + sqlgen::PrimaryKey id; + AccessRestriction min_access_level; + std::string name; + std::string path; +}; + +TEST(duckdb, test_enum_lookup) { + auto documents = std::vector({ + Document{.min_access_level = AccessRestriction::PUBLIC, + .name = "Power Plant Safety Manual", + .path = "/documents/powerplant/safety_manual.txt"}, + Document{.min_access_level = AccessRestriction::INTERNAL, + .name = "Staff Memo", + .path = "/documents/powerplant/staff_memo.txt"}, + Document{.min_access_level = AccessRestriction::CONFIDENTIAL, + .name = "Operations Report", + .path = "/documents/powerplant/operations_report.pdf"}, + Document{.min_access_level = AccessRestriction::PUBLIC, + .name = "Project Plan", + .path = "/documents/powerplant/project_plan.md"}, + Document{.min_access_level = AccessRestriction::INTERNAL, + .name = "Budget Q1", + .path = "/documents/powerplant/budget_q1.pdf"}, + Document{.min_access_level = AccessRestriction::CONFIDENTIAL, + .name = "HR Policies", + .path = "/documents/powerplant/hr_policies.pdf"}, + Document{.min_access_level = AccessRestriction::PUBLIC, + .name = "Team Photo", + .path = "/documents/powerplant/team.jpg"}, + Document{.min_access_level = AccessRestriction::CONFIDENTIAL, + .name = "Executive Summary", + .path = "/documents/powerplant/executive_summary.docx"}, + Document{.min_access_level = AccessRestriction::INTERNAL, + .name = "Release Notes", + .path = "/documents/powerplant/release_notes.txt"}, + }); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto public_documents = + duckdb::connect() + .and_then(drop | if_exists) + .and_then(write(std::ref(documents))) + .and_then(sqlgen::read> | + where("min_access_level"_c == AccessRestriction::PUBLIC) | + order_by("name"_c.desc())) + .value(); + + const auto expected = std::vector({ + Document{.id = 7, + .min_access_level = AccessRestriction::PUBLIC, + .name = "Team Photo", + .path = "/documents/powerplant/team.jpg"}, + Document{.id = 4, + .min_access_level = AccessRestriction::PUBLIC, + .name = "Project Plan", + .path = "/documents/powerplant/project_plan.md"}, + Document{.id = 1, + .min_access_level = AccessRestriction::PUBLIC, + .name = "Power Plant Safety Manual", + .path = "/documents/powerplant/safety_manual.txt"}, + }); + + const auto json1 = rfl::json::write(expected); + const auto json2 = rfl::json::write(public_documents); + EXPECT_EQ(json1, json2); +} + +} // namespace test_enum_lookup + diff --git a/tests/duckdb/test_enum_namespace.cpp b/tests/duckdb/test_enum_namespace.cpp new file mode 100644 index 00000000..920582a7 --- /dev/null +++ b/tests/duckdb/test_enum_namespace.cpp @@ -0,0 +1,59 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_enum_namespace { +namespace first { +enum class IdenticallyNamed { VALUE0, VALUE1, VALUE2 }; + +} +namespace second { +enum class IdenticallyNamed { VALUE3, VALUE4, VALUE5 }; +} + +struct MultiStruct { + sqlgen::PrimaryKey id; + first::IdenticallyNamed enum_one; + second::IdenticallyNamed enum_two; +}; + +TEST(duckdb, test_enum_namespace) { + using namespace sqlgen; + using namespace sqlgen::literals; + + auto objects = std::vector({ + MultiStruct{.enum_one = first::IdenticallyNamed::VALUE0, + .enum_two = second::IdenticallyNamed::VALUE3}, + MultiStruct{.enum_one = first::IdenticallyNamed::VALUE1, + .enum_two = second::IdenticallyNamed::VALUE4}, + MultiStruct{.enum_one = first::IdenticallyNamed::VALUE2, + .enum_two = second::IdenticallyNamed::VALUE5}, + }); + + const auto conn = duckdb::connect(); + conn.and_then(drop | if_exists); + + write(conn, objects); + + const auto read_objects = + sqlgen::read>(conn).value(); + std::vector actual_ids; + for (const auto& obj : read_objects) { + if (obj.enum_one == first::IdenticallyNamed::VALUE0) { + EXPECT_EQ(obj.enum_two, second::IdenticallyNamed::VALUE3); + } else if (obj.enum_one == first::IdenticallyNamed::VALUE1) { + EXPECT_EQ(obj.enum_two, second::IdenticallyNamed::VALUE4); + } else if (obj.enum_one == first::IdenticallyNamed::VALUE2) { + EXPECT_EQ(obj.enum_two, second::IdenticallyNamed::VALUE5); + } else { + FAIL() << "Unexpected enum value"; + } + } +} + +} // namespace test_enum_namespace + From 525d29793e5ade5dcf1dce2657502c16688b01ed Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Mon, 17 Nov 2025 22:01:24 +0100 Subject: [PATCH 41/48] Added support for struct flattening --- include/sqlgen/duckdb/from_chunk_ptrs.hpp | 12 +++--- tests/duckdb/test_flatten.cpp | 45 +++++++++++++++++++++++ 2 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 tests/duckdb/test_flatten.cpp diff --git a/include/sqlgen/duckdb/from_chunk_ptrs.hpp b/include/sqlgen/duckdb/from_chunk_ptrs.hpp index 1586713c..87f88245 100644 --- a/include/sqlgen/duckdb/from_chunk_ptrs.hpp +++ b/include/sqlgen/duckdb/from_chunk_ptrs.hpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -26,11 +27,12 @@ struct FromChunkPtrs, idx_t _i) noexcept { return [&](std::integer_sequence) -> Result { try { - return T{duckdb::parsing::Parser::read( - rfl::get<_is>(_chunk_ptrs).is_not_null(_i) - ? rfl::get<_is>(_chunk_ptrs).data + _i - : nullptr) - .value()...}; + return rfl::from_named_tuple(rfl::named_tuple_t( + duckdb::parsing::Parser::read( + rfl::get<_is>(_chunk_ptrs).is_not_null(_i) + ? rfl::get<_is>(_chunk_ptrs).data + _i + : nullptr) + .value()...)); } catch (const std::exception& e) { return error(e.what()); } diff --git a/tests/duckdb/test_flatten.cpp b/tests/duckdb/test_flatten.cpp new file mode 100644 index 00000000..b3856b86 --- /dev/null +++ b/tests/duckdb/test_flatten.cpp @@ -0,0 +1,45 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_write_and_read { + +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +struct Employee { + static constexpr const char* tablename = "EMPLOYEES"; + + sqlgen::Flatten person; + float salary; +}; + +TEST(duckdb, test_flatten) { + const auto people1 = + std::vector({Employee{.person = Person{.id = 0, + .first_name = "Homer", + .last_name = "Simpson", + .age = 45}, + .salary = 60000.0}}); + + const auto conn = sqlgen::duckdb::connect(); + + sqlgen::write(conn, people1); + + const auto people2 = sqlgen::read>(conn).value(); + + const auto json1 = rfl::json::write(people1); + const auto json2 = rfl::json::write(people2); + + EXPECT_EQ(json1, json2); +} + +} // namespace test_write_and_read From 088d64a0cffb5fd150eb2f7c88165b987a8e9ac1 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Tue, 18 Nov 2025 22:07:22 +0100 Subject: [PATCH 42/48] Added support for JSON --- include/sqlgen/duckdb/parsing/Parser.hpp | 1 + include/sqlgen/duckdb/parsing/Parser_json.hpp | 34 ++++++++++++++ src/sqlgen/duckdb/to_sql.cpp | 2 +- tests/duckdb/test_json.cpp | 45 +++++++++++++++++++ 4 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 include/sqlgen/duckdb/parsing/Parser_json.hpp create mode 100644 tests/duckdb/test_json.cpp diff --git a/include/sqlgen/duckdb/parsing/Parser.hpp b/include/sqlgen/duckdb/parsing/Parser.hpp index 9c76e9fa..453eb4bb 100644 --- a/include/sqlgen/duckdb/parsing/Parser.hpp +++ b/include/sqlgen/duckdb/parsing/Parser.hpp @@ -5,6 +5,7 @@ #include "Parser_date.hpp" #include "Parser_default.hpp" #include "Parser_enum.hpp" +#include "Parser_json.hpp" #include "Parser_optional.hpp" #include "Parser_reflection_type.hpp" #include "Parser_smart_ptr.hpp" diff --git a/include/sqlgen/duckdb/parsing/Parser_json.hpp b/include/sqlgen/duckdb/parsing/Parser_json.hpp new file mode 100644 index 00000000..2ac9fd57 --- /dev/null +++ b/include/sqlgen/duckdb/parsing/Parser_json.hpp @@ -0,0 +1,34 @@ +#ifndef SQLGEN_DUCKDB_PARSING_PARSER_JSON_HPP_ +#define SQLGEN_DUCKDB_PARSING_PARSER_JSON_HPP_ + +#include + +#include +#include +#include + +#include "../../JSON.hpp" +#include "../../Result.hpp" +#include "Parser_base.hpp" +#include "Parser_string.hpp" + +namespace sqlgen::duckdb::parsing { + +template +struct Parser> { + using ResultingType = duckdb_string_t; + + static Result> read(const ResultingType* _r) noexcept { + return Parser::read(_r).and_then( + [&](const auto& _str) { return rfl::json::read(_str); }); + } + + static Result write(const JSON& _t, + duckdb_appender _appender) noexcept { + return Parser::write(rfl::json::write(_t.value()), _appender); + } +}; + +} // namespace sqlgen::duckdb::parsing + +#endif diff --git a/src/sqlgen/duckdb/to_sql.cpp b/src/sqlgen/duckdb/to_sql.cpp index bc4430fb..498e913b 100644 --- a/src/sqlgen/duckdb/to_sql.cpp +++ b/src/sqlgen/duckdb/to_sql.cpp @@ -890,7 +890,7 @@ std::string type_to_sql(const dynamic::Type& _type) noexcept { return "VARCHAR(" + std::to_string(_t.length) + ")"; } else if constexpr (std::is_same_v) { - return "JSONB"; + return "JSON"; } else if constexpr (std::is_same_v) { return "DATE"; diff --git a/tests/duckdb/test_json.cpp b/tests/duckdb/test_json.cpp new file mode 100644 index 00000000..b970301f --- /dev/null +++ b/tests/duckdb/test_json.cpp @@ -0,0 +1,45 @@ +#include + +#include +#include +#include +#include +#include + +namespace test_json { + +struct Person { + std::string first_name; + std::string last_name; + int age; + sqlgen::JSON>> children; +}; + +TEST(duckdb, test_json) { + const auto children = std::vector( + {Person{.first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{.first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + const auto homer = Person{.first_name = "Homer", + .last_name = "Simpson", + .age = 45, + .children = children}; + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto people2 = sqlgen::duckdb::connect() + .and_then(drop | if_exists) + .and_then(write(std::ref(homer))) + .and_then(sqlgen::read>) + .value(); + + const std::string expected = + R"([{"first_name":"Homer","last_name":"Simpson","age":45,"children":[{"first_name":"Bart","last_name":"Simpson","age":10},{"first_name":"Lisa","last_name":"Simpson","age":8},{"first_name":"Maggie","last_name":"Simpson","age":0}]}])"; + + EXPECT_EQ(rfl::json::write(people2), expected); +} + +} // namespace test_json + From 3f85a9b72e961683a394ab4032cf57e299aa8944 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Tue, 18 Nov 2025 22:28:02 +0100 Subject: [PATCH 43/48] Added a test for dynamic types --- include/sqlgen/duckdb/parsing/Parser_json.hpp | 7 +- tests/duckdb/test_dynamic_type.cpp | 129 ++++++++++++++++++ 2 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 tests/duckdb/test_dynamic_type.cpp diff --git a/include/sqlgen/duckdb/parsing/Parser_json.hpp b/include/sqlgen/duckdb/parsing/Parser_json.hpp index 2ac9fd57..03cdd44a 100644 --- a/include/sqlgen/duckdb/parsing/Parser_json.hpp +++ b/include/sqlgen/duckdb/parsing/Parser_json.hpp @@ -25,7 +25,12 @@ struct Parser> { static Result write(const JSON& _t, duckdb_appender _appender) noexcept { - return Parser::write(rfl::json::write(_t.value()), _appender); + try { + return Parser::write(rfl::json::write(_t.value()), + _appender); + } catch (const std::exception& e) { + return error(e.what()); + } } }; diff --git a/tests/duckdb/test_dynamic_type.cpp b/tests/duckdb/test_dynamic_type.cpp new file mode 100644 index 00000000..6ab8954f --- /dev/null +++ b/tests/duckdb/test_dynamic_type.cpp @@ -0,0 +1,129 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace sqlgen::parsing { + +template <> +struct Parser { + using Type = boost::uuids::uuid; + + static Result read( + const std::optional& _str) noexcept { + if (!_str) { + return error("boost::uuids::uuid cannot be NULL."); + } + return boost::lexical_cast(*_str); + } + + static std::optional write( + const boost::uuids::uuid& _u) noexcept { + return boost::uuids::to_string(_u); + } + + static dynamic::Type to_type() noexcept { + return sqlgen::dynamic::types::Dynamic{"TEXT"}; + } +}; + +} // namespace sqlgen::parsing + +namespace sqlgen::duckdb::parsing { + +template <> +struct Parser { + using ResultingType = duckdb_string_t; + + static Result read(const ResultingType* _r) noexcept { + return Parser::read(_r).and_then( + [&](const std::string& _str) -> Result { + try { + return boost::lexical_cast(_str); + } catch (const std::exception& e) { + return error(e.what()); + } + }); + } + + static Result write(const boost::uuids::uuid& _u, + duckdb_appender _appender) noexcept { + return Parser::write(boost::uuids::to_string(_u), _appender); + } +}; + +} // namespace sqlgen::duckdb::parsing + +namespace sqlgen::transpilation { + +template <> +struct ToValue { + dynamic::Value operator()(const boost::uuids::uuid& _u) const { + return dynamic::Value{dynamic::String{.val = boost::uuids::to_string(_u)}}; + } +}; + +} // namespace sqlgen::transpilation + +/// For the JSON serialization - not needed for +/// the actual DB operations. +namespace rfl { + +template <> +struct Reflector { + using ReflType = std::string; + + static boost::uuids::uuid to(const std::string& _str) { + return boost::lexical_cast(_str); + } + + static std::string from(const boost::uuids::uuid& _u) { + return boost::uuids::to_string(_u); + } +}; + +} // namespace rfl + +namespace test_dynamic_type { + +struct Person { + sqlgen::PrimaryKey id = + boost::uuids::uuid(boost::uuids::random_generator()()); + std::string first_name; + std::string last_name; + int age; +}; + +TEST(duckdb, test_dynamic_type) { + const auto people1 = std::vector( + {Person{.first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.first_name = "Lisa", .last_name = "Simpson", .age = 8}, + Person{.first_name = "Maggie", .last_name = "Simpson", .age = 0}}); + + using namespace sqlgen; + using namespace sqlgen::literals; + + const auto conn = duckdb::connect().and_then(drop | if_exists); + + const auto people2 = sqlgen::write(conn, people1) + .and_then(sqlgen::read> | + where("id"_c == people1.front().id())) + .value(); + + const auto json1 = rfl::json::write(std::vector({people1.front()})); + const auto json2 = rfl::json::write(people2); + + EXPECT_EQ(json1, json2); +} + +} // namespace test_dynamic_type + From 2244aa2c5f8d2f5692cffde36857454fd5d93fd4 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Tue, 18 Nov 2025 22:51:29 +0100 Subject: [PATCH 44/48] Added documentation --- README.md | 1 + docs/README.md | 1 + docs/duckdb.md | 142 +++++++++++++++++++++++++++++++++++++++++++++++ docs/mysql.md | 2 +- docs/postgres.md | 2 +- docs/sqlite.md | 2 +- 6 files changed, 147 insertions(+), 3 deletions(-) create mode 100644 docs/duckdb.md diff --git a/README.md b/README.md index 9d6aad34..2b9d1921 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ The following table lists the databases currently supported by sqlgen and the un | Database | Library | Version | License | Remarks | |---------------|--------------------------------------------------------------------------|--------------|---------------| -----------------------------------------------------| +| DuckDB | [duckdb](https://github.com/duckdb/duckdb) | >= 1.4.1 | MIT | | | MySQL/MariaDB | [libmariadb](https://github.com/mariadb-corporation/mariadb-connector-c) | >= 3.4.5 | LGPL | | | PostgreSQL | [libpq](https://github.com/postgres/postgres) | >= 16.4 | PostgreSQL | Will work for all libpq-compatible databases | | sqlite | [sqlite](https://sqlite.org/index.html) | >= 3.49.1 | Public Domain | | diff --git a/docs/README.md b/docs/README.md index 8c98144f..e1e079ac 100644 --- a/docs/README.md +++ b/docs/README.md @@ -59,6 +59,7 @@ Welcome to the sqlgen documentation. This guide provides detailed information ab ## Supported Databases +- [DuckDB](duckdb.md) - How to interact with DuckDB - [MySQL](mysql.md) - How to interact with MariaDB and MySQL - [PostgreSQL](postgres.md) - How to interact with PostgreSQL and compatible databases (Redshift, Aurora, Greenplum, CockroachDB, ...) - [SQLite](sqlite.md) - How to interact with SQLite3 diff --git a/docs/duckdb.md b/docs/duckdb.md new file mode 100644 index 00000000..09ba8089 --- /dev/null +++ b/docs/duckdb.md @@ -0,0 +1,142 @@ +∂# `sqlgen::duckdb` + +The `sqlgen::duckdb` module provides a type-safe and efficient interface for interacting with DuckDB databases. It implements the core database operations through a connection-based API with support for prepared statements, transactions, and efficient data iteration. + +## Usage + +### Basic Connection + +Create a connection to a DuckDB database: + +```cpp +// Connect to an in-memory database +const auto conn = sqlgen::duckdb::connect(); + +// Connect to a file-based database +const auto conn = sqlgen::duckdb::connect("database.db"); +``` + +The type of `conn` is `sqlgen::Result>`, which is useful for error handling: + +```cpp +// Handle connection errors +const auto conn = sqlgen::duckdb::connect("database.db"); +if (!conn) { + // Handle error... + return; +} + +using namespace sqlgen; +using namespace sqlgen::literals; + +const auto query = sqlgen::read> | + where("age"_c < 18 and "first_name"_c != "Hugo"); + +// Use the connection +const auto minors = query(conn); +``` + +### Basic Operations + +Write data to the database: + +```cpp +struct Person { + sqlgen::PrimaryKey id; + std::string first_name; + std::string last_name; + int age; +}; + +const auto people = std::vector{ + Person{.id = 0, .first_name = "Homer", .last_name = "Simpson", .age = 45}, + Person{.id = 1, .first_name = "Bart", .last_name = "Simpson", .age = 10}, + Person{.id = 2, .first_name = "Lisa", .last_name = "Simpson", .age = 8} +}; + +// Write data to database +const auto result = sqlgen::write(conn, people); +``` + +Read data with filtering and ordering: + +```cpp +using namespace sqlgen; +using namespace sqlgen::literals; + +// Read all people ordered by age +const auto all_people = sqlgen::read> | + order_by("age"_c); + +// Read minors only +const auto minors = sqlgen::read> | + where("age"_c < 18) | + order_by("age"_c); + +// Use the queries +const auto result1 = all_people(conn); +const auto result2 = minors(conn); +``` + +### Transactions + +Perform operations within transactions: + +```cpp +using namespace sqlgen; +using namespace sqlgen::literals; + +// Delete a person and update another in a transaction +const auto delete_hugo = delete_from | + where("first_name"_c == "Hugo"); + +const auto update_homer = update("age"_c.set(46)) | + where("first_name"_c == "Homer"); + +const auto result = begin_transaction(conn) + .and_then(delete_hugo) + .and_then(update_homer) + .and_then(commit) + .value(); +``` + +### Update Operations + +Update data in a table: + +```cpp +using namespace sqlgen; +using namespace sqlgen::literals; + +// Update multiple columns +const auto query = update("first_name"_c.set("last_name"_c), "age"_c.set(100)) | + where("first_name"_c == "Hugo"); + +query(conn).value(); +``` + +## Notes + +- The module provides a type-safe interface for DuckDB operations +- All operations return `sqlgen::Result` for error handling +- Prepared statements are used for efficient query execution +- The iterator interface supports batch processing of results +- SQL generation adapts to DuckDB's dialect +- The module supports: + - In-memory and file-based databases + - Transactions (begin, commit, rollback) + - Efficient batch operations + - Type-safe SQL generation + - Error handling through `Result` + - Resource management through `Ref` + - Auto-incrementing primary keys + - Various data types including VARCHAR, TIMESTAMP, DATE + - Complex queries with WHERE clauses, ORDER BY, LIMIT, JOINs + - LIKE and pattern matching operations + - Mathematical operations and string functions + - JSON data types + - Foreign keys and referential integrity + - Unique constraints + - Views and materialized views + - Indexes +``` \ No newline at end of file diff --git a/docs/mysql.md b/docs/mysql.md index 85e1c475..87edb17a 100644 --- a/docs/mysql.md +++ b/docs/mysql.md @@ -23,7 +23,7 @@ const auto creds = sqlgen::mysql::Credentials{ const auto conn = sqlgen::mysql::connect(creds); ``` -The connection is wrapped in a `sqlgen::Result>` for error handling: +The type of `conn` is `sqlgen::Result>`, which is useful for error handling: ```cpp // Handle connection errors diff --git a/docs/postgres.md b/docs/postgres.md index 9cea693b..f8fde585 100644 --- a/docs/postgres.md +++ b/docs/postgres.md @@ -22,7 +22,7 @@ const auto creds = sqlgen::postgres::Credentials{ const auto conn = sqlgen::postgres::connect(creds); ``` -The connection is wrapped in a `sqlgen::Result>` for error handling: +The type of `conn` is `sqlgen::Result>`, which is useful for error handling: ```cpp // Handle connection errors diff --git a/docs/sqlite.md b/docs/sqlite.md index 7ebfeb8a..833d787d 100644 --- a/docs/sqlite.md +++ b/docs/sqlite.md @@ -16,7 +16,7 @@ const auto conn = sqlgen::sqlite::connect(); const auto conn = sqlgen::sqlite::connect("database.db"); ``` -The connection is wrapped in a `sqlgen::Result>` for error handling: +The type of `conn` is `sqlgen::Result>`, which is useful for error handling: ```cpp // Handle connection errors From 11f810ef345781d746ec8806d612b67c75dc1723 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Tue, 18 Nov 2025 22:55:25 +0100 Subject: [PATCH 45/48] Updated the documentation for sqlgen::Dynamic --- docs/dynamic.md | 54 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/docs/dynamic.md b/docs/dynamic.md index 03e51af7..86b25940 100644 --- a/docs/dynamic.md +++ b/docs/dynamic.md @@ -45,6 +45,50 @@ struct Parser { } // namespace sqlgen::parsing ``` +### DuckDB parser specialization + +**Important:** If you're using DuckDB, you must also implement a separate parser specialization in the `sqlgen::duckdb::parsing` namespace. This is required for performance reasons, as DuckDB uses its own native types and appender interface. + +The DuckDB parser has a different interface than the generic parser: + +```cpp +#include +#include +#include +#include +#include + +namespace sqlgen::duckdb::parsing { + +template <> +struct Parser { + using ResultingType = duckdb_string_t; + + static Result read(const ResultingType* _r) noexcept { + return Parser::read(_r).and_then( + [&](const std::string& _str) -> Result { + try { + return boost::lexical_cast(_str); + } catch (const std::exception& e) { + return error(e.what()); + } + }); + } + + static Result write(const boost::uuids::uuid& _u, + duckdb_appender _appender) noexcept { + return Parser::write(boost::uuids::to_string(_u), _appender); + } +}; + +} // namespace sqlgen::duckdb::parsing +``` + +Key differences from the generic parser: +- `read` takes `const ResultingType*` (where `ResultingType = duckdb_string_t`) instead of `const std::optional&` +- `write` takes a `duckdb_appender` parameter and returns `Result` instead of `std::optional` +- No `to_type()` method is required (the generic parser's `to_type()` is used for schema generation) + The second step is to specialize `sqlgen::transpilation::ToValue` for `boost::uuids::uuid` and implement `operator()`: ```cpp @@ -157,6 +201,15 @@ static dynamic::Type to_type() noexcept { } ``` +- DuckDB: +```cpp +static dynamic::Type to_type() noexcept { + return sqlgen::dynamic::types::Dynamic{"TEXT"}; +} +``` + +Note: For DuckDB, you must also implement the `sqlgen::duckdb::parsing::Parser` specialization as shown in the DuckDB parser specialization section above. + ## Parser specialization requirements Specializing `sqlgen::parsing::Parser` requires three methods. These guidelines help ensure correctness and portability: @@ -200,4 +253,5 @@ Additional best practices: - Works with all operations: `create_table`, `insert`, `select`, `update`, `delete` - The type name is passed directly to the database; ensure it is valid for the target dialect - Keep specializations in the `sqlgen::parsing` namespace +- **DuckDB users:** You must implement both `sqlgen::parsing::Parser` and `sqlgen::duckdb::parsing::Parser` specializations for your custom type From b66e12860ae524be5919af00df56bb1f017c9264 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Wed, 19 Nov 2025 20:06:43 +0100 Subject: [PATCH 46/48] Removed relative includes --- include/sqlgen/duckdb/Connection.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index 3894f9ec..6c7b7379 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -16,6 +16,8 @@ #include "../Ref.hpp" #include "../Result.hpp" #include "../Transaction.hpp" +#include "../dynamic/Operation.hpp" +#include "../dynamic/SelectFrom.hpp" #include "../dynamic/Write.hpp" #include "../internal/iterator_t.hpp" #include "../internal/remove_auto_incr_primary_t.hpp" @@ -26,8 +28,6 @@ #include "DuckDBConnection.hpp" #include "DuckDBResult.hpp" #include "Iterator.hpp" -#include "sqlgen/dynamic/Operation.hpp" -#include "sqlgen/dynamic/SelectFrom.hpp" #include "to_sql.hpp" namespace sqlgen::duckdb { From 9bee2c56947e934aebb35ca088ce6249efc6c527 Mon Sep 17 00:00:00 2001 From: "Dr. Patrick Urbanke" Date: Wed, 19 Nov 2025 21:37:57 +0100 Subject: [PATCH 47/48] Get rid of the headers on Windows --- .github/workflows/windows-cxx20-vcpkg.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/windows-cxx20-vcpkg.yaml b/.github/workflows/windows-cxx20-vcpkg.yaml index f8a2f71d..ef97f4af 100644 --- a/.github/workflows/windows-cxx20-vcpkg.yaml +++ b/.github/workflows/windows-cxx20-vcpkg.yaml @@ -14,7 +14,6 @@ jobs: - db: postgres - db: sqlite - db: mysql - - db: headers name: "(windows-${{ matrix.db }})" concurrency: group: ci-${{ github.ref }}-windows-${{ matrix.db }} @@ -34,11 +33,6 @@ jobs: core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - uses: ilammy/msvc-dev-cmd@v1 - uses: lukka/run-vcpkg@v11 - - name: Compile - if: matrix.db == 'headers' - run: | - cmake -S . -B build -DCMAKE_CXX_STANDARD=20 -DSQLGEN_CHECK_HEADERS=ON - cmake --build build --config Release -j4 - name: Compile if: matrix.db == 'postgres' run: | @@ -55,6 +49,5 @@ jobs: cmake -S . -B build -DCMAKE_CXX_STANDARD=20 -DCMAKE_BUILD_TYPE=Release -DSQLGEN_BUILD_TESTS=ON -DSQLGEN_MYSQL=ON -DSQLGEN_POSTGRES=OFF -DSQLGEN_SQLITE3=OFF -DSQLGEN_BUILD_DRY_TESTS_ONLY=ON -DBUILD_SHARED_LIBS=ON -DVCPKG_TARGET_TRIPLET=x64-windows-release cmake --build build --config Release -j4 - name: Run tests - if: matrix.db != 'headers' run: | ctest --test-dir build --output-on-failure From b1009d1586a529724cf8ae4c1a23fbf6e2ca2013 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Thu, 20 Nov 2025 00:07:44 +0100 Subject: [PATCH 48/48] Add SQLGEN_API tag --- include/sqlgen/duckdb/Connection.hpp | 3 ++- include/sqlgen/duckdb/DuckDBAppender.hpp | 3 ++- include/sqlgen/duckdb/DuckDBConnection.hpp | 3 ++- include/sqlgen/duckdb/DuckDBResult.hpp | 3 ++- include/sqlgen/duckdb/to_sql.hpp | 3 ++- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/include/sqlgen/duckdb/Connection.hpp b/include/sqlgen/duckdb/Connection.hpp index 6c7b7379..95eff625 100644 --- a/include/sqlgen/duckdb/Connection.hpp +++ b/include/sqlgen/duckdb/Connection.hpp @@ -23,6 +23,7 @@ #include "../internal/remove_auto_incr_primary_t.hpp" #include "../internal/to_container.hpp" #include "../is_connection.hpp" +#include "../sqlgen_api.hpp" #include "./parsing/Parser_default.hpp" #include "DuckDBAppender.hpp" #include "DuckDBConnection.hpp" @@ -32,7 +33,7 @@ namespace sqlgen::duckdb { -class Connection { +class SQLGEN_API Connection { using ConnPtr = Ref; public: diff --git a/include/sqlgen/duckdb/DuckDBAppender.hpp b/include/sqlgen/duckdb/DuckDBAppender.hpp index 4a42dead..3f5aa912 100644 --- a/include/sqlgen/duckdb/DuckDBAppender.hpp +++ b/include/sqlgen/duckdb/DuckDBAppender.hpp @@ -5,11 +5,12 @@ #include +#include "../sqlgen_api.hpp" #include "DuckDBConnection.hpp" namespace sqlgen::duckdb { -class DuckDBAppender { +class SQLGEN_API DuckDBAppender { using ConnPtr = Ref; public: diff --git a/include/sqlgen/duckdb/DuckDBConnection.hpp b/include/sqlgen/duckdb/DuckDBConnection.hpp index ab9ebe68..be9c0d61 100644 --- a/include/sqlgen/duckdb/DuckDBConnection.hpp +++ b/include/sqlgen/duckdb/DuckDBConnection.hpp @@ -8,10 +8,11 @@ #include "../Ref.hpp" #include "../Result.hpp" +#include "../sqlgen_api.hpp" namespace sqlgen::duckdb { -class DuckDBConnection { +class SQLGEN_API DuckDBConnection { public: static Result> make( const std::optional& _fname); diff --git a/include/sqlgen/duckdb/DuckDBResult.hpp b/include/sqlgen/duckdb/DuckDBResult.hpp index 14c0b396..6479e3a0 100644 --- a/include/sqlgen/duckdb/DuckDBResult.hpp +++ b/include/sqlgen/duckdb/DuckDBResult.hpp @@ -5,11 +5,12 @@ #include +#include "../sqlgen_api.hpp" #include "DuckDBConnection.hpp" namespace sqlgen::duckdb { -class DuckDBResult { +class SQLGEN_API DuckDBResult { using ConnPtr = Ref; public: diff --git a/include/sqlgen/duckdb/to_sql.hpp b/include/sqlgen/duckdb/to_sql.hpp index 36d81945..88656ca3 100644 --- a/include/sqlgen/duckdb/to_sql.hpp +++ b/include/sqlgen/duckdb/to_sql.hpp @@ -4,12 +4,13 @@ #include #include "../dynamic/Statement.hpp" +#include "../sqlgen_api.hpp" #include "../transpilation/to_sql.hpp" namespace sqlgen::duckdb { /// Transpiles a dynamic general SQL statement to the duckdb dialect. -std::string to_sql_impl(const dynamic::Statement& _stmt) noexcept; +SQLGEN_API std::string to_sql_impl(const dynamic::Statement& _stmt) noexcept; /// Transpiles any SQL statement to the duckdb dialect. template