Hydrate fetched rows directly from typed SQLite columns, removing the wstring round-trip (#17)

include/database.h

@@ -22,14 +22,13 @@
 
 #pragma once
 
-#include <stdexcept>
 #include <memory>
 #include <mutex>
+#include <stdexcept>
 #include <string>
 #include <typeinfo>
 #include <vector>
 
-#include "fetch_query_results.h"
 #include "queries.h"
 #include "query_predicates.h"
 #include "reflection.h"
@@ -71,8 +70,7 @@ class REFLECTION_EXPORT Database {
         const auto type_id = typeid(T).name();
         const auto& record = GetRecord(type_id);
         EmptyPredicate empty;
-        const auto& query_result = Fetch(record, &empty);
-        return Hydrate<T>(query_result, record);
+        return FetchRecords<T>(record, &empty);
     }
 
     /// Retrieves all entries of a given record from the database, which match a given predicate.
@@ -81,8 +79,7 @@ class REFLECTION_EXPORT Database {
     std::vector<T> Fetch(const QueryPredicateBase* predicate) const {
         const auto type_id = typeid(T).name();
         const auto& record = GetRecord(type_id);
-        const auto& query_result = Fetch(record, predicate);
-        return Hydrate<T>(query_result, record);
+        return FetchRecords<T>(record, predicate);
     }
 
     /// Retrieves a single entry of a given record from the database, which matches a given id.
@@ -92,11 +89,11 @@ class REFLECTION_EXPORT Database {
         const auto type_id = typeid(T).name();
         const auto& record = GetRecord(type_id);
         Equal equal_id_condition(&T::id, id);
-        const auto& query_result = Fetch(record, &equal_id_condition);
-        if (query_result.row_values.size() != 1) {
+        auto models = FetchRecords<T>(record, &equal_id_condition);
+        if (models.size() != 1) {
             throw std::runtime_error("No record with this id found");
         }
-        return Hydrate<T>(query_result, record)[0];
+        return models[0];
     }
 
     /// Saves a given record in the database.
@@ -197,22 +194,21 @@ class REFLECTION_EXPORT Database {
 private:
     explicit Database(const char* path);
 
-    /// Executes a fetch query (SELECT) for a given record with a given predicate,
-    /// and returns the results in a textual representation
-    FetchQueryResults Fetch(const Reflection& record, const QueryPredicateBase* predicate) const;
-
     /// Returns a record type from its type information, retrieved from typeid(...).name()
     static const Reflection& GetRecord(const std::string& type_id);
 
-    /// Creates concrete record types with initialized members,
-    /// based on the textual representation of results from a fetch query
+    /// Executes a fetch query (SELECT) for a given record with a given predicate, streaming
+    /// each matching row directly from the prepared statement into a newly constructed T -
+    /// no intermediate string materialization of the result set
     template <typename T>
-    std::vector<T> Hydrate(const FetchQueryResults& query_results, const Reflection& record) const {
+    std::vector<T> FetchRecords(const Reflection& record, const QueryPredicateBase* predicate) const {
+        std::lock_guard<std::mutex> lock(db_mutex_);
+        FetchRecordsQuery query(db_, record, predicate);
         std::vector<T> models;
-        for (auto i = 0; i < query_results.row_values.size(); i++) {
+        while (query.StepRow()) {
             T model;
-            FetchRecordsQuery::Hydrate((void*)&model, query_results, record, i);
-            models.emplace_back(model);
+            query.HydrateCurrentRow((void*)&model, record);
+            models.emplace_back(std::move(model));
         }
         return models;
     }

include/queries.h

@@ -139,26 +139,24 @@ class REFLECTION_EXPORT UpdateQuery final : public ExecutionQuery {
     void* p_;
 };
 
-struct FetchQueryResults;
-
 /// A query for retrieving all records from the database, which match a given predicate condition
 /// This maps to SELECT * in SQL
 class REFLECTION_EXPORT FetchRecordsQuery final : public Query {
 public:
     explicit FetchRecordsQuery(sqlite3* db, const Reflection& record, const QueryPredicateBase* predicate);
     ~FetchRecordsQuery() override;
 
-    /// Returns a textual representation of the results of the query
-    FetchQueryResults GetResults();
+    /// Prepares the underlying statement on first use and advances it to the next matching
+    /// row. Returns false once there are no more rows left to fetch.
+    bool StepRow();
 
-    /// Reconstructs all record member values based on their concrete type,
-    /// based on the textual representation of the corresponding row result
-    /// of the fetch query
-    static void Hydrate(void* p, const FetchQueryResults& query_results, const Reflection& record, size_t i);
+    /// Hydrates the type-erased record pointed to by p from the current row of the prepared
+    /// statement, reading each column directly via its typed SQLite accessor, with no
+    /// intermediate string representation of the fetched values
+    void HydrateCurrentRow(void* p, const Reflection& record) const;
 
 protected:
     std::string PrepareSql() const override;
-    std::wstring GetColumnValue(int col) const;
 
     sqlite3_stmt* stmt_;
     const QueryPredicateBase* predicate_;

src/database.cc

@@ -101,12 +101,6 @@ std::shared_ptr<const Database> Database::Instance() {
     return instance_;
 }
 
-FetchQueryResults Database::Fetch(const Reflection& record, const QueryPredicateBase* predicate) const {
-    std::lock_guard<std::mutex> lock(db_mutex_);
-    FetchRecordsQuery query(db_, record, predicate);
-    return query.GetResults();
-}
-
 const Reflection& Database::GetRecord(const std::string& type_id) {
     return GetReflectionRegister().records.at(type_id);
 }

src/internal/string_utilities.h

@@ -31,10 +31,8 @@ namespace sqlite_reflection {
 /// between strings and concrete types used by SQLite
 class REFLECTION_EXPORT StringUtilities {
 public:
-    static int64_t ToInt(const std::wstring& s);
     static std::string FromInt(int64_t value);
 
-    static double ToDouble(const std::wstring& s);
     static std::string FromDouble(double value);
 
     static std::string ToUtf8(const std::wstring& wide_string);

src/queries.cc

@@ -25,14 +25,12 @@
 #include <string.h>
 
 #include <algorithm>
-#include <cstdio>
 #include <cstring>
 #include <iterator>
 #include <stdexcept>
 #include <string>
 #include <vector>
 
-#include "fetch_query_results.h"
 #include "internal/sqlite3.h"
 #include "internal/string_utilities.h"
 
@@ -287,71 +285,77 @@ FetchRecordsQuery::~FetchRecordsQuery() {
     }
 }
 
-FetchQueryResults FetchRecordsQuery::GetResults() {
-    const auto sql = PrepareSql();
-
-    if (sqlite3_prepare_v2(db_, sql.data(), -1, &stmt_, nullptr)) {
-        throw std::runtime_error((sql + ": could not get results").data());
-    }
-    BindValues(stmt_, predicate_->Bindings());
-
-    const auto column_count = sqlite3_column_count(stmt_);
-
-    FetchQueryResults results;
-    results.column_names.reserve(column_count);
-    for (auto i = 0; i < column_count; i++) {
-        results.column_names.emplace_back(sqlite3_column_name(stmt_, i));
-    }
-
-    while (sqlite3_step(stmt_) != SQLITE_DONE) {
-        std::vector<std::wstring> row;
-        row.reserve(column_count);
-        for (auto col = 0; col < column_count; col++) {
-            auto value = GetColumnValue(col);
-            row.emplace_back(value);
+bool FetchRecordsQuery::StepRow() {
+    if (stmt_ == nullptr) {
+        const auto sql = PrepareSql();
+        if (sqlite3_prepare_v2(db_, sql.data(), -1, &stmt_, nullptr)) {
+            throw std::runtime_error((sql + ": could not get results").data());
         }
-        results.row_values.emplace_back(row);
+        BindValues(stmt_, predicate_->Bindings());
     }
-
-    return results;
+    return sqlite3_step(stmt_) != SQLITE_DONE;
 }
 
-void FetchRecordsQuery::Hydrate(void* p, const FetchQueryResults& query_results, const Reflection& record, size_t i) {
-    for (auto j = 0; j < query_results.column_names.size(); j++) {
-        const auto current_storage_class = record.member_metadata[j].storage_class;
-        const auto& content = query_results.row_values[i][j];
-        if (content.empty()) {
+void FetchRecordsQuery::HydrateCurrentRow(void* p, const Reflection& record) const {
+    // Initialize() only ever runs CREATE TABLE IF NOT EXISTS, so a table that predates a
+    // field being added to the reflected struct can have fewer actual columns than
+    // record.member_metadata; indexing sqlite3_column_* past the statement's real column
+    // count is undefined behavior, so bound the loop by whichever is smaller, exactly as
+    // the prior text-based path did via sqlite3_column_count(stmt_)
+    const auto column_count = std::min(static_cast<size_t>(sqlite3_column_count(stmt_)), record.member_metadata.size());
+    for (size_t j = 0; j < column_count; j++) {
+        const auto col = static_cast<int>(j);
+
+        // The prior text-based path only ever produced a non-empty string for INTEGER,
+        // FLOAT, or TEXT columns; NULL and BLOB both fell through to an empty string and
+        // were skipped, regardless of the member's declared storage class. Replicate that
+        // here so a NULL or BLOB value (reachable via UnsafeSql or a foreign database file,
+        // even for a column declared as a different storage class) is never fed to the
+        // wrong typed accessor.
+        const int col_type = sqlite3_column_type(stmt_, col);
+        if (col_type == SQLITE_NULL || col_type == SQLITE_BLOB) {
             continue;
         }
 
+        const auto current_storage_class = record.member_metadata[j].storage_class;
         switch (current_storage_class) {
             case SqliteStorageClass::kInt: {
                 auto& v = *reinterpret_cast<int64_t*>(GetMemberAddress(p, record, j));
-                v = StringUtilities::ToInt(content);
+                v = sqlite3_column_int64(stmt_, col);
                 break;
             }
 
             case SqliteStorageClass::kBool: {
                 auto& v = *reinterpret_cast<bool*>(GetMemberAddress(p, record, j));
-                v = StringUtilities::ToInt(content) == 1;
+                v = sqlite3_column_int64(stmt_, col) == 1;
                 break;
             }
 
             case SqliteStorageClass::kReal: {
                 auto& v = *reinterpret_cast<double*>(GetMemberAddress(p, record, j));
-                v = StringUtilities::ToDouble(content);
+                v = sqlite3_column_double(stmt_, col);
                 break;
             }
 
             case SqliteStorageClass::kText: {
+                const auto byte_count = sqlite3_column_bytes(stmt_, col);
+                if (byte_count == 0) {
+                    continue;
+                }
+                const auto content = reinterpret_cast<const char*>(sqlite3_column_text(stmt_, col));
                 auto& v = *reinterpret_cast<std::wstring*>(GetMemberAddress(p, record, j));
-                v = content;
+                v = StringUtilities::FromUtf8(content, byte_count);
                 break;
             }
 
             case SqliteStorageClass::kDateTime: {
+                const auto byte_count = sqlite3_column_bytes(stmt_, col);
+                if (byte_count == 0) {
+                    continue;
+                }
+                const auto content = reinterpret_cast<const char*>(sqlite3_column_text(stmt_, col));
                 auto& v = *reinterpret_cast<TimePoint*>(GetMemberAddress(p, record, j));
-                v = TimePoint::FromSystemTime(content);
+                v = TimePoint::FromSystemTime(StringUtilities::FromUtf8(content, byte_count));
                 break;
             }
 
@@ -370,28 +374,3 @@ std::string FetchRecordsQuery::PrepareSql() const {
     }
     return sql + ";";
 }
-
-std::wstring FetchRecordsQuery::GetColumnValue(const int col) const {
-    const int col_type = sqlite3_column_type(stmt_, col);
-    switch (col_type) {
-        case SQLITE_INTEGER:
-            return std::to_wstring(sqlite3_column_int64(stmt_, col));
-
-        case SQLITE_FLOAT: {
-            // %.17g round-trips any double exactly; to_wstring's fixed 6-decimal
-            // formatting would silently truncate precision here
-            char buffer[64];
-            std::snprintf(buffer, sizeof(buffer), "%.17g", sqlite3_column_double(stmt_, col));
-            return StringUtilities::FromUtf8(buffer, std::strlen(buffer));
-        }
-
-        case SQLITE_TEXT: {
-            const auto content = reinterpret_cast<const char*>(sqlite3_column_text(stmt_, col));
-            const auto byte_count = sqlite3_column_bytes(stmt_, col);
-            return StringUtilities::FromUtf8(content, byte_count);
-        }
-
-        default:
-            return L"";
-    }
-}

src/string_utilities.cc

@@ -30,26 +30,10 @@
 
 using namespace sqlite_reflection;
 
-int64_t StringUtilities::ToInt(const std::wstring& s) {
-    try {
-        return std::stoll(s);
-    } catch (...) {
-        return 0;
-    }
-}
-
 std::string StringUtilities::FromInt(int64_t value) {
     return std::to_string(value);
 }
 
-double StringUtilities::ToDouble(const std::wstring& s) {
-    try {
-        return std::stod(s);
-    } catch (...) {
-        return 0.0;
-    }
-}
-
 std::string StringUtilities::FromDouble(double value) {
     auto textual_representation = std::to_string(value);
     if (textual_representation.find('.') != std::string::npos) {