Skip to content

Update vendored DuckDB to 0.10.2 #76

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 18, 2024
Merged

Update vendored DuckDB to 0.10.2 #76

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
89d7809
update vendored duckdb
hannes Apr 17, 2024
1767e9c
forgot binding
hannes Apr 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
40 changes: 22 additions & 18 deletions binding.gyp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,9 +166,9 @@
"src/duckdb/src/verification/statement_verifier.cpp",
"src/duckdb/src/verification/unoptimized_statement_verifier.cpp",
"src/duckdb/third_party/fmt/format.cc",
"src/duckdb/third_party/fsst/fsst_avx512.cpp",
"src/duckdb/third_party/fsst/libfsst.cpp",
"src/duckdb/third_party/miniz/miniz.cpp",
"src/duckdb/third_party/re2/re2/bitmap256.cc",
"src/duckdb/third_party/re2/re2/bitstate.cc",
"src/duckdb/third_party/re2/re2/compile.cc",
"src/duckdb/third_party/re2/re2/dfa.cc",
Expand Down Expand Up @@ -271,6 +271,7 @@
"src/duckdb/third_party/zstd/compress/zstd_lazy.cpp",
"src/duckdb/third_party/zstd/compress/zstd_ldm.cpp",
"src/duckdb/third_party/zstd/compress/zstd_opt.cpp",
"src/duckdb/third_party/lz4/lz4.cpp",
"src/duckdb/extension/icu/./icu-table-range.cpp",
"src/duckdb/extension/icu/./icu-makedate.cpp",
"src/duckdb/extension/icu/./icu-list-range.cpp",
Expand Down Expand Up @@ -301,30 +302,32 @@
"include_dirs": [
"<!(node -p \"require('node-addon-api').include_dir\")",
"src/duckdb/src/include",
"src/duckdb/third_party/concurrentqueue",
"src/duckdb/third_party/fast_float",
"src/duckdb/third_party/fastpforlib",
"src/duckdb/third_party/fmt/include",
"src/duckdb/third_party/fsst",
"src/duckdb/third_party/re2",
"src/duckdb/third_party/miniz",
"src/duckdb/third_party/utf8proc/include",
"src/duckdb/third_party/utf8proc",
"src/duckdb/third_party/httplib",
"src/duckdb/third_party/hyperloglog",
"src/duckdb/third_party/skiplist",
"src/duckdb/third_party/fastpforlib",
"src/duckdb/third_party/tdigest",
"src/duckdb/third_party/libpg_query/include",
"src/duckdb/third_party/jaro_winkler",
"src/duckdb/third_party/jaro_winkler/details",
"src/duckdb/third_party/libpg_query",
"src/duckdb/third_party/concurrentqueue",
"src/duckdb/third_party/pcg",
"src/duckdb/third_party/httplib",
"src/duckdb/third_party/fast_float",
"src/duckdb/third_party/libpg_query/include",
"src/duckdb/third_party/lz4",
"src/duckdb/third_party/mbedtls",
"src/duckdb/third_party/mbedtls/include",
"src/duckdb/third_party/mbedtls/library",
"src/duckdb/third_party/jaro_winkler",
"src/duckdb/third_party/jaro_winkler/details",
"src/duckdb/third_party/miniz",
"src/duckdb/third_party/pcg",
"src/duckdb/third_party/re2",
"src/duckdb/third_party/skiplist",
"src/duckdb/third_party/tdigest",
"src/duckdb/third_party/utf8proc",
"src/duckdb/third_party/utf8proc/include",
"src/duckdb/extension/parquet/include",
"src/duckdb/third_party/parquet",
"src/duckdb/third_party/thrift",
"src/duckdb/third_party/lz4",
"src/duckdb/third_party/snappy",
"src/duckdb/third_party/zstd/include",
"src/duckdb/third_party/mbedtls",
Expand Down Expand Up @@ -389,9 +392,10 @@
{
"defines": [
"DUCKDB_BUILD_LIBRARY"
],
],
"libraries": [
"rstrtmgr.lib", "bcrypt.lib"
"rstrtmgr.lib",
"bcrypt.lib"
]
}
]
Expand All @@ -414,4 +418,4 @@
]
}
]
}
}
3 changes: 3 additions & 0 deletions binding.gyp.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,9 @@
{
"defines": [
"DUCKDB_BUILD_LIBRARY"
],
"libraries": [
"rstrtmgr.lib", "bcrypt.lib"
]
}
]
Expand Down
4 changes: 3 additions & 1 deletion src/duckdb/extension/icu/icu-timezone.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#include "duckdb/common/types/timestamp.hpp"
#include "duckdb/common/exception/conversion_exception.hpp"
#include "duckdb/function/cast/cast_function_set.hpp"
#include "duckdb/function/cast_rules.hpp"
#include "duckdb/main/extension_util.hpp"
#include "duckdb/parser/parsed_data/create_scalar_function_info.hpp"
#include "duckdb/parser/parsed_data/create_table_function_info.hpp"
Expand Down Expand Up @@ -183,7 +184,8 @@ struct ICUFromNaiveTimestamp : public ICUDateFunc {
auto &config = DBConfig::GetConfig(db);
auto &casts = config.GetCastFunctions();

casts.RegisterCastFunction(LogicalType::TIMESTAMP, LogicalType::TIMESTAMP_TZ, BindCastFromNaive);
const auto implicit_cost = CastRules::ImplicitCast(LogicalType::TIMESTAMP, LogicalType::TIMESTAMP_TZ);
casts.RegisterCastFunction(LogicalType::TIMESTAMP, LogicalType::TIMESTAMP_TZ, BindCastFromNaive, implicit_cost);
casts.RegisterCastFunction(LogicalType::TIMESTAMP_MS, LogicalType::TIMESTAMP_TZ, BindCastFromNaive);
casts.RegisterCastFunction(LogicalType::TIMESTAMP_NS, LogicalType::TIMESTAMP_TZ, BindCastFromNaive);
casts.RegisterCastFunction(LogicalType::TIMESTAMP_S, LogicalType::TIMESTAMP_TZ, BindCastFromNaive);
Expand Down
8 changes: 6 additions & 2 deletions src/duckdb/extension/icu/icu_extension.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ static void SetICUCalendar(ClientContext &context, SetScope scope, Value &parame
}
}

void IcuExtension::Load(DuckDB &ddb) {
static void LoadInternal(DuckDB &ddb) {
auto &db = *ddb.instance;

// iterate over all the collations
Expand Down Expand Up @@ -275,6 +275,10 @@ void IcuExtension::Load(DuckDB &ddb) {
ExtensionUtil::RegisterFunction(db, cal_names);
}

void IcuExtension::Load(DuckDB &ddb) {
LoadInternal(ddb);
}

std::string IcuExtension::Name() {
return "icu";
}
Expand All @@ -285,7 +289,7 @@ extern "C" {

DUCKDB_EXTENSION_API void icu_init(duckdb::DatabaseInstance &db) { // NOLINT
duckdb::DuckDB db_wrapper(db);
db_wrapper.LoadExtension<duckdb::IcuExtension>();
duckdb::LoadInternal(db_wrapper);
}

DUCKDB_EXTENSION_API const char *icu_version() { // NOLINT
Expand Down
13 changes: 10 additions & 3 deletions src/duckdb/extension/json/buffered_json_reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,10 @@ bool JSONFileHandle::RequestedReadsComplete() {
return requested_reads == actual_reads;
}

bool JSONFileHandle::LastReadRequested() const {
return last_read_requested;
}

idx_t JSONFileHandle::FileSize() const {
return file_size;
}
Expand All @@ -56,6 +60,10 @@ bool JSONFileHandle::CanSeek() const {
return can_seek;
}

bool JSONFileHandle::IsPipe() const {
return file_handle->IsPipe();
}

FileHandle &JSONFileHandle::GetHandle() {
return *file_handle;
}
Expand Down Expand Up @@ -193,9 +201,8 @@ BufferedJSONReader::BufferedJSONReader(ClientContext &context, BufferedJSONReade
void BufferedJSONReader::OpenJSONFile() {
lock_guard<mutex> guard(lock);
if (!IsOpen()) {
auto &file_system = FileSystem::GetFileSystem(context);
auto regular_file_handle = file_system.OpenFile(file_name.c_str(), FileFlags::FILE_FLAGS_READ,
FileLockType::NO_LOCK, options.compression);
auto &fs = FileSystem::GetFileSystem(context);
auto regular_file_handle = fs.OpenFile(file_name, FileFlags::FILE_FLAGS_READ | options.compression);
file_handle = make_uniq<JSONFileHandle>(std::move(regular_file_handle), BufferAllocator::Get(context));
}
Reset();
Expand Down
2 changes: 2 additions & 0 deletions src/duckdb/extension/json/include/buffered_json_reader.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,11 +60,13 @@ struct JSONFileHandle {

void Reset();
bool RequestedReadsComplete();
bool LastReadRequested() const;

idx_t FileSize() const;
idx_t Remaining() const;

bool CanSeek() const;
bool IsPipe() const;

FileHandle &GetHandle();

Expand Down
20 changes: 13 additions & 7 deletions src/duckdb/extension/json/include/json_scan.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,13 @@
#pragma once

#include "buffered_json_reader.hpp"
#include "json_enums.hpp"
#include "duckdb/common/multi_file_reader.hpp"
#include "duckdb/common/mutex.hpp"
#include "duckdb/common/pair.hpp"
#include "duckdb/common/types/type_map.hpp"
#include "duckdb/function/scalar/strftime_format.hpp"
#include "duckdb/function/table_function.hpp"
#include "json_enums.hpp"
#include "json_transform.hpp"

namespace duckdb {
Expand Down Expand Up @@ -226,14 +226,20 @@ struct JSONScanLocalState {

private:
bool ReadNextBuffer(JSONScanGlobalState &gstate);
bool ReadNextBufferInternal(JSONScanGlobalState &gstate, optional_idx &buffer_index, bool &file_done);
bool ReadNextBufferSeek(JSONScanGlobalState &gstate, optional_idx &buffer_index, bool &file_done);
bool ReadNextBufferNoSeek(JSONScanGlobalState &gstate, optional_idx &buffer_index, bool &file_done);
bool ReadNextBufferInternal(JSONScanGlobalState &gstate, AllocatedData &buffer, optional_idx &buffer_index,
bool &file_done);
bool ReadNextBufferSeek(JSONScanGlobalState &gstate, AllocatedData &buffer, optional_idx &buffer_index,
bool &file_done);
bool ReadNextBufferNoSeek(JSONScanGlobalState &gstate, AllocatedData &buffer, optional_idx &buffer_index,
bool &file_done);
AllocatedData AllocateBuffer(JSONScanGlobalState &gstate);
data_ptr_t GetReconstructBuffer(JSONScanGlobalState &gstate);

void SkipOverArrayStart();

void ReadAndAutoDetect(JSONScanGlobalState &gstate, optional_idx &buffer_index);
bool ReconstructFirstObject();
void ParseNextChunk();
void ReadAndAutoDetect(JSONScanGlobalState &gstate, AllocatedData &buffer, optional_idx &buffer_index);
bool ReconstructFirstObject(JSONScanGlobalState &gstate);
void ParseNextChunk(JSONScanGlobalState &gstate);

void ParseJSON(char *const json_start, const idx_t json_size, const idx_t remaining);
void ThrowObjectSizeError(const idx_t object_size);
Expand Down
9 changes: 5 additions & 4 deletions src/duckdb/extension/json/include/json_serializer.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,16 @@ struct JsonSerializer : Serializer {
void PushValue(yyjson_mut_val *val);

public:
explicit JsonSerializer(yyjson_mut_doc *doc, bool skip_if_null, bool skip_if_empty)
explicit JsonSerializer(yyjson_mut_doc *doc, bool skip_if_null, bool skip_if_empty, bool skip_if_default)
: doc(doc), stack({yyjson_mut_obj(doc)}), skip_if_null(skip_if_null), skip_if_empty(skip_if_empty) {
serialize_enum_as_string = true;
serialize_default_values = true;
serialize_default_values = !skip_if_default;
}

template <class T>
static yyjson_mut_val *Serialize(T &value, yyjson_mut_doc *doc, bool skip_if_null, bool skip_if_empty) {
JsonSerializer serializer(doc, skip_if_null, skip_if_empty);
static yyjson_mut_val *Serialize(T &value, yyjson_mut_doc *doc, bool skip_if_null, bool skip_if_empty,
bool skip_if_default) {
JsonSerializer serializer(doc, skip_if_null, skip_if_empty, skip_if_default);
value.Serialize(serializer);
return serializer.GetRootObject();
}
Expand Down
6 changes: 3 additions & 3 deletions src/duckdb/extension/json/include/json_structure.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ struct StrpTimeFormat;
struct JSONStructureNode {
public:
JSONStructureNode();
JSONStructureNode(yyjson_val *key_p, yyjson_val *val_p);
JSONStructureNode(yyjson_val *key_p, yyjson_val *val_p, const bool ignore_errors);

//! Disable copy constructors
JSONStructureNode(const JSONStructureNode &other) = delete;
Expand Down Expand Up @@ -64,7 +64,7 @@ struct JSONStructureDescription {
JSONStructureDescription &operator=(JSONStructureDescription &&) noexcept;

JSONStructureNode &GetOrCreateChild();
JSONStructureNode &GetOrCreateChild(yyjson_val *key, yyjson_val *val);
JSONStructureNode &GetOrCreateChild(yyjson_val *key, yyjson_val *val, const bool ignore_errors);

public:
//! Type of this description
Expand All @@ -80,7 +80,7 @@ struct JSONStructureDescription {

struct JSONStructure {
public:
static void ExtractStructure(yyjson_val *val, JSONStructureNode &node);
static void ExtractStructure(yyjson_val *val, JSONStructureNode &node, const bool ignore_errors);
static LogicalType StructureToType(ClientContext &context, const JSONStructureNode &node, const idx_t max_depth,
const double field_appearance_threshold, idx_t depth = 0,
idx_t sample_count = DConstants::INVALID_INDEX);
Expand Down
9 changes: 7 additions & 2 deletions src/duckdb/extension/json/json_functions.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ namespace duckdb {
using JSONPathType = JSONCommon::JSONPathType;

static JSONPathType CheckPath(const Value &path_val, string &path, size_t &len) {
if (path_val.IsNull()) {
throw InternalException("JSON path cannot be NULL");
}
const auto path_str_val = path_val.DefaultCastAs(LogicalType::VARCHAR);
auto path_str = path_str_val.GetValueUnsafe<string_t>();
len = path_str.GetSize();
Expand Down Expand Up @@ -58,9 +61,11 @@ unique_ptr<FunctionData> JSONReadFunctionData::Bind(ClientContext &context, Scal
size_t len = 0;
JSONPathType path_type = JSONPathType::REGULAR;
if (arguments[1]->IsFoldable()) {
constant = true;
const auto path_val = ExpressionExecutor::EvaluateScalar(context, *arguments[1]);
path_type = CheckPath(path_val, path, len);
if (!path_val.IsNull()) {
constant = true;
path_type = CheckPath(path_val, path, len);
}
}
bound_function.arguments[1] = LogicalType::VARCHAR;
if (path_type == JSONCommon::JSONPathType::WILDCARD) {
Expand Down
20 changes: 15 additions & 5 deletions src/duckdb/extension/json/json_functions/json_serialize_plan.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,19 @@ namespace duckdb {
struct JsonSerializePlanBindData : public FunctionData {
bool skip_if_null = false;
bool skip_if_empty = false;
bool skip_if_default = false;
bool format = false;
bool optimize = false;

JsonSerializePlanBindData(bool skip_if_null_p, bool skip_if_empty_p, bool format_p, bool optimize_p)
: skip_if_null(skip_if_null_p), skip_if_empty(skip_if_empty_p), format(format_p), optimize(optimize_p) {
JsonSerializePlanBindData(bool skip_if_null_p, bool skip_if_empty_p, bool skip_if_default_p, bool format_p,
bool optimize_p)
: skip_if_null(skip_if_null_p), skip_if_empty(skip_if_empty_p), skip_if_default(skip_if_default_p),
format(format_p), optimize(optimize_p) {
}

public:
unique_ptr<FunctionData> Copy() const override {
return make_uniq<JsonSerializePlanBindData>(skip_if_null, skip_if_empty, format, optimize);
return make_uniq<JsonSerializePlanBindData>(skip_if_null, skip_if_empty, skip_if_default, format, optimize);
}
bool Equals(const FunctionData &other_p) const override {
return true;
Expand All @@ -48,6 +51,7 @@ static unique_ptr<FunctionData> JsonSerializePlanBind(ClientContext &context, Sc
// Optional arguments
bool skip_if_null = false;
bool skip_if_empty = false;
bool skip_if_default = false;
bool format = false;
bool optimize = false;

Expand All @@ -69,6 +73,11 @@ static unique_ptr<FunctionData> JsonSerializePlanBind(ClientContext &context, Sc
throw BinderException("json_serialize_plan: 'skip_empty' argument must be a boolean");
}
skip_if_empty = BooleanValue::Get(ExpressionExecutor::EvaluateScalar(context, *arg));
} else if (arg->alias == "skip_default") {
if (arg->return_type.id() != LogicalTypeId::BOOLEAN) {
throw BinderException("json_serialize_plan: 'skip_default' argument must be a boolean");
}
skip_if_default = BooleanValue::Get(ExpressionExecutor::EvaluateScalar(context, *arg));
} else if (arg->alias == "format") {
if (arg->return_type.id() != LogicalTypeId::BOOLEAN) {
throw BinderException("json_serialize_plan: 'format' argument must be a boolean");
Expand All @@ -83,7 +92,7 @@ static unique_ptr<FunctionData> JsonSerializePlanBind(ClientContext &context, Sc
throw BinderException(StringUtil::Format("json_serialize_plan: Unknown argument '%s'", arg->alias.c_str()));
}
}
return make_uniq<JsonSerializePlanBindData>(skip_if_null, skip_if_empty, format, optimize);
return make_uniq<JsonSerializePlanBindData>(skip_if_null, skip_if_empty, skip_if_default, format, optimize);
}

static bool OperatorSupportsSerialization(LogicalOperator &op, string &operator_name) {
Expand Down Expand Up @@ -144,7 +153,8 @@ static void JsonSerializePlanFunction(DataChunk &args, ExpressionState &state, V
throw InvalidInputException("Operator '%s' does not support serialization", operator_name);
}

auto plan_json = JsonSerializer::Serialize(*plan, doc, info.skip_if_null, info.skip_if_empty);
auto plan_json =
JsonSerializer::Serialize(*plan, doc, info.skip_if_null, info.skip_if_empty, info.skip_if_default);
yyjson_mut_arr_append(plans_arr, plan_json);
}

Expand Down
Loading