Skip to content

fix some predefine cases #51695

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eldenmoon merged 1 commit into from
Jun 13, 2025
Merged

fix some predefine cases #51695

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions be/src/service/internal_service.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1155,6 +1155,10 @@ void PInternalService::fetch_remote_tablet_schema(google::protobuf::RpcControlle
bool ret = _heavy_work_pool.try_offer([request, response, done]() {
brpc::ClosureGuard closure_guard(done);
Status st = Status::OK();
std::shared_ptr<MemTrackerLimiter> mem_tracker = MemTrackerLimiter::create_shared(
MemTrackerLimiter::Type::OTHER,
fmt::format("InternalService::fetch_remote_tablet_schema"));
SCOPED_ATTACH_TASK(mem_tracker);
if (request->is_coordinator()) {
// Spawn rpc request to none coordinator nodes, and finally merge them all
PFetchRemoteSchemaRequest remote_request(*request);
Expand Down
24 changes: 22 additions & 2 deletions be/src/vec/common/schema_util.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1164,6 +1164,11 @@ class FieldVisitorToScalarType : public StaticVisitor<size_t> {
}
}
void get_scalar_type(PrimitiveType* type) const {
if (type_indexes.size() == 1) {
// Most cases will have only one type
*type = *type_indexes.begin();
return;
}
DataTypePtr data_type;
get_least_supertype_jsonb(type_indexes, &data_type);
*type = data_type->get_primitive_type();
Expand All @@ -1177,13 +1182,28 @@ class FieldVisitorToScalarType : public StaticVisitor<size_t> {
bool have_nulls = false;
};

void get_precision_and_scale(const Field& field, int* precision, int* scale) {
if (field.get_type() == PrimitiveType::TYPE_ARRAY) {
for (const auto& item : field.get<Array>()) {
if (item.is_null()) {
continue;
}
get_precision_and_scale(item, precision, scale);
return;
}
}
*precision = field.get_precision();
*scale = field.get_scale();
}

template <typename Visitor>
void get_field_info_impl(const Field& field, FieldInfo* info) {
Visitor to_scalar_type_visitor;
apply_visitor(to_scalar_type_visitor, field);
PrimitiveType type_id;
int precision = field.get_precision();
int scale = field.get_scale();
int precision = 0;
int scale = 0;
get_precision_and_scale(field, &precision, &scale);
to_scalar_type_visitor.get_scalar_type(&type_id);
// array item's dimension may missmatch, eg. [1, 2, [1, 2, 3]]
*info = {
Expand Down
3 changes: 2 additions & 1 deletion be/src/vec/functions/function_variant_type.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
// under the License.
#include <glog/logging.h>

#include "util/string_util.h"
#include "vec/columns/column_variant.h"
#include "vec/common/schema_util.h"
#include "vec/functions/simple_function_factory.h"
Expand Down Expand Up @@ -53,7 +54,7 @@ class FunctionVariantType : public IFunction {
}
FieldInfo info;
schema_util::get_field_info(value, &info);
result[key.get_path()] = type_to_string(info.scalar_type_id);
result[key.get_path()] = to_lower(type_to_string(info.scalar_type_id));
}
return result;
}
Expand Down
18 changes: 9 additions & 9 deletions regression-test/data/variant_p0/predefine/load.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,21 +95,21 @@ v1.predefine_col4 text Yes false \N NONE
8 {"PREDEFINE_COL4":"2020-01-01-01"}

-- !sql --
1 {"array_boolean":[1, 0, 1],"array_date":["2021-01-01", "2022-01-01", "2023-01-01"],"array_datetime":["2021-01-01 00:00:00", "2022-01-01 00:00:00", "2023-01-01 00:00:00"],"array_datetimev2":["2021-01-01 00:00:00", "2022-01-01 00:00:00", "2023-01-01 00:00:00"],"array_datev2":["2021-01-01", "2022-01-01", "2023-01-01"],"array_decimal":[1.100000000, 2.200000000, 3.300000000],"array_float":[1.11111],"array_int":[1, 2, 3],"array_ipv4":["127.0.0.1", "172.0.1.1"],"array_ipv6":["ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffe"],"array_string":["a", "b", "c"],"boolean_":1,"date_":"2022-01-01","datetime_":"2022-01-01 11:11:11","datetimev2_":"2022-01-01 11:11:11.999999","datev2_":"2022-01-01","decimal_":188118222.011121920,"float_":128.111,"int_":11111122,"ipv4_":"127.0.0.1","ipv6_":"ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffe","string_":"12111222113","varchar_":"hello world"}
2 {"array_boolean":[1, 0, 1, 0, 1],"array_date":["2021-01-01", "2022-01-01", "2023-01-01"],"array_datetime":["2021-01-01 00:00:00", "2022-01-01 00:00:00", "2023-01-01 00:00:00"],"array_datetimev2":["2021-01-01 00:00:00", "2022-01-01 00:00:00", "2023-01-01 00:00:00"],"array_datev2":["2021-01-01", "2022-01-01", "2023-01-01"],"array_decimal":[1.100000000, 2.200000000, 3.300000000],"array_float":[2.22222],"array_int":[1, 2, 3],"array_ipv4":["127.0.0.1", "172.0.1.1"],"array_ipv6":["ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffe"],"array_string":["a", "b", "c"],"boolean_":1,"date_":"2022-01-01","datetime_":"2022-01-01 11:11:11","datetimev2_":"2022-01-01 11:11:11.999999","datev2_":"2022-01-01","decimal_":219911111111.011121933,"float_":1.1111112,"ipv4_":"127.0.0.1","ipv6_":"ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffe","string_":"12111222113","varchar_":"world hello"}
3 {"array_datetimev2":["2021-01-01 00:00:00", "2022-01-01 00:00:00", "2023-01-01 00:00:00"],"array_int":[1, 2, 3],"array_string":["a", "b", "c"],"boolean_":1,"date_":"2022-01-01","decimal_":219911111111.011121933,"float_":1.1111112,"ipv4_":"127.0.0.1","varchar_":"world hello"}
4 {"array_datetimev2":["2021-01-01 00:00:00", "2022-01-01 00:00:00", "2023-01-01 00:00:00"],"array_int":[1, 2, 3],"array_string":["a", "b", "c"],"ext_1":1.111111,"ext_2":"this is an extra field","ext_3":[1, 2, 3],"float_":1.1111112,"ipv4_":"127.0.0.1","varchar_":"world hello"}
1 {"array_boolean":[1, 0, 1],"array_date":["2021-01-01", "2022-01-01", "2023-01-01"],"array_datetime":["2021-01-01 00:00:00", "2022-01-01 00:00:00", "2023-01-01 00:00:00"],"array_datetimev2":["2021-01-01 00:00:00", "2022-01-01 00:00:00", "2023-01-01 00:00:00"],"array_datev2":["2021-01-01", "2022-01-01", "2023-01-01"],"array_decimal":[1.100000000, 2.200000000, 3.300000000],"array_float":[1.111110000],"array_int":[1, 2, 3],"array_ipv4":["127.0.0.1", "172.0.1.1"],"array_ipv6":["ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffe"],"array_string":["a", "b", "c"],"boolean_":1,"date_":"2022-01-01","datetime_":"2022-01-01 11:11:11","datetimev2_":"2022-01-01 11:11:11.999999","datev2_":"2022-01-01","decimal_":188118222.011121920,"float_":128.111000000,"int_":11111122,"ipv4_":"127.0.0.1","ipv6_":"ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffe","string_":"12111222113","varchar_":"hello world"}
2 {"array_boolean":[1, 0, 1, 0, 1],"array_date":["2021-01-01", "2022-01-01", "2023-01-01"],"array_datetime":["2021-01-01 00:00:00", "2022-01-01 00:00:00", "2023-01-01 00:00:00"],"array_datetimev2":["2021-01-01 00:00:00", "2022-01-01 00:00:00", "2023-01-01 00:00:00"],"array_datev2":["2021-01-01", "2022-01-01", "2023-01-01"],"array_decimal":[1.100000000, 2.200000000, 3.300000000],"array_float":[2.222220000],"array_int":[1, 2, 3],"array_ipv4":["127.0.0.1", "172.0.1.1"],"array_ipv6":["ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffe"],"array_string":["a", "b", "c"],"boolean_":1,"date_":"2022-01-01","datetime_":"2022-01-01 11:11:11","datetimev2_":"2022-01-01 11:11:11.999999","datev2_":"2022-01-01","decimal_":219911111111.011121933,"float_":1.111111111,"ipv4_":"127.0.0.1","ipv6_":"ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffe","string_":"12111222113","varchar_":"world hello"}
3 {"array_datetimev2":["2021-01-01 00:00:00", "2022-01-01 00:00:00", "2023-01-01 00:00:00"],"array_int":[1, 2, 3],"array_string":["a", "b", "c"],"boolean_":1,"date_":"2022-01-01","decimal_":219911111111.011121933,"float_":1.111111111,"ipv4_":"127.0.0.1","varchar_":"world hello"}
4 {"array_datetimev2":["2021-01-01 00:00:00", "2022-01-01 00:00:00", "2023-01-01 00:00:00"],"array_int":[1, 2, 3],"array_string":["a", "b", "c"],"ext_1":1.111111,"ext_2":"this is an extra field","ext_3":[1, 2, 3],"float_":1.111111111,"ipv4_":"127.0.0.1","varchar_":"world hello"}

-- !sql --
id bigint No true \N
v1 variant<MATCH_NAME_GLOB 'array_int':array<int>,MATCH_NAME_GLOB 'array_string':array<text>,MATCH_NAME_GLOB 'array_decimal':array<decimalv3(26,9)>,MATCH_NAME_GLOB 'array_datetime':array<datetimev2(0)>,MATCH_NAME_GLOB 'array_datetimev2':array<datetimev2(0)>,MATCH_NAME_GLOB 'array_date':array<datev2>,MATCH_NAME_GLOB 'array_datev2':array<datev2>,MATCH_NAME_GLOB 'array_ipv4':array<ipv4>,MATCH_NAME_GLOB 'array_ipv6':array<ipv6>,MATCH_NAME_GLOB 'array_float':array<float>,MATCH_NAME_GLOB 'array_boolean':array<boolean>,MATCH_NAME_GLOB 'int_':int,MATCH_NAME_GLOB 'string_':text,MATCH_NAME_GLOB 'decimal_':decimalv3(26,9),MATCH_NAME_GLOB 'datetime_':datetimev2(0),MATCH_NAME_GLOB 'datetimev2_':datetimev2(6),MATCH_NAME_GLOB 'date_':datev2,MATCH_NAME_GLOB 'datev2_':datev2,MATCH_NAME_GLOB 'ipv4_':ipv4,MATCH_NAME_GLOB 'ipv6_':ipv6,MATCH_NAME_GLOB 'float_':float,MATCH_NAME_GLOB 'boolean_':boolean,MATCH_NAME_GLOB 'varchar_':text> Yes false \N NONE
v1 variant<MATCH_NAME_GLOB 'array_int':array<int>,MATCH_NAME_GLOB 'array_string':array<text>,MATCH_NAME_GLOB 'array_decimal':array<decimalv3(26,9)>,MATCH_NAME_GLOB 'array_datetime':array<datetimev2(0)>,MATCH_NAME_GLOB 'array_datetimev2':array<datetimev2(0)>,MATCH_NAME_GLOB 'array_date':array<datev2>,MATCH_NAME_GLOB 'array_datev2':array<datev2>,MATCH_NAME_GLOB 'array_ipv4':array<ipv4>,MATCH_NAME_GLOB 'array_ipv6':array<ipv6>,MATCH_NAME_GLOB 'array_float':array<decimalv3(26,9)>,MATCH_NAME_GLOB 'array_boolean':array<boolean>,MATCH_NAME_GLOB 'int_':int,MATCH_NAME_GLOB 'string_':text,MATCH_NAME_GLOB 'decimal_':decimalv3(26,9),MATCH_NAME_GLOB 'datetime_':datetimev2(0),MATCH_NAME_GLOB 'datetimev2_':datetimev2(6),MATCH_NAME_GLOB 'date_':datev2,MATCH_NAME_GLOB 'datev2_':datev2,MATCH_NAME_GLOB 'ipv4_':ipv4,MATCH_NAME_GLOB 'ipv6_':ipv6,MATCH_NAME_GLOB 'float_':decimalv3(26,9),MATCH_NAME_GLOB 'boolean_':boolean,MATCH_NAME_GLOB 'varchar_':text> Yes false \N NONE
v1.array_boolean array<boolean> Yes false \N NONE
v1.array_date array<date> Yes false \N NONE
v1.array_datetime array<datetime> Yes false \N NONE
v1.array_datetimev2 array<datetime> Yes false \N NONE
v1.array_datev2 array<date> Yes false \N NONE
v1.array_decimal array<decimal(26,9)> Yes false \N NONE
v1.array_float array<float> Yes false \N NONE
v1.array_float array<decimal(26,9)> Yes false \N NONE
v1.array_int array<int> Yes false \N NONE
v1.array_ipv4 array<ipv4> Yes false \N NONE
v1.array_ipv6 array<ipv6> Yes false \N NONE
Expand All @@ -123,7 +123,7 @@ v1.decimal_ decimal(26,9) Yes false \N NONE
v1.ext_1 double Yes false \N NONE
v1.ext_2 text Yes false \N NONE
v1.ext_3 array<tinyint> Yes false \N NONE
v1.float_ float Yes false \N NONE
v1.float_ decimal(26,9) Yes false \N NONE
v1.int_ int Yes false \N NONE
v1.ipv4_ ipv4 Yes false \N NONE
v1.ipv6_ ipv6 Yes false \N NONE
Expand Down Expand Up @@ -190,8 +190,8 @@ v.auto_type json Yes false \N NONE
5 {"array_decimal":[1.100000000, 2.200000000, 3.300000000, 4.400000000]}

-- !sql --
{"a.b.c":"Int32","db":"Float64","dcm":"Decimal128V3","dt":"DateTimeV2"}
{"a.b.c":"tinyint","db":"double","dcm":"decimal128i","dt":"datetimev2"}

-- !sql --
{"a.b.c":"Int32","db":"Float64","dcm":"Decimal128V3","dt":"DateTimeV2"}
{"a.b.c":"tinyint","db":"double","dcm":"decimal128i","dt":"datetimev2"}

6 changes: 3 additions & 3 deletions regression-test/data/variant_p0/predefine/sql/q01.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@

-- !q01_2 --
id bigint No true \N
v1 variant<MATCH_NAME_GLOB 'array_int':array<int>,MATCH_NAME_GLOB 'array_string':array<text>,MATCH_NAME_GLOB 'array_decimal':array<decimalv3(26,9)>,MATCH_NAME_GLOB 'array_datetime':array<datetimev2(0)>,MATCH_NAME_GLOB 'array_datetimev2':array<datetimev2(0)>,MATCH_NAME_GLOB 'array_date':array<datev2>,MATCH_NAME_GLOB 'array_datev2':array<datev2>,MATCH_NAME_GLOB 'array_ipv4':array<ipv4>,MATCH_NAME_GLOB 'array_ipv6':array<ipv6>,MATCH_NAME_GLOB 'array_float':array<float>,MATCH_NAME_GLOB 'array_boolean':array<boolean>,MATCH_NAME_GLOB 'int_':int,MATCH_NAME_GLOB 'string_':text,MATCH_NAME_GLOB 'decimal_':decimalv3(26,9),MATCH_NAME_GLOB 'datetime_':datetimev2(0),MATCH_NAME_GLOB 'datetimev2_':datetimev2(6),MATCH_NAME_GLOB 'date_':datev2,MATCH_NAME_GLOB 'datev2_':datev2,MATCH_NAME_GLOB 'ipv4_':ipv4,MATCH_NAME_GLOB 'ipv6_':ipv6,MATCH_NAME_GLOB 'float_':float,MATCH_NAME_GLOB 'boolean_':boolean,MATCH_NAME_GLOB 'varchar_':text> Yes false \N NONE
v1 variant<MATCH_NAME_GLOB 'array_int':array<int>,MATCH_NAME_GLOB 'array_string':array<text>,MATCH_NAME_GLOB 'array_decimal':array<decimalv3(26,9)>,MATCH_NAME_GLOB 'array_datetime':array<datetimev2(0)>,MATCH_NAME_GLOB 'array_datetimev2':array<datetimev2(0)>,MATCH_NAME_GLOB 'array_date':array<datev2>,MATCH_NAME_GLOB 'array_datev2':array<datev2>,MATCH_NAME_GLOB 'array_ipv4':array<ipv4>,MATCH_NAME_GLOB 'array_ipv6':array<ipv6>,MATCH_NAME_GLOB 'array_float':array<decimalv3(26,9)>,MATCH_NAME_GLOB 'array_boolean':array<boolean>,MATCH_NAME_GLOB 'int_':int,MATCH_NAME_GLOB 'string_':text,MATCH_NAME_GLOB 'decimal_':decimalv3(26,9),MATCH_NAME_GLOB 'datetime_':datetimev2(0),MATCH_NAME_GLOB 'datetimev2_':datetimev2(6),MATCH_NAME_GLOB 'date_':datev2,MATCH_NAME_GLOB 'datev2_':datev2,MATCH_NAME_GLOB 'ipv4_':ipv4,MATCH_NAME_GLOB 'ipv6_':ipv6,MATCH_NAME_GLOB 'float_':decimalv3(26,9),MATCH_NAME_GLOB 'boolean_':boolean,MATCH_NAME_GLOB 'varchar_':text> Yes false \N NONE
v1.array_boolean array<boolean> Yes false \N NONE
v1.array_date array<date> Yes false \N NONE
v1.array_datetime array<datetime> Yes false \N NONE
v1.array_datetimev2 array<datetime> Yes false \N NONE
v1.array_datev2 array<date> Yes false \N NONE
v1.array_decimal array<decimal(26,9)> Yes false \N NONE
v1.array_float array<float> Yes false \N NONE
v1.array_float array<decimal(26,9)> Yes false \N NONE
v1.array_int array<int> Yes false \N NONE
v1.array_ipv4 array<ipv4> Yes false \N NONE
v1.array_ipv6 array<ipv6> Yes false \N NONE
Expand All @@ -25,7 +25,7 @@ v1.decimal_ decimal(26,9) Yes false \N NONE
v1.ext_1 double Yes false \N NONE
v1.ext_2 text Yes false \N NONE
v1.ext_3 array<tinyint> Yes false \N NONE
v1.float_ float Yes false \N NONE
v1.float_ decimal(26,9) Yes false \N NONE
v1.int_ int Yes false \N NONE
v1.ipv4_ ipv4 Yes false \N NONE
v1.ipv6_ ipv6 Yes false \N NONE
Expand Down
Loading
Loading