Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Backport #50280 to 23.3: Fixed type conversion from Date/Date32 to DateTime64 when querying with DateTime64 index #50759

Merged
merged 1 commit into from
Jun 9, 2023
Merged

Backport #50280 to 23.3: Fixed type conversion from Date/Date32 to DateTime64 when querying with DateTime64 index #50759

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
16 changes: 16 additions & 0 deletions src/Interpreters/convertFieldToType.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,22 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID
{
return static_cast<const DataTypeDateTime &>(type).getTimeZone().fromDayNum(DayNum(src.get<Int32>()));
}
else if (which_type.isDateTime64() && which_from_type.isDate())
{
const auto & date_time64_type = static_cast<const DataTypeDateTime64 &>(type);
const auto value = date_time64_type.getTimeZone().fromDayNum(DayNum(src.get<UInt16>()));
return DecimalField(
DecimalUtils::decimalFromComponentsWithMultiplier<DateTime64>(value, 0, date_time64_type.getScaleMultiplier()),
date_time64_type.getScale());
}
else if (which_type.isDateTime64() && which_from_type.isDate32())
{
const auto & date_time64_type = static_cast<const DataTypeDateTime64 &>(type);
const auto value = date_time64_type.getTimeZone().fromDayNum(ExtendedDayNum(static_cast<Int32>(src.get<Int32>())));
return DecimalField(
DecimalUtils::decimalFromComponentsWithMultiplier<DateTime64>(value, 0, date_time64_type.getScaleMultiplier()),
date_time64_type.getScale());
}
else if (type.isValueRepresentedByNumber() && src.getType() != Field::Types::String)
{
if (which_type.isUInt8()) return convertNumericType<UInt8>(src, type);
Expand Down
184 changes: 184 additions & 0 deletions src/Interpreters/tests/gtest_convertFieldToType.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
#include <initializer_list>
#include <limits>
#include <ostream>
#include <Core/Field.h>
#include <Core/iostream_debug_helpers.h>
#include <Interpreters/convertFieldToType.h>
#include <DataTypes/DataTypeFactory.h>

#include <gtest/gtest.h>
#include "base/Decimal.h"
#include "base/types.h"

using namespace DB;

struct ConvertFieldToTypeTestParams
{
const char * from_type; // MUST NOT BE NULL
const Field from_value;
const char * to_type; // MUST NOT BE NULL
const std::optional<Field> expected_value;
};

std::ostream & operator << (std::ostream & ostr, const ConvertFieldToTypeTestParams & params)
{
return ostr << "{"
<< "\n\tfrom_type : " << params.from_type
<< "\n\tfrom_value : " << params.from_value
<< "\n\tto_type : " << params.to_type
<< "\n\texpected : " << (params.expected_value ? *params.expected_value : Field())
<< "\n}";
}

class ConvertFieldToTypeTest : public ::testing::TestWithParam<ConvertFieldToTypeTestParams>
{};

TEST_P(ConvertFieldToTypeTest, convert)
{
const auto & params = GetParam();

ASSERT_NE(nullptr, params.from_type);
ASSERT_NE(nullptr, params.to_type);

const auto & type_factory = DataTypeFactory::instance();
const auto from_type = type_factory.get(params.from_type);
const auto to_type = type_factory.get(params.to_type);

if (params.expected_value)
{
const auto result = convertFieldToType(params.from_value, *to_type, from_type.get());
EXPECT_EQ(*params.expected_value, result);
}
else
{
EXPECT_ANY_THROW(convertFieldToType(params.from_value, *to_type, from_type.get()));
}
}

// Basically, the number of seconds in a day works for UTC here
const Int64 Day = 24 * 60 * 60;

// 123 is arbitrary value here

INSTANTIATE_TEST_SUITE_P(
DateToDateTime64,
ConvertFieldToTypeTest,
::testing::ValuesIn(std::initializer_list<ConvertFieldToTypeTestParams>{
// min value of Date
{
"Date",
Field(0),
"DateTime64(0, 'UTC')",
DecimalField(DateTime64(0), 0)
},
// Max value of Date
{
"Date",
Field(std::numeric_limits<DB::UInt16>::max()),
"DateTime64(0, 'UTC')",
DecimalField(DateTime64(std::numeric_limits<DB::UInt16>::max() * Day), 0)
},
// check that scale is respected
{
"Date",
Field(123),
"DateTime64(0, 'UTC')",
DecimalField(DateTime64(123 * Day), 0)
},
{
"Date",
Field(1),
"DateTime64(1, 'UTC')",
DecimalField(DateTime64(Day * 10), 1)
},
{
"Date",
Field(123),
"DateTime64(3, 'UTC')",
DecimalField(DateTime64(123 * Day * 1000), 3)
},
{
"Date",
Field(123),
"DateTime64(6, 'UTC')",
DecimalField(DateTime64(123 * Day * 1'000'000), 6)
},
})
);

INSTANTIATE_TEST_SUITE_P(
Date32ToDateTime64,
ConvertFieldToTypeTest,
::testing::ValuesIn(std::initializer_list<ConvertFieldToTypeTestParams>{
// min value of Date32: 1st Jan 1900 (see DATE_LUT_MIN_YEAR)
{
"Date32",
Field(-25'567),
"DateTime64(0, 'UTC')",
DecimalField(DateTime64(-25'567 * Day), 0)
},
// max value of Date32: 31 Dec 2299 (see DATE_LUT_MAX_YEAR)
{
"Date32",
Field(120'529),
"DateTime64(0, 'UTC')",
DecimalField(DateTime64(120'529 * Day), 0)
},
// check that scale is respected
{
"Date32",
Field(123),
"DateTime64(0, 'UTC')",
DecimalField(DateTime64(123 * Day), 0)
},
{
"Date32",
Field(123),
"DateTime64(1, 'UTC')",
DecimalField(DateTime64(123 * Day * 10), 1)
},
{
"Date32",
Field(123),
"DateTime64(3, 'UTC')",
DecimalField(DateTime64(123 * Day * 1000), 3)
},
{
"Date32",
Field(123),
"DateTime64(6, 'UTC')",
DecimalField(DateTime64(123 * Day * 1'000'000), 6)
}
})
);

INSTANTIATE_TEST_SUITE_P(
DateTimeToDateTime64,
ConvertFieldToTypeTest,
::testing::ValuesIn(std::initializer_list<ConvertFieldToTypeTestParams>{
{
"DateTime",
Field(1),
"DateTime64(0, 'UTC')",
DecimalField(DateTime64(1), 0)
},
{
"DateTime",
Field(1),
"DateTime64(1, 'UTC')",
DecimalField(DateTime64(1'0), 1)
},
{
"DateTime",
Field(123),
"DateTime64(3, 'UTC')",
DecimalField(DateTime64(123'000), 3)
},
{
"DateTime",
Field(123),
"DateTime64(6, 'UTC')",
DecimalField(DateTime64(123'000'000), 6)
},
})
);
2 changes: 2 additions & 0 deletions tests/queries/1_stateful/00178_query_datetime64_index.reference
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
2023-05-27 00:00:00.000
2023-05-27 00:00:00.000
9 changes: 9 additions & 0 deletions tests/queries/1_stateful/00178_query_datetime64_index.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
DROP TABLE IF EXISTS datetime64_index_tbl;

CREATE TABLE datetime64_index_tbl(ts DateTime64(3, 'UTC')) ENGINE=MergeTree ORDER BY ts;
INSERT INTO datetime64_index_tbl(ts) VALUES(toDateTime64('2023-05-27 00:00:00', 3, 'UTC'));

SELECT ts FROM datetime64_index_tbl WHERE ts < toDate('2023-05-28');
SELECT ts FROM datetime64_index_tbl WHERE ts < toDate32('2023-05-28');

DROP TABLE datetime64_index_tbl;