Skip to content

Commit

Permalink
make timestamp string format cassandra compatible
Browse files Browse the repository at this point in the history 
when we convert timestamp into string it must look like: '2017-12-27T11:57:42.500Z'
it concerns any conversion except JSON timestamp format
JSON string has space as time separator and must look like: '2017-12-27 11:57:42.500Z'
both formats always contain milliseconds and timezone specification

Fixes #14518
Fixes #7997

Closes #14726
  • Loading branch information
@alezzqz @denesb
alezzqz authored and denesb committed Jul 27, 2023
1 parent 1b7bde2 commit ff721ec
Show file tree
Hide file tree
Showing 9 changed files with 47 additions and 40 deletions.
2 changes: 2 additions & 0 deletions concrete_types.hh
View file
Expand Up @@ -118,6 +118,8 @@ struct date_type_impl final : public concrete_type<db_clock::time_point> {

using timestamp_date_base_class = concrete_type<db_clock::time_point>;

sstring timestamp_to_json_string(const timestamp_date_base_class& t, const bytes_view& bv);

struct timeuuid_type_impl final : public concrete_type<utils::UUID> {
timeuuid_type_impl();
static utils::UUID from_sstring(sstring_view s);
Expand Down
2 changes: 1 addition & 1 deletion cql3/type_json.cc
View file
Expand Up @@ -489,7 +489,7 @@ struct to_json_string_visitor {
sstring operator()(const string_type_impl& t) { return quote_json_string(t.to_string(bv)); }
sstring operator()(const bytes_type_impl& t) { return quote_json_string("0x" + t.to_string(bv)); }
sstring operator()(const boolean_type_impl& t) { return t.to_string(bv); }
sstring operator()(const timestamp_date_base_class& t) { return quote_json_string(t.to_string(bv)); }
sstring operator()(const timestamp_date_base_class& t) { return quote_json_string(timestamp_to_json_string(t, bv)); }
sstring operator()(const timeuuid_type_impl& t) { return quote_json_string(t.to_string(bv)); }
sstring operator()(const map_type_impl& t) { return to_json_string_aux(t, bv); }
sstring operator()(const set_type_impl& t) { return to_json_string_aux(t, bv); }
Expand Down
12 changes: 6 additions & 6 deletions test/boost/castas_fcts_test.cc
View file
Expand Up @@ -494,29 +494,29 @@ SEASTAR_TEST_CASE(test_time_casts_in_selection_clause) {
}
{
auto msg = e.execute_cql("SELECT CAST(CAST(a AS timestamp) AS text), CAST(CAST(a AS date) AS text), CAST(CAST(b as date) AS text), CAST(CAST(c AS timestamp) AS text) FROM test").get0();
assert_that(msg).is_rows().with_size(1).with_row({{utf8_type->from_string("2009-12-17T00:26:29.805000")},
assert_that(msg).is_rows().with_size(1).with_row({{utf8_type->from_string("2009-12-17T00:26:29.805Z")},
{utf8_type->from_string("2009-12-17")},
{utf8_type->from_string("2015-05-21")},
{utf8_type->from_string("2015-05-21T00:00:00")}});
{utf8_type->from_string("2015-05-21T00:00:00.000Z")}});
}
{
auto msg = e.execute_cql("SELECT CAST(a AS text), CAST(b as text), CAST(c AS text), CAST(d AS text) FROM test").get0();
assert_that(msg).is_rows().with_size(1).with_row({{utf8_type->from_string("d2177dd0-eaa2-11de-a572-001b779c76e3")},
{utf8_type->from_string("2015-05-21T11:03:02")},
{utf8_type->from_string("2015-05-21T11:03:02.000Z")},
{utf8_type->from_string("2015-05-21")},
{utf8_type->from_string("11:03:02.000000000")}});
}
{
auto msg = e.execute_cql("SELECT CAST(CAST(a AS timestamp) AS ascii), CAST(CAST(a AS date) AS ascii), CAST(CAST(b as date) AS ascii), CAST(CAST(c AS timestamp) AS ascii) FROM test").get0();
assert_that(msg).is_rows().with_size(1).with_row({{ascii_type->from_string("2009-12-17T00:26:29.805000")},
assert_that(msg).is_rows().with_size(1).with_row({{ascii_type->from_string("2009-12-17T00:26:29.805Z")},
{ascii_type->from_string("2009-12-17")},
{ascii_type->from_string("2015-05-21")},
{ascii_type->from_string("2015-05-21T00:00:00")}});
{ascii_type->from_string("2015-05-21T00:00:00.000Z")}});
}
{
auto msg = e.execute_cql("SELECT CAST(a AS ascii), CAST(b as ascii), CAST(c AS ascii), CAST(d AS ascii) FROM test").get0();
assert_that(msg).is_rows().with_size(1).with_row({{ascii_type->from_string("d2177dd0-eaa2-11de-a572-001b779c76e3")},
{ascii_type->from_string("2015-05-21T11:03:02")},
{ascii_type->from_string("2015-05-21T11:03:02.000Z")},
{ascii_type->from_string("2015-05-21")},
{ascii_type->from_string("11:03:02.000000000")}});
}
Expand Down
4 changes: 2 additions & 2 deletions test/boost/expr_test.cc
View file
Expand Up @@ -163,7 +163,7 @@ BOOST_AUTO_TEST_CASE(expr_printer_timestamp_test) {
raw_value::make_value(timestamp_type->from_string("2011-03-02T03:05:00+0000")),
timestamp_type
);
BOOST_REQUIRE_EQUAL(expr_print(timestamp_const), "'2011-03-02T03:05:00+0000'");
BOOST_REQUIRE_EQUAL(expr_print(timestamp_const), "'2011-03-02T03:05:00.000Z'");
}

BOOST_AUTO_TEST_CASE(expr_printer_time_test) {
Expand All @@ -179,7 +179,7 @@ BOOST_AUTO_TEST_CASE(expr_printer_date_test) {
raw_value::make_value(date_type->from_string("2011-02-03+0000")),
date_type
};
BOOST_REQUIRE_EQUAL(expr_print(date_const), "'2011-02-03T00:00:00+0000'");
BOOST_REQUIRE_EQUAL(expr_print(date_const), "'2011-02-03T00:00:00.000Z'");
}

BOOST_AUTO_TEST_CASE(expr_printer_duration_test) {
Expand Down
4 changes: 2 additions & 2 deletions test/boost/json_cql_query_test.cc
View file
Expand Up @@ -95,7 +95,7 @@ SEASTAR_TEST_CASE(test_select_json_types) {
"\"\\\"G\\\"\": \"127.0.0.1\", " // note the double quoting on case-sensitive column names
"\"\\\"H\\\"\": 3, "
"\"\\\"I\\\"\": \"zażółć gęślą jaźń\", "
"\"j\": \"2001-10-18T14:15:55.134000\", "
"\"j\": \"2001-10-18 14:15:55.134Z\", "
"\"k\": \"d2177dd0-eaa2-11de-a572-001b779c76e3\", "
"\"l\": \"d2177dd0-eaa2-11de-a572-001b779c76e3\", "
"\"m\": \"varchar\", "
Expand Down Expand Up @@ -127,7 +127,7 @@ SEASTAR_TEST_CASE(test_select_json_types) {
utf8_type->decompose("\"127.0.0.1\""),
utf8_type->decompose("3"),
utf8_type->decompose("\"zażółć gęślą jaźń\""),
utf8_type->decompose("\"2001-10-18T14:15:55.134000\""),
utf8_type->decompose("\"2001-10-18 14:15:55.134Z\""),
utf8_type->decompose("\"d2177dd0-eaa2-11de-a572-001b779c76e3\""),
utf8_type->decompose("\"d2177dd0-eaa2-11de-a572-001b779c76e3\""),
utf8_type->decompose("\"varchar\""),
Expand Down
16 changes: 8 additions & 8 deletions test/boost/types_test.cc
View file
Expand Up @@ -278,27 +278,27 @@ void test_timestamp_like_string_conversions(data_type timestamp_type) {
BOOST_REQUIRE(timestamp_type->equal(timestamp_type->from_string("2015-07-03T12:30:00+1230"), timestamp_type->decompose(tp)));
BOOST_REQUIRE(timestamp_type->equal(timestamp_type->from_string("2015-07-02T23:00-0100"), timestamp_type->decompose(tp)));

BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "2015-07-03T00:00:00");
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "2015-07-03T00:00:00.000Z");

// test fractional milliseconds
tp = db_clock::time_point(db_clock::duration(1435881600123));
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "2015-07-03T00:00:00.123000");
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "2015-07-03T00:00:00.123Z");

// test time_stamps around the unix epoch time
tp = db_clock::time_point(db_clock::duration(0));
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "1970-01-01T00:00:00");
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "1970-01-01T00:00:00.000Z");
tp = db_clock::time_point(db_clock::duration(456));
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "1970-01-01T00:00:00.456000");
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "1970-01-01T00:00:00.456Z");
tp = db_clock::time_point(db_clock::duration(-456));
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "1969-12-31T23:59:59.544000");
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "1969-12-31T23:59:59.544Z");

// test time_stamps around year 0
tp = db_clock::time_point(db_clock::duration(-62167219200000));
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "0000-01-01T00:00:00");
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "0000-01-01T00:00:00.000Z");
tp = db_clock::time_point(db_clock::duration(-62167219199211));
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "0000-01-01T00:00:00.789000");
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "0000-01-01T00:00:00.789Z");
tp = db_clock::time_point(db_clock::duration(-62167219200789));
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "-0001-12-31T23:59:59.211000");
BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "-0001-12-31T23:59:59.211Z");

auto now = time(nullptr);
::tm local_now;
Expand Down
1 change: 0 additions & 1 deletion test/cql-pytest/cassandra_tests/functions/cast_fcts_test.py
View file
Expand Up @@ -169,7 +169,6 @@ def testNoLossOfPrecisionForCastToDecimal(cql, test_keyspace):
assertRows(execute(cql, table, "SELECT CAST(bigint_clmn AS decimal), CAST(varint_clmn AS decimal) FROM %s"),
row(Decimal("9223372036854775807"), Decimal("1234567890123456789")))

@pytest.mark.xfail(reason="issue #14518")
def testTimeCastsInSelectionClause(cql, test_keyspace):
with create_table(cql, test_keyspace, "(a timeuuid primary key, b timestamp, c date, d time)") as table:
yearMonthDay = "2015-05-21"
Expand Down
11 changes: 10 additions & 1 deletion test/cql-pytest/test_json.py
View file
Expand Up @@ -18,6 +18,7 @@
import pytest
import json
from decimal import Decimal
from datetime import datetime

@pytest.fixture(scope="module")
def type1(cql, test_keyspace):
Expand All @@ -29,7 +30,7 @@ def type1(cql, test_keyspace):
@pytest.fixture(scope="module")
def table1(cql, test_keyspace, type1):
table = test_keyspace + "." + unique_name()
cql.execute(f"CREATE TABLE {table} (p int PRIMARY KEY, v int, bigv bigint, a ascii, b boolean, vi varint, mai map<ascii, int>, tup frozen<tuple<text, int>>, l list<text>, d double, t time, dec decimal, tupmap map<frozen<tuple<text, int>>, int>, t1 frozen<{type1}>, \"CaseSensitive\" int)")
cql.execute(f"CREATE TABLE {table} (p int PRIMARY KEY, v int, bigv bigint, a ascii, b boolean, vi varint, mai map<ascii, int>, tup frozen<tuple<text, int>>, l list<text>, d double, t time, dec decimal, tupmap map<frozen<tuple<text, int>>, int>, t1 frozen<{type1}>, \"CaseSensitive\" int, ts timestamp)")
yield table
cql.execute("DROP TABLE " + table)

Expand Down Expand Up @@ -310,6 +311,14 @@ def test_tojson_time(cql, table1):
cql.execute(stmt, [p, 123])
assert list(cql.execute(f"SELECT toJson(t) from {table1} where p = {p}")) == [('"00:00:00.000000123"',)]

# Check that toJson() returns timestamp string in correct cassandra compatible format (issue #7997)
# with milliseconds and timezone specification
def test_tojson_timestamp(cql, table1):
p = unique_key_int()
stmt = cql.prepare(f"INSERT INTO {table1} (p, ts) VALUES (?, ?)")
cql.execute(stmt, [p, datetime(2014, 1, 1, 12, 15, 45)])
assert list(cql.execute(f"SELECT toJson(ts) from {table1} where p = {p}")) == [('"2014-01-01 12:15:45.000Z"',)]

# The EquivalentJson class wraps a JSON string, and compare equal to other
# strings if both are valid JSON strings which decode to the same object.
# EquivalentJson("....") can be used in assert_rows() checks below, to check
Expand Down
35 changes: 16 additions & 19 deletions types/types.cc
View file
Expand Up @@ -68,7 +68,7 @@ requires requires {
requires std::same_as<typename T::duration, std::chrono::milliseconds>;
}
sstring
time_point_to_string(const T& tp)
time_point_to_string(const T& tp, bool use_time_separator = true)
{
auto count = tp.time_since_epoch().count();
auto d = std::div(int64_t(count), int64_t(1000));
Expand All @@ -78,26 +78,21 @@ time_point_to_string(const T& tp)
return fmt::format("{} milliseconds (out of range)", count);
}

auto to_string = [] (const std::tm& tm) {
auto year_digits = tm.tm_year >= -1900 ? 4 : 5;
return fmt::format("{:-0{}d}-{:02d}-{:02d}T{:02d}:{:02d}:{:02d}",
tm.tm_year + 1900, year_digits, tm.tm_mon + 1, tm.tm_mday,
tm.tm_hour, tm.tm_min, tm.tm_sec);
};

auto millis = d.rem;
if (!millis) {
return fmt::format("{}", to_string(tm));
}
// adjust seconds for time points earlier than posix epoch
// to keep the fractional millis positive
if (millis < 0) {
millis += 1000;
seconds--;
gmtime_r(&seconds, &tm);
}
auto micros = millis * 1000;
return fmt::format("{}.{:06d}", to_string(tm), micros);

const auto time_separator = (use_time_separator) ? "T" : " ";
auto year_digits = tm.tm_year >= -1900 ? 4 : 5;

return fmt::format("{:-0{}d}-{:02d}-{:02d}{}{:02d}:{:02d}:{:02d}.{:03d}Z",
tm.tm_year + 1900, year_digits, tm.tm_mon + 1, tm.tm_mday, time_separator,
tm.tm_hour, tm.tm_min, tm.tm_sec, millis);
}

sstring simple_date_to_string(const uint32_t days_count) {
Expand Down Expand Up @@ -2853,6 +2848,12 @@ static sstring format_if_not_empty(
return f(static_cast<const N&>(*b));
}

sstring timestamp_to_json_string(const timestamp_date_base_class& t, const bytes_view& bv)
{
auto tp = value_cast<const timestamp_date_base_class::native_type>(t.deserialize(bv));
return format_if_not_empty(t, &tp, [](const db_clock::time_point& v) { return time_point_to_string(v, false); });
}

static sstring to_string_impl(const abstract_type& t, const void* v);

namespace {
Expand Down Expand Up @@ -3629,16 +3630,12 @@ sstring data_value::to_parsable_string() const {

abstract_type::kind type_kind = _type->without_reversed().get_kind();

if (type_kind == abstract_type::kind::date || type_kind == abstract_type::kind::timestamp) {
// Put timezone information after a date or timestamp to specify that it's in UTC
// Otherwise it will be parsed as a date in the local timezone.
return fmt::format("'{}+0000'", *this);
}

if (type_kind == abstract_type::kind::utf8
|| type_kind == abstract_type::kind::ascii
|| type_kind == abstract_type::kind::inet
|| type_kind == abstract_type::kind::time
|| type_kind == abstract_type::kind::date
|| type_kind == abstract_type::kind::timestamp
) {
// Put quotes on types that require it
return fmt::format("'{}'", *this);
Expand Down

0 comments on commit ff721ec

Please sign in to comment.