make timestamp string format cassandra compatible

when we convert timestamp into string it must look like: 2017-12-27T11:57:42.500Z
it concerns any conversion and affects JSON timestamp format as well - always contains milliseconds and timezone specification

Fixes scylladb#14518
Fixes scylladb#7997

test/boost/castas_fcts_test.cc

@@ -494,29 +494,29 @@ SEASTAR_TEST_CASE(test_time_casts_in_selection_clause) {
         }
         {
             auto msg = e.execute_cql("SELECT CAST(CAST(a AS timestamp) AS text), CAST(CAST(a AS date) AS text), CAST(CAST(b as date) AS text), CAST(CAST(c AS timestamp) AS text) FROM test").get0();
-            assert_that(msg).is_rows().with_size(1).with_row({{utf8_type->from_string("2009-12-17T00:26:29.805000")},
+            assert_that(msg).is_rows().with_size(1).with_row({{utf8_type->from_string("2009-12-17T00:26:29.805Z")},
                                                               {utf8_type->from_string("2009-12-17")},
                                                               {utf8_type->from_string("2015-05-21")},
-                                                              {utf8_type->from_string("2015-05-21T00:00:00")}});
+                                                              {utf8_type->from_string("2015-05-21T00:00:00.000Z")}});
         }
         {
             auto msg = e.execute_cql("SELECT CAST(a AS text), CAST(b as text), CAST(c AS text), CAST(d AS text) FROM test").get0();
             assert_that(msg).is_rows().with_size(1).with_row({{utf8_type->from_string("d2177dd0-eaa2-11de-a572-001b779c76e3")},
-                                                              {utf8_type->from_string("2015-05-21T11:03:02")},
+                                                              {utf8_type->from_string("2015-05-21T11:03:02.000Z")},
                                                               {utf8_type->from_string("2015-05-21")},
                                                               {utf8_type->from_string("11:03:02.000000000")}});
         }
         {
             auto msg = e.execute_cql("SELECT CAST(CAST(a AS timestamp) AS ascii), CAST(CAST(a AS date) AS ascii), CAST(CAST(b as date) AS ascii), CAST(CAST(c AS timestamp) AS ascii) FROM test").get0();
-            assert_that(msg).is_rows().with_size(1).with_row({{ascii_type->from_string("2009-12-17T00:26:29.805000")},
+            assert_that(msg).is_rows().with_size(1).with_row({{ascii_type->from_string("2009-12-17T00:26:29.805Z")},
                                                               {ascii_type->from_string("2009-12-17")},
                                                               {ascii_type->from_string("2015-05-21")},
-                                                              {ascii_type->from_string("2015-05-21T00:00:00")}});
+                                                              {ascii_type->from_string("2015-05-21T00:00:00.000Z")}});
         }
         {
             auto msg = e.execute_cql("SELECT CAST(a AS ascii), CAST(b as ascii), CAST(c AS ascii), CAST(d AS ascii) FROM test").get0();
             assert_that(msg).is_rows().with_size(1).with_row({{ascii_type->from_string("d2177dd0-eaa2-11de-a572-001b779c76e3")},
-                                                              {ascii_type->from_string("2015-05-21T11:03:02")},
+                                                              {ascii_type->from_string("2015-05-21T11:03:02.000Z")},
                                                               {ascii_type->from_string("2015-05-21")},
                                                               {ascii_type->from_string("11:03:02.000000000")}});
         }

test/boost/expr_test.cc

@@ -163,7 +163,7 @@ BOOST_AUTO_TEST_CASE(expr_printer_timestamp_test) {
         raw_value::make_value(timestamp_type->from_string("2011-03-02T03:05:00+0000")),
         timestamp_type
     );
-    BOOST_REQUIRE_EQUAL(expr_print(timestamp_const), "'2011-03-02T03:05:00+0000'");
+    BOOST_REQUIRE_EQUAL(expr_print(timestamp_const), "'2011-03-02T03:05:00.000Z'");
 }
 
 BOOST_AUTO_TEST_CASE(expr_printer_time_test) {
@@ -179,7 +179,7 @@ BOOST_AUTO_TEST_CASE(expr_printer_date_test) {
         raw_value::make_value(date_type->from_string("2011-02-03+0000")),
         date_type
     };
-    BOOST_REQUIRE_EQUAL(expr_print(date_const), "'2011-02-03T00:00:00+0000'");
+    BOOST_REQUIRE_EQUAL(expr_print(date_const), "'2011-02-03T00:00:00.000Z'");
 }
 
 BOOST_AUTO_TEST_CASE(expr_printer_duration_test) {

test/boost/json_cql_query_test.cc

@@ -95,7 +95,7 @@ SEASTAR_TEST_CASE(test_select_json_types) {
                     "\"\\\"G\\\"\": \"127.0.0.1\", " // note the double quoting on case-sensitive column names
                     "\"\\\"H\\\"\": 3, "
                     "\"\\\"I\\\"\": \"zażółć gęślą jaźń\", "
-                    "\"j\": \"2001-10-18T14:15:55.134000\", "
+                    "\"j\": \"2001-10-18T14:15:55.134Z\", "
                     "\"k\": \"d2177dd0-eaa2-11de-a572-001b779c76e3\", "
                     "\"l\": \"d2177dd0-eaa2-11de-a572-001b779c76e3\", "
                     "\"m\": \"varchar\", "
@@ -127,7 +127,7 @@ SEASTAR_TEST_CASE(test_select_json_types) {
                 utf8_type->decompose("\"127.0.0.1\""),
                 utf8_type->decompose("3"),
                 utf8_type->decompose("\"zażółć gęślą jaźń\""),
-                utf8_type->decompose("\"2001-10-18T14:15:55.134000\""),
+                utf8_type->decompose("\"2001-10-18T14:15:55.134Z\""),
                 utf8_type->decompose("\"d2177dd0-eaa2-11de-a572-001b779c76e3\""),
                 utf8_type->decompose("\"d2177dd0-eaa2-11de-a572-001b779c76e3\""),
                 utf8_type->decompose("\"varchar\""),

test/boost/types_test.cc

@@ -278,27 +278,27 @@ void test_timestamp_like_string_conversions(data_type timestamp_type) {
     BOOST_REQUIRE(timestamp_type->equal(timestamp_type->from_string("2015-07-03T12:30:00+1230"), timestamp_type->decompose(tp)));
     BOOST_REQUIRE(timestamp_type->equal(timestamp_type->from_string("2015-07-02T23:00-0100"), timestamp_type->decompose(tp)));
 
-    BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "2015-07-03T00:00:00");
+    BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "2015-07-03T00:00:00.000Z");
 
     // test fractional milliseconds
     tp = db_clock::time_point(db_clock::duration(1435881600123));
-    BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "2015-07-03T00:00:00.123000");
+    BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "2015-07-03T00:00:00.123Z");
 
     // test time_stamps around the unix epoch time
     tp = db_clock::time_point(db_clock::duration(0));
-    BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "1970-01-01T00:00:00");
+    BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "1970-01-01T00:00:00.000Z");
     tp = db_clock::time_point(db_clock::duration(456));
-    BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "1970-01-01T00:00:00.456000");
+    BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "1970-01-01T00:00:00.456Z");
     tp = db_clock::time_point(db_clock::duration(-456));
-    BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "1969-12-31T23:59:59.544000");
+    BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "1969-12-31T23:59:59.544Z");
 
     // test time_stamps around year 0
     tp = db_clock::time_point(db_clock::duration(-62167219200000));
-    BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "0000-01-01T00:00:00");
+    BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "0000-01-01T00:00:00.000Z");
     tp = db_clock::time_point(db_clock::duration(-62167219199211));
-    BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "0000-01-01T00:00:00.789000");
+    BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "0000-01-01T00:00:00.789Z");
     tp = db_clock::time_point(db_clock::duration(-62167219200789));
-    BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "-0001-12-31T23:59:59.211000");
+    BOOST_REQUIRE_EQUAL(timestamp_type->to_string(timestamp_type->decompose(tp)), "-0001-12-31T23:59:59.211Z");
 
     auto now = time(nullptr);
     ::tm local_now;

test/cql-pytest/cassandra_tests/functions/cast_fcts_test.py

@@ -169,7 +169,6 @@ def testNoLossOfPrecisionForCastToDecimal(cql, test_keyspace):
         assertRows(execute(cql, table, "SELECT CAST(bigint_clmn AS decimal), CAST(varint_clmn AS decimal) FROM %s"),
                    row(Decimal("9223372036854775807"), Decimal("1234567890123456789")))
 
-@pytest.mark.xfail(reason="issue #14518")
 def testTimeCastsInSelectionClause(cql, test_keyspace):
     with create_table(cql, test_keyspace, "(a timeuuid primary key, b timestamp, c date, d time)") as table:
         yearMonthDay = "2015-05-21"

test/cql-pytest/test_json.py

@@ -18,6 +18,7 @@
 import pytest
 import json
 from decimal import Decimal
+from datetime import datetime
 
 @pytest.fixture(scope="module")
 def type1(cql, test_keyspace):
@@ -29,7 +30,7 @@ def type1(cql, test_keyspace):
 @pytest.fixture(scope="module")
 def table1(cql, test_keyspace, type1):
     table = test_keyspace + "." + unique_name()
-    cql.execute(f"CREATE TABLE {table} (p int PRIMARY KEY, v int, bigv bigint, a ascii, b boolean, vi varint, mai map<ascii, int>, tup frozen<tuple<text, int>>, l list<text>, d double, t time, dec decimal, tupmap map<frozen<tuple<text, int>>, int>, t1 frozen<{type1}>, \"CaseSensitive\" int)")
+    cql.execute(f"CREATE TABLE {table} (p int PRIMARY KEY, v int, bigv bigint, a ascii, b boolean, vi varint, mai map<ascii, int>, tup frozen<tuple<text, int>>, l list<text>, d double, t time, dec decimal, tupmap map<frozen<tuple<text, int>>, int>, t1 frozen<{type1}>, \"CaseSensitive\" int, ts timestamp)")
     yield table
     cql.execute("DROP TABLE " + table)
 
@@ -310,6 +311,14 @@ def test_tojson_time(cql, table1):
     cql.execute(stmt, [p, 123])
     assert list(cql.execute(f"SELECT toJson(t) from {table1} where p = {p}")) == [('"00:00:00.000000123"',)]
 
+# Check that toJson() returns timestamp string in correct cassandra compatible format (issue #7997)
+# with milliseconds and timezone specification
+def test_tojson_timestamp(cql, table1):
+    p = unique_key_int()
+    stmt = cql.prepare(f"INSERT INTO {table1} (p, ts) VALUES (?, ?)")
+    cql.execute(stmt, [p, datetime(2014, 1, 1, 12, 15, 45)])
+    assert list(cql.execute(f"SELECT toJson(ts) from {table1} where p = {p}")) == [('"2014-01-01T12:15:45.000Z"',)]
+
 # The EquivalentJson class wraps a JSON string, and compare equal to other
 # strings if both are valid JSON strings which decode to the same object.
 # EquivalentJson("....") can be used in assert_rows() checks below, to check

types/types.cc

@@ -78,26 +78,19 @@ time_point_to_string(const T& tp)
         return fmt::format("{} milliseconds (out of range)", count);
     }
 
-    auto to_string = [] (const std::tm& tm) {
-        auto year_digits = tm.tm_year >= -1900 ? 4 : 5;
-        return fmt::format("{:-0{}d}-{:02d}-{:02d}T{:02d}:{:02d}:{:02d}",
-                tm.tm_year + 1900, year_digits, tm.tm_mon + 1, tm.tm_mday,
-                tm.tm_hour, tm.tm_min, tm.tm_sec);
-    };
-
     auto millis = d.rem;
-    if (!millis) {
-        return fmt::format("{}", to_string(tm));
-    }
     // adjust seconds for time points earlier than posix epoch
     // to keep the fractional millis positive
     if (millis < 0) {
         millis += 1000;
         seconds--;
         gmtime_r(&seconds, &tm);
     }
-    auto micros = millis * 1000;
-    return fmt::format("{}.{:06d}", to_string(tm), micros);
+
+    auto year_digits = tm.tm_year >= -1900 ? 4 : 5;
+    return fmt::format("{:-0{}d}-{:02d}-{:02d}T{:02d}:{:02d}:{:02d}.{:03d}Z",
+            tm.tm_year + 1900, year_digits, tm.tm_mon + 1, tm.tm_mday,
+            tm.tm_hour, tm.tm_min, tm.tm_sec, millis);
 }
 
 sstring simple_date_to_string(const uint32_t days_count) {
@@ -3629,16 +3622,12 @@ sstring data_value::to_parsable_string() const {
 
     abstract_type::kind type_kind = _type->without_reversed().get_kind();
 
-    if (type_kind == abstract_type::kind::date || type_kind == abstract_type::kind::timestamp) {
-        // Put timezone information after a date or timestamp to specify that it's in UTC
-        // Otherwise it will be parsed as a date in the local timezone.
-        return fmt::format("'{}+0000'", *this);
-    }
-
     if (type_kind == abstract_type::kind::utf8
         || type_kind == abstract_type::kind::ascii
         || type_kind == abstract_type::kind::inet
         || type_kind == abstract_type::kind::time
+        || type_kind == abstract_type::kind::date
+        || type_kind == abstract_type::kind::timestamp
     ) {
         // Put quotes on types that require it
         return fmt::format("'{}'", *this);