From 282297b677d0650b5d268f895d1eaa5c233e3eb1 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Tue, 16 May 2023 23:46:01 +0000 Subject: [PATCH 1/4] binary encoding of IPv6 in protobuf --- src/Formats/ProtobufSerializer.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Formats/ProtobufSerializer.cpp b/src/Formats/ProtobufSerializer.cpp index 8bdef0e7d3fe..ee33d2d99915 100644 --- a/src/Formats/ProtobufSerializer.cpp +++ b/src/Formats/ProtobufSerializer.cpp @@ -1,4 +1,5 @@ #include +#include "Common/formatIPv6.h" #if USE_PROTOBUF # include @@ -1852,25 +1853,26 @@ namespace write_function = [this](IPv6 value) { - ipToString(value, text_buffer); + text_buffer = String(IPV6_BINARY_LENGTH, '\0'); + memcpy(text_buffer.data(), &value.toUnderType(), IPV6_BINARY_LENGTH); writeStr(text_buffer); }; read_function = [this]() -> IPv6 { readStr(text_buffer); - return parse(text_buffer); + if (text_buffer.size() != IPV6_BINARY_LENGTH) + throw Exception(ErrorCodes::PROTOBUF_BAD_CAST, + "Could not convert bytes field {} to IPv6 for inserting into column {} - field size {} is not equal to IPv6 size {}", + field_descriptor.full_name(), column_name, text_buffer.size(), IPV6_BINARY_LENGTH); + IPv6 value; + memcpy(&value.toUnderType(), text_buffer.data(), IPV6_BINARY_LENGTH); + return value; }; default_function = [this]() -> IPv6 { return parse(field_descriptor.default_value_string()); }; } - static void ipToString(const IPv6 & ip, String & str) - { - WriteBufferFromString buf{str}; - writeText(ip, buf); - } - std::function write_function; std::function read_function; std::function default_function; From 0a44a69dc82bac8f490399b8197c6220d673e1d3 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Wed, 17 May 2023 00:22:13 +0000 Subject: [PATCH 2/4] remove unnecessary header --- src/Formats/ProtobufSerializer.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Formats/ProtobufSerializer.cpp b/src/Formats/ProtobufSerializer.cpp index ee33d2d99915..f690800d145c 100644 --- a/src/Formats/ProtobufSerializer.cpp +++ b/src/Formats/ProtobufSerializer.cpp @@ -1,5 +1,4 @@ #include -#include "Common/formatIPv6.h" #if USE_PROTOBUF # include From 612b79868b6a5e9f8c91ddf0caea6f72e97c0320 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Wed, 17 May 2023 20:40:51 +0000 Subject: [PATCH 3/4] test added --- .../0_stateless/02751_protobuf_ipv6.reference | 2 ++ tests/queries/0_stateless/02751_protobuf_ipv6.sh | 14 ++++++++++++++ .../format_schemas/02751_protobuf_ipv6.proto | 6 ++++++ 3 files changed, 22 insertions(+) create mode 100644 tests/queries/0_stateless/02751_protobuf_ipv6.reference create mode 100755 tests/queries/0_stateless/02751_protobuf_ipv6.sh create mode 100644 tests/queries/0_stateless/format_schemas/02751_protobuf_ipv6.proto diff --git a/tests/queries/0_stateless/02751_protobuf_ipv6.reference b/tests/queries/0_stateless/02751_protobuf_ipv6.reference new file mode 100644 index 000000000000..0318b49c77e1 --- /dev/null +++ b/tests/queries/0_stateless/02751_protobuf_ipv6.reference @@ -0,0 +1,2 @@ +::ffff:1.2.3.4 +::ffff:1.2.3.4 diff --git a/tests/queries/0_stateless/02751_protobuf_ipv6.sh b/tests/queries/0_stateless/02751_protobuf_ipv6.sh new file mode 100755 index 000000000000..ecf565d9db48 --- /dev/null +++ b/tests/queries/0_stateless/02751_protobuf_ipv6.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +SCHEMADIR=$CURDIR/format_schemas + + +echo 121a1000000000000000000000ffff01020304 | xxd -r -p | $CLICKHOUSE_LOCAL --input-format Protobuf --format_schema="$SCHEMADIR/02751_protobuf_ipv6:Message" --structure="ipv6_bytes IPv6" -q "select * from table" + +$CLICKHOUSE_LOCAL -q "select '::ffff:1.2.3.4'::IPv6 as ipv6_bytes format Protobuf settings format_schema = '$SCHEMADIR/02751_protobuf_ipv6:Message'" | $CLICKHOUSE_LOCAL --input-format Protobuf --format_schema="$SCHEMADIR/02751_protobuf_ipv6:Message" --structure="ipv6_bytes IPv6" -q "select * from table" + diff --git a/tests/queries/0_stateless/format_schemas/02751_protobuf_ipv6.proto b/tests/queries/0_stateless/format_schemas/02751_protobuf_ipv6.proto new file mode 100644 index 000000000000..8e6f115f2d79 --- /dev/null +++ b/tests/queries/0_stateless/format_schemas/02751_protobuf_ipv6.proto @@ -0,0 +1,6 @@ +syntax = "proto3"; + +message Message +{ + bytes ipv6_bytes = 3; +} From 30083351f5769d781a2dacccc8c259e846518956 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Thu, 18 May 2023 14:42:48 +0000 Subject: [PATCH 4/4] test fix --- tests/queries/0_stateless/02751_protobuf_ipv6.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02751_protobuf_ipv6.sh b/tests/queries/0_stateless/02751_protobuf_ipv6.sh index ecf565d9db48..f93963aa6c67 100755 --- a/tests/queries/0_stateless/02751_protobuf_ipv6.sh +++ b/tests/queries/0_stateless/02751_protobuf_ipv6.sh @@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) SCHEMADIR=$CURDIR/format_schemas -echo 121a1000000000000000000000ffff01020304 | xxd -r -p | $CLICKHOUSE_LOCAL --input-format Protobuf --format_schema="$SCHEMADIR/02751_protobuf_ipv6:Message" --structure="ipv6_bytes IPv6" -q "select * from table" +echo -ne '\x12\x1a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\x01\x02\x03\x04' | $CLICKHOUSE_LOCAL --input-format Protobuf --format_schema="$SCHEMADIR/02751_protobuf_ipv6:Message" --structure="ipv6_bytes IPv6" -q "select * from table" $CLICKHOUSE_LOCAL -q "select '::ffff:1.2.3.4'::IPv6 as ipv6_bytes format Protobuf settings format_schema = '$SCHEMADIR/02751_protobuf_ipv6:Message'" | $CLICKHOUSE_LOCAL --input-format Protobuf --format_schema="$SCHEMADIR/02751_protobuf_ipv6:Message" --structure="ipv6_bytes IPv6" -q "select * from table"