Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow to cast IPv6 to IPv4 for address in proper mapping block #49759

Merged
merged 8 commits into from Jun 24, 2023
Merged

Allow to cast IPv6 to IPv4 for address in proper mapping block #49759

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
58bdcc2
allow to cast IPv6 to IPv4 for address in proper mapping block
yakov-olkhovskiy May 10, 2023
108e256
allow to cast IPv4 to IPv6
yakov-olkhovskiy May 11, 2023
24067ea
allow conversion for toIPv4OrDefault
yakov-olkhovskiy May 12, 2023
fc857aa
tests added
yakov-olkhovskiy May 15, 2023
9443457
Merge branch 'master' into cast-ipv6-to-ipv4
yakov-olkhovskiy May 18, 2023
0d4ed32
better exception message
yakov-olkhovskiy May 31, 2023
3af7e0a
better exception message
yakov-olkhovskiy May 31, 2023
09806bc
Merge branch 'master' into cast-ipv6-to-ipv4
yakov-olkhovskiy Jun 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
81 changes: 81 additions & 0 deletions src/Functions/FunctionsCodingIP.h
View file
Open in desktop
Expand Up @@ -2,6 +2,7 @@

#include <type_traits>
#include <Common/formatIPv6.h>
#include <Common/IPv6ToBinary.h>

#include <Columns/ColumnFixedString.h>
#include <Columns/ColumnNullable.h>
Expand All @@ -16,6 +17,7 @@ namespace ErrorCodes
extern const int CANNOT_PARSE_IPV4;
extern const int CANNOT_PARSE_IPV6;
extern const int ILLEGAL_COLUMN;
extern const int CANNOT_CONVERT_TYPE;
}

enum class IPStringToNumExceptionMode : uint8_t
Expand Down Expand Up @@ -296,4 +298,83 @@ ColumnPtr convertToIPv4(ColumnPtr column, const PaddedPODArray<UInt8> * null_map
return col_res;
}

template <IPStringToNumExceptionMode exception_mode, typename ToColumn = ColumnIPv4>
ColumnPtr convertIPv6ToIPv4(ColumnPtr column, const PaddedPODArray<UInt8> * null_map = nullptr)
{
const ColumnIPv6 * column_ipv6 = checkAndGetColumn<ColumnIPv6>(column.get());

if (!column_ipv6)
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column type {}. Expected IPv6.", column->getName());

size_t column_size = column_ipv6->size();

ColumnUInt8::MutablePtr col_null_map_to;
ColumnUInt8::Container * vec_null_map_to = nullptr;

if constexpr (exception_mode == IPStringToNumExceptionMode::Null)
{
col_null_map_to = ColumnUInt8::create(column_size, false);
vec_null_map_to = &col_null_map_to->getData();
}

const uint8_t ip4_cidr[] {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00};

auto col_res = ToColumn::create();
auto & vec_res = col_res->getData();
vec_res.resize(column_size);
const auto & vec_src = column_ipv6->getData();

for (size_t i = 0; i < vec_res.size(); ++i)
{
const uint8_t * src = reinterpret_cast<const uint8_t *>(&vec_src[i]);
uint8_t * dst = reinterpret_cast<uint8_t *>(&vec_res[i]);

if (null_map && (*null_map)[i])
{
std::memset(dst, '\0', IPV4_BINARY_LENGTH);
if constexpr (exception_mode == IPStringToNumExceptionMode::Null)
(*vec_null_map_to)[i] = true;
continue;
}

if (!matchIPv6Subnet(src, ip4_cidr, 96))
{
if constexpr (exception_mode == IPStringToNumExceptionMode::Throw)
{
throw Exception(ErrorCodes::CANNOT_CONVERT_TYPE, "IPv6 in column {} is not in IPv4 mapping block", column->getName());
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe include vec_src[i] in the error message?

}
else if constexpr (exception_mode == IPStringToNumExceptionMode::Default)
{
std::memset(dst, '\0', IPV4_BINARY_LENGTH);
}
else if constexpr (exception_mode == IPStringToNumExceptionMode::Null)
{
(*vec_null_map_to)[i] = true;
std::memset(dst, '\0', IPV4_BINARY_LENGTH);
}
continue;
}

if constexpr (std::endian::native == std::endian::little)
{
dst[0] = src[15];
dst[1] = src[14];
dst[2] = src[13];
dst[3] = src[12];
}
else
{
dst[0] = src[12];
dst[1] = src[13];
dst[2] = src[14];
dst[3] = src[15];
}
}

if constexpr (exception_mode == IPStringToNumExceptionMode::Null)
return ColumnNullable::create(std::move(col_res), std::move(col_null_map_to));

return col_res;
}

}
72 changes: 69 additions & 3 deletions src/Functions/FunctionsConversion.h
View file
Open in desktop
Expand Up @@ -57,6 +57,7 @@
#include <Interpreters/Context.h>
#include <Common/HashTable/HashMap.h>
#include <DataTypes/DataTypeIPv4andIPv6.h>
#include <Common/IPv6ToBinary.h>
#include <Core/Types.h>


Expand Down Expand Up @@ -217,13 +218,13 @@ struct ConvertImpl
}
else if constexpr (
(std::is_same_v<FromDataType, DataTypeIPv4> != std::is_same_v<ToDataType, DataTypeIPv4>)
&& !(is_any_of<FromDataType, DataTypeUInt8, DataTypeUInt16, DataTypeUInt32, DataTypeUInt64> || is_any_of<ToDataType, DataTypeUInt32, DataTypeUInt64, DataTypeUInt128, DataTypeUInt256>)
&& !(is_any_of<FromDataType, DataTypeUInt8, DataTypeUInt16, DataTypeUInt32, DataTypeUInt64, DataTypeIPv6> || is_any_of<ToDataType, DataTypeUInt32, DataTypeUInt64, DataTypeUInt128, DataTypeUInt256, DataTypeIPv6>)
)
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Conversion from {} to {} is not supported",
TypeName<typename FromDataType::FieldType>, TypeName<typename ToDataType::FieldType>);
}
else if constexpr (std::is_same_v<FromDataType, DataTypeIPv6> != std::is_same_v<ToDataType, DataTypeIPv6>)
else if constexpr (std::is_same_v<FromDataType, DataTypeIPv6> != std::is_same_v<ToDataType, DataTypeIPv6> && !(std::is_same_v<ToDataType, DataTypeIPv4> || std::is_same_v<FromDataType, DataTypeIPv4>))
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED,
"Conversion between numeric types and IPv6 is not supported. "
Expand Down Expand Up @@ -304,7 +305,52 @@ struct ConvertImpl
}
else
{
if constexpr (std::is_same_v<ToDataType, DataTypeIPv4> && std::is_same_v<FromDataType, DataTypeUInt64>)
if constexpr (std::is_same_v<ToDataType, DataTypeIPv4> && std::is_same_v<FromDataType, DataTypeIPv6>)
{
const uint8_t ip4_cidr[] {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00};
const uint8_t * src = reinterpret_cast<const uint8_t *>(&vec_from[i].toUnderType());
if (!matchIPv6Subnet(src, ip4_cidr, 96))
throw Exception(ErrorCodes::CANNOT_CONVERT_TYPE, "IPv6 in column {} is not in IPv4 mapping block", named_from.column->getName());

uint8_t * dst = reinterpret_cast<uint8_t *>(&vec_to[i].toUnderType());
if constexpr (std::endian::native == std::endian::little)
{
dst[0] = src[15];
dst[1] = src[14];
dst[2] = src[13];
dst[3] = src[12];
}
else
{
dst[0] = src[12];
dst[1] = src[13];
dst[2] = src[14];
dst[3] = src[15];
}
}
else if constexpr (std::is_same_v<ToDataType, DataTypeIPv6> && std::is_same_v<FromDataType, DataTypeIPv4>)
{
const uint8_t * src = reinterpret_cast<const uint8_t *>(&vec_from[i].toUnderType());
uint8_t * dst = reinterpret_cast<uint8_t *>(&vec_to[i].toUnderType());
std::memset(dst, '\0', IPV6_BINARY_LENGTH);
dst[10] = dst[11] = 0xff;

if constexpr (std::endian::native == std::endian::little)
{
dst[12] = src[3];
dst[13] = src[2];
dst[14] = src[1];
dst[15] = src[0];
}
else
{
dst[12] = src[0];
dst[13] = src[1];
dst[14] = src[2];
dst[15] = src[3];
}
}
else if constexpr (std::is_same_v<ToDataType, DataTypeIPv4> && std::is_same_v<FromDataType, DataTypeUInt64>)
vec_to[i] = static_cast<ToFieldType>(static_cast<IPv4::UnderlyingType>(vec_from[i]));
else
vec_to[i] = static_cast<ToFieldType>(vec_from[i]);
Expand Down Expand Up @@ -3989,6 +4035,26 @@ class FunctionCast final : public FunctionCastBase
return true;
}
}
else if constexpr (WhichDataType(FromDataType::type_id).isIPv6() && WhichDataType(ToDataType::type_id).isIPv4())
{
ret = [cast_ipv4_ipv6_default_on_conversion_error_value, requested_result_is_nullable](
ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const ColumnNullable * column_nullable, size_t)
-> ColumnPtr
{
if (!WhichDataType(result_type).isIPv4())
throw Exception(
ErrorCodes::TYPE_MISMATCH, "Wrong result type {}. Expected IPv4", result_type->getName());

const auto * null_map = column_nullable ? &column_nullable->getNullMapData() : nullptr;
if (cast_ipv4_ipv6_default_on_conversion_error_value || requested_result_is_nullable)
return convertIPv6ToIPv4<IPStringToNumExceptionMode::Default>(arguments[0].column, null_map);
else
return convertIPv6ToIPv4<IPStringToNumExceptionMode::Throw>(arguments[0].column, null_map);
};

return true;
}

if constexpr (WhichDataType(ToDataType::type_id).isStringOrFixedString())
{
if (from_type->getCustomSerialization())
Expand Down
4 changes: 4 additions & 0 deletions tests/queries/0_stateless/02234_cast_to_ip_address.reference
View file
Open in desktop
Expand Up @@ -11,6 +11,10 @@ IPv4 functions
127.0.0.1
127.0.0.1
--
1.2.3.4
1.2.3.4
0.0.0.0
--
127.0.0.1
--
0
Expand Down
7 changes: 7 additions & 0 deletions tests/queries/0_stateless/02234_cast_to_ip_address.sql
View file
Open in desktop
Expand Up @@ -20,6 +20,13 @@ SELECT toIPv4OrNull('127.0.0.1');

SELECT '--';

SELECT toIPv4(toIPv6('::ffff:1.2.3.4'));
SELECT toIPv4(toIPv6('::afff:1.2.3.4')); --{serverError CANNOT_CONVERT_TYPE}
SELECT toIPv4OrDefault(toIPv6('::ffff:1.2.3.4'));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+ test also with toIPv4OrNull? (just for completeness)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

for historical reason toIPv4OrNull and toIPv4OrZero are not implemented with argument other than string - they are implementations of a string parser

SELECT toIPv4OrDefault(toIPv6('::afff:1.2.3.4'));

SELECT '--';

SELECT cast('test' , 'IPv4'); --{serverError CANNOT_PARSE_IPV4}
SELECT cast('127.0.0.1' , 'IPv4');

Expand Down