Skip to content

Commit

Permalink
馃敤 Fix up the code_point / code_unit shenanigans
Browse files Browse the repository at this point in the history 
- Clean up the docs related to this.
- Add default_compile_time_code_point_encoding - this may be used by downstream users, even if we don't have very strict code point types yet.
- Polish any_encoding usage.
  • Loading branch information
@ThePhD
ThePhD committed Feb 26, 2021
1 parent 64a41a6 commit 01d33cd
Show file tree
Hide file tree
Showing 30 changed files with 1,012 additions and 402 deletions.
7 changes: 6 additions & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,12 @@ string(CONCAT ztd-use-cuneicode $<
>)

string(CONCAT --warn-pedantic $<$<BOOL:${ZTD_TEXT_DIAGNOSTIC_PEDANTIC}>:-Wpedantic>)
string(CONCAT --warn-default $<$<BOOL:${ZTD_TEXT_DIAGNOSTIC_DEFAULTS}>:-Wall>)
string(CONCAT --warn-default $<
$<AND:
$<BOOL:${ZTD_TEXT_DIAGNOSTIC_DEFAULTS}>,
$<NOT:$<BOOL:${MSVC}>>,
>:-Wall
>)
string(CONCAT --deny-errors $<$<BOOL:${ZTD_TEXT_DIAGNOSTIC_ERRORS}>:-Werror>)

add_library(ztd.text INTERFACE)
Expand Down
5 changes: 5 additions & 0 deletions documentation/source/api/default_code_point_encoding.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,8 @@ Picks the default encoding for the given code point type. In all cases, this jus
:members:

.. doxygentypedef:: ztd::text::default_code_point_encoding_t

.. doxygenclass:: ztd::text::default_compile_time_code_point_encoding
:members:

.. doxygentypedef:: ztd::text::default_compile_time_code_point_encoding_t
5 changes: 5 additions & 0 deletions documentation/source/api/default_code_unit_encoding.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,3 +70,8 @@ Picks the default encoding for the given code unit type. The default association

.. doxygentypedef:: ztd::text::default_code_unit_encoding_t

.. doxygenclass:: ztd::text::default_compile_time_code_unit_encoding
:members:

.. doxygentypedef:: ztd::text::default_compile_time_code_unit_encoding_t

2 changes: 1 addition & 1 deletion include/ztd/text.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
#include <ztd/text/encode.hpp>
#include <ztd/text/decode.hpp>
#include <ztd/text/transcode.hpp>
#include <ztd/text/count_code_units.hpp>
#include <ztd/text/count_code_points.hpp>
#include <ztd/text/count_code_points.hpp>
#include <ztd/text/validate_code_units.hpp>
#include <ztd/text/validate_code_points.hpp>
Expand Down
113 changes: 59 additions & 54 deletions include/ztd/text/any_encoding.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@
#include <ztd/text/decode.hpp>
#include <ztd/text/validate_code_units.hpp>
#include <ztd/text/validate_code_points.hpp>
#include <ztd/text/count_code_units.hpp>
#include <ztd/text/count_code_points.hpp>
#include <ztd/text/count_code_units.hpp>

#include <ztd/text/detail/transcode_one.hpp>
#include <ztd/text/detail/range.hpp>
Expand Down Expand Up @@ -87,7 +87,7 @@ namespace ztd { namespace text {
///
/// @tparam _EncodeCodeUnits The output of @c encode_one and related operations.
/// @tparam _EncodeCodePoints The input of @c encode_one and related operations.
/// @tparam _DecodeCodeUnits The output of @c decode_one and related operations.
/// @tparam _DecodeCodeUnits The input of @c decode_one and related operations.
/// @tparam _DecodeCodePoints The output of @c decode_one and related operations.
/// @tparam _MaxCodeUnits The maximum number of code units that can be output through a given operation. Directly
/// related to the maximum_code_units inline constexpr variable definition.
Expand Down Expand Up @@ -163,20 +163,20 @@ namespace ztd { namespace text {
static inline constexpr ::std::size_t max_code_units = _MaxCodeUnits;

private:
using __encode_result = encode_result<_EncodeCodePoints, _EncodeCodeUnits, encode_state>;
using __decode_result = decode_result<_DecodeCodeUnits, _DecodeCodePoints, decode_state>;
using __count_code_units_result = count_result<_EncodeCodePoints, encode_state>;
using __count_code_points_result = count_result<_DecodeCodeUnits, decode_state>;
using __encode_result = encode_result<_EncodeCodePoints, _EncodeCodeUnits, encode_state>;
using __count_code_units_result = count_result<_DecodeCodeUnits, decode_state>;
using __count_code_points_result = count_result<_EncodeCodePoints, encode_state>;
using __validate_code_units_result = validate_result<_DecodeCodeUnits, decode_state>;
using __validate_code_points_result = validate_result<_EncodeCodePoints, encode_state>;
using __decode_error_handler = ::std::function<__decode_result(
const any_encoding_with&, __decode_result, const ::ztd::text::span<const code_unit>&)>;
using __encode_error_handler = ::std::function<__encode_result(
const any_encoding_with&, __encode_result, const ::ztd::text::span<const code_point>&)>;
using __count_code_points_error_handler = ::std::function<__decode_result(
const any_encoding_with&, __count_code_points_result, const ::ztd::text::span<const code_unit>&)>;
using __count_code_units_error_handler = ::std::function<__encode_result(
const any_encoding_with&, __count_code_units_result, const ::ztd::text::span<const code_point>&)>;
using __count_code_units_error_handler = ::std::function<__decode_result(
const any_encoding_with&, __decode_result, const ::ztd::text::span<const code_point>&)>;
using __count_code_points_error_handler = ::std::function<__encode_result(
const any_encoding_with&, __encode_result, const ::ztd::text::span<const code_unit>&)>;

struct __erased_state {
virtual ~__erased_state() {
Expand All @@ -198,10 +198,10 @@ namespace ztd { namespace text {
_DecodeCodeUnits __input, decode_state& __state) const = 0;
virtual __validate_code_points_result __validate_code_points_one(
_EncodeCodePoints __input, encode_state& __state) const = 0;
virtual __count_code_points_result __count_code_points_one(_DecodeCodeUnits __input,
__count_code_points_error_handler __error_handler, decode_state& __state) const = 0;
virtual __count_code_units_result __count_code_units_one(_EncodeCodePoints __input,
__count_code_units_error_handler __error_handler, encode_state& __state) const = 0;
virtual __count_code_points_result __count_code_points_one(_EncodeCodePoints __input,
__count_code_points_error_handler __error_handler, encode_state& __state) const = 0;
virtual __count_code_units_result __count_code_units_one(_DecodeCodeUnits __input,
__count_code_units_error_handler __error_handler, decode_state& __state) const = 0;

virtual __decode_result __decode(_DecodeCodeUnits __input, _DecodeCodePoints __output,
__decode_error_handler __error_handler, decode_state& __state) const = 0;
Expand All @@ -211,10 +211,10 @@ namespace ztd { namespace text {
_DecodeCodeUnits __input, decode_state& __state) const = 0;
virtual __validate_code_points_result __validate_code_points(
_EncodeCodePoints __input, encode_state& __state) const = 0;
virtual __count_code_points_result __count_code_points(_DecodeCodeUnits __input,
__count_code_points_error_handler __error_handler, decode_state& __state) const = 0;
virtual __count_code_units_result __count_code_units(_EncodeCodePoints __input,
__count_code_units_error_handler __error_handler, encode_state& __state) const = 0;
virtual __count_code_points_result __count_code_points(_EncodeCodePoints __input,
__count_code_points_error_handler __error_handler, encode_state& __state) const = 0;
virtual __count_code_units_result __count_code_units(_DecodeCodeUnits __input,
__count_code_units_error_handler __error_handler, decode_state& __state) const = 0;

virtual std::unique_ptr<__erased_state> __create_encode_state() const = 0;
virtual std::unique_ptr<__erased_state> __create_decode_state() const = 0;
Expand Down Expand Up @@ -441,43 +441,43 @@ namespace ztd { namespace text {
}
}

virtual __count_code_points_result __count_code_points_one(_DecodeCodeUnits __input,
__count_code_points_error_handler __error_handler, decode_state& __state) const override {
virtual __count_code_units_result __count_code_units_one(_DecodeCodeUnits __input,
__count_code_units_error_handler __error_handler, decode_state& __state) const override {
__real_decode_state& __actual_state = this->_M_get_state(__state);
__detail::__pass_through_handler __pass_handler;
auto& __encoding = this->__base_t::get_value();
if constexpr (__detail::__is_detected_v<__detail::__detect_object_count_code_points_one, _Encoding,
__detail::__pass_through_handler, __real_decode_state>) {
if constexpr (__detail::__is_detected_v<__detail::__detect_object_count_code_units_one, _Encoding,
_DecodeCodeUnits, __detail::__pass_through_handler, __real_decode_state>) {
auto __raw_result
= __encoding.count_code_points_one(::std::move(__input), __pass_handler, __actual_state);
return __count_code_points_result(::std::move(__raw_result.input), __raw_result.count, __state,
= __encoding.count_code_units_one(::std::move(__input), __pass_handler, __actual_state);
return __count_code_units_result(::std::move(__raw_result.input), __raw_result.count, __state,
__raw_result.error_code, __raw_result.handled_error);
}
else {
auto __raw_result = __detail::__basic_count_code_points_one(
auto __raw_result = __detail::__basic_count_code_units_one(
::std::move(__input), __encoding, __pass_handler, __actual_state);
return __count_code_points_result(
::std::move(__raw_result.input), __raw_result.count, __state);
return __count_code_units_result(::std::move(__raw_result.input), __raw_result.count, __state,
__raw_result.error_code, __raw_result.handled_error);
}
}

virtual __count_code_units_result __count_code_units_one(_EncodeCodePoints __input,
__count_code_units_error_handler __error_handler, encode_state& __state) const override {
virtual __count_code_points_result __count_code_points_one(_EncodeCodePoints __input,
__count_code_points_error_handler __error_handler, encode_state& __state) const override {
__real_encode_state& __actual_state = this->_M_get_state(__state);
__detail::__pass_through_handler __pass_handler;
auto& __encoding = this->__base_t::get_value();
if constexpr (__detail::__is_detected_v<__detail::__detect_object_count_code_units_one, _Encoding,
if constexpr (__detail::__is_detected_v<__detail::__detect_object_count_code_points_one, _Encoding,
_EncodeCodePoints, __detail::__pass_through_handler, __real_encode_state>) {
auto __raw_result
= __encoding.count_code_units_one(::std::move(__input), __pass_handler, __actual_state);
return __count_code_units_result(::std::move(__raw_result.input), __raw_result.count, __state,
= __encoding.count_code_points_one(::std::move(__input), __pass_handler, __actual_state);
return __count_code_points_result(::std::move(__raw_result.input), __raw_result.count, __state,
__raw_result.error_code, __raw_result.handled_error);
}
else {
auto __raw_result = __detail::__basic_count_code_units_one(
auto __raw_result = __detail::__basic_count_code_points_one(
::std::move(__input), __encoding, __pass_handler, __actual_state);
return __count_code_units_result(::std::move(__raw_result.input), __raw_result.count, __state,
__raw_result.error_code, __raw_result.handled_error);
return __count_code_points_result(
::std::move(__raw_result.input), __raw_result.count, __state);
}
}

Expand Down Expand Up @@ -518,24 +518,24 @@ namespace ztd { namespace text {
return __validate_code_points_result(::std::move(__raw_result.input), __raw_result.valid, __state);
}

virtual __count_code_points_result __count_code_points(_DecodeCodeUnits __input,
__count_code_points_error_handler __error_handler, decode_state& __state) const override {
virtual __count_code_units_result __count_code_units(_DecodeCodeUnits __input,
__count_code_units_error_handler __error_handler, decode_state& __state) const override {
__real_decode_state& __actual_state = this->_M_get_state(__state);
auto& __encoding = this->__base_t::get_value();
__detail::__pass_through_handler __pass_handler;
auto __raw_result = ::ztd::text::count_code_points(
::std::move(__input), this->__base_t::get_value(), __pass_handler, __actual_state);
return __count_code_points_result(::std::move(__raw_result.input), __raw_result.count, __state,
auto __raw_result = ::ztd::text::count_code_units(
::std::move(__input), __encoding, __pass_handler, __actual_state);
return __count_code_units_result(::std::move(__raw_result.input), __raw_result.count, __state,
__raw_result.error_code, __raw_result.handled_error);
}

virtual __count_code_units_result __count_code_units(_EncodeCodePoints __input,
__count_code_units_error_handler __error_handler, encode_state& __state) const override {
virtual __count_code_points_result __count_code_points(_EncodeCodePoints __input,
__count_code_points_error_handler __error_handler, encode_state& __state) const override {
__real_encode_state& __actual_state = this->_M_get_state(__state);
auto& __encoding = this->__base_t::get_value();
__detail::__pass_through_handler __pass_handler;
auto __raw_result = ::ztd::text::count_code_units(
::std::move(__input), __encoding, __pass_handler, __actual_state);
return __count_code_units_result(::std::move(__raw_result.input), __raw_result.count, __state,
auto __raw_result = ::ztd::text::count_code_points(
::std::move(__input), this->__base_t::get_value(), __pass_handler, __actual_state);
return __count_code_points_result(::std::move(__raw_result.input), __raw_result.count, __state,
__raw_result.error_code, __raw_result.handled_error);
}

Expand Down Expand Up @@ -738,14 +738,14 @@ namespace ztd { namespace text {
return this->_M_storage->__validate_code_points_one(::std::move(__input), __state);
}

__count_code_points_result __count_code_points_one(_DecodeCodeUnits __input,
__count_code_points_error_handler __error_handler, decode_state& __state) const {
__count_code_points_result __count_code_points_one(_EncodeCodePoints __input,
__count_code_points_error_handler __error_handler, encode_state& __state) const {
return this->_M_storage->__count_code_points_one(
::std::move(__input), ::std::move(__error_handler), __state);
}

__count_code_units_result __count_code_units_one(
_EncodeCodePoints __input, __encode_error_handler __error_handler, encode_state& __state) const {
_DecodeCodeUnits __input, __encode_error_handler __error_handler, decode_state& __state) const {
return this->_M_storage->__count_code_units_one(
::std::move(__input), ::std::move(__error_handler), __state);
}
Expand Down Expand Up @@ -775,13 +775,13 @@ namespace ztd { namespace text {
}

__count_code_points_result __count_code_points(
_DecodeCodeUnits __input, __decode_error_handler __error_handler, decode_state& __state) const {
_EncodeCodePoints __input, __decode_error_handler __error_handler, encode_state& __state) const {
return this->_M_storage->__count_code_points(
::std::move(__input), ::std::move(__error_handler), __state);
}

__count_code_units_result __count_code_units(
_EncodeCodePoints __input, __encode_error_handler __error_handler, encode_state& __state) const {
_DecodeCodeUnits __input, __encode_error_handler __error_handler, decode_state& __state) const {
return this->_M_storage->__count_code_units(::std::move(__input), ::std::move(__error_handler), __state);
}

Expand Down Expand Up @@ -856,8 +856,13 @@ namespace ztd { namespace text {
return __encoding.__validate_code_units(::std::move(__input), __state);
}

//////
/// @internal
///
/// @brief Extension point hooks for the implementation-side only.
//////
template <typename _ErrorHandler, typename _State>
constexpr friend auto __text_count_code_points_one(_DecodeCodeUnits __input,
constexpr friend auto __text_count_code_points_one(_EncodeCodePoints __input,
__detail::__type_identity_t<const any_encoding_with&> __encoding, _ErrorHandler&& __error_handler,
_State& __state) {
return __encoding.__count_code_points_one(
Expand All @@ -870,7 +875,7 @@ namespace ztd { namespace text {
/// @brief Extension point hooks for the implementation-side only.
//////
template <typename _ErrorHandler, typename _State>
constexpr friend auto __text_count_code_units_one(_EncodeCodePoints __input,
constexpr friend auto __text_count_code_units_one(_DecodeCodeUnits __input,
__detail::__type_identity_t<const any_encoding_with&> __encoding, _ErrorHandler&& __error_handler,
_State& __state) {
return __encoding.__count_code_units_one(
Expand All @@ -883,7 +888,7 @@ namespace ztd { namespace text {
/// @brief Extension point hooks for the implementation-side only.
//////
template <typename _ErrorHandler, typename _State>
constexpr friend auto __text_count_code_points(_DecodeCodeUnits __input,
constexpr friend auto __text_count_code_points(_EncodeCodePoints __input,
__detail::__type_identity_t<const any_encoding_with&> __encoding, _ErrorHandler&& __error_handler,
_State& __state) {
return __encoding.__count_code_points(::std::move(__input), ::std::move(__error_handler), __state);
Expand All @@ -895,7 +900,7 @@ namespace ztd { namespace text {
/// @brief Extension point hooks for the implementation-side only.
//////
template <typename _ErrorHandler, typename _State>
constexpr friend auto __text_count_code_units(_EncodeCodePoints __input,
constexpr friend auto __text_count_code_units(_DecodeCodeUnits __input,
__detail::__type_identity_t<const any_encoding_with&> __encoding, _ErrorHandler&& __error_handler,
_State& __state) {
return __encoding.__count_code_units(
Expand Down

0 comments on commit 01d33cd

Please sign in to comment.