Skip to content

Commit

Permalink
馃洜 Tear much of the internals apart!
Browse files Browse the repository at this point in the history 
- 馃敤 Properly deal with span-only encodings by never passing non-views to internal methods
- 鉁↖ntroduce a new concept of adding a "range_category" for encode/decode operations so a specific encoding can tag itself. This prevents SFINAE-failures (SFIAE???) due to mandates-like crashes in the code. (MUST STILL BE DOCUMENTED!!)
- 馃敤 VASTLY improve the replacement handler infrastructure for both decoding and encoding!
  • Loading branch information
@ThePhD
ThePhD committed Jul 2, 2021
1 parent 1f7b896 commit 7610526
Show file tree
Hide file tree
Showing 18 changed files with 384 additions and 516 deletions.
2 changes: 2 additions & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,8 +187,10 @@ mark_as_advanced(ZTD_TEXT_SCRATCH)
if (ZTD_TEXT_SCRATCH)
add_executable(scratch main.cpp)
target_link_libraries(scratch PRIVATE ztd::text)
target_include_directories(scratch PRIVATE tests/shared/include)
target_compile_options(scratch
PRIVATE
${--template-debugging-mode}
${--disable-permissive}
${--warn-pedantic}
${--warn-all}
Expand Down
4 changes: 2 additions & 2 deletions examples/documentation/source/runtime_locale_encoding.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ class runtime_locale {
ztd::text::span<code_point> output,
rtl_decode_error_handler error_handler,
decode_state& current // decode-based state
) {
) const {
using empty_span = ztd::text::span<const code_unit, 0>;

if (output.size() < 1) {
Expand Down Expand Up @@ -177,7 +177,7 @@ class runtime_locale {
ztd::text::span<code_unit> output,
rtl_encode_error_handler error_handler,
encode_state& current // encode-based state
) {
) const {
using empty_span = ztd::text::span<const code_point, 0>;

// saved, in case we need to go
Expand Down
2 changes: 1 addition & 1 deletion examples/shift_jis/include/shift_jis.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ struct shift_jis {
static constexpr inline std::size_t max_code_units = 2;

constexpr ztd::text::span<const code_unit, 1> replacement_code_units() const noexcept {
return ztd::text::span<const code_unit, 1>("?" + 0, 1);
return ztd::text::span<const code_unit, 1>(&"?"[0], 1);
}

using is_decode_injective = std::true_type;
Expand Down
2 changes: 1 addition & 1 deletion examples/shift_jis/source/shift_jis_index.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1362,7 +1362,7 @@ namespace detail {

std::optional<std::size_t> shift_jis_code_point_to_index(char32_t code) noexcept {
auto predicate = [&code](const std::pair<std::uint_least32_t, std::uint_least32_t>& value) {
return code == value.second && !(value.first < 8272 && value.first > 8835);
return code == value.second && !(value.first > 8272 && value.first < 8835);
};
auto it = std::find_if(index_codepoint_map.cbegin(), index_codepoint_map.cend(), predicate);
if (it == index_codepoint_map.cend()) {
Expand Down
5 changes: 3 additions & 2 deletions include/ztd/text/count_decodable.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,11 +132,12 @@ namespace ztd { namespace text {
else {
using _CodePoint = code_point_t<_UEncoding>;

_CodePoint __code_point_buf[max_code_points_v<_UEncoding>] {};
_CodePoint __intermediate_storage[max_code_points_v<_UEncoding>] {};
::ztd::text::span<_CodePoint, max_code_points_v<_UEncoding>> __intermediate(__intermediate_storage);

for (;;) {
auto __result = __txt_detail::__basic_count_decodable_one(
__working_input, __encoding, __code_point_buf, __error_handler, __state);
__working_input, __encoding, __error_handler, __state, __intermediate);
if (__result.error_code != encoding_error::ok) {
return _Result(
::std::move(__result.input), __code_point_count, __state, __result.error_code, false);
Expand Down
5 changes: 3 additions & 2 deletions include/ztd/text/count_encodable.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,11 +133,12 @@ namespace ztd { namespace text {
else {
using _CodeUnit = code_unit_t<_UEncoding>;

_CodeUnit __code_unit_buf[max_code_units_v<_UEncoding>] {};
_CodeUnit __intermediate_storage[max_code_units_v<_UEncoding>] {};
::ztd::text::span<_CodeUnit, max_code_units_v<_UEncoding>> __intermediate(__intermediate_storage);

for (;;) {
auto __result = __txt_detail::__basic_count_encodable_one(
::std::move(__working_input), __encoding, __code_unit_buf, __error_handler, __state);
::std::move(__working_input), __encoding, __error_handler, __state, __intermediate);
if (__result.error_code != encoding_error::ok) {
return _Result(
::std::move(__result.input), __code_unit_count, __state, __result.error_code, false);
Expand Down
99 changes: 30 additions & 69 deletions include/ztd/text/decode.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,7 @@ namespace ztd { namespace text {
template <typename _Input, typename _Encoding, typename _Output, typename _ErrorHandler, typename _State>
constexpr auto basic_decode_into(_Input&& __input, _Encoding&& __encoding, _Output&& __output,
_ErrorHandler&& __error_handler, _State& __state) {
using _IntermediateInput
= __txt_detail::__string_view_or_span_or_reconstruct_t<_Input>;
using _IntermediateInput = __txt_detail::__string_view_or_span_or_reconstruct_t<_Input>;
using _IntermediateOutput = __txt_detail::__range_reconstruct_t<_Output>;
using _Result = decltype(__encoding.decode_one(
::std::declval<_IntermediateInput>(), ::std::declval<_IntermediateOutput>(), __error_handler, __state));
Expand All @@ -108,7 +107,8 @@ namespace ztd { namespace text {
"or "
"decode_into(in, encoding, out, handler, ...) explicitly in order to bypass this.");

_WorkingInput __working_input = __txt_detail::__string_view_or_span_or_reconstruct(::std::forward<_Input>(__input));
_WorkingInput __working_input
= __txt_detail::__string_view_or_span_or_reconstruct(::std::forward<_Input>(__input));
_WorkingOutput __working_output(
__txt_detail::__reconstruct(::std::in_place_type<_WorkingOutput>, ::std::forward<_Output>(__output)));
::std::size_t __handled_errors = 0;
Expand Down Expand Up @@ -185,13 +185,12 @@ namespace ztd { namespace text {
? max_code_points_v<_UEncoding>
: ZTD_TEXT_INTERMEDIATE_BUFFER_SIZE_I_;
using _IntermediateValueType = code_point_t<_UEncoding>;
using _IntermediateInput
= __txt_detail::__string_view_or_span_or_reconstruct_t<_Input>;
using _OutputInitial = ::ztd::text::span<_IntermediateValueType, __intermediate_buffer_max>;
using _Output = ::ztd::text::span<_IntermediateValueType>;
using _Result = decltype(__encoding.decode_one(
using _IntermediateInput = __txt_detail::__string_view_or_span_or_reconstruct_t<_Input>;
using _OutputInitial = ::ztd::text::span<_IntermediateValueType, __intermediate_buffer_max>;
using _Output = ::ztd::text::span<_IntermediateValueType>;
using _Result = decltype(__encoding.decode_one(
::std::declval<_IntermediateInput>(), ::std::declval<_Output>(), __error_handler, __state));
using _WorkingInput = __txt_detail::__remove_cvref_t<decltype(::std::declval<_Result>().input)>;
using _WorkingInput = __txt_detail::__remove_cvref_t<decltype(::std::declval<_Result>().input)>;

_WorkingInput __working_input(
__txt_detail::__string_view_or_span_or_reconstruct(::std::forward<_Input>(__input)));
Expand All @@ -201,8 +200,7 @@ namespace ztd { namespace text {
_OutputInitial __intermediate_initial_output(__intermediate_translation_buffer);
auto __result = decode_into(::std::move(__working_input), ::std::forward<_Encoding>(__encoding),
__intermediate_initial_output, ::std::forward<_ErrorHandler>(__error_handler), __state);
_Output __intermediate_output(
__intermediate_initial_output.data(), __result.output.data());
_Output __intermediate_output(__intermediate_initial_output.data(), __result.output.data());
using _SpanIterator = typename _Output::iterator;
if constexpr (__txt_detail::__is_detected_v<__txt_detail::__detect_insert_bulk, _OutputContainer,
_SpanIterator, _SpanIterator>) {
Expand Down Expand Up @@ -350,16 +348,7 @@ namespace ztd { namespace text {
template <typename _OutputContainer, typename _Input, typename _Encoding, typename _ErrorHandler, typename _State>
constexpr auto decode_to(
_Input&& __input, _Encoding&& __encoding, _ErrorHandler&& __error_handler, _State& __state) {
using _UEncoding = __txt_detail::__remove_cvref_t<_Encoding>;
using _BackInserterIterator = decltype(::std::back_inserter(::std::declval<_OutputContainer&>()));
using _Unbounded = unbounded_view<_BackInserterIterator>;
using _UInput = __txt_detail::__remove_cvref_t<_Input>;
using _InputValueType = __txt_detail::__range_value_type_t<_UInput>;
using _IntermediateInput
= __txt_detail::__range_reconstruct_t<::std::conditional_t<::std::is_array_v<_UInput>,
::std::conditional_t<__txt_detail::__is_character_v<_InputValueType>,
::std::basic_string_view<_InputValueType>, ::ztd::text::span<const _InputValueType>>,
_UInput>>;
using _UEncoding = __txt_detail::__remove_cvref_t<_Encoding>;

_OutputContainer __output {};
if constexpr (__txt_detail::__is_detected_v<__txt_detail::__detect_adl_size, _Input>) {
Expand All @@ -371,23 +360,15 @@ namespace ztd { namespace text {
__output.reserve(__output_size_hint);
}
}
if constexpr (__txt_detail::__is_decode_error_handler_callable_v<_Encoding, _IntermediateInput, _Unbounded,
_ErrorHandler, _State>) {
if constexpr (__txt_detail::__is_decode_one_callable_v<_Encoding, _IntermediateInput, _Unbounded,
_ErrorHandler, _State>) {
// We can use the unbounded stuff
_Unbounded __insert_view(::std::back_inserter(__output));
auto __stateful_result
= decode_into(::std::forward<_Input>(__input), ::std::forward<_Encoding>(__encoding),
::std::move(__insert_view), ::std::forward<_ErrorHandler>(__error_handler), __state);
return __txt_detail::__replace_result_output(::std::move(__stateful_result), ::std::move(__output));
}
else {
auto __stateful_result = __txt_detail::__intermediate_decode_to_storage(
::std::forward<_Input>(__input), ::std::forward<_Encoding>(__encoding), __output,
::std::forward<_ErrorHandler>(__error_handler), __state);
return __txt_detail::__replace_result_output(::std::move(__stateful_result), ::std::move(__output));
}
if constexpr (__txt_detail::__is_decode_range_category_output_v<_UEncoding>) {
// We can use the unbounded stuff
using _BackInserterIterator = decltype(::std::back_inserter(::std::declval<_OutputContainer&>()));
using _Unbounded = unbounded_view<_BackInserterIterator>;
_Unbounded __insert_view(::std::back_inserter(__output));
auto __stateful_result
= decode_into(::std::forward<_Input>(__input), ::std::forward<_Encoding>(__encoding),
::std::move(__insert_view), ::std::forward<_ErrorHandler>(__error_handler), __state);
return __txt_detail::__replace_result_output(::std::move(__stateful_result), ::std::move(__output));
}
else {
auto __stateful_result = __txt_detail::__intermediate_decode_to_storage(::std::forward<_Input>(__input),
Expand Down Expand Up @@ -487,16 +468,7 @@ namespace ztd { namespace text {
typename _State>
constexpr auto __decode_dispatch(
_Input&& __input, _Encoding&& __encoding, _ErrorHandler&& __error_handler, _State& __state) {
using _UEncoding = __txt_detail::__remove_cvref_t<_Encoding>;
using _BackInserterIterator = decltype(::std::back_inserter(::std::declval<_OutputContainer&>()));
using _UInput = __txt_detail::__remove_cvref_t<_Input>;
using _InputValueType = __txt_detail::__range_value_type_t<_UInput>;
using _Unbounded = unbounded_view<_BackInserterIterator>;
using _IntermediateInput
= __txt_detail::__range_reconstruct_t<::std::conditional_t<::std::is_array_v<_UInput>,
::std::conditional_t<__txt_detail::__is_character_v<_InputValueType>,
::std::basic_string_view<_InputValueType>, ::ztd::text::span<const _InputValueType>>,
_UInput>>;
using _UEncoding = __txt_detail::__remove_cvref_t<_Encoding>;

_OutputContainer __output {};
if constexpr (__txt_detail::__is_detected_v<__txt_detail::__detect_adl_size, _Input>) {
Expand All @@ -508,27 +480,16 @@ namespace ztd { namespace text {
__output.reserve(__output_size_hint);
}
}
if constexpr (__txt_detail::__is_decode_error_handler_callable_v<_Encoding, _IntermediateInput,
_Unbounded, _ErrorHandler, _State>) {
if constexpr (__txt_detail::__is_decode_one_callable_v<_Encoding, _IntermediateInput, _Unbounded,
_ErrorHandler, _State>) {
// We can use the unbounded stuff
// We can use the unbounded stuff
_Unbounded __insert_view(::std::back_inserter(__output));
auto __stateful_result
= decode_into(::std::forward<_Input>(__input), ::std::forward<_Encoding>(__encoding),
::std::move(__insert_view), ::std::forward<_ErrorHandler>(__error_handler), __state);
// We are explicitly discarding this information with this function call.
(void)__stateful_result;
return __output;
}
else {
auto __stateful_result = __txt_detail::__intermediate_decode_to_storage(
::std::forward<_Input>(__input), ::std::forward<_Encoding>(__encoding), __output,
::std::forward<_ErrorHandler>(__error_handler), __state);
(void)__stateful_result;
return __output;
}
if constexpr (__txt_detail::__is_decode_range_category_output_v<_Encoding>) {
using _BackInserterIterator = decltype(::std::back_inserter(::std::declval<_OutputContainer&>()));
using _Unbounded = unbounded_view<_BackInserterIterator>;
_Unbounded __insert_view(::std::back_inserter(__output));
auto __stateful_result
= decode_into(::std::forward<_Input>(__input), ::std::forward<_Encoding>(__encoding),
::std::move(__insert_view), ::std::forward<_ErrorHandler>(__error_handler), __state);
// We are explicitly discarding this information with this function call.
(void)__stateful_result;
return __output;
}
else {
auto __stateful_result = __txt_detail::__intermediate_decode_to_storage(
Expand Down
14 changes: 0 additions & 14 deletions include/ztd/text/decode_result.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,20 +264,6 @@ namespace ztd { namespace text {
::std::forward<_OutFirst>(__out_first), ::std::forward<_OutLast>(__out_last),
::std::forward<_ArgState>(__state), __error_code);
}

template <typename _Encoding, typename _Input, typename _Output, typename _ErrorHandler, typename _State>
inline constexpr bool __is_decode_error_handler_callable_v = __is_detected_v<__detect_callable_handler,
_ErrorHandler, _Encoding, __reconstruct_decode_result_t<_Input, _Output, _State>,
::ztd::text::span<code_unit_t<__remove_cvref_t<_Encoding>>>>;

template <typename _Encoding, typename _Input, typename _Output, typename _ErrorHandler, typename _State>
inline constexpr bool __is_decode_one_callable_v
= __is_detected_v<__detect_object_decode_one, _Encoding, _Input, _Output, _ErrorHandler, _State>;

template <typename _Encoding, typename _Input, typename _Output, typename _ErrorHandler, typename _State>
inline constexpr bool __is_decode_one_and_error_handler_callable_v
= __is_decode_one_callable_v<_Encoding, _Input, _Output, _ErrorHandler, _State>&&
__is_decode_error_handler_callable_v<_Encoding, _Input, _Output, _ErrorHandler, _State>;
} // namespace __txt_detail

ZTD_TEXT_INLINE_ABI_NAMESPACE_CLOSE_I_
Expand Down
52 changes: 52 additions & 0 deletions include/ztd/text/detail/encoding_range.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@

#include <ztd/text/detail/adl.hpp>
#include <ztd/text/detail/type_traits.hpp>
#include <ztd/text/detail/iterator.hpp>

#include <utility>

Expand Down Expand Up @@ -269,6 +270,57 @@ namespace ztd { namespace text {
template <typename _Handler, typename _Encoding, typename _Result, typename _Progress>
using __detect_callable_handler = decltype(::std::declval<_Handler>()(
::std::declval<const _Encoding&>(), ::std::declval<_Result>(), ::std::declval<_Progress>()));

template <typename _Encoding, typename = void>
struct __range_category {
using type = void;
};

template <typename _Encoding>
struct __range_category<_Encoding, ::std::void_t<typename __remove_cvref_t<_Encoding>::range_category>> {
using type = typename __remove_cvref_t<_Encoding>::range_category;
};

template <typename _Encoding, typename = void>
struct __decode_range_category : public __range_category<_Encoding> { };

template <typename _Encoding>
struct __decode_range_category<_Encoding,
::std::void_t<typename __remove_cvref_t<_Encoding>::decode_range_category>> {
using type = typename __remove_cvref_t<_Encoding>::decode_range_category;
};

template <typename _Encoding, typename = void>
struct __encode_range_category : public __range_category<_Encoding> { };

template <typename _Encoding>
struct __encode_range_category<_Encoding,
::std::void_t<typename __remove_cvref_t<_Encoding>::encode_range_category>> {
using type = typename __remove_cvref_t<_Encoding>::encode_range_category;
};

template <typename _Encoding>
using __decode_range_category_t = typename __decode_range_category<_Encoding>::type;

template <typename _Encoding>
using __encode_range_category_t = typename __encode_range_category<_Encoding>::type;

template <typename _Encoding>
inline constexpr bool __is_decode_range_category_output_v
= ::std::is_base_of_v<__decode_range_category_t<_Encoding>, ::std::output_iterator_tag>;

template <typename _Encoding>
inline constexpr bool __is_encode_range_category_output_v
= ::std::is_base_of_v<__encode_range_category_t<_Encoding>, ::std::output_iterator_tag>;

template <typename _Encoding>
inline constexpr bool __is_encode_range_category_contiguous_v
= ::std::is_base_of_v<__encode_range_category_t<_Encoding>, ::ztd::text::contiguous_iterator_tag>;

template <typename _Encoding>
inline constexpr bool __is_decode_range_category_contiguous_v
= ::std::is_base_of_v<__decode_range_category_t<_Encoding>, ::ztd::text::contiguous_iterator_tag>;

} // namespace __txt_detail
ZTD_TEXT_INLINE_ABI_NAMESPACE_CLOSE_I_
}} // namespace ztd::text
Expand Down
24 changes: 17 additions & 7 deletions include/ztd/text/detail/iterator.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,18 @@
namespace ztd { namespace text {
ZTD_TEXT_INLINE_ABI_NAMESPACE_OPEN_I_

namespace __txt_detail {
class __contiguous_iterator_tag : public ::std::random_access_iterator_tag { };
} // namespace __txt_detail

//////
/// @brief Either a typedef or a polyfill of the contiguous iterator tag, only standardized in C++20.
//////
using contiguous_iterator_tag =
#if ZTD_TEXT_IS_ON(ZTD_TEXT_STD_LIBRARY_CONTIGUOUS_ITERATOR_TAG_I_)
using contiguous_iterator_tag = ::std::contiguous_iterator_tag;
::std::contiguous_iterator_tag;
#else
class contiguous_iterator_tag : public ::std::random_access_iterator_tag { };
__txt_detail::__contiguous_iterator_tag;
#endif

namespace __txt_detail {
Expand Down Expand Up @@ -207,14 +215,16 @@ namespace ztd { namespace text {
= ::std::is_base_of_v<_Tag, __iterator_concept_t<_It>>;

template <typename _It>
inline constexpr bool __is_iterator_contiguous_iterator_v = (
inline constexpr bool __is_iterator_contiguous_iterator_v
= (
#if ZTD_TEXT_IS_ON(ZTD_TEXT_STD_LIBRARY_CONTIGUOUS_ITERATOR_TAG_I_)
__is_iterator_concept_or_better_v<contiguous_iterator_tag, _It>) || (__is_iterator_concept_or_better_v<contiguous_iterator_tag, _It> &&
__is_iterator_concept_or_better_v<contiguous_iterator_tag, _It>)
|| (__is_iterator_concept_or_better_v<contiguous_iterator_tag, _It> &&
#else
::std::is_pointer_v<_It> &&
::std::is_pointer_v<_It> &&
#endif
__is_to_addressable_v<
_It> && ::std::is_lvalue_reference_v<__iterator_reference_t<::std::remove_reference_t<_It>>>);
__is_to_addressable_v<
_It> && ::std::is_lvalue_reference_v<__iterator_reference_t<::std::remove_reference_t<_It>>>);

template <typename _It>
inline constexpr bool __is_iterator_input_iterator_v
Expand Down
4 changes: 4 additions & 0 deletions include/ztd/text/detail/range.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,10 @@ namespace ztd { namespace text {
inline constexpr bool __is_range_input_or_output_range_v
= __is_iterator_input_or_output_iterator_v<__range_iterator_t<_Range>>;

template <typename _Range>
inline constexpr bool __is_range_contiguous_range_v
= __is_iterator_contiguous_iterator_v<__range_iterator_t<_Range>>;

template <typename _Range>
inline constexpr bool __is_sized_range_v
= __is_sized_sentinel_for_v<__range_iterator_t<_Range>, __range_sentinel_t<_Range>>;
Expand Down

0 comments on commit 7610526

Please sign in to comment.