✨ EUC-KR and Latin1 (Windows-1252-strict) encodings added!

— ✨ 📝 A Real quickstart guide and vastly improved documentation! — ✨ New "recode" routine, since that name won the poll! It's really only useful for validation endeavors, but it's present! — ✨ ACTUAL tests for additional encodings, with data files in the right place! — ✨ A real skip_input_error implementation, and usage in replacement_error — ✨ The start of a single_byte_lookup_encoding, which will speed up development of all the single-byte encoding types by just adding more encoding tables when necessary!
soasis · Dec 23, 2022 · d0f6290 · d0f6290
1 parent cdad528
commit d0f6290
Show file tree

Hide file tree

Showing 144 changed files with 10,562 additions and 2,742 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -108,6 +108,7 @@ if(ZTD_TEXT_IS_TOP_LEVEL_PROJECT)
 		check_compiler_flag(warn-errors MSVC /WX GCC -Werror)
 		check_compiler_flag(warn-extra GCC -Wextra Clang -Wextra)
 		check_compiler_diagnostic(alignas-extra-padding MSVC 4324 LANGUAGES CXX)
+		check_compiler_diagnostic(bit-int-extension)
 		# (Wstringop-overflow) - [meta-bug] bogus/missing -Wstringop-overflow warnings
 		#   https://gcc.gnu.org/bugzilla/show_bug.cgi?id=88443
 		# Bogus -Wstringop-overflow warning
@@ -266,7 +267,11 @@ mark_as_advanced(ZTD_TEXT_SCRATCH)
 
 if(ZTD_TEXT_SCRATCH)
 	add_executable(scratch main.cpp)
-	target_link_libraries(scratch PRIVATE ztd::text)
+	target_link_libraries(scratch
+		PRIVATE
+		ztd::text
+		ztd::text::benchmarks::barrier
+	)
 	target_include_directories(scratch PRIVATE tests/shared/include)
 	target_compile_options(scratch
 		PRIVATE

diff --git a/benchmarks/barrier/CMakeLists.txt b/benchmarks/barrier/CMakeLists.txt
@@ -37,7 +37,7 @@ add_library(ztd.text.benchmarks.barrier
 	SHARED
 	${ztd.text.benchmarks.barrier.sources}
 )
-add_library(ztd::cuneicode::benchmarks::barrier
+add_library(ztd::text::benchmarks::barrier
 	ALIAS
 	ztd.text.benchmarks.barrier
 )

diff --git a/benchmarks/conversion_speed/CMakeLists.txt b/benchmarks/conversion_speed/CMakeLists.txt
@@ -45,7 +45,7 @@ function (generate_converion_speed_benchmark_targets name data_name title)
 			ztd::platform
 			ztd::cuneicode
 			ztd::cuneicode::shared
-			ztd::cuneicode::benchmarks::barrier
+			ztd::text::benchmarks::barrier
 			ztd::cuneicode::shared::simdutf
 			ztd::text
 			ctre::ctre

diff --git a/benchmarks/conversion_speed/include/ztd/text/benchmarks/fast_transcode.ztd.text.hpp b/benchmarks/conversion_speed/include/ztd/text/benchmarks/fast_transcode.ztd.text.hpp
diff --git a/benchmarks/conversion_speed/source/ztd.text.cpp b/benchmarks/conversion_speed/source/ztd.text.cpp
@@ -53,8 +53,8 @@
 			const from_char_t* input = input_data.data();                                                                  \
 			size_t output_size       = output_data.size();                                                                 \
 			to_char_t* output        = output_data.data();                                                                 \
-			auto err                 = ztd::text::transcode_into(                                                          \
-			                     ztd::span(input, input_size), ztd::text::utf##TO_N, ztd::span(output, output_size));      \
+			auto err                 = ztd::text::transcode_into_raw(                                                      \
+                    ztd::span(input, input_size), ztd::text::utf##TO_N, ztd::span(output, output_size));      \
 			if (err.error_code != ztd::text::encoding_error::ok) {                                                         \
 				result = false;                                                                                           \
 			}                                                                                                              \
@@ -117,7 +117,7 @@
 			const from_char_t* input = input_data.data();                                                                  \
 			to_char_t* output        = output_data.data();                                                                 \
 			auto err                 = ztd::text::transcode_into(                                                          \
-			                     ztd::span(input, input_size), ztd::text::utf##TO_N, ztd::ranges::unbounded_view(output)); \
+                    ztd::span(input, input_size), ztd::text::utf##TO_N, ztd::ranges::unbounded_view(output)); \
 			if (err.error_code != ztd::text::encoding_error::ok) {                                                         \
 				result = false;                                                                                           \
 			}                                                                                                              \

diff --git a/benchmarks/conversion_speed/source/ztd.text.unchecked.cpp b/benchmarks/conversion_speed/source/ztd.text.unchecked.cpp
@@ -53,7 +53,7 @@
 			const from_char_t* input = input_data.data();                                                      \
 			size_t output_size       = output_data.size();                                                     \
 			to_char_t* output        = output_data.data();                                                     \
-			auto err = ztd::text::transcode_into(ztd::span(input, input_size), ztd::text::utf##FROM_N,         \
+			auto err = ztd::text::transcode_into_raw(ztd::span(input, input_size), ztd::text::utf##FROM_N,         \
 			     ztd::span(output, output_size), ztd::text::utf##TO_N, ztd::text::assume_valid_handler,        \
 			     ztd::text::assume_valid_handler);                                                             \
 			if (err.error_code != ztd::text::encoding_error::ok) {                                             \
@@ -118,7 +118,7 @@
 			size_t input_size        = input_data.size();                                                      \
 			const from_char_t* input = input_data.data();                                                      \
 			to_char_t* output        = output_data.data();                                                     \
-			auto err = ztd::text::transcode_into(ztd::span(input, input_size), ztd::text::utf##FROM_N,         \
+			auto err = ztd::text::transcode_into_raw(ztd::span(input, input_size), ztd::text::utf##FROM_N,         \
 			     ztd::ranges::unbounded_view(output), ztd::text::utf##TO_N, ztd::text::assume_valid_handler,   \
 			     ztd::text::assume_valid_handler);                                                             \
 			if (err.error_code != ztd::text::encoding_error::ok) {                                             \

diff --git a/benchmarks/function_form/CMakeLists.txt b/benchmarks/function_form/CMakeLists.txt
@@ -58,7 +58,7 @@ function (generate_function_form_benchmark_targets name data_name title)
 	)
 	target_link_libraries(ztd.text.benchmarks.function_form.${name}
 		PRIVATE
-		ztd::cuneicode::benchmarks::barrier
+		ztd::text::benchmarks::barrier
 		ztd::cuneicode
 		benchmark::benchmark
 		${CMAKE_DL_LIBS}