Skip to content

Commit

Permalink
✨ Posix and Windows wrappers moved here!
Browse files Browse the repository at this point in the history
— ✨encoding_detection.h C function to determine if the encoding is something unicode-y!
  • Loading branch information
ThePhD committed Nov 20, 2021
1 parent 53e37ef commit 8911056
Show file tree
Hide file tree
Showing 15 changed files with 677 additions and 162 deletions.
10 changes: 10 additions & 0 deletions include/ztd/idk/charN_t.h
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
/// @{
//////

typedef char ztd_char_t;
typedef ZTD_CHAR8_T_I_ ztd_char8_t;

// clang-format off
Expand Down Expand Up @@ -92,6 +93,15 @@ typedef ZTD_CHAR8_T_I_ ztd_char8_t;
// clang-format on


//////
/// @typedef ztd_char_t
///
/// @brief An alias to `char`.
///
/// @remarks This is simply for consistency with the other character types in ztd. It's not strictly necessary, but it's
/// nice!
//////

//////
/// @typedef ztd_wchar_t
///
Expand Down
249 changes: 156 additions & 93 deletions include/ztd/idk/detail/bit.intrinsic.h

Large diffs are not rendered by default.

32 changes: 16 additions & 16 deletions include/ztd/idk/detail/bit.load_store.h
Original file line number Diff line number Diff line change
Expand Up @@ -51,38 +51,38 @@
ZTD_EXTERN_C_OPEN_I_

#define ZTDC_GENERATE_LOAD8_STORE8_DECLARATIONS(_N) \
ZTD_IDK_API_LINKAGE_I_ uint_least##_N##_t ztdc_load8_leu##_N( \
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ uint_least##_N##_t ztdc_load8_leu##_N( \
const unsigned char __ptr[ZTD_PTR_EXTENT(_N / CHAR_BIT)]) ZTD_CXX_NOEXCEPT_I_; \
ZTD_IDK_API_LINKAGE_I_ uint_least##_N##_t ztdc_load8_beu##_N( \
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ uint_least##_N##_t ztdc_load8_beu##_N( \
const unsigned char __ptr[ZTD_PTR_EXTENT(_N / CHAR_BIT)]) ZTD_CXX_NOEXCEPT_I_; \
ZTD_IDK_API_LINKAGE_I_ uint_least##_N##_t ztdc_load8_aligned_leu##_N( \
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ uint_least##_N##_t ztdc_load8_aligned_leu##_N( \
const unsigned char __ptr[ZTD_PTR_EXTENT(_N / CHAR_BIT)]) ZTD_CXX_NOEXCEPT_I_; \
ZTD_IDK_API_LINKAGE_I_ uint_least##_N##_t ztdc_load8_aligned_beu##_N( \
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ uint_least##_N##_t ztdc_load8_aligned_beu##_N( \
const unsigned char __ptr[ZTD_PTR_EXTENT(_N / CHAR_BIT)]) ZTD_CXX_NOEXCEPT_I_; \
ZTD_IDK_API_LINKAGE_I_ int_least##_N##_t ztdc_load8_les##_N( \
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ int_least##_N##_t ztdc_load8_les##_N( \
const unsigned char __ptr[ZTD_PTR_EXTENT(_N / CHAR_BIT)]) ZTD_CXX_NOEXCEPT_I_; \
ZTD_IDK_API_LINKAGE_I_ int_least##_N##_t ztdc_load8_bes##_N( \
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ int_least##_N##_t ztdc_load8_bes##_N( \
const unsigned char __ptr[ZTD_PTR_EXTENT(_N / CHAR_BIT)]) ZTD_CXX_NOEXCEPT_I_; \
ZTD_IDK_API_LINKAGE_I_ int_least##_N##_t ztdc_load8_aligned_les##_N( \
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ int_least##_N##_t ztdc_load8_aligned_les##_N( \
const unsigned char __ptr[ZTD_PTR_EXTENT(_N / CHAR_BIT)]) ZTD_CXX_NOEXCEPT_I_; \
ZTD_IDK_API_LINKAGE_I_ int_least##_N##_t ztdc_load8_aligned_bes##_N( \
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ int_least##_N##_t ztdc_load8_aligned_bes##_N( \
const unsigned char __ptr[ZTD_PTR_EXTENT(_N / CHAR_BIT)]) ZTD_CXX_NOEXCEPT_I_; \
\
ZTD_IDK_API_LINKAGE_I_ void ztdc_store8_leu##_N( \
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ void ztdc_store8_leu##_N( \
const uint_least##_N##_t __value, unsigned char __ptr[ZTD_PTR_EXTENT(_N / CHAR_BIT)]) ZTD_CXX_NOEXCEPT_I_; \
ZTD_IDK_API_LINKAGE_I_ void ztdc_store8_beu##_N( \
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ void ztdc_store8_beu##_N( \
const uint_least##_N##_t __value, unsigned char __ptr[ZTD_PTR_EXTENT(_N / CHAR_BIT)]) ZTD_CXX_NOEXCEPT_I_; \
ZTD_IDK_API_LINKAGE_I_ void ztdc_store8_aligned_leu##_N( \
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ void ztdc_store8_aligned_leu##_N( \
const uint_least##_N##_t __value, unsigned char __ptr[ZTD_PTR_EXTENT(_N / CHAR_BIT)]) ZTD_CXX_NOEXCEPT_I_; \
ZTD_IDK_API_LINKAGE_I_ void ztdc_store8_aligned_beu##_N( \
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ void ztdc_store8_aligned_beu##_N( \
const uint_least##_N##_t __value, unsigned char __ptr[ZTD_PTR_EXTENT(_N / CHAR_BIT)]) ZTD_CXX_NOEXCEPT_I_; \
ZTD_IDK_API_LINKAGE_I_ void ztdc_store8_les##_N( \
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ void ztdc_store8_les##_N( \
const int_least##_N##_t __value, unsigned char __ptr[ZTD_PTR_EXTENT(_N / CHAR_BIT)]) ZTD_CXX_NOEXCEPT_I_; \
ZTD_IDK_API_LINKAGE_I_ void ztdc_store8_bes##_N( \
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ void ztdc_store8_bes##_N( \
const int_least##_N##_t __value, unsigned char __ptr[ZTD_PTR_EXTENT(_N / CHAR_BIT)]) ZTD_CXX_NOEXCEPT_I_; \
ZTD_IDK_API_LINKAGE_I_ void ztdc_store8_aligned_les##_N( \
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ void ztdc_store8_aligned_les##_N( \
const int_least##_N##_t __value, unsigned char __ptr[ZTD_PTR_EXTENT(_N / CHAR_BIT)]) ZTD_CXX_NOEXCEPT_I_; \
ZTD_IDK_API_LINKAGE_I_ void ztdc_store8_aligned_bes##_N( \
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ void ztdc_store8_aligned_bes##_N( \
const int_least##_N##_t __value, unsigned char __ptr[ZTD_PTR_EXTENT(_N / CHAR_BIT)]) ZTD_CXX_NOEXCEPT_I_; \
ztd_static_assert(((_N % 8) == 0), "👍")

Expand Down
19 changes: 10 additions & 9 deletions include/ztd/idk/detail/bit.memreverse.h
Original file line number Diff line number Diff line change
Expand Up @@ -51,30 +51,31 @@
ZTD_EXTERN_C_OPEN_I_

#if (CHAR_BIT % 8 == 0)
ZTD_IDK_API_LINKAGE_I_ void ztdc_memreverse8(size_t __n, unsigned char __ptr[ZTD_PTR_EXTENT(__n)]) ZTD_CXX_NOEXCEPT_I_;
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ void ztdc_memreverse8(
size_t __n, unsigned char __ptr[ZTD_PTR_EXTENT(__n)]) ZTD_CXX_NOEXCEPT_I_;
#if defined(UINT8_MAX)
ZTD_IDK_API_LINKAGE_I_ uint8_t ztdc_memreverse8u8(uint8_t __value) ZTD_CXX_NOEXCEPT_I_;
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ uint8_t ztdc_memreverse8u8(uint8_t __value) ZTD_CXX_NOEXCEPT_I_;
#endif // 8 bits
#if defined(UINT16_MAX)
ZTD_IDK_API_LINKAGE_I_ uint16_t ztdc_memreverse8u16(uint16_t __value) ZTD_CXX_NOEXCEPT_I_;
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ uint16_t ztdc_memreverse8u16(uint16_t __value) ZTD_CXX_NOEXCEPT_I_;
#endif // 16 bits
#if defined(UINT24_MAX)
ZTD_IDK_API_LINKAGE_I_ uint24_t ztdc_memreverse824(uint24_t __value) ZTD_CXX_NOEXCEPT_I_;
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ uint24_t ztdc_memreverse824(uint24_t __value) ZTD_CXX_NOEXCEPT_I_;
#endif // 24 bits
#if defined(UINT32_MAX)
ZTD_IDK_API_LINKAGE_I_ uint32_t ztdc_memreverse8u32(uint32_t __value) ZTD_CXX_NOEXCEPT_I_;
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ uint32_t ztdc_memreverse8u32(uint32_t __value) ZTD_CXX_NOEXCEPT_I_;
#endif // 32 bits
#if defined(UINT40_MAX)
ZTD_IDK_API_LINKAGE_I_ uint40_t ztdc_memreverse8u40(uint40_t __value) ZTD_CXX_NOEXCEPT_I_;
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ uint40_t ztdc_memreverse8u40(uint40_t __value) ZTD_CXX_NOEXCEPT_I_;
#endif // 40 bits
#if defined(UINT48_MAX)
ZTD_IDK_API_LINKAGE_I_ uint48_t ztdc_memreverse8u48(uint48_t __value) ZTD_CXX_NOEXCEPT_I_;
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ uint48_t ztdc_memreverse8u48(uint48_t __value) ZTD_CXX_NOEXCEPT_I_;
#endif // 48 bits
#if defined(UINT56_MAX)
ZTD_IDK_API_LINKAGE_I_ uint56_t ztdc_memreverse8u56(uint56_t __value) ZTD_CXX_NOEXCEPT_I_;
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ uint56_t ztdc_memreverse8u56(uint56_t __value) ZTD_CXX_NOEXCEPT_I_;
#endif // 48 bits
#if defined(UINT64_MAX)
ZTD_IDK_API_LINKAGE_I_ uint64_t ztdc_memreverse8u64(uint64_t __value) ZTD_CXX_NOEXCEPT_I_;
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ uint64_t ztdc_memreverse8u64(uint64_t __value) ZTD_CXX_NOEXCEPT_I_;
#endif // 64 bits
#endif

Expand Down
91 changes: 91 additions & 0 deletions include/ztd/idk/detail/posix.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
// =============================================================================
//
// ztd.idk
// Copyright © 2021 JeanHeyd "ThePhD" Meneide and Shepherd's Oasis, LLC
// Contact: opensource@soasis.org
//
// Commercial License Usage
// Licensees holding valid commercial ztd.idk licenses may use this file in
// accordance with the commercial license agreement provided with the
// Software or, alternatively, in accordance with the terms contained in
// a written agreement between you and Shepherd's Oasis, LLC.
// For licensing terms and conditions see your agreement. For
// further information contact opensource@soasis.org.
//
// Apache License Version 2 Usage
// Alternatively, this file may be used under the terms of Apache License
// Version 2.0 (the "License") for non-commercial use; you may not use this
// file except in compliance with the License. You may obtain a copy of the
// License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// ============================================================================>

#pragma once

#ifndef ZTD_IDK_DETAIL_POSIX_HPP
#define ZTD_IDK_DETAIL_POSIX_HPP

#include <ztd/idk/version.hpp>

#include <ztd/idk/encoding_name.hpp>

#if ZTD_IS_ON(ZTD_PLATFORM_UNIX_I_) || ZTD_IS_ON(ZTD_PLATFORM_POSIX_I_)

// clang-format off
#include <clocale>
#if ZTD_IS_ON(ZTD_LANGINFO_I_)
extern "C" {
#include <langinfo.h>
}
#elif ZTD_IS_ON(ZTD_NL_LANGINFO_I_)
// IBM-specific??
extern "C" {
#include <nl_langinfo.h>
}
#endif

#if ZTD_IS_ON(ZTD_LIBICONV_LOAD_I_)
#if ZTD_IS_ON(ZTD_PLATFORM_POSIX_I_) && ZTD_IS_ON(ZTD_DLFCN_H_I_)
#include <dlfcn.h>
#endif
#endif
// clang-format on

#include <ztd/prologue.hpp>

namespace ztd {

ZTD_IDK_INLINE_ABI_NAMESPACE_OPEN_I_

namespace __idk_detail { namespace __posix {

inline __idk_detail::__encoding_id __determine_active_code_page() noexcept {
#if ZTD_IS_ON(ZTD_LANGINFO_I_) || ZTD_IS_ON(ZTD_NL_LANGINFO_I_)
const char* __name = nl_langinfo(LC_CTYPE);
return __idk_detail::__to_encoding_id(__name);
#else
// fallback to stdlib I guess?
const char* __ctype_name = setlocale(LC_CTYPE, nullptr);
return __idk_detail::__to_encoding_id(__ctype_name);
#endif
}

}} // namespace __idk_detail::__posix

ZTD_IDK_INLINE_ABI_NAMESPACE_CLOSE_I_

} // namespace ztd

#endif // POSIX

#include <ztd/epilogue.hpp>

#endif // ZTD_IDK_DETAIL_POSIX_HPP
40 changes: 38 additions & 2 deletions include/ztd/idk/detail/windows.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,11 @@

#if ZTD_IS_ON(ZTD_PLATFORM_WINDOWS_I_)

#if ZTD_IS_ON(ZTD_COMPILER_VCXX_I_)
#pragma push_macro("NOMINMAX")
#pragma push_macro("WIN32_LEAN_AND_MEAN")
#pragma push_macro("VC_EXTRALEAN")
#endif

#define NOMINMAX 1
#define WIN32_LEAN_AND_MEAN 1
Expand All @@ -59,17 +61,51 @@ ZTD_EXTERN_C_CLOSE_I_

#include <ztd/prologue.hpp>

namespace ztd { namespace idk {
namespace ztd {
ZTD_IDK_INLINE_ABI_NAMESPACE_OPEN_I_
namespace __idk_detail { namespace __windows {

inline int __determine_active_code_page() noexcept {
#if defined(_STL_LANG) || defined(_YVALS_CORE_H) || defined(_STDEXT)
// Removed in later versions of VC++
if (___lc_codepage_func() == CP_UTF8) {
return CP_UTF8;
}
#endif // VC++ stuff

#if !defined(_KERNELX) && !defined(_ONECORE)
if (!::AreFileApisANSI()) {
return CP_OEMCP;
}
#endif // !defined(_KERNELX) && !defined(_ONECORE)

return CP_ACP;
}

inline bool __is_unicode_code_page(int __codepage_id) {
switch (__codepage_id) {
case CP_UTF7:
case CP_UTF8:
case 1200: // UTF-16, Little Endian ("utf-16")
case 1201: // UTF-16, Big Endian ("unicodeFFFE")
case 12000: // UTF-16, Little Endian ("utf-32")
case 12001: // UTF-16, Big Endian ("utf-32BE")
case 54936: // GB18030, 4 bytes long
return true;
default:
return false;
}
}

}} // namespace __idk_detail::__windows
ZTD_IDK_INLINE_ABI_NAMESPACE_CLOSE_I_
}} // namespace ztd::idk
} // namespace ztd

#if ZTD_IS_ON(ZTD_COMPILER_VCXX_I_)
#pragma pop_macro("VC_EXTRALEAN")
#pragma pop_macro("WIN32_LEAN_AND_MEAN")
#pragma pop_macro("NOMINMAX")
#endif

#endif // Windows nightmare

Expand Down
46 changes: 46 additions & 0 deletions include/ztd/idk/encoding_detection.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
// =============================================================================
//
// ztd.cuneicode
// Copyright © 2021 JeanHeyd "ThePhD" Meneide and Shepherd's Oasis, LLC
// Contact: opensource@soasis.org
//
// Commercial License Usage
// Licensees holding valid commercial ztd.cuneicode licenses may use this file
// in accordance with the commercial license agreement provided with the
// Software or, alternatively, in accordance with the terms contained in
// a written agreement between you and Shepherd's Oasis, LLC.
// For licensing terms and conditions see your agreement. For
// further information contact opensource@soasis.org.
//
// Apache License Version 2 Usage
// Alternatively, this file may be used under the terms of Apache License
// Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// ============================================================================>

#ifndef ZTD_IDK_ENCODING_DETECTION_H
#define ZTD_IDK_ENCODING_DETECTION_H

#pragma once

#include <ztd/idk/version.h>

#if ZTD_IS_ON(ZTD_C_I_)
#include <stdbool.h>
#endif // C++

ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ bool ztdc_is_execution_encoding_unicode(
void) ZTD_NOEXCEPT_IF_CXX_I_;
ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_API_LINKAGE_I_ bool ztdc_is_wide_execution_encoding_unicode(
void) ZTD_NOEXCEPT_IF_CXX_I_;

#endif // ZTD_IDK_ENCODING_DETECTION_H
37 changes: 37 additions & 0 deletions include/ztd/idk/encoding_name.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,43 @@ namespace ztd {
return "unknown";
}
}

inline ztd_char8_t __normalize_ascii_letter(ztd_char8_t __c0) {
if (__c0 <= 'Z' && __c0 >= 'A') {
// A is equivalent to a, etc. etc.
// ASCII character, make sure lowercase
// add to 5th bit (0-based) to get lowercase
__c0 &= static_cast<ztd_char8_t>(0x20);
}
return __c0;
}

template <typename _Target>
void __inplace_id_normalize(_Target& __target) {
// FIXME: full unicode case folding at some point in my short,
// miserable existence
// FIXME: full normalization form D case folding at some point in my
// pathetic mortality
size_t __target_size = __target.size();
for (size_t __idx = 0; __idx < __target_size;) {
ztd_char8_t __c0 = __target[__idx];
int __seq_len = __ztd_idk_detail_utf8_sequence_length(__c0);
bool __is_ascii = __seq_len < 2 && ((__c0 & 0x7F) == 0);
if (__is_ascii) {
if (__c0 == '-' || __c0 == '_' || __c0 == ' ' || __c0 == '\r' || __c0 == '\n'
|| __c0 == '\t') {
// skip dash, underscore, related whitespace
__target.erase(__target.begin() + __idx);
continue;
}
__target[__idx] = __normalize_ascii_letter(__c0);
++__idx;
continue;
}

__idx += __seq_len;
}
}
} // namespace __idk_detail

template <typename _Left, typename _Right>
Expand Down
12 changes: 12 additions & 0 deletions include/ztd/idk/version.h
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,18 @@
#define ZTD_IDK_API_LINKAGE_I_ ZTD_API_LINKAGE_I_
#endif

#if defined(ZTD_IDK_C_LANGUAGE_LINKAGE)
#define ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_IDK_C_LANGUAGE_LINKAGE
#else
#define ZTD_IDK_C_LANGUAGE_LINKAGE_I_ ZTD_C_LANGUAGE_LINKAGE_I_
#endif

#if defined(ZTD_IDK_CXX_LANGUAGE_LINKAGE)
#define ZTD_IDK_CXX_LANGUAGE_LINKAGE_I_ ZTD_IDK_CXX_LANGUAGE_LINKAGE
#else
#define ZTD_IDK_CXX_LANGUAGE_LINKAGE_I_ ZTD_CXX_LANGUAGE_LINKAGE_I_
#endif

#define ZTD_IDK_VERSION_MAJOR 0
#define ZTD_IDK_VERSION_MINOR 0
#define ZTD_IDK_VERSION_PATCH 0
Expand Down

0 comments on commit 8911056

Please sign in to comment.