diff --git a/.appveyor.yml b/.appveyor.yml index 88719fcf..f305b5a8 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -112,11 +112,11 @@ environment: B2_CXXSTD: 11,1z B2_TOOLSET: gcc - - FLAVOR: mingw32 - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 + - FLAVOR: mingw64 (32-bit) + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 + ADDPATH: C:\mingw-w64\i686-8.1.0-posix-dwarf-rt_v6-rev0\mingw32\bin; B2_ADDRESS_MODEL: 32 - ADDPATH: C:\mingw\bin; - B2_CXXSTD: 11,14,1z + B2_CXXSTD: 03,11,14,17,2a B2_TOOLSET: gcc - FLAVOR: mingw64 diff --git a/.drone.star b/.drone.star new file mode 100644 index 00000000..43028dab --- /dev/null +++ b/.drone.star @@ -0,0 +1,67 @@ +# Copyright 2020 Rene Rivera +# Copyright 2022-2023 Alexander Grund +# +# Distributed under the Boost Software License, Version 1.0. +# https://www.boost.org/LICENSE_1_0.txt + +# For Drone CI we use the Starlark scripting language to reduce duplication. +# As the yaml syntax for Drone CI is rather limited. + +# Base environment for all jobs +globalenv={'B2_CI_VERSION': '1', 'B2_VARIANT': 'debug,release'} + +# Wrapper function to apply the globalenv to all jobs +def job( + # job specific environment options + env={}, + **kwargs): + real_env = dict(globalenv) + real_env.update(env) + return job_impl(env=real_env, **kwargs) + +def main(ctx): + return [ + job(compiler='clang-13', cxxstd='11,14,17,20,2b', os='ubuntu-22.04', install='libicu-dev'), + job(compiler='clang-14', cxxstd='11,14,17,20,2b', os='ubuntu-22.04', install='libicu-dev'), + job(compiler='clang-15', cxxstd='11,14,17,20,2b', os='ubuntu-22.04', install='libicu-dev', add_llvm=True), + + job(compiler='gcc-11', cxxstd='11,14,17,20,2b', os='ubuntu-22.04', install='libicu-dev'), + job(compiler='gcc-12', cxxstd='11,14,17,20,2b', os='ubuntu-22.04', install='libicu-dev'), + + # Sanitizers + job(name='ASAN', asan=True, + compiler='gcc-12', cxxstd='11,14,17,20', os='ubuntu-22.04', install='libicu-dev'), + job(name='UBSAN', ubsan=True, + compiler='gcc-12', cxxstd='11,14,17,20', os='ubuntu-22.04', install='libicu-dev'), + job(name='Clang 14 w/ sanitizers', asan=True, ubsan=True, + compiler='clang-14', cxxstd='11,14,17,20', os='ubuntu-22.04', install='libicu-dev'), + + # libc++ + job(compiler='clang-15', cxxstd='11,14,17,20', os='ubuntu-22.04', stdlib='libc++', install='libicu-dev libc++-15-dev libc++abi-15-dev', add_llvm=True), + + # FreeBSD + job(compiler='clang-10', cxxstd='11,14,17,20', os='freebsd-13.1'), + job(compiler='clang-15', cxxstd='11,14,17,20', os='freebsd-13.1'), + # ICU is linked against libc++, so either don't use ICU or use libc++. The latter doesn't seem to work well (segfaults) + job(compiler='gcc-11', cxxstd='11,14,17,20', os='freebsd-13.1', testflags='boost.locale.icu=off', linkflags='-Wl,-rpath=/usr/local/lib/gcc11'), + # OSX + job(compiler='clang', cxxstd='11,14,17,2a', os='osx-xcode-10.1'), + job(compiler='clang', cxxstd='11,14,17,2a', os='osx-xcode-10.3'), + job(compiler='clang', cxxstd='11,14,17,2a', os='osx-xcode-12'), + job(compiler='clang', cxxstd='11,14,17,20', os='osx-xcode-12.5.1'), + job(compiler='clang', cxxstd='11,14,17,20', os='osx-xcode-13.4.1'), + # Don't work yet -> #206 ? + # job(compiler='clang', cxxstd='11,14,17,20,2b', os='osx-xcode-14.3.1'), + # job(compiler='clang', cxxstd='11,14,17,20,2b', os='osx-xcode-15.0.1'), + # ARM64 + job(compiler='clang-12', cxxstd='11,14,17,20', os='ubuntu-20.04', arch='arm64', add_llvm=True), + job(compiler='gcc-11', cxxstd='11,14,17,20', os='ubuntu-20.04', arch='arm64'), + # S390x + job(compiler='clang-12', cxxstd='11,14,17,20', os='ubuntu-20.04', arch='s390x', add_llvm=True), + job(compiler='gcc-11', cxxstd='11,14,17,20', os='ubuntu-20.04', arch='s390x'), + # Windows + job(compiler='msvc-14.3', cxxstd='14,17,20,latest', os='windows'), + ] + +# from https://github.com/boostorg/boost-ci +load("@boost_ci//ci/drone/:functions.star", "linux_cxx", "windows_cxx", "osx_cxx", "freebsd_cxx", "job_impl") diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3347b87d..9d76c182 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: CheckFormatting: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: DoozyX/clang-format-lint-action@v0.14 with: exclude: './doc' @@ -59,9 +59,9 @@ jobs: Create_Boost_Documentation: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Fetch Boost.CI - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: boostorg/boost-ci ref: master @@ -305,9 +305,21 @@ jobs: run: ci/build.sh env: {B2_FLAGS: -a boost.locale.icu=on boost.locale.iconv=off} - - name: Upload coverage + - name: Collect coverage if: matrix.coverage run: ci/codecov.sh "upload" + env: + BOOST_CI_CODECOV_IO_UPLOAD: skip + + - name: Upload coverage + if: matrix.coverage + uses: codecov/codecov-action@v4 + with: + disable_search: true + file: coverage.info + name: Github Actions + token: ${{secrets.CODECOV_TOKEN}} + verbose: true - name: Run coverity if: matrix.coverity && github.event_name == 'push' && (github.ref_name == 'develop' || github.ref_name == 'master') @@ -333,10 +345,10 @@ jobs: env: {ICU_VERSION: '71.1'} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Fetch Boost.CI - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: boostorg/boost-ci ref: master @@ -351,7 +363,7 @@ jobs: run: ci\github\install.bat - name: Get cached ICU - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-icu with: path: ICU @@ -421,9 +433,13 @@ jobs: - name: Upload coverage if: matrix.coverage - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: - files: __out/cobertura.xml + disable_search: true + file: __out/cobertura.xml + name: Github Actions + token: ${{secrets.CODECOV_TOKEN}} + verbose: true MSYS2: defaults: @@ -439,7 +455,7 @@ jobs: runs-on: windows-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup MSYS2 environment uses: msys2/setup-msys2@v2 @@ -450,7 +466,7 @@ jobs: pacboy: gcc:p cmake:p ninja:p - name: Fetch Boost.CI - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: boostorg/boost-ci ref: master @@ -503,9 +519,9 @@ jobs: runs-on: ${{matrix.os}} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Fetch Boost.CI - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: boostorg/boost-ci ref: master @@ -536,7 +552,7 @@ jobs: echo "LD_LIBRARY_PATH=$ICU_ROOT/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV - name: Get cached ICU if: matrix.icu - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-icu with: path: ICU diff --git a/CMakeLists.txt b/CMakeLists.txt index eae5fcf2..f47ad394 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,7 @@ add_library(boost_locale src/boost/locale/shared/localization_backend.cpp src/boost/locale/shared/message.cpp src/boost/locale/shared/mo_lambda.cpp + src/boost/locale/shared/std_collate_adapter.hpp src/boost/locale/util/codecvt_converter.cpp src/boost/locale/util/default_locale.cpp src/boost/locale/util/encoding.cpp diff --git a/README.md b/README.md index 2cc87678..ea18e02d 100644 --- a/README.md +++ b/README.md @@ -42,10 +42,10 @@ Distributed under the [Boost Software License, Version 1.0](https://www.boost.or ### Build Status -Branch | GH Actions | Appveyor | codecov.io | Deps | Docs | Tests | -:-------------: | ---------- | -------- | ---------- | ---- | ---- | ----- | -[`master`](https://github.com/boostorg/locale/tree/master) | [![CI](https://github.com/boostorg/locale/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/boostorg/locale/actions/workflows/ci.yml) | [![Build status](https://ci.appveyor.com/api/projects/status/github/boostorg/locale?branch=master&svg=true)](https://ci.appveyor.com/project/Flamefire/locale/branch/master) | [![codecov](https://codecov.io/gh/boostorg/locale/branch/master/graph/badge.svg)](https://codecov.io/gh/boostorg/locale/branch/master) | [![Deps](https://img.shields.io/badge/deps-master-brightgreen.svg)](https://pdimov.github.io/boostdep-report/master/locale.html) | [![Documentation](https://img.shields.io/badge/docs-master-brightgreen.svg)](https://www.boost.org/doc/libs/master/libs/locale/doc/html/index.html) | [![Enter the Matrix](https://img.shields.io/badge/matrix-master-brightgreen.svg)](http://www.boost.org/development/tests/master/developer/locale.html) -[`develop`](https://github.com/boostorg/locale/tree/develop) | [![CI](https://github.com/boostorg/locale/actions/workflows/ci.yml/badge.svg?branch=develop)](https://github.com/boostorg/locale/actions/workflows/ci.yml) | [![Build status](https://ci.appveyor.com/api/projects/status/github/boostorg/locale?branch=develop&svg=true)](https://ci.appveyor.com/project/Flamefire/locale/branch/develop) | [![codecov](https://codecov.io/gh/boostorg/locale/branch/develop/graph/badge.svg)](https://codecov.io/gh/boostorg/locale/branch/develop) | [![Deps](https://img.shields.io/badge/deps-develop-brightgreen.svg)](https://pdimov.github.io/boostdep-report/develop/locale.html) | [![Documentation](https://img.shields.io/badge/docs-develop-brightgreen.svg)](https://www.boost.org/doc/libs/develop/libs/locale/doc/html/index.html) | [![Enter the Matrix](https://img.shields.io/badge/matrix-develop-brightgreen.svg)](http://www.boost.org/development/tests/develop/developer/locale.html) +Branch | GH Actions | Appveyor | Drone | codecov.io | Deps | Docs | Tests | +:-------------: | ---------- | -------- | ----- | ---------- | ---- | ---- | ----- | +[`master`](https://github.com/boostorg/locale/tree/master) | [![CI](https://github.com/boostorg/locale/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/boostorg/locale/actions/workflows/ci.yml) | [![Build status](https://ci.appveyor.com/api/projects/status/github/boostorg/locale?branch=master&svg=true)](https://ci.appveyor.com/project/Flamefire/locale/branch/master) | [![Build Status](https://drone.cpp.al/api/badges/boostorg/locale/status.svg?ref=refs/heads/master)](https://drone.cpp.al/boostorg/locale) | [![codecov](https://codecov.io/gh/boostorg/locale/branch/master/graph/badge.svg)](https://codecov.io/gh/boostorg/locale/branch/master) | [![Deps](https://img.shields.io/badge/deps-master-brightgreen.svg)](https://pdimov.github.io/boostdep-report/master/locale.html) | [![Documentation](https://img.shields.io/badge/docs-master-brightgreen.svg)](https://www.boost.org/doc/libs/master/libs/locale/doc/html/index.html) | [![Enter the Matrix](https://img.shields.io/badge/matrix-master-brightgreen.svg)](http://www.boost.org/development/tests/master/developer/locale.html) +[`develop`](https://github.com/boostorg/locale/tree/develop) | [![CI](https://github.com/boostorg/locale/actions/workflows/ci.yml/badge.svg?branch=develop)](https://github.com/boostorg/locale/actions/workflows/ci.yml) | [![Build status](https://ci.appveyor.com/api/projects/status/github/boostorg/locale?branch=develop&svg=true)](https://ci.appveyor.com/project/Flamefire/locale/branch/develop) | [![Build Status](https://drone.cpp.al/api/badges/boostorg/locale/status.svg?ref=refs/heads/develop)](https://drone.cpp.al/boostorg/locale) | [![codecov](https://codecov.io/gh/boostorg/locale/branch/develop/graph/badge.svg)](https://codecov.io/gh/boostorg/locale/branch/develop) | [![Deps](https://img.shields.io/badge/deps-develop-brightgreen.svg)](https://pdimov.github.io/boostdep-report/develop/locale.html) | [![Documentation](https://img.shields.io/badge/docs-develop-brightgreen.svg)](https://www.boost.org/doc/libs/develop/libs/locale/doc/html/index.html) | [![Enter the Matrix](https://img.shields.io/badge/matrix-develop-brightgreen.svg)](http://www.boost.org/development/tests/develop/developer/locale.html) ### Directories diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index f1e2d05d..b7268c58 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -192,6 +192,13 @@ ICU_OPTS = exe has_icu : $(TOP)/build/has_icu_test.cpp : $(ICU_OPTS) ; explicit has_icu ; +# Separate pair of obj & exe rules so the CPP file is built with the changed include paths +obj has_iconv_with_icu_obj : $(TOP)/build/has_iconv.cpp : $(ICU_OPTS) ; +exe has_iconv_with_icu : has_external_iconv_with_icu_obj : $(ICU_OPTS) ; +explicit has_iconv_with_icu ; +obj has_external_iconv_with_icu_obj : $(TOP)/build/has_iconv.cpp iconv : $(ICU_OPTS) ; +exe has_external_iconv_with_icu : has_external_iconv_with_icu_obj iconv : $(ICU_OPTS) ; +explicit has_external_iconv_with_icu ; # This function is called whenever the 'boost_locale' metatarget # below is generated and figures out what external components we have, @@ -212,25 +219,8 @@ rule configure-full ( properties * : flags-only ) } } - local found-iconv ; - - if ! off in $(properties) - { - # See if iconv is bundled with standard library. - if [ configure.builds has_iconv : $(properties) : "iconv (libc)" ] - { - found-iconv = true ; - } else if [ configure.builds has_external_iconv : $(properties) : "iconv (separate)" ] - { - found-iconv = true ; - result += iconv ; - } - } - if $(found-iconv) - { - flags-result += BOOST_LOCALE_WITH_ICONV=1 ; - } - + local internal_iconv_target = has_iconv ; + local external_iconv_target = has_external_iconv ; local found-icu ; if ! off in $(properties) @@ -256,9 +246,31 @@ rule configure-full ( properties * : flags-only ) result += icu/$(ICU_SOURCES).cpp /boost/thread//boost_thread ; + # ICU might introduce an own iconv, so check for that. + internal_iconv_target = has_iconv_with_icu ; + external_iconv_target = has_external_iconv_with_icu ; } } + local found-iconv ; + + if ! off in $(properties) + { + # See if iconv is bundled with standard library. + if [ configure.builds $(internal_iconv_target) : $(properties) : "iconv (libc)" ] + { + found-iconv = true ; + } else if [ configure.builds $(external_iconv_target) : $(properties) : "iconv (separate)" ] + { + found-iconv = true ; + result += iconv ; + } + } + if $(found-iconv) + { + flags-result += BOOST_LOCALE_WITH_ICONV=1 ; + } + if ! $(found-iconv) && ! $(found-icu) && ! windows in $(properties) && ! cygwin in $(properties) { ECHO "- Boost.Locale needs either iconv or ICU library to be built." ; diff --git a/doc/changelog.txt b/doc/changelog.txt index 267a5fc7..4d539b50 100644 --- a/doc/changelog.txt +++ b/doc/changelog.txt @@ -1,6 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) -// Copyright (c) 2021-2023 Alexander Grund +// Copyright (c) 2021-2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -8,6 +8,9 @@ /*! \page changelog Changelog +- 1.85.0 + - Breaking changes + - `collator` does no longer derive from `std::collator` avoiding possible type confusion - 1.84.0 - Breaking changes - `to_title` for the WinAPI backend returns the string unchanged instead of an empty string diff --git a/include/boost/locale/collator.hpp b/include/boost/locale/collator.hpp index b45f1547..693d33a6 100644 --- a/include/boost/locale/collator.hpp +++ b/include/boost/locale/collator.hpp @@ -8,6 +8,7 @@ #define BOOST_LOCALE_COLLATOR_HPP_INCLUDED #include +#include #include #ifdef BOOST_MSVC @@ -43,17 +44,16 @@ namespace boost { namespace locale { /// \brief Collation facet. /// - /// It reimplements standard C++ std::collate, - /// allowing usage of std::locale for direct string comparison + /// It reimplements standard C++ std::collate with support for collation levels template - class collator : public std::collate { + class BOOST_SYMBOL_VISIBLE collator : public std::locale::facet, public detail::facet_id> { public: /// Type of the underlying character typedef CharType char_type; /// Type of string used with this facet typedef std::basic_string string_type; - /// Compare two strings in rage [b1,e1), [b2,e2) according using a collation level \a level. Calls do_compare + /// Compare two strings in range [b1,e1), [b2,e2) according to collation level \a level. Calls do_compare /// /// Returns -1 if the first of the two strings sorts before the seconds, returns 1 if sorts after and 0 if /// they considered equal. @@ -66,6 +66,13 @@ namespace boost { namespace locale { return do_compare(level, b1, e1, b2, e2); } + /// Default compare function as-in std::collate that does not take collation level into account. + /// Uses identical level + int compare(const char_type* b1, const char_type* e1, const char_type* b2, const char_type* e2) const + { + return compare(collate_level::identical, b1, e1, b2, e2); + } + /// Create a binary string that can be compared to other in order to get collation order. The string is created /// for text in range [b,e). It is useful for collation of multiple strings for text. /// @@ -80,6 +87,13 @@ namespace boost { namespace locale { return do_transform(level, b, e); } + /// Default transform function as-in std::collate that does not take collation level into account. + /// Uses identical level + string_type transform(const char_type* b, const char_type* e) const + { + return transform(collate_level::identical, b, e); + } + /// Calculate a hash of a text in range [b,e). The value can be used for collation sensitive string comparison. /// /// If compare(level,b1,e1,b2,e2) == 0 then hash(level,b1,e1) == hash(level,b2,e2) @@ -87,6 +101,10 @@ namespace boost { namespace locale { /// Calls do_hash long hash(collate_level level, const char_type* b, const char_type* e) const { return do_hash(level, b, e); } + /// Default hash function as-in std::collate that does not take collation level into account. + /// Uses identical level + long hash(const char_type* b, const char_type* e) const { return hash(collate_level::identical, b, e); } + /// Compare two strings \a l and \a r using collation level \a level /// /// Returns -1 if the first of the two strings sorts before the seconds, returns 1 if sorts after and 0 if @@ -107,7 +125,7 @@ namespace boost { namespace locale { /// Create a binary string from string \a s, that can be compared to other, useful for collation of multiple /// strings. /// - /// The transformation follows these rules: + /// The transformation follows this rule: /// \code /// compare(level,s1,s2) == sign( transform(level,s1).compare(transform(level,s2)) ); /// \endcode @@ -118,29 +136,7 @@ namespace boost { namespace locale { protected: /// constructor of the collator object - collator(size_t refs = 0) : std::collate(refs) {} - - /// This function is used to override default collation function that does not take in account collation level. - /// Uses primary level - int - do_compare(const char_type* b1, const char_type* e1, const char_type* b2, const char_type* e2) const override - { - return do_compare(collate_level::identical, b1, e1, b2, e2); - } - - /// This function is used to override default collation function that does not take in account collation level. - /// Uses primary level - string_type do_transform(const char_type* b, const char_type* e) const override - { - return do_transform(collate_level::identical, b, e); - } - - /// This function is used to override default collation function that does not take in account collation level. - /// Uses primary level - long do_hash(const char_type* b, const char_type* e) const override - { - return do_hash(collate_level::identical, b, e); - } + collator(size_t refs = 0) : std::locale::facet(refs) {} /// Actual function that performs comparison between the strings. For details see compare member function. Can /// be overridden. @@ -157,7 +153,7 @@ namespace boost { namespace locale { }; /// \brief This class can be used in STL algorithms and containers for comparison of strings - /// with a level other than primary + /// with a level other than identical /// /// For example: /// @@ -169,21 +165,22 @@ namespace boost { namespace locale { template struct comparator { public: - /// Create a comparator class for locale \a l and with collation leval \a level + /// Create a comparator class for locale \a l and with collation level \a level /// /// \throws std::bad_cast: \a l does not have \ref collator facet installed comparator(const std::locale& l = std::locale(), collate_level level = default_level) : - locale_(l), level_(level) + locale_(l), collator_(std::use_facet>(locale_)), level_(level) {} /// Compare two strings -- equivalent to return left < right according to collation rules bool operator()(const std::basic_string& left, const std::basic_string& right) const { - return std::use_facet>(locale_).compare(level_, left, right) < 0; + return collator_.compare(level_, left, right) < 0; } private: std::locale locale_; + const collator& collator_; collate_level level_; }; diff --git a/include/boost/locale/detail/any_string.hpp b/include/boost/locale/detail/any_string.hpp new file mode 100644 index 00000000..c0cc7ffb --- /dev/null +++ b/include/boost/locale/detail/any_string.hpp @@ -0,0 +1,71 @@ +// +// Copyright (c) 2023 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#ifndef BOOST_LOCALE_DETAIL_ANY_STRING_HPP_INCLUDED +#define BOOST_LOCALE_DETAIL_ANY_STRING_HPP_INCLUDED + +#include +#include +#include +#include +#include +#include + +/// \cond INTERNAL +namespace boost { namespace locale { namespace detail { + /// Type-erased std::basic_string + class any_string { + struct BOOST_SYMBOL_VISIBLE base { + virtual ~base() = default; + virtual base* clone() const = 0; + + protected: + base() = default; + base(const base&) = default; + base(base&&) = delete; + base& operator=(const base&) = default; + base& operator=(base&&) = delete; + }; + template + struct BOOST_SYMBOL_VISIBLE impl : base { + explicit impl(const boost::basic_string_view value) : s(value) {} + impl* clone() const override { return new impl(*this); } + std::basic_string s; + }; + + std::unique_ptr s_; + + public: + any_string() = default; + any_string(const any_string& other) : s_(other.s_ ? other.s_->clone() : nullptr) {} + any_string(any_string&&) = default; + any_string& operator=(any_string other) // Covers the copy and move assignment + { + s_.swap(other.s_); + return *this; + } + + template + void set(const boost::basic_string_view s) + { + BOOST_ASSERT(!s.empty()); + s_.reset(new impl(s)); + } + + template + std::basic_string get() const + { + if(!s_) + throw std::bad_cast(); + return dynamic_cast&>(*s_).s; + } + }; + +}}} // namespace boost::locale::detail + +/// \endcond + +#endif diff --git a/include/boost/locale/formatting.hpp b/include/boost/locale/formatting.hpp index d14e6f69..e3c8619e 100644 --- a/include/boost/locale/formatting.hpp +++ b/include/boost/locale/formatting.hpp @@ -8,15 +8,13 @@ #ifndef BOOST_LOCALE_FORMATTING_HPP_INCLUDED #define BOOST_LOCALE_FORMATTING_HPP_INCLUDED +#include #include -#include -#include #include #include #include #include #include -#include #ifdef BOOST_MSVC # pragma warning(push) @@ -130,13 +128,13 @@ namespace boost { namespace locale { template void date_time_pattern(const std::basic_string& str) { - date_time_pattern_set().set(str); + datetime_.set(str); } /// Get date/time pattern (strftime like) template std::basic_string date_time_pattern() const { - return date_time_pattern_set().get(); + return datetime_.get(); } /// \cond INTERNAL @@ -144,51 +142,10 @@ namespace boost { namespace locale { /// \endcond private: - class string_set; - - const string_set& date_time_pattern_set() const; - string_set& date_time_pattern_set(); - - class BOOST_LOCALE_DECL string_set { - public: - string_set(); - ~string_set(); - string_set(const string_set& other); - string_set& operator=(string_set other); - void swap(string_set& other); - - template - void set(const boost::basic_string_view s) - { - BOOST_ASSERT(!s.empty()); - delete[] ptr; - ptr = nullptr; - type = &typeid(Char); - size = sizeof(Char) * s.size(); - ptr = size ? new char[size] : nullptr; - memcpy(ptr, s.data(), size); - } - - template - std::basic_string get() const - { - if(type == nullptr || *type != typeid(Char)) - throw std::bad_cast(); - std::basic_string result(size / sizeof(Char), Char(0)); - memcpy(&result.front(), ptr, size); - return result; - } - - private: - const std::type_info* type; - size_t size; - char* ptr; - }; - uint64_t flags_; int domain_id_; std::string time_zone_; - string_set datetime_; + detail::any_string datetime_; }; /// \brief This namespace includes all manipulators that can be used on IO streams diff --git a/src/boost/locale/encoding/iconv_converter.hpp b/src/boost/locale/encoding/iconv_converter.hpp index 38a8f485..2877dbca 100644 --- a/src/boost/locale/encoding/iconv_converter.hpp +++ b/src/boost/locale/encoding/iconv_converter.hpp @@ -10,6 +10,7 @@ #include #include "boost/locale/util/encoding.hpp" #include "boost/locale/util/iconv.hpp" +#include #include #include @@ -47,6 +48,7 @@ namespace boost { namespace locale { namespace conv { namespace impl { if(in_left == 0) is_unshifting = true; + const auto old_in_left = in_left; const size_t res = (!is_unshifting) ? conv(&begin, &in_left, &out_ptr, &out_left) : conv(nullptr, nullptr, &out_ptr, &out_left); @@ -60,6 +62,7 @@ namespace boost { namespace locale { namespace conv { namespace impl { if(res == (size_t)(-1)) { const int err = errno; + BOOST_ASSERT_MSG(err == EILSEQ || err == EINVAL || err == E2BIG, "Invalid error code from IConv"); if(err == EILSEQ || err == EINVAL) { if(how_ == stop) throw conversion_error(); @@ -70,9 +73,11 @@ namespace boost { namespace locale { namespace conv { namespace impl { break; } else break; - } else if(err == E2BIG) - continue; - else // Invalid error code, shouldn't ever happen or iconv has a bug + } else if(err == E2BIG) { + if(in_left != old_in_left || out_ptr != out_start) // Check to avoid infinite loop + continue; + throw std::runtime_error("No progress, IConv is faulty!"); // LCOV_EXCL_LINE + } else // Invalid error code, shouldn't ever happen or iconv has a bug throw conversion_error(); // LCOV_EXCL_LINE } if(is_unshifting) diff --git a/src/boost/locale/icu/collator.cpp b/src/boost/locale/icu/collator.cpp index 86f5e51e..72ade74f 100644 --- a/src/boost/locale/icu/collator.cpp +++ b/src/boost/locale/icu/collator.cpp @@ -11,8 +11,10 @@ #include "boost/locale/icu/icu_util.hpp" #include "boost/locale/icu/uconv.hpp" #include "boost/locale/shared/mo_hash.hpp" +#include "boost/locale/shared/std_collate_adapter.hpp" #include #include +#include #include #include #if BOOST_LOCALE_ICU_VERSION >= 402 @@ -51,7 +53,7 @@ namespace boost { namespace locale { namespace impl_icu { { icu::StringPiece left(b1, e1 - b1); icu::StringPiece right(b2, e2 - b2); - return get_collator(level)->compareUTF8(left, right, status); + return get_collator(level).compareUTF8(left, right, status); } #endif @@ -64,7 +66,7 @@ namespace boost { namespace locale { namespace impl_icu { { icu::UnicodeString left = cvt_.icu(b1, e1); icu::UnicodeString right = cvt_.icu(b2, e2); - return get_collator(level)->compare(left, right, status); + return get_collator(level).compare(left, right, status); } int do_real_compare(collate_level level, @@ -101,11 +103,11 @@ namespace boost { namespace locale { namespace impl_icu { icu::UnicodeString str = cvt_.icu(b, e); std::vector tmp; tmp.resize(str.length() + 1u); - icu::Collator* collate = get_collator(level); - const int len = collate->getSortKey(str, tmp.data(), tmp.size()); + icu::Collator& collate = get_collator(level); + const int len = collate.getSortKey(str, tmp.data(), tmp.size()); if(len > int(tmp.size())) { tmp.resize(len); - collate->getSortKey(str, tmp.data(), tmp.size()); + collate.getSortKey(str, tmp.data(), tmp.size()); } else tmp.resize(len); return tmp; @@ -126,7 +128,7 @@ namespace boost { namespace locale { namespace impl_icu { collate_impl(const cdata& d) : cvt_(d.encoding()), locale_(d.locale()), is_utf8_(d.is_utf8()) {} - icu::Collator* get_collator(collate_level level) const + icu::Collator& get_collator(collate_level level) const { const int lvl_idx = level_to_int(level); constexpr icu::Collator::ECollationStrength levels[level_count] = {icu::Collator::PRIMARY, @@ -136,18 +138,17 @@ namespace boost { namespace locale { namespace impl_icu { icu::Collator::IDENTICAL}; icu::Collator* col = collates_[lvl_idx].get(); - if(col) - return col; - - UErrorCode status = U_ZERO_ERROR; - - collates_[lvl_idx].reset(icu::Collator::createInstance(locale_, status)); - - if(U_FAILURE(status)) - throw std::runtime_error(std::string("Creation of collate failed:") + u_errorName(status)); - - collates_[lvl_idx]->setStrength(levels[lvl_idx]); - return collates_[lvl_idx].get(); + if(!col) { + UErrorCode status = U_ZERO_ERROR; + std::unique_ptr tmp_col(icu::Collator::createInstance(locale_, status)); + if(U_FAILURE(status)) + throw std::runtime_error(std::string("Creation of collate failed:") + u_errorName(status)); + + tmp_col->setStrength(levels[lvl_idx]); + col = tmp_col.release(); + collates_[lvl_idx].reset(col); + } + return *col; } private: @@ -173,21 +174,21 @@ namespace boost { namespace locale { namespace impl_icu { return do_ustring_compare(level, b1, e1, b2, e2, status); } #endif - std::locale create_collate(const std::locale& in, const cdata& cd, char_facet_t type) { switch(type) { case char_facet_t::nochar: break; - case char_facet_t::char_f: return std::locale(in, new collate_impl(cd)); - case char_facet_t::wchar_f: return std::locale(in, new collate_impl(cd)); + case char_facet_t::char_f: return impl::create_collators(in, cd); + case char_facet_t::wchar_f: return impl::create_collators(in, cd); #ifdef __cpp_char8_t - case char_facet_t::char8_f: break; // std-facet not available (yet) + case char_facet_t::char8_f: + return std::locale(in, new collate_impl(cd)); // std-facet not available (yet) #endif #ifdef BOOST_LOCALE_ENABLE_CHAR16_T - case char_facet_t::char16_f: return std::locale(in, new collate_impl(cd)); + case char_facet_t::char16_f: return impl::create_collators(in, cd); #endif #ifdef BOOST_LOCALE_ENABLE_CHAR32_T - case char_facet_t::char32_f: return std::locale(in, new collate_impl(cd)); + case char_facet_t::char32_f: return impl::create_collators(in, cd); #endif } return in; diff --git a/src/boost/locale/shared/formatting.cpp b/src/boost/locale/shared/formatting.cpp index 489d1fd5..457ba782 100644 --- a/src/boost/locale/shared/formatting.cpp +++ b/src/boost/locale/shared/formatting.cpp @@ -7,43 +7,8 @@ #include #include #include "boost/locale/shared/ios_prop.hpp" -#include -#include namespace boost { namespace locale { - - ios_info::string_set::string_set() : type(nullptr), size(0), ptr(nullptr) {} - ios_info::string_set::~string_set() - { - delete[] ptr; - } - ios_info::string_set::string_set(const string_set& other) - { - if(other.ptr != nullptr) { - ptr = new char[other.size]; - size = other.size; - type = other.type; - memcpy(ptr, other.ptr, size); - } else { - ptr = nullptr; - size = 0; - type = nullptr; - } - } - - void ios_info::string_set::swap(string_set& other) - { - std::swap(type, other.type); - std::swap(size, other.size); - std::swap(ptr, other.ptr); - } - - ios_info::string_set& ios_info::string_set::operator=(string_set other) - { - swap(other); - return *this; - } - ios_info::ios_info() : flags_(0), domain_id_(0), time_zone_(time_zone::global()) {} ios_info::~ios_info() = default; @@ -105,16 +70,6 @@ namespace boost { namespace locale { return time_zone_; } - const ios_info::string_set& ios_info::date_time_pattern_set() const - { - return datetime_; - } - - ios_info::string_set& ios_info::date_time_pattern_set() - { - return datetime_; - } - ios_info& ios_info::get(std::ios_base& ios) { return impl::ios_prop::get(ios); diff --git a/src/boost/locale/shared/iconv_codecvt.cpp b/src/boost/locale/shared/iconv_codecvt.cpp index d34cb738..2ed618a0 100644 --- a/src/boost/locale/shared/iconv_codecvt.cpp +++ b/src/boost/locale/shared/iconv_codecvt.cpp @@ -42,7 +42,7 @@ namespace boost { namespace locale { insize = 1; outsize = sizeof(obuf); call_iconv(d, nullptr, nullptr, nullptr, nullptr); - size_t res = call_iconv(d, ibuf, &insize, reinterpret_cast(obuf), &outsize); + const size_t res = call_iconv(d, ibuf, &insize, reinterpret_cast(obuf), &outsize); // Now if this single byte starts a sequence we add incomplete // to know to ask that we need two bytes, otherwise it may only be illegal @@ -118,15 +118,14 @@ namespace boost { namespace locale { const utf::code_point inbuf[2] = {cp, 0}; size_t insize = sizeof(inbuf); char outseq[3] = {0}; - size_t outsize = 3; + size_t outsize = sizeof(outseq); call_iconv(from_utf_, reinterpret_cast(inbuf), &insize, outseq, &outsize); - if(insize != 0 || outsize > 1) + if(insize != 0 || outsize == sizeof(outseq)) return illegal; - const size_t len = 2 - outsize; - const size_t reminder = end - begin; - if(reminder < len) + const size_t len = sizeof(outseq) - outsize - 1; // Skip trailing NULL + if(static_cast(len) > end - begin) return incomplete; for(unsigned i = 0; i < len; i++) *begin++ = outseq[i]; diff --git a/src/boost/locale/shared/ids.cpp b/src/boost/locale/shared/ids.cpp index 95a64ff1..819de7f9 100644 --- a/src/boost/locale/shared/ids.cpp +++ b/src/boost/locale/shared/ids.cpp @@ -1,5 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) +// Copyright (c) 2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -24,6 +25,7 @@ namespace boost { namespace locale { BOOST_LOCALE_DEFINE_ID(calendar_facet); #define BOOST_LOCALE_INSTANTIATE(CHARTYPE) \ + BOOST_LOCALE_DEFINE_ID(collator); \ BOOST_LOCALE_DEFINE_ID(converter); \ BOOST_LOCALE_DEFINE_ID(message_format); \ BOOST_LOCALE_DEFINE_ID(boundary::boundary_indexing); @@ -48,6 +50,7 @@ namespace boost { namespace locale { void init_by(const std::locale& l) { init_facet>(l); + init_facet>(l); init_facet>(l); init_facet>(l); } diff --git a/src/boost/locale/shared/mo_lambda.cpp b/src/boost/locale/shared/mo_lambda.cpp index a6d2eb6a..7b37ca91 100644 --- a/src/boost/locale/shared/mo_lambda.cpp +++ b/src/boost/locale/shared/mo_lambda.cpp @@ -22,7 +22,7 @@ namespace boost { namespace locale { namespace gnu_gettext { namespace lambda { namespace { // anon template - expr_ptr make_expr(Ts... ts) + expr_ptr make_expr(Ts&&... ts) { return expr_ptr(new TExp(std::forward(ts)...)); } diff --git a/src/boost/locale/shared/std_collate_adapter.hpp b/src/boost/locale/shared/std_collate_adapter.hpp new file mode 100644 index 00000000..ee4ff43b --- /dev/null +++ b/src/boost/locale/shared/std_collate_adapter.hpp @@ -0,0 +1,58 @@ +// +// Copyright (c) 2024 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#ifndef BOOST_LOCALE_STD_COLLATE_ADAPTER_HPP +#define BOOST_LOCALE_STD_COLLATE_ADAPTER_HPP + +#include +#include +#include + +namespace boost { namespace locale { namespace impl { + + template + class BOOST_SYMBOL_VISIBLE std_collate_adapter : public std::collate { + public: + using typename std::collate::string_type; + + template + explicit std_collate_adapter(TArgs&&... args) : base_(std::forward(args)...) + {} + + protected: + int do_compare(const CharT* beg1, const CharT* end1, const CharT* beg2, const CharT* end2) const override + { + return base_.compare(collate_level::identical, beg1, end1, beg2, end2); + } + + string_type do_transform(const CharT* beg, const CharT* end) const override + { + return base_.transform(collate_level::identical, beg, end); + } + long do_hash(const CharT* beg, const CharT* end) const override + { + return base_.hash(collate_level::identical, beg, end); + } + Base base_; + }; + + template + static std::locale create_collators(const std::locale& in, TArgs&&... args) + { + static_assert(std::is_base_of, CollatorImpl>::value, "Must be a collator implementation"); + std::locale res(in, new CollatorImpl(args...)); + return std::locale(res, new std_collate_adapter(args...)); + } + + template class CollatorImpl, typename... TArgs> + static std::locale create_collators(const std::locale& in, TArgs&&... args) + { + return create_collators>(in, args...); + } + +}}} // namespace boost::locale::impl + +#endif diff --git a/src/boost/locale/win32/all_generator.hpp b/src/boost/locale/win32/all_generator.hpp index c175d5be..3cf7da94 100644 --- a/src/boost/locale/win32/all_generator.hpp +++ b/src/boost/locale/win32/all_generator.hpp @@ -12,7 +12,7 @@ namespace boost { namespace locale { namespace impl_win { - class winlocale; + struct winlocale; std::locale create_convert(const std::locale& in, const winlocale& lc, char_facet_t type); diff --git a/src/boost/locale/win32/api.hpp b/src/boost/locale/win32/api.hpp index aeda7ae6..33791ef9 100644 --- a/src/boost/locale/win32/api.hpp +++ b/src/boost/locale/win32/api.hpp @@ -27,6 +27,7 @@ #include #include +#include namespace boost { namespace locale { namespace impl_win { @@ -45,18 +46,16 @@ namespace boost { namespace locale { namespace impl_win { case collate_level::quaternary: case collate_level::identical: return 0; } - return 0; + return 0; // LCOV_EXCL_LINE } - class winlocale { - public: - winlocale() : lcid(0) {} + struct winlocale { + explicit winlocale(unsigned locale_id = 0) : lcid(locale_id) {} + explicit winlocale(const std::string& name) { lcid = locale_to_lcid(name); } - winlocale(const std::string& name) { lcid = locale_to_lcid(name); } + bool is_c() const { return lcid == 0; } unsigned lcid; - - bool is_c() const { return lcid == 0; } }; //////////////////////////////////////////////////////////////////////// @@ -87,7 +86,7 @@ namespace boost { namespace locale { namespace impl_win { || GetLocaleInfoW(lcid, LOCALE_SDECIMAL, de, de_size) == 0 || GetLocaleInfoW(lcid, LOCALE_SGROUPING, gr, gr_size) == 0) { - return res; + return res; // LCOV_EXCL_LINE } res.decimal_point = de; res.thousands_sep = th; @@ -118,7 +117,7 @@ namespace boost { namespace locale { namespace impl_win { throw std::length_error("String to long for int type"); int len = LCMapStringW(l.lcid, flags, begin, static_cast(end - begin), 0, 0); if(len == 0) - return res; + return res; // LCOV_EXCL_LINE if(len == std::numeric_limits::max()) throw std::length_error("String to long for int type"); std::vector buf(len + 1); @@ -149,6 +148,7 @@ namespace boost { namespace locale { namespace impl_win { static_cast(le - lb), rb, static_cast(re - rb)); + BOOST_ASSERT_MSG(result != 0, "CompareStringW failed"); return result - 2; // Subtract 2 to get the meaning of <0, ==0, and >0 } @@ -195,8 +195,7 @@ namespace boost { namespace locale { namespace impl_win { inline std::wstring wcsfold(const wchar_t* begin, const wchar_t* end) { - winlocale l; - l.lcid = 0x0409; // en-US + const winlocale l(0x0409); // en-US return win_map_string_l(LCMAP_LOWERCASE, begin, end, l); } @@ -214,9 +213,9 @@ namespace boost { namespace locale { namespace impl_win { if(end - begin > std::numeric_limits::max()) throw std::length_error("String to long for int type"); - int len = FoldStringW(flags, begin, static_cast(end - begin), 0, 0); + int len = FoldStringW(flags, begin, static_cast(end - begin), nullptr, 0); if(len == 0) - return std::wstring(); + return std::wstring(); // LCOV_EXCL_LINE if(len == std::numeric_limits::max()) throw std::length_error("String to long for int type"); std::vector v(len + 1); @@ -226,9 +225,7 @@ namespace boost { namespace locale { namespace impl_win { inline std::wstring wcsxfrm_l(collate_level level, const wchar_t* begin, const wchar_t* end, const winlocale& l) { - int flag = LCMAP_SORTKEY | collation_level_to_flag(level); - - return win_map_string_l(flag, begin, end, l); + return win_map_string_l(LCMAP_SORTKEY | collation_level_to_flag(level), begin, end, l); } inline std::wstring towupper_l(const wchar_t* begin, const wchar_t* end, const winlocale& l) diff --git a/src/boost/locale/win32/collate.cpp b/src/boost/locale/win32/collate.cpp index 107fed08..16fe38d1 100644 --- a/src/boost/locale/win32/collate.cpp +++ b/src/boost/locale/win32/collate.cpp @@ -7,33 +7,71 @@ #include #include #include "boost/locale/shared/mo_hash.hpp" +#include "boost/locale/shared/std_collate_adapter.hpp" #include "boost/locale/win32/api.hpp" #include #include #include +#include namespace boost { namespace locale { namespace impl_win { + template + using enable_if_sizeof_wchar_t = typename std::enable_if::type; + template + using disable_if_sizeof_wchar_t = typename std::enable_if::type; - class utf8_collator : public collator { - public: - utf8_collator(winlocale lc, size_t refs = 0) : collator(refs), lc_(lc) {} - int - do_compare(collate_level level, const char* lb, const char* le, const char* rb, const char* re) const override + namespace { + template + enable_if_sizeof_wchar_t compare_impl(collate_level level, + const CharT* lb, + const CharT* le, + const CharT* rb, + const CharT* re, + const winlocale& wl) + { + return wcscoll_l(level, + reinterpret_cast(lb), + reinterpret_cast(le), + reinterpret_cast(rb), + reinterpret_cast(re), + wl); + } + + template + disable_if_sizeof_wchar_t compare_impl(collate_level level, + const CharT* lb, + const CharT* le, + const CharT* rb, + const CharT* re, + const winlocale& wl) { const std::wstring l = conv::utf_to_utf(lb, le); const std::wstring r = conv::utf_to_utf(rb, re); - return wcscoll_l(level, l.c_str(), l.c_str() + l.size(), r.c_str(), r.c_str() + r.size(), lc_); + return wcscoll_l(level, l.c_str(), l.c_str() + l.size(), r.c_str(), r.c_str() + r.size(), wl); } - long do_hash(collate_level level, const char* b, const char* e) const override + + template + enable_if_sizeof_wchar_t + normalize_impl(collate_level level, const CharT* b, const CharT* e, const winlocale& l) { - const std::string key = do_transform(level, b, e); - return gnu_gettext::pj_winberger_hash_function(key.c_str(), key.c_str() + key.size()); + return wcsxfrm_l(level, reinterpret_cast(b), reinterpret_cast(e), l); } - std::string do_transform(collate_level level, const char* b, const char* e) const override + + template + disable_if_sizeof_wchar_t + normalize_impl(collate_level level, const CharT* b, const CharT* e, const winlocale& l) { const std::wstring tmp = conv::utf_to_utf(b, e); - const std::wstring wkey = wcsxfrm_l(level, tmp.c_str(), tmp.c_str() + tmp.size(), lc_); - std::string key; + return wcsxfrm_l(level, tmp.c_str(), tmp.c_str() + tmp.size(), l); + } + + template + typename std::enable_if>::type + transform_impl(collate_level level, const CharT* b, const CharT* e, const winlocale& l) + { + const std::wstring wkey = normalize_impl(level, b, e, l); + + std::basic_string key; BOOST_LOCALE_START_CONST_CONDITION if(sizeof(wchar_t) == 2) key.reserve(wkey.size() * 2); @@ -42,46 +80,61 @@ namespace boost { namespace locale { namespace impl_win { for(const wchar_t c : wkey) { if(sizeof(wchar_t) == 2) { const uint16_t tv = static_cast(c); - key += char(tv >> 8); - key += char(tv & 0xFF); + key += CharT(tv >> 8); + key += CharT(tv & 0xFF); } else { // 4 const uint32_t tv = static_cast(c); // 21 bit - key += char((tv >> 16) & 0xFF); - key += char((tv >> 8) & 0xFF); - key += char(tv & 0xFF); + key += CharT((tv >> 16) & 0xFF); + key += CharT((tv >> 8) & 0xFF); + key += CharT(tv & 0xFF); } } BOOST_LOCALE_END_CONST_CONDITION return key; } - private: - winlocale lc_; - }; + template + typename std::enable_if::value, std::wstring>::type + transform_impl(collate_level level, const CharT* b, const CharT* e, const winlocale& l) + { + return normalize_impl(level, b, e, l); + } + + template + typename std::enable_if= sizeof(wchar_t) && !std::is_same::value, + std::basic_string>::type + transform_impl(collate_level level, const CharT* b, const CharT* e, const winlocale& l) + { + const std::wstring wkey = normalize_impl(level, b, e, l); + return std::basic_string(wkey.begin(), wkey.end()); + } + } // namespace - class utf16_collator : public collator { + template + class utf_collator : public collator { public: - typedef std::collate wfacet; - utf16_collator(winlocale lc, size_t refs = 0) : collator(refs), lc_(lc) {} + using typename collator::string_type; + + explicit utf_collator(winlocale lc) : collator(), lc_(lc) {} + int do_compare(collate_level level, - const wchar_t* lb, - const wchar_t* le, - const wchar_t* rb, - const wchar_t* re) const override + const CharT* lb, + const CharT* le, + const CharT* rb, + const CharT* re) const override { - return wcscoll_l(level, lb, le, rb, re, lc_); + return compare_impl(level, lb, le, rb, re, lc_); } - long do_hash(collate_level level, const wchar_t* b, const wchar_t* e) const override + long do_hash(collate_level level, const CharT* b, const CharT* e) const override { - std::wstring key = do_transform(level, b, e); - const char* begin = reinterpret_cast(key.c_str()); - const char* end = begin + key.size() * sizeof(wchar_t); - return gnu_gettext::pj_winberger_hash_function(begin, end); + const std::wstring key = normalize_impl(level, b, e, lc_); + return gnu_gettext::pj_winberger_hash_function(reinterpret_cast(key.c_str()), + reinterpret_cast(key.c_str() + key.size())); } - std::wstring do_transform(collate_level level, const wchar_t* b, const wchar_t* e) const override + string_type do_transform(collate_level level, const CharT* b, const CharT* e) const override { - return wcsxfrm_l(level, b, e, lc_); + return transform_impl(level, b, e, lc_); } private: @@ -108,16 +161,17 @@ namespace boost { namespace locale { namespace impl_win { } else { switch(type) { case char_facet_t::nochar: break; - case char_facet_t::char_f: return std::locale(in, new utf8_collator(lc)); - case char_facet_t::wchar_f: return std::locale(in, new utf16_collator(lc)); + case char_facet_t::char_f: return impl::create_collators>(in, lc); + case char_facet_t::wchar_f: return impl::create_collators>(in, lc); #ifdef __cpp_char8_t - case char_facet_t::char8_f: break; // std-facet not available (yet) + case char_facet_t::char8_f: + return std::locale(in, new utf_collator(lc)); // std-facet not available (yet) #endif #ifdef BOOST_LOCALE_ENABLE_CHAR16_T - case char_facet_t::char16_f: break; + case char_facet_t::char16_f: return impl::create_collators>(in, lc); #endif #ifdef BOOST_LOCALE_ENABLE_CHAR32_T - case char_facet_t::char32_f: break; + case char_facet_t::char32_f: return impl::create_collators>(in, lc); #endif } } diff --git a/test/boostLocale/test/unit_test.hpp b/test/boostLocale/test/unit_test.hpp index 322547c5..9c9ac7aa 100644 --- a/test/boostLocale/test/unit_test.hpp +++ b/test/boostLocale/test/unit_test.hpp @@ -153,14 +153,15 @@ std::string to_string(const std::vector& v) } /// Put the char into the stream making sure it is readable -/// Fallback to the unicode representation of it (e.g. U+00A0) +/// Fallback to the Unicode representation of it (e.g. U+00A0) template void stream_char(std::ostream& s, const Char c) { if((c >= '!' && c <= '~') || c == ' ') s << static_cast(c); else - s << "U+" << std::hex << std::uppercase << std::setw(sizeof(Char)) << static_cast(c); + s << "U+" << std::hex << std::uppercase << std::setw(sizeof(Char)) << std::setfill('0') + << static_cast(c); } template diff --git a/test/test_encoding.cpp b/test/test_encoding.cpp index c0e852a0..f8701d9d 100644 --- a/test/test_encoding.cpp +++ b/test/test_encoding.cpp @@ -19,6 +19,26 @@ const bool test_iso_8859_8 = hasWinCodepage(28598); #endif +#if defined(BOOST_LOCALE_WITH_ICONV) +// Reproduce issue #206 to detect faulty IConv +static bool isFaultyIconv() +{ + namespace blc = boost::locale::conv; + auto from_utf = blc::detail::make_utf_decoder("ISO-2022-CN", blc::skip, blc::detail::conv_backend::IConv); + try { + from_utf->convert("实"); + } catch(const std::runtime_error& e) { // LCOV_EXCL_LINE + return std::string(e.what()).find("IConv is faulty") != std::string::npos; // LCOV_EXCL_LINE + } + return false; +} +#else +constexpr bool isFaultyIconv() +{ + return false; +} +#endif + constexpr boost::locale::conv::detail::conv_backend all_conv_backends[] = { #ifdef BOOST_LOCALE_WITH_ICONV boost::locale::conv::detail::conv_backend::IConv, @@ -49,10 +69,13 @@ template void test_to_utf_for_impls(const std::string& source, const std::basic_string& target, const std::string& encoding, - const bool expectSuccess = true) + const bool expectSuccess = true, + const bool test_default = true) { - boost::locale::conv::utf_encoder conv(encoding); - TEST_EQ(conv(source), target); + if(test_default) { + boost::locale::conv::utf_encoder conv(encoding); + TEST_EQ(conv(source), target); + } for(const auto impl : all_conv_backends) { std::cout << "----- " << impl << '\n'; using boost::locale::conv::invalid_charset_error; @@ -61,7 +84,8 @@ void test_to_utf_for_impls(const std::string& source, boost::locale::conv::detail::make_utf_encoder(encoding, boost::locale::conv::skip, impl); TEST_EQ(convPtr->convert(source), target); } catch(invalid_charset_error&) { - continue; // LCOV_EXCL_LINE + std::cout << "--- Charset not supported\n"; // LCOV_EXCL_LINE + continue; // LCOV_EXCL_LINE } if(!expectSuccess) { auto convPtr = @@ -83,10 +107,13 @@ template void test_from_utf_for_impls(const std::basic_string& source, const std::string& target, const std::string& encoding, - const bool expectSuccess = true) + const bool expectSuccess = true, + const bool test_default = true) { - boost::locale::conv::utf_decoder conv(encoding); - TEST_EQ(conv(source), target); + if(test_default) { + boost::locale::conv::utf_decoder conv(encoding); + TEST_EQ(conv(source), target); + } for(const auto impl : all_conv_backends) { std::cout << "----- " << impl << '\n'; using boost::locale::conv::invalid_charset_error; @@ -95,7 +122,8 @@ void test_from_utf_for_impls(const std::basic_string& source, boost::locale::conv::detail::make_utf_decoder(encoding, boost::locale::conv::skip, impl); TEST_EQ(convPtr->convert(source), target); } catch(invalid_charset_error&) { - continue; // LCOV_EXCL_LINE + std::cout << "--- Charset not supported\n"; // LCOV_EXCL_LINE + continue; // LCOV_EXCL_LINE } if(!expectSuccess) { auto convPtr = @@ -114,18 +142,23 @@ void test_from_utf_for_impls(const std::basic_string& source, } template -void test_to_from_utf(std::string source, std::basic_string target, std::string encoding) +void test_to_from_utf(const std::string& source, + const std::basic_string& target, + const std::string& encoding, + const bool test_default = true) { std::cout << "-- " << encoding << std::endl; - TEST_EQ(boost::locale::conv::to_utf(source, encoding), target); - TEST_EQ(boost::locale::conv::from_utf(target, encoding), source); - test_to_utf_for_impls(source, target, encoding); - test_from_utf_for_impls(target, source, encoding); + if(test_default) { + TEST_EQ(boost::locale::conv::to_utf(source, encoding), target); + TEST_EQ(boost::locale::conv::from_utf(target, encoding), source); + } + test_to_utf_for_impls(source, target, encoding, true, test_default); + test_from_utf_for_impls(target, source, encoding, true, test_default); } template -void test_error_to_utf(std::string source, std::basic_string target, std::string encoding) +void test_error_to_utf(const std::string& source, const std::basic_string& target, const std::string& encoding) { using boost::locale::conv::to_utf; using boost::locale::conv::stop; @@ -146,7 +179,7 @@ void test_error_to_utf(std::string source, std::basic_string target, std:: } template -void test_error_from_utf(std::basic_string source, std::string target, std::string encoding) +void test_error_from_utf(const std::basic_string& source, const std::string& target, const std::string& encoding) { using boost::locale::conv::from_utf; using boost::locale::conv::stop; @@ -305,21 +338,30 @@ void test_utf_for() } catch(const invalid_charset_error&) { // LCOV_EXCL_LINE std::cout << "--- not supported\n"; // LCOV_EXCL_LINE } - // Testing a codepage which may crash with IConv on macOS, see issue #196 - test_to_from_utf("\xa1\xad\xa1\xad", utf("……"), "gbk"); + if(!isFaultyIconv()) { + // Testing a codepage which may crash with IConv on macOS, see issue #196 + test_to_from_utf("\xa1\xad\xa1\xad", utf("……"), "gbk", false); + // This might cause a bogus E2BIG on macOS, see issue #206 + test_to_from_utf("\x1b\x24\x29\x41\x0e\x4a\x35\xf", utf("实"), "ISO-2022-CN", false); + } std::cout << "- Testing correct invalid bytes skipping\n"; { std::cout << "-- UTF-8" << std::endl; - // At start + std::cout << "--- At start single" << std::endl; test_error_to_utf("\xFFgrüßen", utf("grüßen"), "UTF-8"); + std::cout << "--- At start multiple" << std::endl; test_error_to_utf("\xFF\xFFgrüßen", utf("grüßen"), "UTF-8"); - // Middle + + std::cout << "--- At middle single" << std::endl; test_error_to_utf("g\xFFrüßen", utf("grüßen"), "UTF-8"); + std::cout << "--- At middle multiple" << std::endl; test_error_to_utf("g\xFF\xFF\xFFrüßen", utf("grüßen"), "UTF-8"); - // End + + std::cout << "--- At end single" << std::endl; test_error_to_utf("grüßen\xFF", utf("grüßen"), "UTF-8"); + std::cout << "--- At end multiple" << std::endl; test_error_to_utf("grüßen\xFF\xFF", utf("grüßen"), "UTF-8"); try { @@ -340,16 +382,21 @@ void test_utf_for() } catch(const invalid_charset_error&) { // LCOV_EXCL_LINE std::cout << "--- not supported\n"; // LCOV_EXCL_LINE } - std::cout << "-- Error for encoding at start, middle and end" << std::endl; + std::cout << "-- Error for encoding at start" << std::endl; test_error_from_utf(utf("שלום hello"), " hello", "ISO8859-1"); + std::cout << "-- Error for encoding at middle and end" << std::endl; test_error_from_utf(utf("hello שלום world"), "hello world", "ISO8859-1"); + std::cout << "-- Error for encoding at end" << std::endl; test_error_from_utf(utf("hello שלום"), "hello ", "ISO8859-1"); - std::cout << "-- Error for decoding" << std::endl; + std::cout << "-- Error for decoding to UTF-8" << std::endl; test_error_from_utf(utfutf::bad(), utfutf::ok(), "UTF-8"); + std::cout << "-- Error for decoding to Latin1" << std::endl; test_error_from_utf(utfutf::bad(), to(utfutf::ok()), "Latin1"); - std::cout << "-- Error decoding string of only invalid chars" << std::endl; + const std::basic_string onlyInvalidUtf(2, utfutf::bad_char()); + std::cout << "-- Error decoding string of only invalid chars to UTF-8" << std::endl; test_error_from_utf(onlyInvalidUtf, "", "UTF-8"); + std::cout << "-- Error decoding string of only invalid chars to Latin1" << std::endl; test_error_from_utf(onlyInvalidUtf, "", "Latin1"); } diff --git a/test/test_generator.cpp b/test/test_generator.cpp index f138061d..084101bf 100644 --- a/test/test_generator.cpp +++ b/test/test_generator.cpp @@ -37,6 +37,21 @@ namespace boost { namespace locale { namespace test { { return std::has_facet(l) && is_facet(&std::use_facet(l)); } + + template + bool has_not_facet(const std::locale& l) + { + const Facet* f; + try { + f = &std::use_facet(l); + } catch(const std::bad_cast&) { + return !std::has_facet(l); + } + // This mustn't be reached, checks for debugging + TEST(is_facet(f)); // LCOV_EXCL_LINE + TEST(!std::has_facet(l)); // LCOV_EXCL_LINE + return false; // LCOV_EXCL_LINE + } }}} // namespace boost::locale::test namespace blt = boost::locale::test; @@ -232,6 +247,49 @@ void test_install_chartype(const std::string& backendName) } } +template +struct dummy_collate : std::collate {}; + +template +bool has_dummy_collate(const std::locale& l) +{ + const auto& col = std::use_facet>(l); // Implicitely require existance of std::collate + return blt::is_facet>(&col); +} + +void test_std_collate_replaced(const std::string& /*backendName*/) +{ + std::locale origLocale = std::locale::classic(); + origLocale = std::locale(origLocale, new dummy_collate); + origLocale = std::locale(origLocale, new dummy_collate); +#ifdef BOOST_LOCALE_ENABLE_CHAR16_T + origLocale = std::locale(origLocale, new dummy_collate); +#endif +#ifdef BOOST_LOCALE_ENABLE_CHAR32_T + origLocale = std::locale(origLocale, new dummy_collate); +#endif + + // Use ASCII and UTF-8 encoding + for(const std::string localeName : {"C", "en_US.UTF-8"}) { + std::cout << "--- Locale: " << localeName << std::endl; + bl::generator g; + g.categories(boost::locale::category_t::collation); + const std::locale l = g.generate(origLocale, localeName); + TEST(has_dummy_collate(origLocale)); + TEST(!has_dummy_collate(l)); + TEST(has_dummy_collate(origLocale)); + TEST(!has_dummy_collate(l)); +#ifdef BOOST_LOCALE_ENABLE_CHAR16_T + TEST(has_dummy_collate(origLocale)); + TEST(!has_dummy_collate(l)); +#endif +#ifdef BOOST_LOCALE_ENABLE_CHAR32_T + TEST(has_dummy_collate(origLocale)); + TEST(!has_dummy_collate(l)); +#endif + } +} + void test_main(int /*argc*/, char** /*argv*/) { { @@ -266,25 +324,30 @@ void test_main(int /*argc*/, char** /*argv*/) std::cout << "-- Locale: " << localeName << std::endl; const std::locale l = g(localeName); #ifdef __cpp_char8_t -# define TEST_HAS_FACET_CHAR8(facet, l) TEST(blt::has_facet>(l)) +# define TEST_FOR_CHAR8(check) TEST(check) #else -# define TEST_HAS_FACET_CHAR8(facet, l) (void)0 +# define TEST_FOR_CHAR8(check) (void)0 #endif #ifndef BOOST_LOCALE_NO_CXX20_STRING8 -# define TEST_HAS_FACET_STRING8(facet, l) TEST(blt::has_facet>(l)) +# define TEST_FOR_STRING8(check) TEST(check) #else -# define TEST_HAS_FACET_STRING8(facet, l) (void)0 +# define TEST_FOR_STRING8(check) (void)0 #endif #ifdef BOOST_LOCALE_ENABLE_CHAR16_T -# define TEST_HAS_FACET_CHAR16(facet, l) TEST(blt::has_facet>(l)) +# define TEST_FOR_CHAR16(check) TEST(check) #else -# define TEST_HAS_FACET_CHAR16(facet, l) (void)0 +# define TEST_FOR_CHAR16(check) (void)0 #endif #ifdef BOOST_LOCALE_ENABLE_CHAR32_T -# define TEST_HAS_FACET_CHAR32(facet, l) TEST(blt::has_facet>(l)) +# define TEST_FOR_CHAR32(check) TEST(check) #else -# define TEST_HAS_FACET_CHAR32(facet, l) (void)0 +# define TEST_FOR_CHAR32(check) (void)0 #endif +#define TEST_HAS_FACET_CHAR8(facet, l) TEST_FOR_CHAR8(blt::has_facet>(l)) +#define TEST_HAS_FACET_CHAR16(facet, l) TEST_FOR_CHAR16(blt::has_facet>(l)) +#define TEST_HAS_FACET_CHAR32(facet, l) TEST_FOR_CHAR32(blt::has_facet>(l)) +#define TEST_HAS_FACET_STRING8(facet, l) TEST_FOR_STRING8(blt::has_facet>(l)) + #define TEST_HAS_FACETS(facet, l) \ do { \ TEST(blt::has_facet>(l)); \ @@ -296,7 +359,18 @@ void test_main(int /*argc*/, char** /*argv*/) // Convert TEST_HAS_FACETS(bl::converter, l); TEST_HAS_FACET_STRING8(bl::converter, l); + // Collator TEST_HAS_FACETS(std::collate, l); + if(backendName == "icu" || (backendName == "winapi" && std::use_facet(l).utf8())) { + TEST_HAS_FACETS(bl::collator, l); + TEST_HAS_FACET_STRING8(bl::collator, l); + } else { + TEST(blt::has_not_facet>(l)); + TEST(blt::has_not_facet>(l)); + TEST_FOR_STRING8(blt::has_not_facet>(l)); + TEST_FOR_CHAR16(blt::has_not_facet>(l)); + TEST_FOR_CHAR32(blt::has_not_facet>(l)); + } // Formatting TEST_HAS_FACETS(std::num_put, l); TEST_HAS_FACETS(std::time_put, l); @@ -396,6 +470,7 @@ void test_main(int /*argc*/, char** /*argv*/) TEST(!std::use_facet(g("en_US.ISO8859-1")).utf8()); test_install_chartype(backendName); + test_std_collate_replaced(backendName); } std::cout << "Test special locales" << std::endl; test_special_locales(); diff --git a/test/test_ios_info.cpp b/test/test_ios_info.cpp index 9b63aaed..79179a8f 100644 --- a/test/test_ios_info.cpp +++ b/test/test_ios_info.cpp @@ -105,18 +105,6 @@ void test_member_methods() info.date_time_pattern(std::string("Pattern")); TEST_EQ(info.date_time_pattern(), "Pattern"); - - info.date_time_pattern(ascii_to("WChar Pattern")); - TEST_EQ(info.date_time_pattern(), ascii_to("WChar Pattern")); - TEST_THROWS(info.date_time_pattern(), std::bad_cast); - - info.date_time_pattern(ascii_to("Char16 Pattern")); - TEST_THROWS(info.date_time_pattern(), std::bad_cast); - TEST_EQ(info.date_time_pattern(), ascii_to("Char16 Pattern")); - - info.date_time_pattern(ascii_to("Char32 Pattern")); - TEST_THROWS(info.date_time_pattern(), std::bad_cast); - TEST_EQ(info.date_time_pattern(), ascii_to("Char32 Pattern")); } } @@ -212,8 +200,60 @@ void test_manipulators() TEST_EQ(info2.date_time_pattern(), L"My TZ"); } +void test_any_string() +{ + boost::locale::detail::any_string s; + TEST_THROWS(s.get(), std::bad_cast); + TEST_THROWS(s.get(), std::bad_cast); + + s.set("Char Pattern"); + TEST_EQ(s.get(), "Char Pattern"); + TEST_THROWS(s.get(), std::bad_cast); + + s.set(ascii_to("WChar Pattern")); + TEST_EQ(s.get(), ascii_to("WChar Pattern")); + TEST_THROWS(s.get(), std::bad_cast); + + s.set(ascii_to("Char16 Pattern")); + TEST_EQ(s.get(), ascii_to("Char16 Pattern")); + TEST_THROWS(s.get(), std::bad_cast); + + s.set(ascii_to("Char32 Pattern")); + TEST_EQ(s.get(), ascii_to("Char32 Pattern")); + TEST_THROWS(s.get(), std::bad_cast); + +#ifndef BOOST_LOCALE_NO_CXX20_STRING8 + s.set(ascii_to("Char8 Pattern")); + TEST_EQ(s.get(), ascii_to("Char8 Pattern")); + TEST_THROWS(s.get(), std::bad_cast); +#endif + + boost::locale::detail::any_string s1, s2, empty; + s1.set("Char"); + s2.set(ascii_to("WChar")); + // Copy ctor + boost::locale::detail::any_string s3(s1); + TEST_EQ(s3.get(), "Char"); + TEST_EQ(s1.get(), "Char"); + // Ensure deep copy + s3.set("Foo"); + TEST_EQ(s3.get(), "Foo"); + TEST_EQ(s1.get(), "Char"); + // Copy assign + s3 = s2; + TEST_EQ(s3.get(), ascii_to("WChar")); + TEST_EQ(s2.get(), ascii_to("WChar")); + // Move assign + s3 = std::move(s1); + TEST_EQ(s3.get(), "Char"); + // From empty + s3 = empty; + TEST_THROWS(s3.get(), std::bad_cast); +} + void test_main(int /*argc*/, char** /*argv*/) { + test_any_string(); test_member_methods(); test_manipulators(); } diff --git a/test/test_message.cpp b/test/test_message.cpp index 19323761..304d66c7 100644 --- a/test/test_message.cpp +++ b/test/test_message.cpp @@ -115,7 +115,7 @@ template using has_ctype = decltype(is_complete_impl(std::declval*>())); template -typename std::enable_if::value, bool>::type stream_translate(std::basic_ostream& ss, Ts... args) +typename std::enable_if::value, bool>::type stream_translate(std::basic_ostream& ss, Ts&&... args) { ss << bl::translate(args...); return true; @@ -124,7 +124,7 @@ typename std::enable_if::value, bool>::type stream_translate(std // Required for char types not fully supported by the standard library // e.g.: error: implicit instantiation of undefined template 'std::ctype' template -typename std::enable_if::value, bool>::type stream_translate(std::basic_ostream&, Ts...) +typename std::enable_if::value, bool>::type stream_translate(std::basic_ostream&, Ts&&...) { return false; // LCOV_EXCL_LINE } diff --git a/test/test_std_convert.cpp b/test/test_std_convert.cpp index ae446fcb..bc8126e3 100644 --- a/test/test_std_convert.cpp +++ b/test/test_std_convert.cpp @@ -42,7 +42,16 @@ void test_char() std::cout << "Testing " << name << std::endl; l = gen(name); test_one(l, "Hello World", "hello world", "HELLO WORLD"); - test_one(l, "Façade", "façade", "FAÇADE"); +#if BOOST_LOCALE_USE_WIN32_API + name = "English_United States"; +#endif + // Check that ç can be converted to Ç by the stdlib (fails on e.g. FreeBSD libstd++) + if(std::toupper('\xe7', std::locale(name)) == '\xc7') + test_one(l, "Façade", "façade", "FAÇADE"); + else { + std::cout << "- en_US.ISO8859-1 (" << name << ") not well supported. "; // LCOV_EXCL_LINE + std::cout << "Skipping conv test" << std::endl; // LCOV_EXCL_LINE + } boost::locale::case_convert_test::test_no_op_title_case(l, "Hello world i"); } else std::cout << "- en_US.ISO8859-1 is not supported, skipping" << std::endl; // LCOV_EXCL_LINE