Merge pull request #504 from LeonineKing1199/unicode-charset

Unicode charset
boostorg · May 7, 2019 · 3d0dafe · 3d0dafe
2 parents d407f9a + 017478b
commit 3d0dafe
Show file tree

Hide file tree

Showing 2 changed files with 68 additions and 2 deletions.
diff --git a/test/x3/char1.cpp b/test/x3/char1.cpp
@@ -1,15 +1,23 @@
 /*=============================================================================
     Copyright (c) 2001-2015 Joel de Guzman
     Copyright (c) 2001-2011 Hartmut Kaiser
+    Copyright (c)      2019 Christian Mazakas
 
     Distributed under the Boost Software License, Version 1.0. (See accompanying
     file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
 =============================================================================*/
+
+#define BOOST_SPIRIT_X3_UNICODE
+
 #include <boost/detail/lightweight_test.hpp>
-#include <boost/spirit/home/x3/core.hpp>
-#include <boost/spirit/home/x3/char.hpp>
+#include <boost/spirit/home/x3.hpp>
+
+#include <boost/utility/string_view.hpp>
 
 #include <iostream>
+#include <vector>
+#include <algorithm>
+
 #include "test.hpp"
 
 int
@@ -92,6 +100,53 @@ main()
         BOOST_TEST(!test(L"z", ~~char_(L'b', L'y')));
     }
 
+    // unicode (normal ASCII)
+    {
+        using namespace boost::spirit::x3::unicode;
+
+        BOOST_TEST(test(U"abcd", +char_(U"abcd")));
+        BOOST_TEST(!test(U"abcd", +char_(U"qwer")));
+
+        auto const sub_delims = char_(U"!$&'()*+,;=");
+
+        auto const delims =
+            std::vector<boost::u32string_view>{U"!", U"$", U"&", U"'", U"(", U")", U"*", U"+",
+                                               U",", U";", U"="};
+
+        auto const matched_all_sub_delims =
+            std::all_of(delims.begin(), delims.end(), [&](auto const delim) -> bool {
+                return test(delim, sub_delims);
+            });
+
+        BOOST_TEST(matched_all_sub_delims);
+    }
+
+    // unicode (escaped Unicode char literals)
+    {
+        using namespace boost::spirit::x3::unicode;
+
+        auto const chars = char_(U"\u0024\u00a2\u0939\u20ac\U00010348");
+
+        auto const test_strings =
+            std::vector<boost::u32string_view>{U"\u0024", U"\u00a2", U"\u0939", U"\u20ac",
+                                               U"\U00010348"};
+
+        auto const bad_test_strings = std::vector<boost::u32string_view>{U"a", U"B", U"c", U"\u0409"};
+
+        auto const all_matched =
+            std::all_of(test_strings.begin(), test_strings.end(), [&](auto const test_str) -> bool {
+                return test(test_str, chars);
+            });
+
+        auto const none_matched =
+            std::all_of(bad_test_strings.begin(), bad_test_strings.end(), [&](auto const bad_test_str) -> bool {
+                return !test(bad_test_str, chars);
+            });
+
+        BOOST_TEST(all_matched);
+        BOOST_TEST(none_matched);
+    }
+
 
     {   // single char strings!
         namespace ascii = boost::spirit::x3::ascii;

diff --git a/test/x3/test.hpp b/test/x3/test.hpp
@@ -8,6 +8,7 @@
 #define BOOST_SPIRIT_TEST_FEBRUARY_01_2007_0605PM
 
 #include <boost/spirit/home/x3/core/parse.hpp>
+#include <boost/utility/string_view.hpp>
 #include <iostream>
 
 namespace spirit_test
@@ -22,6 +23,16 @@ namespace spirit_test
             && (!full_match || (in == last));
     }
 
+    template <typename Char, typename Parser>
+    bool test(boost::basic_string_view<Char, std::char_traits<Char>> in,
+              Parser const& p, bool full_match = true)
+    {
+        auto const last = in.end();
+        auto pos        = in.begin();
+
+        return boost::spirit::x3::parse(pos, last, p) && (!full_match || (pos == last));
+    }
+
     template <typename Char, typename Parser, typename Skipper>
     bool test(Char const* in, Parser const& p
         , Skipper const& s, bool full_match = true)