Merge pull request #14 from niklas88/driveby_cleanups

Support splitting/stripping UTF-8 string
ad-freiburg · Jul 4, 2017 · 18618a9 · 18618a9
2 parents 36aef6d + d34197a
commit 18618a9
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 3 deletions.
diff --git a/src/util/StringUtils.h b/src/util/StringUtils.h
@@ -399,7 +399,7 @@ vector<string> splitAny(const string& orig, const string& seps) {
     size_t from = 0;
     size_t i = 0;
     while (i < orig.size()) {
-      if (chars[orig[i]]) {
+      if (chars[static_cast<unsigned char>(orig[i])]) {
         if (from < i) {
           result.emplace_back(orig.substr(from, i - from));
         }
@@ -447,7 +447,7 @@ inline string lstrip(const string& text, string s) {
     chars[s[i]] = true;
   }
   size_t i = 0;
-  while (i < text.size() && chars[text[i]]) {
+  while (i < text.size() && chars[static_cast<unsigned char>(text[i])]) {
     ++i;
   }
   return text.substr(i);
@@ -476,7 +476,7 @@ inline string rstrip(const string& text, string s) {
     chars[s[i]] = true;
   }
   size_t i = text.size();
-  while (i > 0 && chars[text[i - 1]]) {
+  while (i > 0 && chars[static_cast<unsigned char>(text[i - 1])]) {
     --i;
   }
   return text.substr(0, i);

diff --git a/test/StringUtilsTest.cpp b/test/StringUtilsTest.cpp
@@ -112,6 +112,12 @@ TEST(StringUtilsTest, split) {
   ASSERT_EQ(size_t(1), v3.size());
   auto v4 = split(s3, ' ');
   ASSERT_EQ(size_t(3), v4.size());
+  // and with unicode
+  string s5 = u8"Spaß ❤ 漢字";
+  auto v5 = split(s5, ' ');
+  ASSERT_EQ(u8"Spaß", v5[0]);
+  ASSERT_EQ(u8"❤", v5[1]);
+  ASSERT_EQ(u8"漢字", v5[2]);
 }
 
 TEST(StringUtilsTest, join) {
@@ -171,6 +177,12 @@ TEST(StringUtilsTest, splitAny) {
 
   auto v8 = splitAny(s1, "sih\tt");
   ASSERT_EQ(size_t(0), v8.size());
+  // and with unicode
+  string s9 = u8"Spaß ❤\t漢字";
+  auto v9 = splitAny(s9, " \t");
+  ASSERT_EQ(u8"Spaß", v9[0]);
+  ASSERT_EQ(u8"❤", v9[1]);
+  ASSERT_EQ(u8"漢字", v9[2]);
 }
 
 TEST(StringUtilsTest, strip) {
@@ -189,6 +201,10 @@ TEST(StringUtilsTest, strip) {
   ASSERT_EQ("abc", strip(s5, 'x'));
 
   ASSERT_EQ("bc", strip("xxaxaxaxabcaaaxxx", "xa"));
+  // And with unicode
+  ASSERT_EQ(u8"äcaaaxxx", lstrip(u8"xxaxaxaxaäcaaaxxx", "xa"));
+  ASSERT_EQ(u8"äö", strip(u8"xxaxaxaxaäöaaaxxx", "xa"));
+  ASSERT_EQ("xxaxaxaxa♥", rstrip("xxaxaxaxa♥aaaxxx", "xa"));
 }
 }  // namespace