[cpp] Implement UTF-8 check with the Bjoern DFA

cpp/NINJA_subgraph.py

@@ -85,8 +85,17 @@ def NinjaGraph(ru):
         srcs=[
             'cpp/data_lang.cc',
         ],
+        deps=[
+            '//mycpp/runtime',
+        ],
     )
 
+    ru.cc_binary('cpp/data_lang_test.cc',
+                 deps=[
+                     '//cpp/data_lang',
+                 ],
+                 matrix=ninja_lib.COMPILERS_VARIANTS)
+
     # Note: depends on code generated by re2c
     ru.cc_library(
         '//cpp/frontend_match',

cpp/TEST.sh

@@ -63,6 +63,8 @@ unit() {
 
     run-test-in-dir  cpp/core_test '' $variant  # has testdata
 
+    run-one-test     cpp/data_lang_test '' $variant
+
     run-one-test     cpp/qsn_test '' $variant
 
     run-one-test     cpp/frontend_flag_spec_test '' $variant
@@ -97,6 +99,8 @@ coverage() {
 
   run-test-in-dir  cpp/core_test $compiler $variant  # has testdata
 
+  run-one-test     cpp/data_lang_test $compiler $variant
+
   run-one-test     cpp/qsn_test $compiler $variant
 
   run-one-test     cpp/frontend_flag_spec_test $compiler $variant

cpp/data_lang.cc

@@ -2,11 +2,25 @@
 
 #include "cpp/data_lang.h"
 
+#include "data_lang/utf8_impls/bjoern_dfa.h"
+
 namespace pyj8 {
 
 bool PartIsUtf8(BigStr* s, int start, int end) {
-  // TODO: use Bjoern DFA
-  return true;
+  uint32_t codepoint;
+  uint32_t state = UTF8_ACCEPT;
+
+  for (int i = start; i < end; ++i) {
+    // This var or a static_cast<> is necessary.  Should really change BigStr*
+    // to use unsigned type
+    uint8_t c = s->data_[i];
+    decode(&state, &codepoint, c);
+    if (state == UTF8_REJECT) {
+      return false;
+    }
+  }
+
+  return state == UTF8_ACCEPT;
 }
 
 }  // namespace pyj8

cpp/data_lang_test.cc

@@ -0,0 +1,33 @@
+#include "cpp/data_lang.h"
+
+#include "vendor/greatest.h"
+
+TEST part_is_utf8_test() {
+  BigStr* s = StrFromC("hi");
+
+  ASSERT(pyj8::PartIsUtf8(s, 0, 2));
+
+  // empty string is trivially UTF-8
+  ASSERT(pyj8::PartIsUtf8(s, 0, 0));
+
+  BigStr* binary = StrFromC("h\xff");
+  ASSERT(!pyj8::PartIsUtf8(binary, 0, len(binary)));
+
+  // first byte is UTF-8
+  ASSERT(pyj8::PartIsUtf8(binary, 0, 1));
+  // second byte isn't
+  ASSERT(!pyj8::PartIsUtf8(binary, 1, 2));
+
+  PASS();
+}
+
+GREATEST_MAIN_DEFS();
+
+int main(int argc, char** argv) {
+  GREATEST_MAIN_BEGIN();
+
+  RUN_TEST(part_is_utf8_test);
+
+  GREATEST_MAIN_END();
+  return 0;
+}