Redesign translation data format

The current format for translation data is problematic. Design and
implement a new format which is more time-efficient and space-efficient.
See the documentation in tools/compile-translations.go for details.

This commit should not change behavior.

Benchmarks before:

    --------------------------------------------------------------------------------------
    Benchmark                                            Time             CPU   Iterations
    --------------------------------------------------------------------------------------
    benchmark_translate_from_source_code              4.20 ns         4.20 ns    166058970
    benchmark_translate_from_translation_hit          21.5 ns         21.5 ns     32483955
    benchmark_translate_from_translation_miss         14.3 ns         14.3 ns     48922995
    benchmark_load_translations/c                     24.5 ns         24.5 ns     28542187
    benchmark_load_translations/en                    28.5 ns         28.5 ns     24432980
    benchmark_load_translations/en_loud               42.8 ns         42.8 ns     16316674
    benchmark_load_translations/en_us                 51.6 ns         51.6 ns     13472161
    benchmark_load_translations/en_us_loud            71.6 ns         71.6 ns      9730601
    benchmark_load_translations/en_us_utf8            94.9 ns         94.9 ns      7243153
    benchmark_load_translations/en_us_utf8_loud        119 ns          119 ns      5856172
    benchmark_load_translations/posix                 32.3 ns         32.3 ns     21791104
    benchmark_load_translations_and_find_hit          56.3 ns         56.3 ns     12252757
    benchmark_load_translations_and_find_miss         52.1 ns         52.1 ns     13305961

Benchmarks after:

    --------------------------------------------------------------------------------------
    Benchmark                                            Time             CPU   Iterations
    --------------------------------------------------------------------------------------
    benchmark_translate_from_source_code              3.11 ns         3.11 ns    223754407
    benchmark_translate_from_translation_hit          1.73 ns         1.73 ns    404250404
    benchmark_translate_from_translation_miss         2.17 ns         2.17 ns    320953691
    benchmark_load_translations/c                     30.1 ns         30.1 ns     23174741
    benchmark_load_translations/en                    34.1 ns         34.1 ns     20494803
    benchmark_load_translations/en_loud               39.3 ns         39.3 ns     17804728
    benchmark_load_translations/en_us                 63.1 ns         63.1 ns     10996089
    benchmark_load_translations/en_us_loud            67.1 ns         67.1 ns     10363768
    benchmark_load_translations/en_us_utf8             115 ns          115 ns      6080506
    benchmark_load_translations/en_us_utf8_loud        126 ns          126 ns      5536004
    benchmark_load_translations/posix                 37.2 ns         37.2 ns     18806537
    benchmark_load_translations_and_find_hit          39.9 ns         39.9 ns     17542497
    benchmark_load_translations_and_find_miss         40.2 ns         40.2 ns     17409687

Author: strager

CMakeLists.txt

@@ -66,7 +66,6 @@ add_subdirectory(plugin/vim)
 add_subdirectory(plugin/emacs)
 add_subdirectory(plugin/vscode)
 add_subdirectory(src)
-add_subdirectory(tools)
 add_subdirectory(website/wasm)
 add_subdirectory(completions)
 include(CTest)

benchmark/benchmark-translation.cpp

@@ -3,8 +3,6 @@
 
 #include <benchmark/benchmark.h>
 #include <quick-lint-js/assert.h>
-#include <quick-lint-js/gmo.h>
-#include <quick-lint-js/translation-data.h>
 #include <quick-lint-js/translation.h>
 
 namespace quick_lint_js {
@@ -14,55 +12,55 @@ void benchmark_translate_from_source_code(benchmark::State &state) {
   messages.use_messages_from_source_code();
   for (auto _ : state) {
     ::benchmark::DoNotOptimize(messages.translate(
-        "variable assigned before its declaration"_gmo_message));
+        "variable assigned before its declaration"_translatable));
     ::benchmark::DoNotOptimize(
-        messages.translate("variable declared here"_gmo_message));
+        messages.translate("variable declared here"_translatable));
     ::benchmark::DoNotOptimize(messages.translate(
-        "~~~ invalid string, do not use outside benchmark ~~~"_gmo_message));
+        "~~~ invalid string, do not use outside benchmark ~~~"_translatable));
   }
 }
 BENCHMARK(benchmark_translate_from_source_code);
 
 void benchmark_translate_from_translation_hit(benchmark::State &state) {
-  static constexpr gmo_message messages_to_translate[] = {
-      "variable assigned before its declaration"_gmo_message,
-      "variable declared here"_gmo_message,
+  static constexpr translatable_message messages_to_translate[] = {
+      "variable assigned before its declaration"_translatable,
+      "variable declared here"_translatable,
   };
 
   translatable_messages messages;
-  bool have_translation =
-      messages.use_messages_from_locale("en@loud", gmo_files);
+  bool have_translation = messages.use_messages_from_locale("en@loud");
   QLJS_ALWAYS_ASSERT(have_translation);
-  for (const gmo_message &message : messages_to_translate) {
+  for (const translatable_message &message : messages_to_translate) {
     // Messages should be translated.
-    QLJS_ALWAYS_ASSERT(messages.translate(message) != message.message());
+    QLJS_ALWAYS_ASSERT(
+        std::strcmp(messages.translate(message), message.c_str()) != 0);
   }
 
   for (auto _ : state) {
-    for (const gmo_message &message : messages_to_translate) {
+    for (const translatable_message &message : messages_to_translate) {
       ::benchmark::DoNotOptimize(messages.translate(message));
     }
   }
 }
 BENCHMARK(benchmark_translate_from_translation_hit);
 
 void benchmark_translate_from_translation_miss(benchmark::State &state) {
-  static constexpr gmo_message messages_to_translate[] = {
-      "~~~ invalid string, do not use outside benchmark ~~~"_gmo_message,
-      "another invalid string, do not use outside benchmark"_gmo_message,
+  static constexpr translatable_message messages_to_translate[] = {
+      "~~~ invalid string, do not use outside benchmark ~~~"_translatable,
+      "another invalid string, do not use outside benchmark"_translatable,
   };
 
   translatable_messages messages;
-  bool have_translation =
-      messages.use_messages_from_locale("en@loud", gmo_files);
+  bool have_translation = messages.use_messages_from_locale("en@loud");
   QLJS_ALWAYS_ASSERT(have_translation);
-  for (const gmo_message &message : messages_to_translate) {
+  for (const translatable_message &message : messages_to_translate) {
     // Messages should not be translated.
-    QLJS_ALWAYS_ASSERT(messages.translate(message) == message.message());
+    QLJS_ALWAYS_ASSERT(
+        std::strcmp(messages.translate(message), message.c_str()) == 0);
   }
 
   for (auto _ : state) {
-    for (const gmo_message &message : messages_to_translate) {
+    for (const translatable_message &message : messages_to_translate) {
       ::benchmark::DoNotOptimize(messages.translate(message));
     }
   }
@@ -72,8 +70,7 @@ BENCHMARK(benchmark_translate_from_translation_miss);
 void benchmark_load_translations(benchmark::State &state, const char *locale) {
   for (auto _ : state) {
     translatable_messages messages;
-    bool have_translation =
-        messages.use_messages_from_locale(locale, gmo_files);
+    bool have_translation = messages.use_messages_from_locale(locale);
     ::benchmark::DoNotOptimize(have_translation);
     ::benchmark::DoNotOptimize(messages);
   }
@@ -89,49 +86,45 @@ BENCHMARK_CAPTURE(benchmark_load_translations, en_us_utf8_loud,
 BENCHMARK_CAPTURE(benchmark_load_translations, posix, "POSIX");
 
 void benchmark_load_translations_and_find_hit(benchmark::State &state) {
-  static constexpr gmo_message message_to_translate =
-      "variable assigned before its declaration"_gmo_message;
+  static constexpr translatable_message message_to_translate =
+      "variable assigned before its declaration"_translatable;
 
   const char *locale = "en@loud";
   {
     // Message should be translated.
     translatable_messages messages;
-    bool have_translation =
-        messages.use_messages_from_locale(locale, gmo_files);
+    bool have_translation = messages.use_messages_from_locale(locale);
     QLJS_ALWAYS_ASSERT(have_translation);
-    QLJS_ALWAYS_ASSERT(messages.translate(message_to_translate) !=
-                       message_to_translate.message());
+    QLJS_ALWAYS_ASSERT(std::strcmp(messages.translate(message_to_translate),
+                                   message_to_translate.c_str()) != 0);
   }
 
   for (auto _ : state) {
     translatable_messages messages;
-    bool have_translation =
-        messages.use_messages_from_locale(locale, gmo_files);
+    bool have_translation = messages.use_messages_from_locale(locale);
     ::benchmark::DoNotOptimize(have_translation);
     ::benchmark::DoNotOptimize(messages.translate(message_to_translate));
   }
 }
 BENCHMARK(benchmark_load_translations_and_find_hit);
 
 void benchmark_load_translations_and_find_miss(benchmark::State &state) {
-  static constexpr gmo_message message_to_translate =
-      "~~~ invalid string, do not use outside benchmark ~~~"_gmo_message;
+  static constexpr translatable_message message_to_translate =
+      "~~~ invalid string, do not use outside benchmark ~~~"_translatable;
 
   const char *locale = "en@loud";
   {
     // Message should not be translated.
     translatable_messages messages;
-    bool have_translation =
-        messages.use_messages_from_locale(locale, gmo_files);
+    bool have_translation = messages.use_messages_from_locale(locale);
     QLJS_ALWAYS_ASSERT(have_translation);
-    QLJS_ALWAYS_ASSERT(messages.translate(message_to_translate) ==
-                       message_to_translate.message());
+    QLJS_ALWAYS_ASSERT(std::strcmp(messages.translate(message_to_translate),
+                                   message_to_translate.c_str()) == 0);
   }
 
   for (auto _ : state) {
     translatable_messages messages;
-    bool have_translation =
-        messages.use_messages_from_locale(locale, gmo_files);
+    bool have_translation = messages.use_messages_from_locale(locale);
     ::benchmark::DoNotOptimize(have_translation);
     ::benchmark::DoNotOptimize(messages.translate(message_to_translate));
   }

src/CMakeLists.txt

@@ -69,7 +69,6 @@ quick_lint_js_add_library(
   file.cpp
   global-variables-browser.cpp
   global-variables.cpp
-  gmo.cpp
   integer.cpp
   json.cpp
   language.cpp
@@ -105,7 +104,6 @@ quick_lint_js_add_library(
   quick-lint-js/cli-location.h
   quick-lint-js/configuration-loader.h
   quick-lint-js/configuration.h
-  quick-lint-js/constant-divider.h
   quick-lint-js/cpp.h
   quick-lint-js/crash.h
   quick-lint-js/diagnostic-formatter.h
@@ -127,7 +125,7 @@ quick_lint_js_add_library(
   quick-lint-js/file.h
   quick-lint-js/force-inline.h
   quick-lint-js/global-variables.h
-  quick-lint-js/gmo.h
+  quick-lint-js/hash-fnv.h
   quick-lint-js/have.h
   quick-lint-js/integer.h
   quick-lint-js/json.h
@@ -159,7 +157,8 @@ quick_lint_js_add_library(
   quick-lint-js/simdjson.h
   quick-lint-js/temporary-directory.h
   quick-lint-js/text-error-reporter.h
-  quick-lint-js/translation-data.h
+  quick-lint-js/translation-table-generated.h
+  quick-lint-js/translation-table.h
   quick-lint-js/translation.h
   quick-lint-js/unreachable.h
   quick-lint-js/uri.h
@@ -174,7 +173,7 @@ quick_lint_js_add_library(
   simdjson.cpp
   temporary-directory.cpp
   text-error-reporter.cpp
-  translation-data.cpp
+  translation-table-generated.cpp
   translation.cpp
   uri.cpp
   utf-16.cpp

src/diagnostic.cpp

@@ -5,7 +5,6 @@
 #include <quick-lint-js/cpp.h>
 #include <quick-lint-js/diagnostic.h>
 #include <quick-lint-js/error.h>
-#include <quick-lint-js/gmo.h>
 #include <quick-lint-js/lex.h>
 #include <quick-lint-js/narrow-cast.h>
 #include <utility>
@@ -28,7 +27,7 @@ class diagnostic_info_builder {
   // Each of Args must be a diagnostic_message_arg_info.
   template <class... Args>
   constexpr diagnostic_info_builder add(diagnostic_severity sev,
-                                        const gmo_message& message,
+                                        const translatable_message& message,
                                         const Args&... arg_infos) {
     diagnostic_message_info& message_info =
         this->info_.messages[this->current_message_index_++];