Redesign translation data format

The current format for translation data is problematic. Design and implement a new format which is more time-efficient and space-efficient. See the documentation in tools/compile-translations.go for details. This commit should not change behavior. Benchmarks before: -------------------------------------------------------------------------------------- Benchmark Time CPU Iterations -------------------------------------------------------------------------------------- benchmark_translate_from_source_code 4.20 ns 4.20 ns 166058970 benchmark_translate_from_translation_hit 21.5 ns 21.5 ns 32483955 benchmark_translate_from_translation_miss 14.3 ns 14.3 ns 48922995 benchmark_load_translations/c 24.5 ns 24.5 ns 28542187 benchmark_load_translations/en 28.5 ns 28.5 ns 24432980 benchmark_load_translations/en_loud 42.8 ns 42.8 ns 16316674 benchmark_load_translations/en_us 51.6 ns 51.6 ns 13472161 benchmark_load_translations/en_us_loud 71.6 ns 71.6 ns 9730601 benchmark_load_translations/en_us_utf8 94.9 ns 94.9 ns 7243153 benchmark_load_translations/en_us_utf8_loud 119 ns 119 ns 5856172 benchmark_load_translations/posix 32.3 ns 32.3 ns 21791104 benchmark_load_translations_and_find_hit 56.3 ns 56.3 ns 12252757 benchmark_load_translations_and_find_miss 52.1 ns 52.1 ns 13305961 Benchmarks after: -------------------------------------------------------------------------------------- Benchmark Time CPU Iterations -------------------------------------------------------------------------------------- benchmark_translate_from_source_code 3.11 ns 3.11 ns 223754407 benchmark_translate_from_translation_hit 1.73 ns 1.73 ns 404250404 benchmark_translate_from_translation_miss 2.17 ns 2.17 ns 320953691 benchmark_load_translations/c 30.1 ns 30.1 ns 23174741 benchmark_load_translations/en 34.1 ns 34.1 ns 20494803 benchmark_load_translations/en_loud 39.3 ns 39.3 ns 17804728 benchmark_load_translations/en_us 63.1 ns 63.1 ns 10996089 benchmark_load_translations/en_us_loud 67.1 ns 67.1 ns 10363768 benchmark_load_translations/en_us_utf8 115 ns 115 ns 6080506 benchmark_load_translations/en_us_utf8_loud 126 ns 126 ns 5536004 benchmark_load_translations/posix 37.2 ns 37.2 ns 18806537 benchmark_load_translations_and_find_hit 39.9 ns 39.9 ns 17542497 benchmark_load_translations_and_find_miss 40.2 ns 40.2 ns 17409687
quick-lint · Nov 29, 2021 · 1dcedfe · 1dcedfe
1 parent 5a2c71f
commit 1dcedfe
Show file tree

Hide file tree

Showing 29 changed files with 3,893 additions and 6,312 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -66,7 +66,6 @@ add_subdirectory(plugin/vim)
 add_subdirectory(plugin/emacs)
 add_subdirectory(plugin/vscode)
 add_subdirectory(src)
-add_subdirectory(tools)
 add_subdirectory(website/wasm)
 add_subdirectory(completions)
 include(CTest)

diff --git a/benchmark/benchmark-translation.cpp b/benchmark/benchmark-translation.cpp
@@ -3,8 +3,6 @@
 
 #include <benchmark/benchmark.h>
 #include <quick-lint-js/assert.h>
-#include <quick-lint-js/gmo.h>
-#include <quick-lint-js/translation-data.h>
 #include <quick-lint-js/translation.h>
 
 namespace quick_lint_js {
@@ -14,55 +12,55 @@ void benchmark_translate_from_source_code(benchmark::State &state) {
   messages.use_messages_from_source_code();
   for (auto _ : state) {
     ::benchmark::DoNotOptimize(messages.translate(
-        "variable assigned before its declaration"_gmo_message));
+        "variable assigned before its declaration"_translatable));
     ::benchmark::DoNotOptimize(
-        messages.translate("variable declared here"_gmo_message));
+        messages.translate("variable declared here"_translatable));
     ::benchmark::DoNotOptimize(messages.translate(
-        "~~~ invalid string, do not use outside benchmark ~~~"_gmo_message));
+        "~~~ invalid string, do not use outside benchmark ~~~"_translatable));
   }
 }
 BENCHMARK(benchmark_translate_from_source_code);
 
 void benchmark_translate_from_translation_hit(benchmark::State &state) {
-  static constexpr gmo_message messages_to_translate[] = {
-      "variable assigned before its declaration"_gmo_message,
-      "variable declared here"_gmo_message,
+  static constexpr translatable_message messages_to_translate[] = {
+      "variable assigned before its declaration"_translatable,
+      "variable declared here"_translatable,
   };
 
   translatable_messages messages;
-  bool have_translation =
-      messages.use_messages_from_locale("en@loud", gmo_files);
+  bool have_translation = messages.use_messages_from_locale("en@loud");
   QLJS_ALWAYS_ASSERT(have_translation);
-  for (const gmo_message &message : messages_to_translate) {
+  for (const translatable_message &message : messages_to_translate) {
     // Messages should be translated.
-    QLJS_ALWAYS_ASSERT(messages.translate(message) != message.message());
+    QLJS_ALWAYS_ASSERT(
+        std::strcmp(messages.translate(message), message.c_str()) != 0);
   }
 
   for (auto _ : state) {
-    for (const gmo_message &message : messages_to_translate) {
+    for (const translatable_message &message : messages_to_translate) {
       ::benchmark::DoNotOptimize(messages.translate(message));
     }
   }
 }
 BENCHMARK(benchmark_translate_from_translation_hit);
 
 void benchmark_translate_from_translation_miss(benchmark::State &state) {
-  static constexpr gmo_message messages_to_translate[] = {
-      "~~~ invalid string, do not use outside benchmark ~~~"_gmo_message,
-      "another invalid string, do not use outside benchmark"_gmo_message,
+  static constexpr translatable_message messages_to_translate[] = {
+      "~~~ invalid string, do not use outside benchmark ~~~"_translatable,
+      "another invalid string, do not use outside benchmark"_translatable,
   };
 
   translatable_messages messages;
-  bool have_translation =
-      messages.use_messages_from_locale("en@loud", gmo_files);
+  bool have_translation = messages.use_messages_from_locale("en@loud");
   QLJS_ALWAYS_ASSERT(have_translation);
-  for (const gmo_message &message : messages_to_translate) {
+  for (const translatable_message &message : messages_to_translate) {
     // Messages should not be translated.
-    QLJS_ALWAYS_ASSERT(messages.translate(message) == message.message());
+    QLJS_ALWAYS_ASSERT(
+        std::strcmp(messages.translate(message), message.c_str()) == 0);
   }
 
   for (auto _ : state) {
-    for (const gmo_message &message : messages_to_translate) {
+    for (const translatable_message &message : messages_to_translate) {
       ::benchmark::DoNotOptimize(messages.translate(message));
     }
   }
@@ -72,8 +70,7 @@ BENCHMARK(benchmark_translate_from_translation_miss);
 void benchmark_load_translations(benchmark::State &state, const char *locale) {
   for (auto _ : state) {
     translatable_messages messages;
-    bool have_translation =
-        messages.use_messages_from_locale(locale, gmo_files);
+    bool have_translation = messages.use_messages_from_locale(locale);
     ::benchmark::DoNotOptimize(have_translation);
     ::benchmark::DoNotOptimize(messages);
   }
@@ -89,49 +86,45 @@ BENCHMARK_CAPTURE(benchmark_load_translations, en_us_utf8_loud,
 BENCHMARK_CAPTURE(benchmark_load_translations, posix, "POSIX");
 
 void benchmark_load_translations_and_find_hit(benchmark::State &state) {
-  static constexpr gmo_message message_to_translate =
-      "variable assigned before its declaration"_gmo_message;
+  static constexpr translatable_message message_to_translate =
+      "variable assigned before its declaration"_translatable;
 
   const char *locale = "en@loud";
   {
     // Message should be translated.
     translatable_messages messages;
-    bool have_translation =
-        messages.use_messages_from_locale(locale, gmo_files);
+    bool have_translation = messages.use_messages_from_locale(locale);
     QLJS_ALWAYS_ASSERT(have_translation);
-    QLJS_ALWAYS_ASSERT(messages.translate(message_to_translate) !=
-                       message_to_translate.message());
+    QLJS_ALWAYS_ASSERT(std::strcmp(messages.translate(message_to_translate),
+                                   message_to_translate.c_str()) != 0);
   }
 
   for (auto _ : state) {
     translatable_messages messages;
-    bool have_translation =
-        messages.use_messages_from_locale(locale, gmo_files);
+    bool have_translation = messages.use_messages_from_locale(locale);
     ::benchmark::DoNotOptimize(have_translation);
     ::benchmark::DoNotOptimize(messages.translate(message_to_translate));
   }
 }
 BENCHMARK(benchmark_load_translations_and_find_hit);
 
 void benchmark_load_translations_and_find_miss(benchmark::State &state) {
-  static constexpr gmo_message message_to_translate =
-      "~~~ invalid string, do not use outside benchmark ~~~"_gmo_message;
+  static constexpr translatable_message message_to_translate =
+      "~~~ invalid string, do not use outside benchmark ~~~"_translatable;
 
   const char *locale = "en@loud";
   {
     // Message should not be translated.
     translatable_messages messages;
-    bool have_translation =
-        messages.use_messages_from_locale(locale, gmo_files);
+    bool have_translation = messages.use_messages_from_locale(locale);
     QLJS_ALWAYS_ASSERT(have_translation);
-    QLJS_ALWAYS_ASSERT(messages.translate(message_to_translate) ==
-                       message_to_translate.message());
+    QLJS_ALWAYS_ASSERT(std::strcmp(messages.translate(message_to_translate),
+                                   message_to_translate.c_str()) == 0);
   }
 
   for (auto _ : state) {
     translatable_messages messages;
-    bool have_translation =
-        messages.use_messages_from_locale(locale, gmo_files);
+    bool have_translation = messages.use_messages_from_locale(locale);
     ::benchmark::DoNotOptimize(have_translation);
     ::benchmark::DoNotOptimize(messages.translate(message_to_translate));
   }

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -69,7 +69,6 @@ quick_lint_js_add_library(
   file.cpp
   global-variables-browser.cpp
   global-variables.cpp
-  gmo.cpp
   integer.cpp
   json.cpp
   language.cpp
@@ -105,7 +104,6 @@ quick_lint_js_add_library(
   quick-lint-js/cli-location.h
   quick-lint-js/configuration-loader.h
   quick-lint-js/configuration.h
-  quick-lint-js/constant-divider.h
   quick-lint-js/cpp.h
   quick-lint-js/crash.h
   quick-lint-js/diagnostic-formatter.h
@@ -127,7 +125,7 @@ quick_lint_js_add_library(
   quick-lint-js/file.h
   quick-lint-js/force-inline.h
   quick-lint-js/global-variables.h
-  quick-lint-js/gmo.h
+  quick-lint-js/hash-fnv.h
   quick-lint-js/have.h
   quick-lint-js/integer.h
   quick-lint-js/json.h
@@ -159,7 +157,8 @@ quick_lint_js_add_library(
   quick-lint-js/simdjson.h
   quick-lint-js/temporary-directory.h
   quick-lint-js/text-error-reporter.h
-  quick-lint-js/translation-data.h
+  quick-lint-js/translation-table-generated.h
+  quick-lint-js/translation-table.h
   quick-lint-js/translation.h
   quick-lint-js/unreachable.h
   quick-lint-js/uri.h
@@ -174,7 +173,7 @@ quick_lint_js_add_library(
   simdjson.cpp
   temporary-directory.cpp
   text-error-reporter.cpp
-  translation-data.cpp
+  translation-table-generated.cpp
   translation.cpp
   uri.cpp
   utf-16.cpp

diff --git a/src/diagnostic.cpp b/src/diagnostic.cpp
@@ -5,7 +5,6 @@
 #include <quick-lint-js/cpp.h>
 #include <quick-lint-js/diagnostic.h>
 #include <quick-lint-js/error.h>
-#include <quick-lint-js/gmo.h>
 #include <quick-lint-js/lex.h>
 #include <quick-lint-js/narrow-cast.h>
 #include <utility>
@@ -28,7 +27,7 @@ class diagnostic_info_builder {
   // Each of Args must be a diagnostic_message_arg_info.
   template <class... Args>
   constexpr diagnostic_info_builder add(diagnostic_severity sev,
-                                        const gmo_message& message,
+                                        const translatable_message& message,
                                         const Args&... arg_infos) {
     diagnostic_message_info& message_info =
         this->info_.messages[this->current_message_index_++];