diff --git a/app/jobs/scheduled/automatic_translation_backfill.rb b/app/jobs/scheduled/automatic_translation_backfill.rb index 12ca0093..0366a50c 100644 --- a/app/jobs/scheduled/automatic_translation_backfill.rb +++ b/app/jobs/scheduled/automatic_translation_backfill.rb @@ -14,34 +14,41 @@ def execute(args = nil) def fetch_untranslated_model_ids(model, content_column, limit, target_locale) m = model.name.downcase + + # Query selects every model (post/topic) *except* those who are **both** + # already locale detected and translated DB.query_single(<<~SQL, target_locale: target_locale, limit: limit) - SELECT m.id - FROM #{model.table_name} m - #{limit_to_public_clause(model)} - WHERE m.deleted_at IS NULL - AND m.#{content_column} != '' - AND m.user_id > 0 - #{max_age_clause} - AND ( - NOT EXISTS ( - SELECT 1 - FROM discourse_translator_#{m}_locales - WHERE #{m}_id = m.id + SELECT * FROM + ( + ( -- every post / topic + SELECT m.id + FROM #{model.table_name} m + #{limit_to_public_clause(model)} + WHERE m.deleted_at IS NULL + AND m.#{content_column} != '' + AND m.user_id > 0 + #{max_age_clause} + ORDER BY m.updated_at DESC + ) + EXCEPT + ( + ( -- locale detected + SELECT + #{m}_id + FROM + discourse_translator_#{m}_locales + WHERE + detected_locale = :target_locale ) - OR EXISTS ( - SELECT 1 - FROM discourse_translator_#{m}_locales - WHERE #{m}_id = m.id - AND detected_locale != :target_locale + INTERSECT + ( -- translated + SELECT #{m}_id + FROM discourse_translator_#{m}_translations + WHERE + locale = :target_locale ) ) - AND NOT EXISTS ( - SELECT 1 - FROM discourse_translator_#{m}_translations - WHERE #{m}_id = m.id - AND locale = :target_locale - ) - ORDER BY m.updated_at DESC + ) AS subquery LIMIT :limit SQL end @@ -50,14 +57,10 @@ def fetch_untranslated_model_ids(model, content_column, limit, target_locale) def should_backfill? return false if SiteSetting.automatic_translation_target_languages.blank? - return false if SiteSetting.automatic_translation_backfill_maximum_translations_per_hour == 0 + return false if SiteSetting.automatic_translation_backfill_rate == 0 true end - def translations_per_run - [(SiteSetting.automatic_translation_backfill_maximum_translations_per_hour / 12), 1].max - end - def backfill_locales @backfill_locales ||= SiteSetting.automatic_translation_target_languages.split("|").map { |l| l.gsub("_", "-") } @@ -83,23 +86,12 @@ def translate_records(type, record_ids, target_locale) end def process_batch - models_translated = [Post, Topic].size - avg_translations_per_model_per_language = [ - translations_per_run / models_translated / backfill_locales.size, - 1, - ].max - - records_to_translate = avg_translations_per_model_per_language + records_to_translate = SiteSetting.automatic_translation_backfill_rate backfill_locales.each_with_index do |target_locale, i| topic_ids = fetch_untranslated_model_ids(Topic, "title", records_to_translate, target_locale) post_ids = fetch_untranslated_model_ids(Post, "raw", records_to_translate, target_locale) - # if we end up translating fewer records than records_to_translate, - # add to the value so that the next locales can have more quota - records_to_translate = - avg_translations_per_model_per_language + - ((records_to_translate - topic_ids.size - post_ids.size) / backfill_locales.size - i) next if topic_ids.empty? && post_ids.empty? DiscourseTranslator::VerboseLogger.log( diff --git a/config/locales/server.de.yml b/config/locales/server.de.yml index d492813a..9f9fd557 100644 --- a/config/locales/server.de.yml +++ b/config/locales/server.de.yml @@ -25,7 +25,7 @@ de: restrict_translation_by_poster_group: "Erlaube nur die Übersetzung von Beiträgen von Benutzern aus zulässigen Gruppen. Wenn das Feld leer ist, wird die Übersetzung von Beiträgen aller Benutzer erlaubt." experimental_anon_language_switcher: "Aktiviere die experimentelle Sprachumschaltung. Damit können nicht eingeloggte Besucher zwischen übersetzten Versionen von Discourse und von Benutzern beigetragenen Inhalten in Themen wechseln." errors: - needs_nonzero_backfill: "Für die automatische Sprachübersetzung muss die versteckte Einstellung 'automatic_translation_backfill_maximum_translations_per_hour' ungleich Null sein. Bitte wende dich an deinen Website-Administrator, um dieses Limit zu erhöhen." + needs_nonzero_backfill: "Für die automatische Sprachübersetzung muss die versteckte Einstellung 'automatic_translation_backfill_rate' ungleich Null sein. Bitte wende dich an deinen Website-Administrator, um dieses Limit zu erhöhen." experimental_anon_language_switcher_requirements: "Die experimentelle Sprachumschaltung setzt voraus, dass die Website-Einstellung `set locale from cookie` aktiviert ist und `automatic translation target languages` mindestens eine Sprache enthält." experimental_inline_translation: "Aktiviere die experimentelle Übersetzungsfunktion mit Überschreiben. Diese ersetzt die bisherige parallele Übersetzung und ermöglicht es Besuchern der Website, die nicht das Standardgebietsschema verwenden, Inhalte direkt in ihrer Sprache zu sehen." automatic_translation_target_languages: "Die Sprachen, in die Benutzerinhalte (Beiträge, Themen) automatisch übersetzt werden sollen. Wenn nichts ausgewählt ist, wird in keine Sprachen automatisch übersetzt." diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index 801f6958..da4ced5b 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -19,7 +19,7 @@ en: restrict_translation_by_poster_group: "Only allow translation of posts made by users in allowed groups. If empty, allow translations of posts from all users." experimental_anon_language_switcher: "Enable experimental language switcher. This will allow site visitors who are not logged in to switch between translated versions of Discourse and user-contributed content in topics." errors: - needs_nonzero_backfill: "Automatic language translation requires the 'automatic_translation_backfill_maximum_translations_per_hour' hidden setting to be a non-zero value. Please approach your site administrator to increase this limit." + needs_nonzero_backfill: "Automatic language translation requires the 'automatic_translation_backfill_rate' hidden setting to be a non-zero value. Please approach your site administrator to increase this limit." experimental_anon_language_switcher_requirements: "The experimental language switcher requires the `set locale from cookie` site setting to be enabled, and the `automatic translation target languages` to have at least one language." experimental_inline_translation: "Enable experimental inline translation feature. This replaces existing parallel translation, allowing site visitors with a non-default locale to view content in their language." automatic_translation_target_languages: "The languages to automatically translate user content (posts, topics) to. If empty, no languages will be automatically translated." diff --git a/config/locales/server.he.yml b/config/locales/server.he.yml index 6ca6b7b2..8391afeb 100644 --- a/config/locales/server.he.yml +++ b/config/locales/server.he.yml @@ -25,7 +25,7 @@ he: restrict_translation_by_poster_group: "לאפשר תרגום של פוסטים שנוצרו על ידי משתמשים בקבוצות מורשות בלבד. אם ריק, לאפשר תרגומים של פוסטים מכל המשתמשים." experimental_anon_language_switcher: "הפעלת בורר שפות ניסיוני. כך יוכלו מבקרי האתר שלא נכנסו לחשבונות המשתמש שלהם לעבור בין הגרסאות המתורגמות של Discourse ותוכן שנתרם על ידי המשתמשים בנושאים." errors: - needs_nonzero_backfill: "תרגום שפה אוטומטי דורש שהערך של ההגדרה הנסתרת ‚automatic_translation_backfill_maximum_translations_per_hour’ יהיה שונה מאפס. נא לגשת להנהלת האתר שלך כדי להגדיל את המגבלה הזאת." + needs_nonzero_backfill: "תרגום שפה אוטומטי דורש שהערך של ההגדרה הנסתרת ‚automatic_translation_backfill_rate’ יהיה שונה מאפס. נא לגשת להנהלת האתר שלך כדי להגדיל את המגבלה הזאת." experimental_anon_language_switcher_requirements: "בורר השפה הניסיוני דורש את הפעלת הגדרת האתר `set locale from cookie` (הגדרת השפה מהעוגיות), ושתחת `automatic translation target languages` (שפות יעד לתרגום אוטומטי) תהיה לפחות שפה אחת." experimental_inline_translation: "הפעלת יכולת ניסיונית לתרגום פנימי. מחליפה תרגום מקבילי קיים, מה שמאפשר למבקרי האתר עם שפה ששונה מברירת המחדל לצפות בתוכן בשפה שלהם." automatic_translation_target_languages: "השפה אליה לתרגם את תוכן המשתמשים אוטומטית (פוסטים, נושאים). אם השדה ריק, אף שפה לא תתורגם אוטומטית." diff --git a/config/locales/server.ro.yml b/config/locales/server.ro.yml index cbee5e1f..f869a7e7 100644 --- a/config/locales/server.ro.yml +++ b/config/locales/server.ro.yml @@ -24,7 +24,7 @@ ro: restrict_translation_by_group: "Doar grupurile permise pot traduce" restrict_translation_by_poster_group: "Permite traducerea numai a articolelor făcute de utilizatorii din grupurile permise. Dacă este gol, permite traducerea articolelor de la toți utilizatorii." errors: - needs_nonzero_backfill: "Traducerea automată a limbii necesită ca setarea ascunsă „automatic_translation_backfill_maximum_translations_per_hour” să aibă o valoare diferită de zero. Te rugăm să te adresezi administratorului site-ului tău pentru a crește această limită." + needs_nonzero_backfill: "Traducerea automată a limbii necesită ca setarea ascunsă „automatic_translation_backfill_rate” să aibă o valoare diferită de zero. Te rugăm să te adresezi administratorului site-ului tău pentru a crește această limită." experimental_inline_translation: "Activează funcția experimentală de traducere în-linie. Aceasta înlocuiește traducerea paralelă existentă, permițând vizitatorilor site-ului cu o locală care nu este implicită să vadă conținutul în limba lor." automatic_translation_target_languages: "Limbile pentru a traduce automat conținutul utilizatorului (postări, subiecte). Dacă este gol, nicio limbă nu va fi tradusă automat." translator: diff --git a/config/settings.yml b/config/settings.yml index d67b06df..bb543940 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -22,7 +22,7 @@ discourse_translator: allow_any: false choices: "DiscourseTranslator::TranslatableLanguagesSetting.values" validator: "DiscourseTranslator::Validators::TranslatableLanguagesValidator" - automatic_translation_backfill_maximum_translations_per_hour: + automatic_translation_backfill_rate: default: 0 client: false hidden: true diff --git a/lib/discourse_translator/validators/translatable_languages_validator.rb b/lib/discourse_translator/validators/translatable_languages_validator.rb index 6fbccd41..0e3df8d9 100644 --- a/lib/discourse_translator/validators/translatable_languages_validator.rb +++ b/lib/discourse_translator/validators/translatable_languages_validator.rb @@ -10,7 +10,7 @@ def initialize(opts = {}) def valid_value?(val) return true if val.blank? - SiteSetting.automatic_translation_backfill_maximum_translations_per_hour > 0 + SiteSetting.automatic_translation_backfill_rate > 0 end def error_message diff --git a/spec/jobs/automatic_translation_backfill_spec.rb b/spec/jobs/automatic_translation_backfill_spec.rb index 7ed90e18..56dc0369 100644 --- a/spec/jobs/automatic_translation_backfill_spec.rb +++ b/spec/jobs/automatic_translation_backfill_spec.rb @@ -59,15 +59,15 @@ def expect_google_translate(text) end it "does not backfill if backfill limit is set to 0" do - SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1 + SiteSetting.automatic_translation_backfill_rate = 1 SiteSetting.automatic_translation_target_languages = "de" - SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 0 + SiteSetting.automatic_translation_backfill_rate = 0 expect_any_instance_of(Jobs::AutomaticTranslationBackfill).not_to receive(:process_batch) end describe "with two locales ['de', 'es']" do before do - SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 100 + SiteSetting.automatic_translation_backfill_rate = 100 SiteSetting.automatic_translation_target_languages = "de|es" SiteSetting.automatic_translation_backfill_limit_to_public_content = true SiteSetting.automatic_translation_backfill_max_age_days = 5 @@ -159,7 +159,7 @@ def expect_google_translate(text) describe "with just one locale ['de']" do before do - SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 100 + SiteSetting.automatic_translation_backfill_rate = 100 SiteSetting.automatic_translation_target_languages = "de" expect_google_check_language end diff --git a/spec/jobs/translate_translatable_spec.rb b/spec/jobs/translate_translatable_spec.rb index 8ef8b83f..320bd471 100644 --- a/spec/jobs/translate_translatable_spec.rb +++ b/spec/jobs/translate_translatable_spec.rb @@ -8,7 +8,7 @@ before do SiteSetting.translator_enabled = true SiteSetting.translator_provider = "Google" - SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 100 + SiteSetting.automatic_translation_backfill_rate = 100 SiteSetting.automatic_translation_target_languages = "es|fr" allow(DiscourseTranslator::Google).to receive(:translate) end diff --git a/spec/lib/guardian_extension_spec.rb b/spec/lib/guardian_extension_spec.rb index d5af8bd6..ce68e852 100644 --- a/spec/lib/guardian_extension_spec.rb +++ b/spec/lib/guardian_extension_spec.rb @@ -180,7 +180,7 @@ before do SiteSetting.experimental_inline_translation = true - SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1 + SiteSetting.automatic_translation_backfill_rate = 1 SiteSetting.automatic_translation_target_languages = "pt" end diff --git a/spec/models/post_spec.rb b/spec/models/post_spec.rb index 8db5354a..9e298b92 100644 --- a/spec/models/post_spec.rb +++ b/spec/models/post_spec.rb @@ -139,7 +139,7 @@ fab!(:user) it "enqueues translate_translatable job when post cooked" do - SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 100 + SiteSetting.automatic_translation_backfill_rate = 100 SiteSetting.automatic_translation_target_languages = "es" post = Fabricate(:post, user: user) CookedPostProcessor.new(post).post_process @@ -154,7 +154,7 @@ end it "does not enqueue translate_translatable job for bot posts" do - SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1 + SiteSetting.automatic_translation_backfill_rate = 1 SiteSetting.automatic_translation_target_languages = "es" post = Fabricate(:post, user: Discourse.system_user) CookedPostProcessor.new(post).post_process diff --git a/spec/serializers/basic_topic_serializer_spec.rb b/spec/serializers/basic_topic_serializer_spec.rb index 9a64fc46..de92ade4 100644 --- a/spec/serializers/basic_topic_serializer_spec.rb +++ b/spec/serializers/basic_topic_serializer_spec.rb @@ -21,7 +21,7 @@ SiteSetting.experimental_inline_translation = true I18n.locale = "ja" - SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1 + SiteSetting.automatic_translation_backfill_rate = 1 SiteSetting.automatic_translation_target_languages = "ja" end diff --git a/spec/serializers/post_serializer_spec.rb b/spec/serializers/post_serializer_spec.rb index 0ee949f3..32805f9b 100644 --- a/spec/serializers/post_serializer_spec.rb +++ b/spec/serializers/post_serializer_spec.rb @@ -77,7 +77,7 @@ it "returns true when there is a translation for the user's locale in target languages" do SiteSetting.translator_enabled = true SiteSetting.experimental_inline_translation = true - SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1 + SiteSetting.automatic_translation_backfill_rate = 1 SiteSetting.automatic_translation_target_languages = "ja" I18n.locale = "ja" post.set_detected_locale("en") @@ -90,7 +90,7 @@ it "returns false when there is a translation for the user's locale not in target languages" do SiteSetting.translator_enabled = true SiteSetting.experimental_inline_translation = true - SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1 + SiteSetting.automatic_translation_backfill_rate = 1 SiteSetting.automatic_translation_target_languages = "es" I18n.locale = "ja" post.set_detected_locale("en") @@ -103,7 +103,7 @@ it "returns false when there is no translation for the current locale in target languages" do SiteSetting.translator_enabled = true SiteSetting.experimental_inline_translation = true - SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1 + SiteSetting.automatic_translation_backfill_rate = 1 SiteSetting.automatic_translation_target_languages = "ja" I18n.locale = "ja" post.set_translation("es", "Hola") @@ -133,7 +133,7 @@ def serialize_post(guardian_user: user, params: {}) it "does not return translated_cooked when show=original param is present" do I18n.locale = "ja" - SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1 + SiteSetting.automatic_translation_backfill_rate = 1 SiteSetting.automatic_translation_target_languages = "ja" post.set_translation("ja", "こんにちは") @@ -150,7 +150,7 @@ def serialize_post(guardian_user: user, params: {}) end it "returns translated content based on locale presence in target languages" do - SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1 + SiteSetting.automatic_translation_backfill_rate = 1 post.set_translation("ja", "こんにちは") post.set_translation("es", "Hola") I18n.locale = "ja" diff --git a/spec/serializers/topic_view_serializer_spec.rb b/spec/serializers/topic_view_serializer_spec.rb index d23294f0..6cef8676 100644 --- a/spec/serializers/topic_view_serializer_spec.rb +++ b/spec/serializers/topic_view_serializer_spec.rb @@ -59,7 +59,7 @@ before do SiteSetting.experimental_inline_translation = true - SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1 + SiteSetting.automatic_translation_backfill_rate = 1 SiteSetting.automatic_translation_target_languages = "es" SiteSetting.default_locale = "en" @@ -159,7 +159,7 @@ def serialize_topic(guardian_user: user, params: {}) end it "returns translated title in fancy_title when translation exists for current locale" do - SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1 + SiteSetting.automatic_translation_backfill_rate = 1 SiteSetting.automatic_translation_target_languages = "ja" topic.set_translation("ja", jap_title) expect(serialize_topic.fancy_title).to eq("<h1>フス・ロ・ダ・ア</h1>") @@ -170,7 +170,7 @@ def serialize_topic(guardian_user: user, params: {}) fab!(:user) before do - SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1 + SiteSetting.automatic_translation_backfill_rate = 1 SiteSetting.automatic_translation_target_languages = "ja" end diff --git a/spec/system/anon_language_switcher_spec.rb b/spec/system/anon_language_switcher_spec.rb index cfa4ec2e..bbdfba3d 100644 --- a/spec/system/anon_language_switcher_spec.rb +++ b/spec/system/anon_language_switcher_spec.rb @@ -12,7 +12,7 @@ SiteSetting.translator_enabled = true SiteSetting.allow_user_locale = true SiteSetting.set_locale_from_cookie = true - SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1 + SiteSetting.automatic_translation_backfill_rate = 1 SiteSetting.automatic_translation_target_languages = "es|ja" SiteSetting.experimental_anon_language_switcher = true visit("/") diff --git a/spec/system/full_page_translation_spec.rb b/spec/system/full_page_translation_spec.rb index 2bbbf49c..d0282143 100644 --- a/spec/system/full_page_translation_spec.rb +++ b/spec/system/full_page_translation_spec.rb @@ -34,7 +34,7 @@ SiteSetting.set_locale_from_cookie = true SiteSetting.set_locale_from_param = true SiteSetting.experimental_inline_translation = true - SiteSetting.automatic_translation_backfill_maximum_translations_per_hour = 1 + SiteSetting.automatic_translation_backfill_rate = 1 SiteSetting.automatic_translation_target_languages = "ja" end