Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

FEATURE: translate topic #130

Merged
merged 1 commit into from Dec 28, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Expand Up @@ -11,6 +11,7 @@ function translatePost(post) {
post.setProperties({
translated_text: res.translation,
detected_lang: res.detected_lang,
translated_title: res.title_translation,
});
});
}
Expand All @@ -29,7 +30,8 @@ function initializeTranslation(api) {
api.includePostAttributes(
"can_translate",
"translated_text",
"detected_lang"
"detected_lang",
"translated_title"
);

api.decorateWidget("post-menu:before", (dec) => {
Expand All @@ -44,8 +46,17 @@ function initializeTranslation(api) {
const language = dec.attrs.detected_lang;
const translator = siteSettings.translator;

let titleElements = [];

if (dec.attrs.translated_title) {
titleElements = [
dec.h("div.topic-attribution", dec.attrs.translated_title),
];
}

return dec.h("div.post-translation", [
dec.h("hr"),
...titleElements,
dec.h(
"div.post-attribution",
I18n.t("translator.translated_from", { language, translator })
Expand Down
5 changes: 5 additions & 0 deletions assets/stylesheets/common/post.scss
@@ -1,3 +1,8 @@
.topic-attribution {
font-weight: bold;
margin-bottom: 1em;
margin-top: 1em;
}
.post-attribution {
color: #8899a6;
font-size: 12px;
Expand Down
2 changes: 1 addition & 1 deletion config/settings.yml
Expand Up @@ -94,6 +94,6 @@ discourse_translator:
client: true
type: group_list
restrict_translation_by_poster_group:
default: ""
default: ""
client: true
type: group_list
23 changes: 22 additions & 1 deletion plugin.rb
Expand Up @@ -73,16 +73,24 @@ def translate
end

begin
title_json = {}
detected_lang, translation =
"DiscourseTranslator::#{SiteSetting.translator}".constantize.translate(post)
render json: { translation: translation, detected_lang: detected_lang }, status: 200
if post.is_first_post?
_, title_translation =
"DiscourseTranslator::#{SiteSetting.translator}".constantize.translate(post.topic)
title_json = { title_translation: title_translation }
end
render json: { translation: translation, detected_lang: detected_lang }.merge(title_json),
status: 200
rescue ::DiscourseTranslator::TranslatorError => e
render_json_error e.message, status: 422
end
end
end

Post.register_custom_field_type(::DiscourseTranslator::TRANSLATED_CUSTOM_FIELD, :json)
Topic.register_custom_field_type(::DiscourseTranslator::TRANSLATED_CUSTOM_FIELD, :json)

class ::Post < ActiveRecord::Base
before_update :clear_translator_custom_fields, if: :raw_changed?
Expand All @@ -97,6 +105,19 @@ def clear_translator_custom_fields
end
end

class ::Topic < ActiveRecord::Base
before_update :clear_translator_custom_fields, if: :title_changed?

private

def clear_translator_custom_fields
return if !SiteSetting.translator_enabled

self.custom_fields.delete(DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD)
self.custom_fields.delete(DiscourseTranslator::TRANSLATED_CUSTOM_FIELD)
end
end

module ::Jobs
class TranslatorMigrateToAzurePortal < ::Jobs::Onceoff
def execute_onceoff(args)
Expand Down
14 changes: 7 additions & 7 deletions services/discourse_translator/amazon.rb
Expand Up @@ -92,8 +92,8 @@ def self.access_token_key
"aws-translator"
end

def self.detect(post)
text = post.cooked.truncate(MAXLENGTH, omission: nil)
def self.detect(topic_or_post)
text = get_text(topic_or_post).truncate(MAXLENGTH, omission: nil)

return if text.blank?

Expand All @@ -106,21 +106,21 @@ def self.detect(post)
},
)&.source_language_code

assign_lang_custom_field(post, detected_lang)
assign_lang_custom_field(topic_or_post, detected_lang)
end

def self.translate(post)
from_custom_fields(post) do
def self.translate(topic_or_post)
from_custom_fields(topic_or_post) do
result =
client.translate_text(
{
text: post.cooked.truncate(MAXLENGTH, omission: nil),
text: get_text(topic_or_post).truncate(MAXLENGTH, omission: nil),
source_language_code: "auto",
target_language_code: SUPPORTED_LANG_MAPPING[I18n.locale],
},
)

detected_lang = assign_lang_custom_field(post, result.source_language_code)
detected_lang = assign_lang_custom_field(topic_or_post, result.source_language_code)

[detected_lang, result.translated_text]
end
Expand Down
23 changes: 16 additions & 7 deletions services/discourse_translator/base.rb
Expand Up @@ -31,22 +31,31 @@ def self.access_token
raise "Not Implemented"
end

def self.from_custom_fields(post)
post_translated_custom_field =
post.custom_fields[DiscourseTranslator::TRANSLATED_CUSTOM_FIELD] || {}
translated_text = post_translated_custom_field[I18n.locale]
def self.from_custom_fields(topic_or_post)
translated_custom_field =
topic_or_post.custom_fields[DiscourseTranslator::TRANSLATED_CUSTOM_FIELD] || {}
translated_text = translated_custom_field[I18n.locale]

if translated_text.nil?
translated_text = yield

post.custom_fields[
topic_or_post.custom_fields[
DiscourseTranslator::TRANSLATED_CUSTOM_FIELD
] = post_translated_custom_field.merge(I18n.locale => translated_text)
] = translated_custom_field.merge(I18n.locale => translated_text)

post.save!
topic_or_post.save!
end

translated_text
end

def self.get_text(topic_or_post)
case topic_or_post.class.name
when "Post"
text = topic_or_post.cooked
when "Topic"
topic_or_post.title
end
end
nattsw marked this conversation as resolved.
Show resolved Hide resolved
end
end
14 changes: 7 additions & 7 deletions services/discourse_translator/google.rb
Expand Up @@ -74,10 +74,10 @@ def self.access_token
(raise TranslatorError.new("NotFound: Google Api Key not set."))
end

def self.detect(post)
post.custom_fields[DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD] ||= result(
def self.detect(topic_or_post)
topic_or_post.custom_fields[DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD] ||= result(
DETECT_URI,
q: post.cooked.truncate(MAXLENGTH, omission: nil),
q: get_text(topic_or_post).truncate(MAXLENGTH, omission: nil),
)[
"detections"
][
Expand All @@ -92,17 +92,17 @@ def self.translate_supported?(source, target)
res["languages"].any? { |obj| obj["language"] == source }
end

def self.translate(post)
detected_lang = detect(post)
def self.translate(topic_or_post)
detected_lang = detect(topic_or_post)

raise I18n.t("translator.failed") unless translate_supported?(detected_lang, I18n.locale)

translated_text =
from_custom_fields(post) do
from_custom_fields(topic_or_post) do
res =
result(
TRANSLATE_URI,
q: post.cooked.truncate(MAXLENGTH, omission: nil),
q: get_text(topic_or_post).truncate(MAXLENGTH, omission: nil),
source: detected_lang,
target: SUPPORTED_LANG_MAPPING[I18n.locale],
)
Expand Down
18 changes: 10 additions & 8 deletions services/discourse_translator/libretranslate.rb
Expand Up @@ -80,21 +80,23 @@ def self.access_token
SiteSetting.translator_libretranslate_api_key
end

def self.detect(post)
def self.detect(topic_or_post)
res =
result(
detect_uri,
q:
ActionController::Base
.helpers
.strip_tags(post.cooked)
.strip_tags(get_text(topic_or_post))
.truncate(MAXLENGTH, omission: nil),
)

if !res.empty?
post.custom_fields[DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD] ||= res[0]["language"]
topic_or_post.custom_fields[DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD] ||= res[0][
"language"
]
else
post.custom_fields[DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD] ||= "en"
topic_or_post.custom_fields[DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD] ||= "en"
end
end

Expand All @@ -104,17 +106,17 @@ def self.translate_supported?(source, target)
res.any? { |obj| obj["code"] == source } && res.any? { |obj| obj["code"] == lang }
end

def self.translate(post)
detected_lang = detect(post)
def self.translate(topic_or_post)
detected_lang = detect(topic_or_post)

raise I18n.t("translator.failed") unless translate_supported?(detected_lang, I18n.locale)

translated_text =
from_custom_fields(post) do
from_custom_fields(topic_or_post) do
res =
result(
translate_uri,
q: post.cooked.truncate(MAXLENGTH, omission: nil),
q: get_text(topic_or_post).truncate(MAXLENGTH, omission: nil),
source: detected_lang,
target: SUPPORTED_LANG_MAPPING[I18n.locale],
format: "html",
Expand Down
18 changes: 10 additions & 8 deletions services/discourse_translator/microsoft.rb
Expand Up @@ -93,9 +93,9 @@ def self.access_token_key
"microsoft-translator"
end

def self.detect(post)
post.custom_fields[DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD] ||= begin
text = post.raw.truncate(LENGTH_LIMIT, omission: nil)
def self.detect(topic_or_post)
topic_or_post.custom_fields[DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD] ||= begin
text = get_text(topic_or_post).truncate(LENGTH_LIMIT, omission: nil)

body = [{ "Text" => text }].to_json

Expand All @@ -108,21 +108,23 @@ def self.detect(post)
end
end

def self.translate(post)
detected_lang = detect(post)
def self.translate(topic_or_post)
detected_lang = detect(topic_or_post)

if !SUPPORTED_LANG_MAPPING.keys.include?(detected_lang.to_sym) &&
!SUPPORTED_LANG_MAPPING.values.include?(detected_lang.to_s)
raise TranslatorError.new(I18n.t("translator.failed"))
end

raise TranslatorError.new(I18n.t("translator.too_long")) if post.cooked.length > LENGTH_LIMIT
if get_text(topic_or_post).length > LENGTH_LIMIT
raise TranslatorError.new(I18n.t("translator.too_long"))
end

translated_text =
from_custom_fields(post) do
from_custom_fields(topic_or_post) do
query = default_query.merge("from" => detected_lang, "to" => locale, "textType" => "html")

body = [{ "Text" => post.cooked }].to_json
body = [{ "Text" => get_text(topic_or_post) }].to_json

uri = URI(translate_endpoint)
uri.query = URI.encode_www_form(query)
Expand Down
14 changes: 7 additions & 7 deletions services/discourse_translator/yandex.rb
Expand Up @@ -123,9 +123,9 @@ def self.access_token
(raise TranslatorError.new("NotFound: Yandex API Key not set."))
end

def self.detect(post)
post.custom_fields[DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD] ||= begin
query = default_query.merge("text" => post.raw)
def self.detect(topic_or_post)
topic_or_post.custom_fields[DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD] ||= begin
query = default_query.merge("text" => get_text(topic_or_post))

uri = URI(DETECT_URI)
uri.query = URI.encode_www_form(query)
Expand All @@ -136,20 +136,20 @@ def self.detect(post)
end
end

def self.translate(post)
detected_lang = detect(post)
def self.translate(topic_or_post)
detected_lang = detect(topic_or_post)

if !SUPPORTED_LANG_MAPPING.keys.include?(detected_lang.to_sym) &&
!SUPPORTED_LANG_MAPPING.values.include?(detected_lang.to_s)
raise TranslatorError.new(I18n.t("translator.failed"))
end

translated_text =
from_custom_fields(post) do
from_custom_fields(topic_or_post) do
query =
default_query.merge(
"lang" => "#{detected_lang}-#{locale}",
"text" => post.cooked,
"text" => get_text(topic_or_post),
"format" => "html",
)

Expand Down
7 changes: 6 additions & 1 deletion spec/controllers/translator_controller_spec.rb
Expand Up @@ -16,11 +16,16 @@
shared_examples "translation_successful" do
it "returns the translated text" do
DiscourseTranslator::Microsoft.expects(:translate).with(reply).returns(%w[ja ニャン猫])
if reply.is_first_post?
DiscourseTranslator::Microsoft.expects(:translate).with(reply.topic).returns(%w[ja タイトル])
end

post :translate, params: { post_id: reply.id }, format: :json

expect(response).to have_http_status(:ok)
expect(response.body).to eq({ translation: "ニャン猫", detected_lang: "ja" }.to_json)
expect(response.body).to eq(
{ translation: "ニャン猫", detected_lang: "ja", title_translation: "タイトル" }.to_json,
)
end
end

Expand Down
32 changes: 32 additions & 0 deletions spec/models/topic_spec.rb
@@ -0,0 +1,32 @@
# frozen_string_literal: true

require "rails_helper"

RSpec.describe Topic do
describe "translator custom fields" do
fab!(:topic) do
Fabricate(
:topic,
title: "this is a sample title",
custom_fields: {
::DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD => "en",
::DiscourseTranslator::TRANSLATED_CUSTOM_FIELD => {
"en" => "lol",
},
},
)
end

before { SiteSetting.translator_enabled = true }

after { SiteSetting.translator_enabled = false }

it "should reset custom fields when topic title has been updated" do
topic.update!(title: "this is an updated title")

expect(topic.custom_fields[::DiscourseTranslator::DETECTED_LANG_CUSTOM_FIELD]).to be_nil

expect(topic.custom_fields[::DiscourseTranslator::TRANSLATED_CUSTOM_FIELD]).to be_nil
end
end
end