Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add LibreTranslate machine translator, documentation and its test cases. #753

Closed
wants to merge 8 commits into from
17 changes: 17 additions & 0 deletions docs/how-to/integrations/machine-translation.md
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,23 @@ WAGTAILLOCALIZE_MACHINE_TRANSLATOR = {
}
```

## LibreTranslate

Website: [https://libretranslate.com/](https://libretranslate.com/)

Note that You will need a subscription to get an API key. Or you can host your own instance.
More details are available on the github page [https://github.com/LibreTranslate/LibreTranslate](https://github.com/LibreTranslate/LibreTranslate).
drivard marked this conversation as resolved.
Show resolved Hide resolved

```python
WAGTAILLOCALIZE_MACHINE_TRANSLATOR = {
"CLASS": "wagtail_localize.machine_translators.libretranslate.LibreTranslator",
"OPTIONS": {
"LIBRETRANSLATE_URL": "https://libretranslate.org",
drivard marked this conversation as resolved.
Show resolved Hide resolved
"API_KEY": "<Your LibreTranslate api key here>", # Optional on self-hosted instance by providing a random string
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
"API_KEY": "<Your LibreTranslate api key here>", # Optional on self-hosted instance by providing a random string
"API_KEY": "<Your LibreTranslate api key here>", # Use any random string with self-hosted instances.

Copy link
Contributor Author

@drivard drivard Dec 18, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe something like this would be more appropriate
# On self-hosted instances, if you did not setup the api keys support use any random string.

},
}
```

## Dummy

The dummy translator exists primarily for testing Wagtail Localize and it only reverses the strings that are passed to
Expand Down
47 changes: 47 additions & 0 deletions wagtail_localize/machine_translators/libretranslate.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
import json

import requests

from wagtail_localize.machine_translators.base import BaseMachineTranslator
from wagtail_localize.strings import StringValue


class LibreTranslator(BaseMachineTranslator):
"""
A machine translator that uses the LibreTranslate API.
drivard marked this conversation as resolved.
Show resolved Hide resolved
"""

display_name = "LibreTranslate"

def get_api_endpoint(self):
return self.options["LIBRETRANSLATE_URL"]

def language_code(self, code):
return code.split("-")[0]

def translate(self, source_locale, target_locale, strings):
translations = [item.data for item in list(strings)]
response = requests.post(
self.get_api_endpoint() + "/translate",
data=json.dumps(
{
"q": translations,
"source": self.language_code(source_locale.language_code),
"target": self.language_code(target_locale.language_code),
"api_key": self.options["API_KEY"],
}
),
headers={"Content-Type": "application/json"},
timeout=10,
)
response.raise_for_status()

return {
string: StringValue(translation)
for string, translation in zip(strings, response.json()["translatedText"])
}

def can_translate(self, source_locale, target_locale):
return self.language_code(source_locale.language_code) != self.language_code(
target_locale.language_code
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
from django.test import TestCase, override_settings
from wagtail.models import Locale

from wagtail_localize.machine_translators import get_machine_translator
from wagtail_localize.machine_translators.libretranslate import LibreTranslator


LIBRETRANSLATE_SETTINGS_ENDPOINT = {
"CLASS": "wagtail_localize.machine_translators.libretranslate.LibreTranslator",
"OPTIONS": {
"LIBRETRANSLATE_URL": "https://libretranslate.org",
"API_KEY": "test-api-key",
},
}


class TestLibreTranslator(TestCase):
@override_settings(
WAGTAILLOCALIZE_MACHINE_TRANSLATOR=LIBRETRANSLATE_SETTINGS_ENDPOINT
)
def setUp(self):
self.english_locale = Locale.objects.get()
self.french_locale = Locale.objects.create(language_code="fr")
drivard marked this conversation as resolved.
Show resolved Hide resolved
self.translator = get_machine_translator()

def test_api_endpoint(self):
self.assertIsInstance(self.translator, LibreTranslator)
api_endpoint = self.translator.get_api_endpoint()
self.assertEqual(api_endpoint, "https://libretranslate.org")

# This probably requires a request to use the API but the test works against my local instance
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

rather than making an actual API call, I'd test 2 things:

  1. that translate will raise for status (so, mock the post call to that effect)
  2. that we get a dict of string: StringValue(translation) from the response, if there are translated strings. Again, using a mocked response

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

could I have a bit of pointer on mocking a post request please?

# def test_translate_text(self):
# self.assertIsInstance(self.translator, LibreTranslator)

# translations = self.translator.translate(
# self.english_locale,
# self.french_locale,
# {
# StringValue("Hello world!"),
# StringValue("This is a sentence. This is another sentence."),
# },
# )

# self.assertEqual(
# translations,
# {
# StringValue("Hello world!"): StringValue("Bonjour !"),
# StringValue(
# "This is a sentence. This is another sentence."
# ): StringValue("C'est une phrase. C'est une autre phrase."),
# },
# )

# This has been commented out because after a while the public API started
# to return different results for the same input.
# This probably requires a request to use the API but the test works against my local instance
# def test_translate_html(self):
# self.assertIsInstance(self.translator, LibreTranslator)

# string, attrs = StringValue.from_source_html(
# '<a href="https://en.wikipedia.org/wiki/World">Hello !</a>. <b>This is a test</b>.'
# )

# translations = self.translator.translate(
# self.english_locale, self.french_locale, [string]
# )

# self.assertEqual(
# translations[string].render_html(attrs),
# "Bonjour ! C'est un test enregistré/b.",
# )

def test_can_translate(self):
self.assertIsInstance(self.translator, LibreTranslator)

french_locale = Locale.objects.get(language_code="fr")

self.assertTrue(
self.translator.can_translate(self.english_locale, self.french_locale)
)
self.assertTrue(
self.translator.can_translate(self.english_locale, french_locale)
)

# Can't translate the same language
self.assertFalse(
self.translator.can_translate(self.english_locale, self.english_locale)
)

# Can't translate two variants of the same language
self.assertFalse(
self.translator.can_translate(self.french_locale, french_locale)
)