Added feature to treat couples of languages as equal when searching f…

…or subtitles

* Add 'Language-equals' support

This feature will treat couples of languages as equal for list-subtitles
operations. It's optional; its methods won't do anything if an empy list
is set. See more info at docstrings from 'subliminal_patch.core'.

For example, let's say I only want to have "Spanish (es.srt)" subtitles
and I don't care about the differences between Spain and LATAM spanish.
This feature will allow me to always get European Spanish even from
LATAM Spanish providers like Argenteam and Subdivx.

Example for config.ini:

language_equals = ['spa-MX:spa']

(Which means all Latam Spanish subtitles from every provider will be
converted to European Spanish)

* Add PT and ZH language tests

* Add HI and Forced parsing for language pairs

Format example: ["en@HI:en", "es-MX@forced:es-MX"]

* Update languages.py

* Update API definition to reflect the previous change

* Add language equals table to the UI (test only)

* Add global language selector and get language from code3 utilities

* Add unit tests for language equal feature

* Add encode function to language equal feature

* Add CRUD methods to the language equals panel

* Add equals description

* Add parsing support for alpha3 custom languages

* no log: add more tests

* Add forced and hi support to the language equal target

---------

Co-authored-by: morpheus65535 <louis_vezina@hotmail.com>
Co-authored-by: LASER-Yi <liangyi0007@gmail.com>

bazarr/api/system/languages.py

@@ -4,7 +4,7 @@
 from operator import itemgetter
 
 from app.database import TableHistory, TableHistoryMovie, TableSettingsLanguages
-from languages.get_languages import alpha2_from_alpha3, language_from_alpha2
+from languages.get_languages import alpha2_from_alpha3, language_from_alpha2, alpha3_from_alpha2
 
 from ..utils import authenticate, False_Keys
 
@@ -46,6 +46,7 @@ def get(self):
                     try:
                         languages_dicts.append({
                             'code2': code2,
+                            'code3': alpha3_from_alpha2(code2),
                             'name': language_from_alpha2(code2),
                             # Compatibility: Use false temporarily
                             'enabled': False
@@ -55,6 +56,7 @@ def get(self):
         else:
             languages_dicts = TableSettingsLanguages.select(TableSettingsLanguages.name,
                                                             TableSettingsLanguages.code2,
+                                                            TableSettingsLanguages.code3,
                                                             TableSettingsLanguages.enabled)\
                 .order_by(TableSettingsLanguages.name).dicts()
             languages_dicts = list(languages_dicts)

bazarr/app/config.py

@@ -83,7 +83,8 @@ def base_url_slash_cleaner(uri):
         'default_und_audio_lang': '',
         'default_und_embedded_subtitles_lang': '',
         'parse_embedded_audio_track': 'False',
-        'skip_hashing': 'False'
+        'skip_hashing': 'False',
+        'language_equals': '[]',
     },
     'auth': {
         'type': 'None',
@@ -300,7 +301,8 @@ def base_url_slash_cleaner(uri):
               'excluded_series_types',
               'enabled_providers',
               'path_mappings',
-              'path_mappings_movie']
+              'path_mappings_movie',
+              'language_equals']
 
 str_keys = ['chmod']
 

bazarr/app/get_providers.py

@@ -20,6 +20,7 @@
 
 from app.get_args import args
 from app.config import settings, get_array_from
+from languages.get_languages import CustomLanguage
 from app.event_handler import event_stream
 from utilities.binaries import get_binary
 from radarr.blacklist import blacklist_log_movie
@@ -115,6 +116,49 @@ def provider_pool():
     return subliminal_patch.core.SZProviderPool
 
 
+def _lang_from_str(content: str):
+    " Formats: es-MX en@hi es-MX@forced "
+    extra_info = content.split("@")
+    if len(extra_info) > 1:
+        kwargs = {extra_info[-1]: True}
+    else:
+        kwargs = {}
+
+    content = extra_info[0]
+
+    try:
+        code, country = content.split("-")
+    except ValueError:
+        lang = CustomLanguage.from_value(content)
+        if lang is not None:
+            lang = lang.subzero_language()
+            return lang.rebuild(lang, **kwargs)
+
+        code, country = content, None
+
+    return subliminal_patch.core.Language(code, country, **kwargs)
+
+
+def get_language_equals(settings_=None):
+    settings_ = settings_ or settings
+
+    equals = get_array_from(settings_.general.language_equals)
+    if not equals:
+        return []
+
+    items = []
+    for equal in equals:
+        try:
+            from_, to_ = equal.split(":")
+            from_, to_ = _lang_from_str(from_), _lang_from_str(to_)
+        except Exception as error:
+            logging.info("Invalid equal value: '%s' [%s]", equal, error)
+        else:
+            items.append((from_, to_))
+
+    return items
+
+
 def get_providers():
     providers_list = []
     existing_providers = provider_registry.names()

bazarr/subtitles/pool.py

@@ -8,7 +8,7 @@
 
 from radarr.blacklist import get_blacklist_movie
 from sonarr.blacklist import get_blacklist
-from app.get_providers import get_providers, get_providers_auth, provider_throttle, provider_pool
+from app.get_providers import get_providers, get_providers_auth, provider_throttle, provider_pool, get_language_equals
 
 from .utils import get_ban_list
 
@@ -19,10 +19,11 @@ def _init_pool(media_type, profile_id=None, providers=None):
     return pool(
         providers=providers or get_providers(),
         provider_configs=get_providers_auth(),
-        blacklist=get_blacklist() if media_type == 'series' else get_blacklist_movie(),
+        blacklist=get_blacklist() if media_type == "series" else get_blacklist_movie(),
         throttle_callback=provider_throttle,
         ban_list=get_ban_list(profile_id),
         language_hook=None,
+        language_equals=get_language_equals(),
     )
 
 
@@ -54,8 +55,19 @@ def _update_pool(media_type, profile_id=None):
     return pool.update(
         get_providers(),
         get_providers_auth(),
-        get_blacklist() if media_type == 'series' else get_blacklist_movie(),
+        get_blacklist() if media_type == "series" else get_blacklist_movie(),
         get_ban_list(profile_id),
+        get_language_equals(),
+    )
+
+
+def _pool_update(pool, media_type, profile_id=None):
+    return pool.update(
+        get_providers(),
+        get_providers_auth(),
+        get_blacklist() if media_type == "series" else get_blacklist_movie(),
+        get_ban_list(profile_id),
+        get_language_equals(),
     )
 
 

frontend/src/components/bazarr/LanguageSelector.tsx

@@ -0,0 +1,34 @@
+import { useLanguages } from "@/apis/hooks";
+import { Selector, SelectorProps } from "@/components/inputs";
+import { useSelectorOptions } from "@/utilities";
+import { FunctionComponent, useMemo } from "react";
+
+interface LanguageSelectorProps
+  extends Omit<SelectorProps<Language.Server>, "options" | "getkey"> {
+  enabled?: boolean;
+}
+
+const LanguageSelector: FunctionComponent<LanguageSelectorProps> = ({
+  enabled = false,
+  ...selector
+}) => {
+  const { data } = useLanguages();
+
+  const filteredData = useMemo(() => {
+    if (enabled) {
+      return data?.filter((value) => value.enabled);
+    } else {
+      return data;
+    }
+  }, [data, enabled]);
+
+  const options = useSelectorOptions(
+    filteredData ?? [],
+    (value) => value.name,
+    (value) => value.code3
+  );
+
+  return <Selector {...options} searchable {...selector}></Selector>;
+};
+
+export default LanguageSelector;

frontend/src/pages/Settings/Languages/equals.test.ts

@@ -0,0 +1,196 @@
+import {
+  decodeEqualData,
+  encodeEqualData,
+  LanguageEqualData,
+  LanguageEqualImmediateData,
+} from "@/pages/Settings/Languages/equals";
+import { describe, expect, it } from "vitest";
+
+describe("Equals Parser", () => {
+  it("should parse from string correctly", () => {
+    interface TestData {
+      text: string;
+      expected: LanguageEqualImmediateData;
+    }
+
+    function testParsedResult(
+      text: string,
+      expected: LanguageEqualImmediateData
+    ) {
+      const result = decodeEqualData(text);
+
+      if (result === undefined) {
+        expect(false, `Cannot parse '${text}' as language equal data`);
+        return;
+      }
+
+      expect(
+        result,
+        `${text} does not match with the expected equal data`
+      ).toStrictEqual(expected);
+    }
+
+    const testValues: TestData[] = [
+      {
+        text: "spa-MX:spa",
+        expected: {
+          source: {
+            content: "spa-MX",
+            hi: false,
+            forced: false,
+          },
+          target: {
+            content: "spa",
+            hi: false,
+            forced: false,
+          },
+        },
+      },
+      {
+        text: "zho@hi:zht",
+        expected: {
+          source: {
+            content: "zho",
+            hi: true,
+            forced: false,
+          },
+          target: {
+            content: "zht",
+            hi: false,
+            forced: false,
+          },
+        },
+      },
+      {
+        text: "es-MX@forced:es-MX",
+        expected: {
+          source: {
+            content: "es-MX",
+            hi: false,
+            forced: true,
+          },
+          target: {
+            content: "es-MX",
+            hi: false,
+            forced: false,
+          },
+        },
+      },
+      {
+        text: "en:en@hi",
+        expected: {
+          source: {
+            content: "en",
+            hi: false,
+            forced: false,
+          },
+          target: {
+            content: "en",
+            hi: true,
+            forced: false,
+          },
+        },
+      },
+    ];
+
+    testValues.forEach((data) => {
+      testParsedResult(data.text, data.expected);
+    });
+  });
+
+  it("should encode to string correctly", () => {
+    interface TestData {
+      source: LanguageEqualData;
+      expected: string;
+    }
+
+    const testValues: TestData[] = [
+      {
+        source: {
+          source: {
+            content: {
+              name: "Abkhazian",
+              code2: "ab",
+              code3: "abk",
+              enabled: false,
+            },
+            hi: false,
+            forced: false,
+          },
+          target: {
+            content: {
+              name: "Aragonese",
+              code2: "an",
+              code3: "arg",
+              enabled: false,
+            },
+            hi: false,
+            forced: false,
+          },
+        },
+        expected: "abk:arg",
+      },
+      {
+        source: {
+          source: {
+            content: {
+              name: "Abkhazian",
+              code2: "ab",
+              code3: "abk",
+              enabled: false,
+            },
+            hi: true,
+            forced: false,
+          },
+          target: {
+            content: {
+              name: "Aragonese",
+              code2: "an",
+              code3: "arg",
+              enabled: false,
+            },
+            hi: false,
+            forced: false,
+          },
+        },
+        expected: "abk@hi:arg",
+      },
+      {
+        source: {
+          source: {
+            content: {
+              name: "Abkhazian",
+              code2: "ab",
+              code3: "abk",
+              enabled: false,
+            },
+            hi: false,
+            forced: true,
+          },
+          target: {
+            content: {
+              name: "Aragonese",
+              code2: "an",
+              code3: "arg",
+              enabled: false,
+            },
+            hi: false,
+            forced: false,
+          },
+        },
+        expected: "abk@forced:arg",
+      },
+    ];
+
+    function testEncodeResult({ source, expected }: TestData) {
+      const encoded = encodeEqualData(source);
+
+      expect(
+        encoded,
+        `Encoded result '${encoded}' is not matched to '${expected}'`
+      ).toEqual(expected);
+    }
+
+    testValues.forEach(testEncodeResult);
+  });
+});