From cd4cde89e080c88cac49976fa43d494bf44668a7 Mon Sep 17 00:00:00 2001 From: soloam <11949987+soloam@users.noreply.github.com> Date: Thu, 27 May 2021 17:54:33 +0100 Subject: [PATCH 1/7] [add] translations filter: Filter by translations --- flexget/plugins/filter/translations.py | 542 ++++++++++++++++++++++ flexget/tests/test_translations_filter.py | 182 ++++++++ 2 files changed, 724 insertions(+) create mode 100644 flexget/plugins/filter/translations.py create mode 100644 flexget/tests/test_translations_filter.py diff --git a/flexget/plugins/filter/translations.py b/flexget/plugins/filter/translations.py new file mode 100644 index 0000000000..a547f75282 --- /dev/null +++ b/flexget/plugins/filter/translations.py @@ -0,0 +1,542 @@ +import re +from unicodedata import normalize +from typing import Union, List +from babelfish.language import LANGUAGES + +from guessit.api import GuessItApi, GuessitException +from loguru import logger +import babelfish + +from flexget import plugin +from flexget.event import event +from flexget.config_schema import one_or_more + +PLUGIN_NAME = 'translations' +logger = logger.bind(name=PLUGIN_NAME) + +UNKNOWN = 'unknown' +DEFAULT = 'default' +NATIVE = 'native' +NONE = 'none' +ACTION_ACCEPT = 'accept' +ACTION_REJECT = 'reject' +ACTION_SKIP = 'do_nothing' + +Language = babelfish.Language + + +class MyCodeConverter(babelfish.LanguageEquivalenceConverter): + """ + This Class will allow to declare all the match that babelfish is not able to match + """ + + SYMBOLS = {} + + +babelfish.language_converters['mycode'] = MyCodeConverter() + + +class Translations: + """ + Take action on translated content + + Example: + + translations: + action: reject + + translations: + languages: + - "imdb_languages" + - "trakt_language" + - "trakt_series_language" + dubbed: reject + + translations: + languages_synonyms: + portuguese: + - tuga + languages: + - "imdb_languages" + - "trakt_language" + - "trakt_series_language" + dubbed: + portuguese: "accept" + default: "reject" + subbed: + portuguese: "accept" + default: "reject" + """ + + schema = { + 'oneOf': [ + {'type': 'string', 'enum': [ACTION_ACCEPT, ACTION_REJECT]}, + {'type': 'boolean'}, + { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'source': {'type': 'string', 'default': 'title'}, + 'languages': one_or_more({'type': 'string'}), + 'languages_synonyms': { + "type": 'object', + 'additionalProperties': { + 'type': 'array', + 'items': {'type': 'string'}, + 'minItems': 1, + }, + }, + 'dubbed': { + 'oneOf': [ + { + "type": 'string', + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_SKIP], + 'default': ACTION_SKIP, + }, + { + "type": 'object', + 'properties': { + NATIVE: { + 'type': 'string', + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_SKIP], + 'default': ACTION_REJECT, + }, + DEFAULT: { + 'type': 'string', + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_SKIP], + 'default': ACTION_REJECT, + }, + UNKNOWN: { + 'type': 'string', + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_SKIP], + 'default': ACTION_REJECT, + }, + }, + 'additionalProperties': { + 'type': 'string', + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_SKIP], + }, + }, + ] + }, + 'subbed': { + 'oneOf': [ + { + "type": 'string', + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_SKIP], + 'default': ACTION_SKIP, + }, + { + "type": 'object', + 'properties': { + DEFAULT: { + 'type': 'string', + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_SKIP], + 'default': ACTION_REJECT, + }, + UNKNOWN: { + 'type': 'string', + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_SKIP], + 'default': ACTION_REJECT, + }, + }, + 'additionalProperties': { + 'type': 'string', + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_SKIP], + }, + }, + ] + }, + }, + }, + ] + } + + def clean_symbols(self, text: str) -> str: + """Replaces common symbols with spaces. Also normalize unicode strings in decomposed form. + + Args: + text (str): Text to clean + + Returns: + str: Cleaned text + """ + result = text + if isinstance(result, str): + result = normalize('NFKD', result) + result = re.sub(r'[ \(\)\-_\[\]\.]+', ' ', result).lower() + + # Leftovers + result = re.sub(r"[^a-zA-Z0-9 ]", "", result) + + return result + + def _is_language(self, lang: Union[str, Language]) -> bool: + """Checks if is a valid language + + Args: + lang (Union[str, Language]): Language to check + + Returns: + bool: is language + """ + + if isinstance(lang, Language): + return True + + if ( + not isinstance(lang, str) + or lang == '' + or lang == 'und' + or lang == UNKNOWN + or lang == NATIVE + or lang == DEFAULT + or lang == NONE + ): + return False + + try: + mycode = Language.frommycode(lang) + if mycode: + lang = mycode.name + except babelfish.LanguageReverseError: + pass + + try: + language = Language.fromietf(lang) + except ValueError: + try: + language = Language.fromcode(lang, 'name') + except ValueError: + return False + except babelfish.LanguageReverseError: + return False + + if isinstance(language, Language): + return True + + return False + + def _get_language(self, lang: Union[str, List[str], Language, List[Language]]) -> List[str]: + """Returns the language in text format + + Args: + lang (Union[str, Language]): Language to return + + Returns: + str: Language text + """ + + if not isinstance(lang, list): + languages = [lang] + else: + languages = lang + + for key, lang in enumerate(languages): + if isinstance(lang, Language): + lang = lang.name + + if not isinstance(lang, str) or lang == '' or lang == 'und' or lang == UNKNOWN: + languages[key] = UNKNOWN + continue + + if lang is None or lang == NONE: + languages[key] = NONE + continue + + if lang == NATIVE: + languages[key] = NATIVE + continue + + if lang == DEFAULT: + languages[key] = DEFAULT + continue + + try: + mycode = Language.frommycode(lang) + if mycode: + lang = mycode.name + except babelfish.LanguageReverseError: + pass + + try: + language = Language.fromietf(lang) + except ValueError: + try: + language = Language.fromcode(lang, 'name') + except ValueError: + language = lang + except babelfish.LanguageReverseError: + language = lang + + if isinstance(language, Language): + language = language.name + + languages[key] = language.lower() + + return languages + + def _process_config(self, config: dict) -> dict: + """Processes Config to plugin standard + + Args: + config (dict): Config + + Raises: + plugin.PluginError: Plugin Error + + Returns: + dict: Sanitized config + """ + + _config = {} + + # Simple Actions + if isinstance(config, bool): + config = ACTION_ACCEPT if config else ACTION_REJECT + + if isinstance(config, str): + config = {'dubbed': config, 'subbed': config, 'one_entry': True} + + # Source of the filed to parse + _config['source'] = config.get('source', 'title') + + # The Languages synonums + _languages_synonyms = config.get('languages_synonyms', []) + _config['languages_synonyms'] = {} + for lang in _languages_synonyms: + if not self._is_language(lang): + raise plugin.PluginError(f'`{lang}` in languages_synonyms is not a valid language') + lang = self._get_language(lang) + language = Language.fromcode(lang[0], 'name') + _config['languages_synonyms'][language.alpha3] = _languages_synonyms[lang[0]] + MyCodeConverter.SYMBOLS[language.alpha3] = _languages_synonyms[lang[0]] + + # The actual language of the content, or fields to get it + _config['languages'] = config.get('languages', []) + if not isinstance(_config['languages'], list): + _config['languages'] = [_config['languages']] + elif not _config['languages']: + _config['languages'] = [UNKNOWN] + + # Tag as aprove if one is ok + _config['one_entry'] = config.get('one_entry', False) + + # Actions to dubbed + _dubbed = config.get('dubbed', ACTION_SKIP) + if isinstance(_dubbed, str): + if _dubbed == ACTION_SKIP: + _dubbed = {DEFAULT: ACTION_SKIP, NATIVE: ACTION_SKIP, UNKNOWN: ACTION_SKIP} + else: + _dubbed = { + DEFAULT: _dubbed, + NATIVE: ACTION_ACCEPT if _dubbed == ACTION_REJECT else ACTION_REJECT, + UNKNOWN: _dubbed, + } + + _config['dubbed'] = {} + for key in _dubbed: + key = key.lower() + if not key in [UNKNOWN, DEFAULT, NATIVE] and not self._is_language(key): + raise plugin.PluginError(f'`{key}` in dubbed is not a valid language for dubbed') + + lang = self._get_language(key) + _config['dubbed'][lang[0]] = _dubbed[key] + + # Subbed traslations + _subbed = config.get('subbed', ACTION_SKIP) + if isinstance(_subbed, str): + if _subbed == ACTION_SKIP: + _subbed = {DEFAULT: ACTION_SKIP, UNKNOWN: ACTION_SKIP, NONE: ACTION_SKIP} + else: + _subbed = { + DEFAULT: _subbed, + UNKNOWN: _subbed, + NONE: ACTION_ACCEPT if _subbed == ACTION_REJECT else ACTION_REJECT, + } + + _config['subbed'] = {} + for key in _subbed: + key = key.lower() + if not key in [UNKNOWN, DEFAULT, NATIVE, NONE] and not self._is_language(key): + raise plugin.PluginError(f'`{key}` in subbed is not a valid language for subbed') + + lang = self._get_language(key) + _config['subbed'][lang[0]] = _subbed[key] + + return _config + + def _language_to_action( + self, languages: List[str], stream_languages: List[str], config: dict + ) -> str: + """Gets the action to preform to a given language + + Args: + lang (str): Language to process + stream_languages (List[str]): List of the streamed languages for the media + config (dict): Plugin config + + Returns: + str: Action to preform + """ + + if NATIVE in languages: + for stream_language in stream_languages: + if not self._is_language(stream_language): + continue + + if stream_language in config: + return config.get(stream_language), stream_language + + for lang in languages: + if config.get(lang) and config[lang] != ACTION_SKIP: + return config.get(lang), lang + + if lang in stream_languages: + lang = NATIVE + + action = config.get(lang, config.get(DEFAULT, ACTION_SKIP)), lang + if action[0] != ACTION_SKIP: + return action + + return ACTION_SKIP, stream_languages + + def on_task_filter(self, task, config): + guessit_api = GuessItApi() + guessit_api.config = {} + guessit_api.config['synonyms'] = {'nordic': ['nordic']} + + my_config = self._process_config(config) + + synonyms = {} + for synonym in my_config['languages_synonyms']: + synonyms[synonym] = my_config['languages_synonyms'][synonym] + + for entry in task.entries: + guess = {} + + source = my_config['source'] + + if not my_config['source'] in entry: + raise plugin.PluginError(f'No field {source} in entry') + + title = entry.get(my_config['source']) + title_clean = self.clean_symbols(title) + real_title = title + + if entry.get('series_name'): + real_title = entry['series_name'] + guess['type'] = 'episode' + guess['expected_title'] = [self.clean_symbols(real_title)] + elif entry.get('movie_name'): + real_title = entry['movie_name'] + guess['type'] = 'movie' + guess['expected_title'] = [self.clean_symbols(real_title)] + + if 'alternate_names' in entry: + guess['expected_title'] = [ + self.clean_symbols(name) for name in entry['alternate_names'] + ] + + guess['single_value'] = False + guess['advanced_config'] = {'language': {'synonyms': synonyms}} + + try: + guess_result = guessit_api.guessit(title_clean, options=guess) + except GuessitException as e: + logger.warning('Parsing `{}` with guessit failed: {}', title_clean, e) + continue + + if 'language' in guess_result: + file_language = self._get_language(guess_result.get('language')) + logger.debug('`{}` is in language `{}`', title, file_language) + else: + file_language = [NATIVE] + logger.debug('`{}` is assumed not dubbed', title) + + if 'subtitle_language' in guess_result: + file_subtitles = self._get_language(guess_result.get('subtitle_language')) + logger.debug('`{}` is in language `{}`', title, file_subtitles) + else: + file_subtitles = [NONE] + logger.debug('`{}` is assumed not subbed', title) + + stream_languages = {} + for source in my_config['languages']: + if self._is_language(source): + logger.debug('Using `{}` as native language for {}', source, real_title) + language = self._get_language(source) + stream_languages[language[0]] = True + continue + + if source == UNKNOWN: + stream_languages[UNKNOWN] = True + continue + + if not source in entry: + logger.warning('Entry does not contain a field called `{}`', source) + continue + + languages = entry.get(source) + if not isinstance(languages, list): + languages = [languages] + + for lang in languages: + if not lang: + continue + + language = self._get_language(lang) + stream_languages[language[0]] = True + + stream_languages = list(stream_languages.keys()) + + logger.debug('Processing `{}` with native language `{}`', real_title, stream_languages) + + # Check Dubbed + action, f_language = self._language_to_action( + file_language, stream_languages, my_config['dubbed'] + ) + + accept = '' + reject = '' + + if action == ACTION_SKIP: + logger.debug( + 'Skiping dubbed check on `{}` because is in language is `{}`', + title, + f_language, + ) + elif action == ACTION_REJECT: + reject = f'`{title}` is `{f_language}` language' + elif action == ACTION_ACCEPT: + accept = f'`{title}` is `{f_language}` language' + + # Check Subbed + action, f_subtitles = self._language_to_action( + file_subtitles, stream_languages, my_config['subbed'] + ) + + if action == ACTION_SKIP: + logger.debug( + 'Skiping subbed check on `{}` because is subbed in `{}`', title, f_subtitles + ) + elif action == ACTION_REJECT: + if reject: + reject += ' and ' + reject = f'`{title}` is `{f_subtitles}` subbed' + elif action == ACTION_ACCEPT: + if accept: + accept += ' and ' + accept += f'`{title}` is `{f_subtitles}` subbed' + + if accept and my_config['one_entry']: + entry.accept(accept) + elif reject: + entry.reject(reject) + elif accept: + entry.accept(accept) + + +@event('plugin.register') +def register_plugin(): + plugin.register(Translations, PLUGIN_NAME, api_ver=2) diff --git a/flexget/tests/test_translations_filter.py b/flexget/tests/test_translations_filter.py new file mode 100644 index 0000000000..3678139d62 --- /dev/null +++ b/flexget/tests/test_translations_filter.py @@ -0,0 +1,182 @@ +from flexget import plugin +from flexget.entry import Entry + + +class TestTranslationsFilter: + config = """ + tasks: + dubbed1: + mock: + - { title: "Attack on Titan S01E01 DUBFrench DUBItalian 720p WEBRip", url: "http://mock.url/file2.torrent" } + - { title: "Attack on Titan S01E01 French 720p WEBRip", url: "http://mock.url/file2.torrent" } + - { title: "Attack on Titan S01E01 720p WEBRip", url: "http://mock.url/file2.torrent" } + - { title: "Attack on Titan S01E01 720p Japanese", url: "http://mock.url/file2.torrent" } + + + translations: + languages: + - "japanese" + dubbed: + japanese: "accept" + default: "reject" + + series: + - Attack on Titan: + identified_by: ep + + + + dubbed2: + mock: + - { title: "Show S01E01 Tuga 720p", trakt_series_language:"english", url:"http://mock.url/file2.torrent" } + - { title: "Show S01E01 720p", trakt_series_language:"english", url:"http://mock.url/file2.torrent" } + + translations: + languages_synonyms: + portuguese: + - tuga + languages: + - "trakt_series_language" + dubbed: + portuguese: "accept" + default: "reject" + + series: + - Show: + identified_by: ep + + + dubbed3: + mock: + - { title: "Movie French 720p", movie_name: "Movie", trakt_language: "english", url:"http://mock.url/file2.torrent" } + - { title: "Movie Spanish 720p", movie_name: "Movie", trakt_language: "english", url:"http://mock.url/file2.torrent" } + - { title: "Movie English 720p", movie_name: "Movie", trakt_language: "english", url:"http://mock.url/file2.torrent" } + - { title: "Movie 720p", movie_name: "Movie", trakt_language: "english", url:"http://mock.url/file2.torrent" } + - { title: "Movie En 720p", movie_name: "Movie", trakt_language: "english", url:"http://mock.url/file2.torrent" } + + translations: + languages: + - "trakt_language" + dubbed: reject + + dubbed4: + mock: + - { title: "Movie Dub 720p", movie_name: "Movie", url:"http://mock.url/file2.torrent" } + - { title: "Movie Dubbed 720p", movie_name: "Movie", url:"http://mock.url/file2.torrent" } + - { title: "Movie 720p", movie_name: "Movie", url:"http://mock.url/file2.torrent" } + - { title: "Movie PT_BR 720p", movie_name: "Movie", url:"http://mock.url/file2.torrent" } + - { title: "Movie French 720p", movie_name: "Movie", url:"http://mock.url/file2.torrent" } + + translations: accept + + subbed1: + mock: + - { title: "Attack on Titan S01E01 SubFrench SubItalian 720p WEBRip", url: "http://mock.url/file2.torrent" } + - { title: "Attack on Titan S01E01 SubEnglish 720p WEBRip", url: "http://mock.url/file2.torrent" } + - { title: "Attack on Titan S01E01 720p WEBRip", url: "http://mock.url/file2.torrent" } + - { title: "Attack on Titan S01E01 720p SubJapanese", url: "http://mock.url/file2.torrent" } + - { title: "Attack on Titan S01E01 720p Subbed", url: "http://mock.url/file2.torrent" } + + + translations: + languages: + - "japanese" + dubbed: reject + subbed: + english: "accept" + default: "reject" + + series: + - Attack on Titan: + identified_by: ep + + subbed2: + mock: + - { title: "Attack on Titan S01E01 SubFrench SubItalian 720p WEBRip", url: "http://mock.url/file2.torrent" } + - { title: "Attack on Titan S01E01 SubEnglish 720p WEBRip", url: "http://mock.url/file2.torrent" } + - { title: "Attack on Titan S01E01 720p WEBRip", url: "http://mock.url/file2.torrent" } + - { title: "Attack on Titan S01E01 720p SubJapanese", url: "http://mock.url/file2.torrent" } + - { title: "Attack on Titan S01E01 720p Subbed", url: "http://mock.url/file2.torrent" } + + + translations: + languages: + - "japanese" + dubbed: reject + subbed: reject + + series: + - Attack on Titan: + identified_by: ep + + subbed3: + mock: + - { title: "Attack on Titan S01E01 SubFrench SubItalian 720p WEBRip", url: "http://mock.url/file2.torrent" } + - { title: "Attack on Titan S01E01 SubEnglish 720p WEBRip", url: "http://mock.url/file2.torrent" } + - { title: "Attack on Titan S01E01 720p WEBRip", url: "http://mock.url/file2.torrent" } + - { title: "Attack on Titan S01E01 720p SubJapanese", url: "http://mock.url/file2.torrent" } + - { title: "Attack on Titan S01E01 720p Subbed", url: "http://mock.url/file2.torrent" } + + + translations: + languages: + - "japanese" + dubbed: reject + subbed: accept + + series: + - Attack on Titan: + identified_by: ep + """ + + def test_force_native(self, execute_task): + task = execute_task('dubbed1') + assert len(task.accepted) == 2 + assert task.accepted[0]['title'] == 'Attack on Titan S01E01 720p WEBRip' + assert task.accepted[1]['title'] == 'Attack on Titan S01E01 720p Japanese' + + def test_force_synonym(self, execute_task): + task = execute_task('dubbed2') + assert len(task.accepted) == 1 + assert task.accepted[0]['title'] == 'Show S01E01 Tuga 720p' + + def test_get_native(self, execute_task): + task = execute_task('dubbed3') + assert len(task.accepted) == 3 + + expected = ['Movie English 720p', 'Movie 720p', 'Movie En 720p'] + + assert task.accepted[0]['title'] in expected + assert task.accepted[1]['title'] in expected + assert task.accepted[2]['title'] in expected + + def test_get_dubbed(self, execute_task): + task = execute_task('dubbed4') + assert len(task.accepted) == 4 + + expected = ['Movie Dub 720p', 'Movie Dubbed 720p', 'Movie PT_BR 720p', 'Movie French 720p'] + + assert task.accepted[0]['title'] in expected + assert task.accepted[1]['title'] in expected + assert task.accepted[2]['title'] in expected + assert task.accepted[3]['title'] in expected + + def test_subbed_language(self, execute_task): + task = execute_task('subbed1') + assert len(task.accepted) == 1 + assert task.accepted[0]['title'] == 'Attack on Titan S01E01 SubEnglish 720p WEBRip' + + def test_not_subbed(self, execute_task): + task = execute_task('subbed2') + assert len(task.accepted) == 1 + assert task.accepted[0]['title'] == 'Attack on Titan S01E01 720p WEBRip' + + def test_subbed(self, execute_task): + task = execute_task('subbed3') + assert len(task.accepted) == 4 + assert ( + task.accepted[0]['title'] == 'Attack on Titan S01E01 SubFrench SubItalian 720p WEBRip' + ) + assert task.accepted[1]['title'] == 'Attack on Titan S01E01 SubEnglish 720p WEBRip' + assert task.accepted[2]['title'] == 'Attack on Titan S01E01 720p SubJapanese' + assert task.accepted[3]['title'] == 'Attack on Titan S01E01 720p Subbed' From 5ec048cec09e08048896044970a55e9542cd8578 Mon Sep 17 00:00:00 2001 From: soloam <11949987+soloam@users.noreply.github.com> Date: Fri, 28 May 2021 12:30:29 +0100 Subject: [PATCH 2/7] [add] translations filter: Specific do_nothing --- flexget/plugins/filter/translations.py | 45 ++++++++++++++--------- flexget/tests/test_translations_filter.py | 29 +++++++++++++++ 2 files changed, 57 insertions(+), 17 deletions(-) diff --git a/flexget/plugins/filter/translations.py b/flexget/plugins/filter/translations.py index a547f75282..cbdab02879 100644 --- a/flexget/plugins/filter/translations.py +++ b/flexget/plugins/filter/translations.py @@ -20,7 +20,8 @@ NONE = 'none' ACTION_ACCEPT = 'accept' ACTION_REJECT = 'reject' -ACTION_SKIP = 'do_nothing' +ACTION_SKIP = 'skip' +ACTION_NOTHING = 'do_nothing' Language = babelfish.Language @@ -90,7 +91,7 @@ class Translations: 'oneOf': [ { "type": 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_SKIP], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], 'default': ACTION_SKIP, }, { @@ -98,23 +99,23 @@ class Translations: 'properties': { NATIVE: { 'type': 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_SKIP], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], 'default': ACTION_REJECT, }, DEFAULT: { 'type': 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_SKIP], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], 'default': ACTION_REJECT, }, UNKNOWN: { 'type': 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_SKIP], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], 'default': ACTION_REJECT, }, }, 'additionalProperties': { 'type': 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_SKIP], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], }, }, ] @@ -123,7 +124,7 @@ class Translations: 'oneOf': [ { "type": 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_SKIP], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], 'default': ACTION_SKIP, }, { @@ -131,18 +132,18 @@ class Translations: 'properties': { DEFAULT: { 'type': 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_SKIP], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], 'default': ACTION_REJECT, }, UNKNOWN: { 'type': 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_SKIP], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], 'default': ACTION_REJECT, }, }, 'additionalProperties': { 'type': 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_SKIP], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], }, }, ] @@ -395,11 +396,11 @@ def _language_to_action( if lang in stream_languages: lang = NATIVE - action = config.get(lang, config.get(DEFAULT, ACTION_SKIP)), lang - if action[0] != ACTION_SKIP: - return action + action = config.get(lang, None) + if action and action != ACTION_SKIP: + return action, lang - return ACTION_SKIP, stream_languages + return config.get(DEFAULT, ACTION_SKIP), languages def on_task_filter(self, task, config): guessit_api = GuessItApi() @@ -499,6 +500,7 @@ def on_task_filter(self, task, config): accept = '' reject = '' + nothing = '' if action == ACTION_SKIP: logger.debug( @@ -506,6 +508,8 @@ def on_task_filter(self, task, config): title, f_language, ) + elif action == ACTION_NOTHING: + nothing = f'Doing nothing on `{title}` because is in language is `{f_language}`' elif action == ACTION_REJECT: reject = f'`{title}` is `{f_language}` language' elif action == ACTION_ACCEPT: @@ -517,9 +521,14 @@ def on_task_filter(self, task, config): ) if action == ACTION_SKIP: - logger.debug( - 'Skiping subbed check on `{}` because is subbed in `{}`', title, f_subtitles - ) + if NONE not in f_subtitles: + logger.debug( + 'Skiping subbed check on `{}` because is subbed in `{}`', + title, + f_subtitles, + ) + elif action == ACTION_NOTHING: + nothing += f'Doing nothing on `{title}` because is in subbed in `{f_subtitles}`' elif action == ACTION_REJECT: if reject: reject += ' and ' @@ -535,6 +544,8 @@ def on_task_filter(self, task, config): entry.reject(reject) elif accept: entry.accept(accept) + elif nothing: + logger.debug(nothing) @event('plugin.register') diff --git a/flexget/tests/test_translations_filter.py b/flexget/tests/test_translations_filter.py index 3678139d62..56ae794139 100644 --- a/flexget/tests/test_translations_filter.py +++ b/flexget/tests/test_translations_filter.py @@ -127,6 +127,30 @@ class TestTranslationsFilter: series: - Attack on Titan: identified_by: ep + + + do_nothing_test: + mock: + - {'title':'Movie1.720p.PT.ENG.TEST','url':'mock://teste1'} + - {'title':'Movie2.720p.Portugues.Ingles.TEST','url':'mock://teste2'} + - {'title':'Movie3.1080p.ENG.PT.BluRay.TEST','url':'mock://teste3'} + translations: + languages_synonyms: + portuguese: + - tuga + - portuga + - portugues + english: + - ingles + - ing + spanish: + - espanhol + french: + - frances + dubbed: + portuguese: "do_nothing" + default: "reject" + """ def test_force_native(self, execute_task): @@ -180,3 +204,8 @@ def test_subbed(self, execute_task): assert task.accepted[1]['title'] == 'Attack on Titan S01E01 SubEnglish 720p WEBRip' assert task.accepted[2]['title'] == 'Attack on Titan S01E01 720p SubJapanese' assert task.accepted[3]['title'] == 'Attack on Titan S01E01 720p Subbed' + + def test_do_nothing_test(self, execute_task): + task = execute_task('do_nothing_test') + assert len(task.accepted) == 0 + assert len(task.rejected) == 0 From 5af60cc8417f912b57c5b1a0cf91fbf61c05b0dd Mon Sep 17 00:00:00 2001 From: soloam <11949987+soloam@users.noreply.github.com> Date: Fri, 28 May 2021 15:15:13 +0100 Subject: [PATCH 3/7] [add] translations filter: Default actions --- flexget/plugins/filter/translations.py | 111 ++++++++++++---------- flexget/tests/test_translations_filter.py | 86 +++++++++++++---- 2 files changed, 129 insertions(+), 68 deletions(-) diff --git a/flexget/plugins/filter/translations.py b/flexget/plugins/filter/translations.py index cbdab02879..be727e0f9f 100644 --- a/flexget/plugins/filter/translations.py +++ b/flexget/plugins/filter/translations.py @@ -14,14 +14,15 @@ PLUGIN_NAME = 'translations' logger = logger.bind(name=PLUGIN_NAME) -UNKNOWN = 'unknown' -DEFAULT = 'default' -NATIVE = 'native' -NONE = 'none' -ACTION_ACCEPT = 'accept' -ACTION_REJECT = 'reject' -ACTION_SKIP = 'skip' -ACTION_NOTHING = 'do_nothing' +UNKNOWN = 'unknown' # It's a language but is unknown +DEFAULT = 'default' # Default when none applies +NATIVE = 'native' # Native language +NONE = 'none' # No language +OTHER = 'other' # It's a valid language, but not matched +ACTION_ACCEPT = 'accept' # Accept entry +ACTION_REJECT = 'reject' # Reject entry +ACTION_SKIP = 'skip' # Ignore check (internal) +ACTION_NOTHING = 'do_nothing' # Mark entry as undecided Language = babelfish.Language @@ -71,7 +72,7 @@ class Translations: schema = { 'oneOf': [ - {'type': 'string', 'enum': [ACTION_ACCEPT, ACTION_REJECT]}, + {'type': 'string', 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING]}, {'type': 'boolean'}, { 'type': 'object', @@ -92,25 +93,25 @@ class Translations: { "type": 'string', 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], - 'default': ACTION_SKIP, }, { "type": 'object', 'properties': { + OTHER: { + 'type': 'string', + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], + }, NATIVE: { 'type': 'string', 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], - 'default': ACTION_REJECT, }, DEFAULT: { 'type': 'string', 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], - 'default': ACTION_REJECT, }, UNKNOWN: { 'type': 'string', 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], - 'default': ACTION_REJECT, }, }, 'additionalProperties': { @@ -125,20 +126,25 @@ class Translations: { "type": 'string', 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], - 'default': ACTION_SKIP, }, { "type": 'object', 'properties': { + OTHER: { + 'type': 'string', + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], + }, + NONE: { + 'type': 'string', + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], + }, DEFAULT: { 'type': 'string', 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], - 'default': ACTION_REJECT, }, UNKNOWN: { 'type': 'string', 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], - 'default': ACTION_REJECT, }, }, 'additionalProperties': { @@ -185,15 +191,7 @@ def _is_language(self, lang: Union[str, Language]) -> bool: if isinstance(lang, Language): return True - if ( - not isinstance(lang, str) - or lang == '' - or lang == 'und' - or lang == UNKNOWN - or lang == NATIVE - or lang == DEFAULT - or lang == NONE - ): + if not isinstance(lang, str) or lang in ['', 'und', UNKNOWN, NATIVE, DEFAULT, NONE, OTHER]: return False try: @@ -237,7 +235,7 @@ def _get_language(self, lang: Union[str, List[str], Language, List[Language]]) - if isinstance(lang, Language): lang = lang.name - if not isinstance(lang, str) or lang == '' or lang == 'und' or lang == UNKNOWN: + if not isinstance(lang, str) or lang in ['', 'und', UNKNOWN]: languages[key] = UNKNOWN continue @@ -245,12 +243,8 @@ def _get_language(self, lang: Union[str, List[str], Language, List[Language]]) - languages[key] = NONE continue - if lang == NATIVE: - languages[key] = NATIVE - continue - - if lang == DEFAULT: - languages[key] = DEFAULT + if lang in [NATIVE, DEFAULT, OTHER]: + languages[key] = lang continue try: @@ -297,7 +291,7 @@ def _process_config(self, config: dict) -> dict: config = ACTION_ACCEPT if config else ACTION_REJECT if isinstance(config, str): - config = {'dubbed': config, 'subbed': config, 'one_entry': True} + config = {'dubbed': config, 'subbed': config} # Source of the filed to parse _config['source'] = config.get('source', 'title') @@ -320,25 +314,23 @@ def _process_config(self, config: dict) -> dict: elif not _config['languages']: _config['languages'] = [UNKNOWN] - # Tag as aprove if one is ok - _config['one_entry'] = config.get('one_entry', False) - # Actions to dubbed _dubbed = config.get('dubbed', ACTION_SKIP) if isinstance(_dubbed, str): - if _dubbed == ACTION_SKIP: - _dubbed = {DEFAULT: ACTION_SKIP, NATIVE: ACTION_SKIP, UNKNOWN: ACTION_SKIP} + if _dubbed in [ACTION_SKIP, ACTION_NOTHING]: + _dubbed = {DEFAULT: _dubbed, NATIVE: _dubbed, UNKNOWN: _dubbed} else: _dubbed = { - DEFAULT: _dubbed, - NATIVE: ACTION_ACCEPT if _dubbed == ACTION_REJECT else ACTION_REJECT, + DEFAULT: ACTION_SKIP, + OTHER: _dubbed, + NATIVE: ACTION_SKIP, UNKNOWN: _dubbed, } _config['dubbed'] = {} for key in _dubbed: key = key.lower() - if not key in [UNKNOWN, DEFAULT, NATIVE] and not self._is_language(key): + if not key in [UNKNOWN, DEFAULT, NATIVE, OTHER] and not self._is_language(key): raise plugin.PluginError(f'`{key}` in dubbed is not a valid language for dubbed') lang = self._get_language(key) @@ -347,24 +339,33 @@ def _process_config(self, config: dict) -> dict: # Subbed traslations _subbed = config.get('subbed', ACTION_SKIP) if isinstance(_subbed, str): - if _subbed == ACTION_SKIP: - _subbed = {DEFAULT: ACTION_SKIP, UNKNOWN: ACTION_SKIP, NONE: ACTION_SKIP} + if _subbed in [ACTION_SKIP, ACTION_NOTHING]: + _subbed = {DEFAULT: _subbed, UNKNOWN: _subbed, NONE: _subbed} else: _subbed = { - DEFAULT: _subbed, + DEFAULT: ACTION_SKIP, + OTHER: _subbed, + NONE: ACTION_SKIP, UNKNOWN: _subbed, - NONE: ACTION_ACCEPT if _subbed == ACTION_REJECT else ACTION_REJECT, } _config['subbed'] = {} for key in _subbed: key = key.lower() - if not key in [UNKNOWN, DEFAULT, NATIVE, NONE] and not self._is_language(key): + if not key in [UNKNOWN, DEFAULT, NONE, OTHER] and not self._is_language(key): raise plugin.PluginError(f'`{key}` in subbed is not a valid language for subbed') lang = self._get_language(key) _config['subbed'][lang[0]] = _subbed[key] + for field in [UNKNOWN, DEFAULT, NATIVE, OTHER]: + if field not in _config['dubbed']: + _config['dubbed'][field] = ACTION_SKIP + + for field in [UNKNOWN, DEFAULT, NONE, OTHER]: + if field not in _config['subbed']: + _config['subbed'][field] = ACTION_SKIP + return _config def _language_to_action( @@ -381,6 +382,9 @@ def _language_to_action( str: Action to preform """ + if not stream_languages: + stream_languages = [] + if NATIVE in languages: for stream_language in stream_languages: if not self._is_language(stream_language): @@ -397,9 +401,16 @@ def _language_to_action( lang = NATIVE action = config.get(lang, None) + if not action and self._is_language(lang): + # If it's a language and not defined, use OTHER config + action = config.get(OTHER, None) + if action and action != ACTION_SKIP: return action, lang + if self._is_language(lang): + action = config.get(OTHER, None) + return config.get(DEFAULT, ACTION_SKIP), languages def on_task_filter(self, task, config): @@ -517,7 +528,7 @@ def on_task_filter(self, task, config): # Check Subbed action, f_subtitles = self._language_to_action( - file_subtitles, stream_languages, my_config['subbed'] + file_subtitles, None, my_config['subbed'] ) if action == ACTION_SKIP: @@ -532,15 +543,13 @@ def on_task_filter(self, task, config): elif action == ACTION_REJECT: if reject: reject += ' and ' - reject = f'`{title}` is `{f_subtitles}` subbed' + reject += f'`{title}` is `{f_subtitles}` subbed' elif action == ACTION_ACCEPT: if accept: accept += ' and ' accept += f'`{title}` is `{f_subtitles}` subbed' - if accept and my_config['one_entry']: - entry.accept(accept) - elif reject: + if reject: entry.reject(reject) elif accept: entry.accept(accept) diff --git a/flexget/tests/test_translations_filter.py b/flexget/tests/test_translations_filter.py index 56ae794139..87c1783e58 100644 --- a/flexget/tests/test_translations_filter.py +++ b/flexget/tests/test_translations_filter.py @@ -59,7 +59,7 @@ class TestTranslationsFilter: - "trakt_language" dubbed: reject - dubbed4: + dubbed_accept: mock: - { title: "Movie Dub 720p", movie_name: "Movie", url:"http://mock.url/file2.torrent" } - { title: "Movie Dubbed 720p", movie_name: "Movie", url:"http://mock.url/file2.torrent" } @@ -69,6 +69,16 @@ class TestTranslationsFilter: translations: accept + dubbed_reject: + mock: + - { title: "Movie Dub 720p", movie_name: "Movie", url:"http://mock.url/file2.torrent" } + - { title: "Movie Dubbed 720p", movie_name: "Movie", url:"http://mock.url/file2.torrent" } + - { title: "Movie 720p", movie_name: "Movie", url:"http://mock.url/file2.torrent" } + - { title: "Movie PT_BR 720p", movie_name: "Movie", url:"http://mock.url/file2.torrent" } + - { title: "Movie French 720p", movie_name: "Movie", url:"http://mock.url/file2.torrent" } + + translations: reject + subbed1: mock: - { title: "Attack on Titan S01E01 SubFrench SubItalian 720p WEBRip", url: "http://mock.url/file2.torrent" } @@ -83,11 +93,12 @@ class TestTranslationsFilter: - "japanese" dubbed: reject subbed: - english: "accept" + english: "do_nothing" default: "reject" series: - Attack on Titan: + parse_only: yes identified_by: ep subbed2: @@ -107,11 +118,12 @@ class TestTranslationsFilter: series: - Attack on Titan: + parse_only: yes identified_by: ep subbed3: mock: - - { title: "Attack on Titan S01E01 SubFrench SubItalian 720p WEBRip", url: "http://mock.url/file2.torrent" } + - { title: "Attack on Titan S01E01 SubFrench SubItalian 720p", url: "http://mock.url/file2.torrent" } - { title: "Attack on Titan S01E01 SubEnglish 720p WEBRip", url: "http://mock.url/file2.torrent" } - { title: "Attack on Titan S01E01 720p WEBRip", url: "http://mock.url/file2.torrent" } - { title: "Attack on Titan S01E01 720p SubJapanese", url: "http://mock.url/file2.torrent" } @@ -122,10 +134,13 @@ class TestTranslationsFilter: languages: - "japanese" dubbed: reject - subbed: accept + subbed: + none: reject + default: accept series: - Attack on Titan: + parse_only: yes identified_by: ep @@ -156,27 +171,37 @@ class TestTranslationsFilter: def test_force_native(self, execute_task): task = execute_task('dubbed1') assert len(task.accepted) == 2 + assert len(task.rejected) == 2 + assert len(task.undecided) == 0 assert task.accepted[0]['title'] == 'Attack on Titan S01E01 720p WEBRip' assert task.accepted[1]['title'] == 'Attack on Titan S01E01 720p Japanese' def test_force_synonym(self, execute_task): task = execute_task('dubbed2') assert len(task.accepted) == 1 + assert len(task.rejected) == 1 + assert len(task.undecided) == 0 assert task.accepted[0]['title'] == 'Show S01E01 Tuga 720p' def test_get_native(self, execute_task): task = execute_task('dubbed3') - assert len(task.accepted) == 3 - + assert len(task.accepted) == 0 + assert len(task.rejected) == 2 + assert len(task.undecided) == 3 expected = ['Movie English 720p', 'Movie 720p', 'Movie En 720p'] - assert task.accepted[0]['title'] in expected - assert task.accepted[1]['title'] in expected - assert task.accepted[2]['title'] in expected + assert task.undecided[0]['title'] in expected + assert task.undecided[1]['title'] in expected + assert task.undecided[2]['title'] in expected + + assert task.rejected[0]['title'] == 'Movie French 720p' + assert task.rejected[1]['title'] == 'Movie Spanish 720p' def test_get_dubbed(self, execute_task): - task = execute_task('dubbed4') + task = execute_task('dubbed_accept') assert len(task.accepted) == 4 + assert len(task.rejected) == 0 + assert len(task.undecided) == 1 expected = ['Movie Dub 720p', 'Movie Dubbed 720p', 'Movie PT_BR 720p', 'Movie French 720p'] @@ -185,27 +210,54 @@ def test_get_dubbed(self, execute_task): assert task.accepted[2]['title'] in expected assert task.accepted[3]['title'] in expected + assert task.undecided[0]['title'] == 'Movie 720p' + + def test_dont_get_dubbed(self, execute_task): + task = execute_task('dubbed_reject') + assert len(task.accepted) == 0 + assert len(task.rejected) == 4 + assert len(task.undecided) == 1 + + expected = ['Movie Dub 720p', 'Movie Dubbed 720p', 'Movie PT_BR 720p', 'Movie French 720p'] + + assert task.rejected[0]['title'] in expected + assert task.rejected[1]['title'] in expected + assert task.rejected[2]['title'] in expected + assert task.rejected[3]['title'] in expected + + assert task.undecided[0]['title'] == 'Movie 720p' + def test_subbed_language(self, execute_task): task = execute_task('subbed1') - assert len(task.accepted) == 1 - assert task.accepted[0]['title'] == 'Attack on Titan S01E01 SubEnglish 720p WEBRip' + assert len(task.accepted) == 0 + assert len(task.rejected) == 4 + assert len(task.undecided) == 1 + + assert task.undecided[0]['title'] == 'Attack on Titan S01E01 SubEnglish 720p WEBRip' def test_not_subbed(self, execute_task): task = execute_task('subbed2') - assert len(task.accepted) == 1 - assert task.accepted[0]['title'] == 'Attack on Titan S01E01 720p WEBRip' + assert len(task.accepted) == 0 + assert len(task.rejected) == 4 + assert len(task.undecided) == 1 + assert task.undecided[0]['title'] == 'Attack on Titan S01E01 720p WEBRip' def test_subbed(self, execute_task): + task = execute_task('subbed3') assert len(task.accepted) == 4 - assert ( - task.accepted[0]['title'] == 'Attack on Titan S01E01 SubFrench SubItalian 720p WEBRip' - ) + assert len(task.rejected) == 1 + assert len(task.undecided) == 0 + + assert task.accepted[0]['title'] == 'Attack on Titan S01E01 SubFrench SubItalian 720p' assert task.accepted[1]['title'] == 'Attack on Titan S01E01 SubEnglish 720p WEBRip' assert task.accepted[2]['title'] == 'Attack on Titan S01E01 720p SubJapanese' assert task.accepted[3]['title'] == 'Attack on Titan S01E01 720p Subbed' + assert task.rejected[0]['title'] == 'Attack on Titan S01E01 720p WEBRip' + def test_do_nothing_test(self, execute_task): task = execute_task('do_nothing_test') assert len(task.accepted) == 0 assert len(task.rejected) == 0 + assert len(task.undecided) == 3 From aebaebf2ab4e52506e1e2fde0ca2524cbfdf8c5e Mon Sep 17 00:00:00 2001 From: soloam <11949987+soloam@users.noreply.github.com> Date: Tue, 16 Nov 2021 13:45:07 +0000 Subject: [PATCH 4/7] [add] check translation convertion --- flexget/plugins/filter/translations.py | 118 ++++++++++++++-------- flexget/tests/test_translations_filter.py | 28 +++++ 2 files changed, 103 insertions(+), 43 deletions(-) diff --git a/flexget/plugins/filter/translations.py b/flexget/plugins/filter/translations.py index be727e0f9f..1adc98c3d1 100644 --- a/flexget/plugins/filter/translations.py +++ b/flexget/plugins/filter/translations.py @@ -21,8 +21,7 @@ OTHER = 'other' # It's a valid language, but not matched ACTION_ACCEPT = 'accept' # Accept entry ACTION_REJECT = 'reject' # Reject entry -ACTION_SKIP = 'skip' # Ignore check (internal) -ACTION_NOTHING = 'do_nothing' # Mark entry as undecided +ACTION_DO_NOTHING = 'do_nothing' # Ignore check (internal) Language = babelfish.Language @@ -72,7 +71,7 @@ class Translations: schema = { 'oneOf': [ - {'type': 'string', 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING]}, + {'type': 'string', 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_DO_NOTHING]}, {'type': 'boolean'}, { 'type': 'object', @@ -92,31 +91,31 @@ class Translations: 'oneOf': [ { "type": 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_DO_NOTHING], }, { "type": 'object', 'properties': { OTHER: { 'type': 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_DO_NOTHING], }, NATIVE: { 'type': 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_DO_NOTHING], }, DEFAULT: { 'type': 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_DO_NOTHING], }, UNKNOWN: { 'type': 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_DO_NOTHING], }, }, 'additionalProperties': { 'type': 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_DO_NOTHING], }, }, ] @@ -125,31 +124,31 @@ class Translations: 'oneOf': [ { "type": 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_DO_NOTHING], }, { "type": 'object', 'properties': { OTHER: { 'type': 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_DO_NOTHING], }, NONE: { 'type': 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_DO_NOTHING], }, DEFAULT: { 'type': 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_DO_NOTHING], }, UNKNOWN: { 'type': 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_DO_NOTHING], }, }, 'additionalProperties': { 'type': 'string', - 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_NOTHING], + 'enum': [ACTION_ACCEPT, ACTION_REJECT, ACTION_DO_NOTHING], }, }, ] @@ -203,7 +202,7 @@ def _is_language(self, lang: Union[str, Language]) -> bool: try: language = Language.fromietf(lang) - except ValueError: + except (ValueError, babelfish.LanguageReverseError): try: language = Language.fromcode(lang, 'name') except ValueError: @@ -256,7 +255,7 @@ def _get_language(self, lang: Union[str, List[str], Language, List[Language]]) - try: language = Language.fromietf(lang) - except ValueError: + except (ValueError, babelfish.LanguageReverseError): try: language = Language.fromcode(lang, 'name') except ValueError: @@ -315,15 +314,15 @@ def _process_config(self, config: dict) -> dict: _config['languages'] = [UNKNOWN] # Actions to dubbed - _dubbed = config.get('dubbed', ACTION_SKIP) + _dubbed = config.get('dubbed', ACTION_DO_NOTHING) if isinstance(_dubbed, str): - if _dubbed in [ACTION_SKIP, ACTION_NOTHING]: + if _dubbed in [ACTION_DO_NOTHING]: _dubbed = {DEFAULT: _dubbed, NATIVE: _dubbed, UNKNOWN: _dubbed} else: _dubbed = { - DEFAULT: ACTION_SKIP, + DEFAULT: ACTION_DO_NOTHING, OTHER: _dubbed, - NATIVE: ACTION_SKIP, + NATIVE: ACTION_DO_NOTHING, UNKNOWN: _dubbed, } @@ -337,15 +336,15 @@ def _process_config(self, config: dict) -> dict: _config['dubbed'][lang[0]] = _dubbed[key] # Subbed traslations - _subbed = config.get('subbed', ACTION_SKIP) + _subbed = config.get('subbed', ACTION_DO_NOTHING) if isinstance(_subbed, str): - if _subbed in [ACTION_SKIP, ACTION_NOTHING]: + if _subbed in [ACTION_DO_NOTHING]: _subbed = {DEFAULT: _subbed, UNKNOWN: _subbed, NONE: _subbed} else: _subbed = { - DEFAULT: ACTION_SKIP, + DEFAULT: ACTION_DO_NOTHING, OTHER: _subbed, - NONE: ACTION_SKIP, + NONE: ACTION_DO_NOTHING, UNKNOWN: _subbed, } @@ -358,13 +357,21 @@ def _process_config(self, config: dict) -> dict: lang = self._get_language(key) _config['subbed'][lang[0]] = _subbed[key] + ## Dubbed default + # if not NATIVE in _config['dubbed']: + # _config['dubbed'][NATIVE] = ACTION_DO_NOTHING + for field in [UNKNOWN, DEFAULT, NATIVE, OTHER]: if field not in _config['dubbed']: - _config['dubbed'][field] = ACTION_SKIP + _config['dubbed'][field] = _config['dubbed'].get(DEFAULT, ACTION_DO_NOTHING) + + ## Subbed default + # if not NATIVE in _config['dubbed']: + # _config['subbed'][NATIVE] = ACTION_DO_NOTHING for field in [UNKNOWN, DEFAULT, NONE, OTHER]: if field not in _config['subbed']: - _config['subbed'][field] = ACTION_SKIP + _config['subbed'][field] = _config['subbed'].get(DEFAULT, ACTION_DO_NOTHING) return _config @@ -393,9 +400,12 @@ def _language_to_action( if stream_language in config: return config.get(stream_language), stream_language + final_action = {} + for lang in languages: - if config.get(lang) and config[lang] != ACTION_SKIP: - return config.get(lang), lang + if config.get(lang) and config[lang] != ACTION_DO_NOTHING: + final_action[config.get(lang)] = config[lang], lang + continue if lang in stream_languages: lang = NATIVE @@ -405,13 +415,22 @@ def _language_to_action( # If it's a language and not defined, use OTHER config action = config.get(OTHER, None) - if action and action != ACTION_SKIP: - return action, lang + if action: + final_action[action] = action, lang + continue if self._is_language(lang): action = config.get(OTHER, None) - return config.get(DEFAULT, ACTION_SKIP), languages + # Handle Dual Language Files TODO: Add multi language exeption? + if ACTION_ACCEPT in final_action: + return final_action[ACTION_ACCEPT] + elif ACTION_DO_NOTHING in final_action: + return final_action[ACTION_DO_NOTHING] + elif ACTION_REJECT in final_action: + return final_action[ACTION_REJECT] + + return config.get(DEFAULT, ACTION_DO_NOTHING), languages def on_task_filter(self, task, config): guessit_api = GuessItApi() @@ -478,14 +497,18 @@ def on_task_filter(self, task, config): if self._is_language(source): logger.debug('Using `{}` as native language for {}', source, real_title) language = self._get_language(source) - stream_languages[language[0]] = True + if language[0] not in stream_languages: + stream_languages[language[0]] = 0 + + stream_languages[language[0]] += 1 continue if source == UNKNOWN: - stream_languages[UNKNOWN] = True + if UNKNOWN not in stream_languages: + stream_languages[UNKNOWN] = 1 continue - if not source in entry: + if source not in entry: logger.warning('Entry does not contain a field called `{}`', source) continue @@ -498,9 +521,22 @@ def on_task_filter(self, task, config): continue language = self._get_language(lang) - stream_languages[language[0]] = True - stream_languages = list(stream_languages.keys()) + if language[0] not in stream_languages: + stream_languages[language[0]] = 0 + + stream_languages[language[0]] += 1 + + if not stream_languages: + stream_languages[UNKNOWN] = 1 + + stream_languages = sorted( + stream_languages.items(), key=lambda item: item[1], reverse=True + ) + + # Detect Main Language (If more than one source select the main one) + main_language = stream_languages[0] + stream_languages = [(lan[0]) for lan in stream_languages if lan[1] >= main_language[1]] logger.debug('Processing `{}` with native language `{}`', real_title, stream_languages) @@ -513,14 +549,12 @@ def on_task_filter(self, task, config): reject = '' nothing = '' - if action == ACTION_SKIP: + if action == ACTION_DO_NOTHING: logger.debug( 'Skiping dubbed check on `{}` because is in language is `{}`', title, f_language, ) - elif action == ACTION_NOTHING: - nothing = f'Doing nothing on `{title}` because is in language is `{f_language}`' elif action == ACTION_REJECT: reject = f'`{title}` is `{f_language}` language' elif action == ACTION_ACCEPT: @@ -531,15 +565,13 @@ def on_task_filter(self, task, config): file_subtitles, None, my_config['subbed'] ) - if action == ACTION_SKIP: + if action == ACTION_DO_NOTHING: if NONE not in f_subtitles: logger.debug( 'Skiping subbed check on `{}` because is subbed in `{}`', title, f_subtitles, ) - elif action == ACTION_NOTHING: - nothing += f'Doing nothing on `{title}` because is in subbed in `{f_subtitles}`' elif action == ACTION_REJECT: if reject: reject += ' and ' diff --git a/flexget/tests/test_translations_filter.py b/flexget/tests/test_translations_filter.py index 87c1783e58..03714d7ebc 100644 --- a/flexget/tests/test_translations_filter.py +++ b/flexget/tests/test_translations_filter.py @@ -166,6 +166,28 @@ class TestTranslationsFilter: portuguese: "do_nothing" default: "reject" + + do_one_language: + mock: + - {'title':'Movie1.720p.PT.ENG.TEST','url':'mock://teste1'} + - {'title':'Movie2.720p.Portugues.Ingles.TEST','url':'mock://teste2'} + - {'title':'Movie3.1080p.ENG.PT.BluRay.TEST','url':'mock://teste3'} + translations: + languages_synonyms: + portuguese: + - tuga + - portuga + - portugues + english: + - ingles + - ing + spanish: + - espanhol + french: + - frances + dubbed: + portuguese: "accept" + default: "reject" """ def test_force_native(self, execute_task): @@ -261,3 +283,9 @@ def test_do_nothing_test(self, execute_task): assert len(task.accepted) == 0 assert len(task.rejected) == 0 assert len(task.undecided) == 3 + + def test_do_one_language(self, execute_task): + task = execute_task('do_one_language') + assert len(task.accepted) == 3 + assert len(task.rejected) == 0 + assert len(task.undecided) == 0 \ No newline at end of file From f7755136db83ea5997b0a661deb65b75b0b694ef Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Nov 2021 18:55:48 +0000 Subject: [PATCH 5/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- flexget/plugins/filter/translations.py | 8 ++++---- flexget/tests/test_translations_filter.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flexget/plugins/filter/translations.py b/flexget/plugins/filter/translations.py index 1adc98c3d1..b4f470fc78 100644 --- a/flexget/plugins/filter/translations.py +++ b/flexget/plugins/filter/translations.py @@ -1,15 +1,15 @@ import re +from typing import List, Union from unicodedata import normalize -from typing import Union, List -from babelfish.language import LANGUAGES +import babelfish +from babelfish.language import LANGUAGES from guessit.api import GuessItApi, GuessitException from loguru import logger -import babelfish from flexget import plugin -from flexget.event import event from flexget.config_schema import one_or_more +from flexget.event import event PLUGIN_NAME = 'translations' logger = logger.bind(name=PLUGIN_NAME) diff --git a/flexget/tests/test_translations_filter.py b/flexget/tests/test_translations_filter.py index 03714d7ebc..f934d628ba 100644 --- a/flexget/tests/test_translations_filter.py +++ b/flexget/tests/test_translations_filter.py @@ -288,4 +288,4 @@ def test_do_one_language(self, execute_task): task = execute_task('do_one_language') assert len(task.accepted) == 3 assert len(task.rejected) == 0 - assert len(task.undecided) == 0 \ No newline at end of file + assert len(task.undecided) == 0 From 7ba2a309ee3830e6da17e1deb483501247bbd5f4 Mon Sep 17 00:00:00 2001 From: soloam <11949987+soloam@users.noreply.github.com> Date: Tue, 23 May 2023 13:17:32 +0100 Subject: [PATCH 6/7] [add] translations filter: Filter by translations --- flexget/plugins/filter/translations.py | 38 +++++++++++++---------- flexget/tests/test_translations_filter.py | 4 +-- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/flexget/plugins/filter/translations.py b/flexget/plugins/filter/translations.py index b4f470fc78..d8275f6411 100644 --- a/flexget/plugins/filter/translations.py +++ b/flexget/plugins/filter/translations.py @@ -1,15 +1,15 @@ import re -from typing import List, Union from unicodedata import normalize - -import babelfish +from typing import Union, List from babelfish.language import LANGUAGES + from guessit.api import GuessItApi, GuessitException from loguru import logger +import babelfish from flexget import plugin -from flexget.config_schema import one_or_more from flexget.event import event +from flexget.config_schema import one_or_more PLUGIN_NAME = 'translations' logger = logger.bind(name=PLUGIN_NAME) @@ -79,6 +79,7 @@ class Translations: 'properties': { 'source': {'type': 'string', 'default': 'title'}, 'languages': one_or_more({'type': 'string'}), + 'language_fields': one_or_more({'type': 'string'}), 'languages_synonyms': { "type": 'object', 'additionalProperties': { @@ -198,6 +199,7 @@ def _is_language(self, lang: Union[str, Language]) -> bool: if mycode: lang = mycode.name except babelfish.LanguageReverseError: + logger.debug('`{}` is not a language', lang) pass try: @@ -306,11 +308,18 @@ def _process_config(self, config: dict) -> dict: _config['languages_synonyms'][language.alpha3] = _languages_synonyms[lang[0]] MyCodeConverter.SYMBOLS[language.alpha3] = _languages_synonyms[lang[0]] - # The actual language of the content, or fields to get it + # fields to get the language + _config['language_fields'] = config.get('language_fields', []) + if not isinstance(_config['language_fields'], list): + _config['language_fields'] = [_config['language_fields']] + elif not _config['language_fields']: + _config['language_fields'] = [] + + # The actual language of the content _config['languages'] = config.get('languages', []) if not isinstance(_config['languages'], list): _config['languages'] = [_config['languages']] - elif not _config['languages']: + elif not _config['languages'] and not _config['language_fields']: _config['languages'] = [UNKNOWN] # Actions to dubbed @@ -358,17 +367,11 @@ def _process_config(self, config: dict) -> dict: _config['subbed'][lang[0]] = _subbed[key] ## Dubbed default - # if not NATIVE in _config['dubbed']: - # _config['dubbed'][NATIVE] = ACTION_DO_NOTHING - for field in [UNKNOWN, DEFAULT, NATIVE, OTHER]: if field not in _config['dubbed']: _config['dubbed'][field] = _config['dubbed'].get(DEFAULT, ACTION_DO_NOTHING) ## Subbed default - # if not NATIVE in _config['dubbed']: - # _config['subbed'][NATIVE] = ACTION_DO_NOTHING - for field in [UNKNOWN, DEFAULT, NONE, OTHER]: if field not in _config['subbed']: _config['subbed'][field] = _config['subbed'].get(DEFAULT, ACTION_DO_NOTHING) @@ -493,21 +496,22 @@ def on_task_filter(self, task, config): logger.debug('`{}` is assumed not subbed', title) stream_languages = {} - for source in my_config['languages']: - if self._is_language(source): - logger.debug('Using `{}` as native language for {}', source, real_title) - language = self._get_language(source) + for lang in my_config['languages']: + if self._is_language(lang): + logger.debug('Using `{}` as native language for {}', lang, real_title) + language = self._get_language(lang) if language[0] not in stream_languages: stream_languages[language[0]] = 0 stream_languages[language[0]] += 1 continue - if source == UNKNOWN: + if lang == UNKNOWN: if UNKNOWN not in stream_languages: stream_languages[UNKNOWN] = 1 continue + for source in my_config['language_fields']: if source not in entry: logger.warning('Entry does not contain a field called `{}`', source) continue diff --git a/flexget/tests/test_translations_filter.py b/flexget/tests/test_translations_filter.py index f934d628ba..0dc414cd49 100644 --- a/flexget/tests/test_translations_filter.py +++ b/flexget/tests/test_translations_filter.py @@ -35,7 +35,7 @@ class TestTranslationsFilter: languages_synonyms: portuguese: - tuga - languages: + language_fields: - "trakt_series_language" dubbed: portuguese: "accept" @@ -55,7 +55,7 @@ class TestTranslationsFilter: - { title: "Movie En 720p", movie_name: "Movie", trakt_language: "english", url:"http://mock.url/file2.torrent" } translations: - languages: + language_fields: - "trakt_language" dubbed: reject From 272f2220070981c33276b4257352fdaeb3dad650 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 12:18:03 +0000 Subject: [PATCH 7/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- flexget/plugins/filter/translations.py | 13 ++++++------- flexget/tests/test_translations_filter.py | 5 ----- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/flexget/plugins/filter/translations.py b/flexget/plugins/filter/translations.py index d8275f6411..75c53ab26b 100644 --- a/flexget/plugins/filter/translations.py +++ b/flexget/plugins/filter/translations.py @@ -1,15 +1,14 @@ import re +from typing import List, Union from unicodedata import normalize -from typing import Union, List -from babelfish.language import LANGUAGES +import babelfish from guessit.api import GuessItApi, GuessitException from loguru import logger -import babelfish from flexget import plugin -from flexget.event import event from flexget.config_schema import one_or_more +from flexget.event import event PLUGIN_NAME = 'translations' logger = logger.bind(name=PLUGIN_NAME) @@ -338,7 +337,7 @@ def _process_config(self, config: dict) -> dict: _config['dubbed'] = {} for key in _dubbed: key = key.lower() - if not key in [UNKNOWN, DEFAULT, NATIVE, OTHER] and not self._is_language(key): + if key not in [UNKNOWN, DEFAULT, NATIVE, OTHER] and not self._is_language(key): raise plugin.PluginError(f'`{key}` in dubbed is not a valid language for dubbed') lang = self._get_language(key) @@ -360,7 +359,7 @@ def _process_config(self, config: dict) -> dict: _config['subbed'] = {} for key in _subbed: key = key.lower() - if not key in [UNKNOWN, DEFAULT, NONE, OTHER] and not self._is_language(key): + if key not in [UNKNOWN, DEFAULT, NONE, OTHER] and not self._is_language(key): raise plugin.PluginError(f'`{key}` in subbed is not a valid language for subbed') lang = self._get_language(key) @@ -451,7 +450,7 @@ def on_task_filter(self, task, config): source = my_config['source'] - if not my_config['source'] in entry: + if my_config['source'] not in entry: raise plugin.PluginError(f'No field {source} in entry') title = entry.get(my_config['source']) diff --git a/flexget/tests/test_translations_filter.py b/flexget/tests/test_translations_filter.py index 0dc414cd49..487f80d427 100644 --- a/flexget/tests/test_translations_filter.py +++ b/flexget/tests/test_translations_filter.py @@ -1,7 +1,3 @@ -from flexget import plugin -from flexget.entry import Entry - - class TestTranslationsFilter: config = """ tasks: @@ -265,7 +261,6 @@ def test_not_subbed(self, execute_task): assert task.undecided[0]['title'] == 'Attack on Titan S01E01 720p WEBRip' def test_subbed(self, execute_task): - task = execute_task('subbed3') assert len(task.accepted) == 4 assert len(task.rejected) == 1