release: v0.0.30

CHANGELOG.md

@@ -1,3 +1,8 @@
+* [v0.0.30](https://github.com/newt-sc/a4kSubtitles/releases/tag/service.subtitles.a4ksubtitles%2Fservice.subtitles.a4ksubtitles-0.0.30):
+  * Add service names in ad detection
+  * Removed the option to disable cleaning of ads
+  * Add remote fetching of addic7ed data
+
 * [v0.0.29](https://github.com/newt-sc/a4kSubtitles/releases/tag/service.subtitles.a4ksubtitles%2Fservice.subtitles.a4ksubtitles-0.0.29):
   * Fix Podnadpisi download of results missing filename meta
   * Fix last results cache not invalidated on language preferences change

a4kSubtitles/core.py

@@ -24,18 +24,21 @@
     utils,
     video,
 )
-from .services import services
-from .search import search
-from .download import download
-from .data import data
 
 core = sys.modules[__name__]
+utils.core = core
+
 api_mode_enabled = True
 handle = None
 
 progress_dialog = None
 progress_text = ''
 
+from .services import services
+from .search import search
+from .download import download
+from .data import data
+
 def main(handle, paramstring):  # pragma: no cover
     core.api_mode_enabled = False
     core.handle = handle

a4kSubtitles/data/__init__.py

@@ -1,8 +1,9 @@
 # -*- coding: utf-8 -*-
 
 import importlib
-from a4kSubtitles.lib import utils
+from a4kSubtitles.lib import utils, kodi
 
+kodi.xbmcvfs.mkdirs(utils.data_dir)
 __all = utils.get_all_relative_entries(__file__, ext='')
 
 data = {}

a4kSubtitles/data/addic7ed/__init__.py

@@ -1,6 +1,54 @@
 # -*- coding: utf-8 -*-
 
-from a4kSubtitles.lib import utils
+from a4kSubtitles.lib import utils, request, kodi
 
-languages = utils.get_relative_json(__file__, 'languages')
-tvshows = utils.get_relative_json(__file__, 'tvshows')
+__languages_filename = 'languages.json'
+__tvshows_filename = 'tvshows.json'
+__etags_filename = 'etags.json'
+__github_url = 'https://raw.githubusercontent.com/a4k-openproject/a4kSubtitles/master/a4kSubtitles/data/addic7ed/%s'
+__github_tvshows_url = __github_url % __tvshows_filename
+
+__dirname = utils.os.path.dirname(__file__)
+__service_name = utils.os.path.basename(__dirname)
+__remote_data_dir = utils.os.path.join(utils.data_dir, __service_name)
+kodi.xbmcvfs.mkdirs(__remote_data_dir)
+
+__embedded_tvshows_path = utils.os.path.join(__dirname, __tvshows_filename)
+__embedded_tvshows_mtime = utils.os.path.getmtime(__embedded_tvshows_path)
+__remote_tvshows_path = utils.os.path.join(__remote_data_dir, __tvshows_filename)
+__remote_etags_path = utils.os.path.join(__remote_data_dir, __etags_filename)
+
+if not utils.os.path.exists(__remote_tvshows_path) or utils.os.path.getmtime(__remote_tvshows_path) < __embedded_tvshows_mtime:
+    utils.shutil.copyfile(__embedded_tvshows_path, __remote_tvshows_path)
+
+languages = utils.get_json(__file__, __languages_filename)
+try:
+    tvshows = utils.get_json(__remote_data_dir, __tvshows_filename)
+except:
+    tvshows = utils.get_json(__file__, __tvshows_filename)
+
+def __download_data(url, etag, destpath):
+    response = request.execute(utils.core, {
+        'method': 'GET',
+        'url': url,
+        'headers': {
+            'If-None-Match': etag
+        }
+    })
+    if response.status_code != 200:
+        return
+
+    with open(destpath, 'wb') as f_out:
+        f_out.write(response.content)
+
+    etags = {'tvshows': response.headers['etag']}
+    with open(__remote_etags_path, 'w') as f_out:
+        f_out.write(utils.json.dumps(etags, indent=2))
+
+__tvshows_etag = ''
+if utils.os.path.exists(__remote_etags_path):
+    __etags = utils.get_json(__remote_data_dir, __etags_filename)
+    __tvshows_etag = __etags.get('tvshows', '')
+
+utils.core.threading.Thread(target=__download_data,
+                            args=(__github_tvshows_url, __tvshows_etag, __remote_tvshows_path)).start()

a4kSubtitles/download.py

@@ -64,10 +64,9 @@ def __postprocess(core, filepath):
         except: pass
 
         try:
-            if core.kodi.get_bool_setting('general.remove_ads'):
-                text = core.utils.cleanup_subtitles(text)
-                if len(text) < len(text) / 2:
-                    return
+            clean_text = core.utils.cleanup_subtitles(core, text)
+            if len(clean_text) > len(text) / 2:
+                text = clean_text
         except: pass
 
         with open(filepath, 'wb') as f:

a4kSubtitles/lib/request.py

@@ -10,7 +10,9 @@
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 def execute(core, request):
-    request.setdefault('timeout', get_int_setting('general.timeout'))
+    try: default_timeout = get_int_setting('general.timeout')
+    except: default_timeout = 10
+    request.setdefault('timeout', default_timeout)
 
     if core.progress_dialog and not core.progress_dialog.dialog:
         core.progress_dialog.open()

a4kSubtitles/lib/utils.py

@@ -5,6 +5,7 @@
 import re
 import json
 import string
+import shutil
 from . import kodi
 from . import logger
 
@@ -19,7 +20,7 @@
     import queue
     unicode = lambda v, e: v
 
-__url_regex = r'(([a-z0-9][a-z0-9-]{1,5}[a-z0-9]\.[a-z0-9]{2,20})|(opensubtitles))\.[a-z]{2,5}'
+__url_regex = r'[a-z0-9][a-z0-9-]{0,5}[a-z0-9]\.[a-z0-9]{2,20}\.[a-z]{2,5}'
 __credit_part_regex = r'(sync|synced|fix|fixed|corrected|corrections)'
 __credit_regex = __credit_part_regex + r' ?&? ?' + __credit_part_regex + r'? by'
 
@@ -32,6 +33,7 @@
 py3 = not py2
 
 temp_dir = os.path.join(kodi.addon_profile, 'temp')
+data_dir = os.path.join(kodi.addon_profile, 'data')
 
 class DictAsObject(dict):
     def __getattr__(self, name):
@@ -69,7 +71,14 @@ def wait_threads(threads):
     for thread in threads:
         thread.join()
 
-def cleanup_subtitles(sub_contents):
+def get_any_of_regex(array):
+    regex = r'('
+    for target in array:
+        regex += re.escape(target) + r'|'
+    return regex[:-1] + r')'
+
+def cleanup_subtitles(core, sub_contents):
+    service_names_regex = get_any_of_regex(core.services.keys())
     all_lines = sub_contents.split('\n')
     cleaned_lines = []
     buffer = []
@@ -96,7 +105,13 @@ def cleanup_subtitles(sub_contents):
                 buffer = []
             continue
 
-        if re.search(__url_regex, line, re.IGNORECASE) or re.search(__credit_regex, line, re.IGNORECASE):
+        line_contains_ad = (
+            re.search(service_names_regex, line, re.IGNORECASE) or
+            re.search(__url_regex, line, re.IGNORECASE) or
+            re.search(__credit_regex, line, re.IGNORECASE)
+        )
+
+        if line_contains_ad:
             logger.notice('(detected ad) %s' % line.encode('ascii', errors='ignore'))
             if not re.match(r'^\{\d+\}\{\d+\}', line):
                 garbage = True
@@ -110,8 +125,12 @@ def cleanup_subtitles(sub_contents):
 
     return '\n'.join(cleaned_lines)
 
-def get_relative_json(relative_file, filename):
-    json_path = os.path.join(os.path.dirname(relative_file), filename + '.json')
+def get_json(path, filename):
+    path = path if os.path.isdir(path) else os.path.dirname(path)
+    if not filename.endswith('.json'):
+        filename += '.json'
+
+    json_path = os.path.join(path, filename)
     with open(json_path) as json_result:
         return json.load(json_result)
 

addon.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
 <addon id="service.subtitles.a4ksubtitles"
        name="a4kSubtitles"
-       version="0.0.29"
+       version="0.0.30"
        provider-name="Unknown">
     <requires>
       <import addon="xbmc.python" version="2.25.0"/>
@@ -24,6 +24,11 @@ Supports: OpenSubtitles, BSPlayer, Podnadpisi.NET, SubDB, Subscene, Addic7ed
             <icon>icon.png</icon>
         </assets>
         <news>
+[v0.0.30]:
+  * Add service names in ad detection
+  * Removed the option to disable cleaning of ads
+  * Add remote fetching of addic7ed data
+
 [v0.0.29]:
   * Fix Podnadpisi download of results missing filename meta
   * Fix last results cache not invalidated on language preferences change

packages/addons.xml

@@ -4,7 +4,7 @@
 <addons>
 <addon id="service.subtitles.a4ksubtitles"
        name="a4kSubtitles"
-       version="0.0.29"
+       version="0.0.30"
        provider-name="Unknown">
     <requires>
       <import addon="xbmc.python" version="2.25.0"/>
@@ -27,6 +27,11 @@ Supports: OpenSubtitles, BSPlayer, Podnadpisi.NET, SubDB, Subscene, Addic7ed
             <icon>icon.png</icon>
         </assets>
         <news>
+[v0.0.30]:
+  * Add service names in ad detection
+  * Removed the option to disable cleaning of ads
+  * Add remote fetching of addic7ed data
+
 [v0.0.29]:
   * Fix Podnadpisi download of results missing filename meta
   * Fix last results cache not invalidated on language preferences change

packages/addons.xml.crc

@@ -1 +1 @@
-26c4a8e2f77fb73ca686575258576f62f10ddbbf
+0d19fefdccc6fb52065e1e73d2ff156d4ce22c60

resources/language/resource.language.en_gb/strings.po

@@ -28,10 +28,6 @@ msgctxt "#33102"
 msgid "Results Limit"
 msgstr ""
 
-msgctxt "#33103"
-msgid "Remove ads"
-msgstr ""
-
 # Services
 msgctxt "#33201"
 msgid "OpenSubtitles"

resources/settings.xml

@@ -2,9 +2,8 @@
 <settings>
     <!-- General -->
     <category label="33001">
-        <setting id="general.timeout"       label="33101" type="slider" default="15" option="int" range="10,20"/>
+        <setting id="general.timeout"       label="33101" type="slider" default="15" option="int" range="5,20"/>
         <setting id="general.results_limit" label="33102" type="slider" default="20" option="int" range="10,100"/>
-        <setting id="general.remove_ads"    label="33103" type="bool"   default="true"/>
     </category>
     <!-- Services -->
     <category label="33002">

tests/test_suite.py

@@ -81,13 +81,14 @@ def __search(a4ksubtitles_api, settings={}, video_meta={}):
     search.settings = {
         'general.timeout': '20',
         'general.results_limit': '20',
-        'general.remove_ads': 'true',
         'opensubtitles.enabled': 'false',
         'opensubtitles.username': '',
         'opensubtitles.password': '',
         'bsplayer.enabled': 'false',
         'podnadpisi.enabled': 'false',
         'subdb.enabled': 'false',
+        'subscene.enabled': 'false',
+        'addic7ed.enabled': 'false',
     }
     search.settings.update(settings)
 
@@ -187,22 +188,11 @@ def test_opensubtitles():
         'action_args': item['action_args']
     }
 
-    search.settings['general.remove_ads'] = 'false'
     filepath = a4ksubtitles_api.download(params, search.settings)
 
     assert filepath != ''
 
     # remove_ads
-    with open(filepath, 'r') as f:
-        sub_contents = f.read()
-
-    assert re.match(r'.*OpenSubtitles.*', sub_contents, re.DOTALL) is not None
-
-    search.settings['general.remove_ads'] = 'true'
-    filepath = a4ksubtitles_api.download(params, search.settings)
-
-    assert filepath != ''
-
     with open(filepath, 'r') as f:
         sub_contents = f.read()