| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,286 @@ | ||
| # -*- coding: UTF-8 -*- | ||
|
|
||
| # ---------------------------------------------------- | ||
| # Purpose: MythTV Python Bindings for TheTVDB v4 API | ||
| # Copyright: (c) 2021 Roland Ernst | ||
| # License: GPL v2 or later, see COPYING for details | ||
| # ---------------------------------------------------- | ||
|
|
||
| """ | ||
| Parse openapi specification for ttvdb v4. | ||
| See https://github.com/thetvdb/v4-api/blob/main/docs/swagger.yml | ||
| Create definitions and api for the files | ||
| - definitions.py | ||
| - ttvdbv4_api.py | ||
| """ | ||
|
|
||
| import sys | ||
| import os | ||
| import yaml | ||
| import re | ||
|
|
||
|
|
||
| # default strings for api functions | ||
| func_str = \ | ||
| "def {fname}({fparams}):\n" | ||
| param_str = \ | ||
| " params = {}\n" | ||
| param_item_str = \ | ||
| " if %s is not None:\n\ | ||
| params['%s'] = %s\n" | ||
| query_str = \ | ||
| " res = _query_api(%s)\n" | ||
| query_str_params = \ | ||
| " res = _query_api(%s, params)\n" | ||
| query_yielded_str = \ | ||
| " return _query_yielded(%s, %s, params, %s)\n" | ||
| res_str = \ | ||
| " data = res['data'] if res.get('data') is not None else None\n" | ||
| array_str = \ | ||
| "[%s(x) for x in %s] if data is not None else []" | ||
| item_str = \ | ||
| "%s(%s) if data is not None else None" | ||
|
|
||
|
|
||
| # default values for basic types | ||
| defaults = {'string': "''", | ||
| 'integer': 0, | ||
| 'number': 0.0, | ||
| 'boolean': False | ||
| } | ||
|
|
||
|
|
||
| # global | ||
| pathlist = [] | ||
|
|
||
|
|
||
| def read_yaml_from_file(api_spec): | ||
| with open(api_spec) as f: | ||
| data = yaml.safe_load(f) | ||
| return data | ||
|
|
||
|
|
||
| def print_api(title, version, pathlist, api_calls): | ||
| print('"""Generated API for thetvdb.com {} v {}"""'.format(title, version)) | ||
| print("\n") | ||
| for i in pathlist: | ||
| print(i) | ||
| print("\n\n") | ||
| print(api_calls) | ||
|
|
||
|
|
||
| def print_api_gets(pathname, pathdata): | ||
| global pathlist | ||
| paged = False | ||
| pstring = "" | ||
| # get the function definition for 'operationId' | ||
| if pathdata.get('get') is not None: | ||
| operationId = pathdata['get']['operationId'] | ||
| plist = [] # for the 'path' string | ||
| pitems = [] # for the function parameter string | ||
| params = {} # for the query parameters | ||
| if pathdata['get'].get('parameters') is not None: | ||
| # required params are member of the path | ||
| # optional params are attached as 'params' dict | ||
| for param in pathdata['get']['parameters']: | ||
| # search for paged requests | ||
| pkey = param.get('name').replace('-', '_') | ||
| pvalue = pkey | ||
| required = param.get('required', False) | ||
| if pkey == 'page': | ||
| paged = True | ||
| continue | ||
| if pkey == 'id': | ||
| pvalue = 'str(id)' | ||
| if required: | ||
| plist.append("%s=%s" % (pkey, pvalue)) | ||
| pitems.append(pkey) | ||
| else: | ||
| params[pkey] = pvalue | ||
| pitems.append("%s=None" % pvalue) | ||
| if paged: | ||
| params['page'] = 'page' | ||
| pitems.append("%s=%s" % ('page', 0)) | ||
| pitems.append("%s=%s" % ('yielded', False)) | ||
| pstring += func_str.format(fname=operationId, | ||
| fparams=", ".join(pitems)) | ||
|
|
||
| # define the parameter | ||
| if params: | ||
| pstring += param_str | ||
| for k, v in params.items(): | ||
| pstring += param_item_str % (k, k, v) | ||
|
|
||
| # define the url | ||
| path = "%s_path.format(%s)" % (operationId, ", ".join(plist)) | ||
| pstring += " path = %s\n" % path | ||
|
|
||
| # evaluate response properties['data']: | ||
| content = pathdata['get']['responses']['200']['content'] | ||
| data = content['application/json']['schema']['properties']['data'] | ||
|
|
||
| # look for references ('$ref') starting with '#/components/schemas/' | ||
| ref_str = '#/components/schemas/' | ||
| tmplst = [] | ||
| tmpref = "" | ||
| listname = "listname=None" | ||
| if data.get('type') is not None: | ||
| if data.get('items'): | ||
| ref = data['items']['$ref'] | ||
| if ref.startswith(ref_str): | ||
| if data.get('type') == 'array': | ||
| tmpref = ref.split('/')[-1] | ||
| tmplst.append("%s" % (array_str % (tmpref, 'data'))) | ||
| elif data.get('properties'): | ||
| for prop in data['properties'].keys(): | ||
| if data['properties'][prop].get('$ref') is not None: | ||
| ref = data['properties'][prop]['$ref'] | ||
| if ref.startswith(ref_str): | ||
| tmpref = ref.split('/')[-1] | ||
| tmplst.append(item_str % (tmpref, "data['%s']" % prop)) | ||
| elif data['properties'][prop].get('items') is not None: | ||
| if data['properties'][prop].get('type') == 'array': | ||
| ref = data['properties'][prop]['items']['$ref'] | ||
| if ref.startswith(ref_str): | ||
| listname = "listname='%s'" % prop | ||
| tmpref = ref.split('/')[-1] | ||
| tmplst.append(array_str % (tmpref, "data['%s']" % prop)) | ||
|
|
||
| elif data.get('$ref') is not None: | ||
| if data['$ref'].startswith(ref_str): | ||
| pref = data['$ref'].split('/')[-1] | ||
| tmplst.append("%s" % (item_str % (pref, 'data'))) | ||
|
|
||
| # format output | ||
| add_ident = "" | ||
| if params: | ||
| if paged: | ||
| add_ident = " " | ||
| pstring += add_ident + "if yielded:\n" | ||
| pstring += add_ident + query_yielded_str % (tmpref, "path", listname) | ||
| pstring += add_ident + "else:\n" | ||
|
|
||
| pstring += add_ident + query_str_params % ("path") | ||
| pstring += add_ident + res_str | ||
| else: | ||
| pstring += query_str % ("path") | ||
| pstring += res_str | ||
|
|
||
| pstring += \ | ||
| ' %sreturn( %s )' % (add_ident, | ||
| (",\n " + add_ident).join(tmplst)) | ||
|
|
||
| # update pathlist as well | ||
| # replace ('-', '_') in parameters identified within '{}' | ||
| pattern = re.compile(r'{[a-z]+-[a-z]+}') | ||
| s = '%s_path = TTVDBV4_path + "%s"' % (operationId, pathname) | ||
| for match in pattern.findall(s): | ||
| s = s.replace(match, match.replace('-', '_')) | ||
| pathlist.append(s) | ||
|
|
||
| return pstring | ||
|
|
||
|
|
||
| def print_definition(name, defitem, setdefaults=False): | ||
|
|
||
| # string definitions | ||
| defstr = '#/components/schemas' # openapi 3.0 | ||
| classstr = \ | ||
| 'class %s(object):\n' % name + \ | ||
| ' """%s"""\n' % defitem['description'] + \ | ||
| ' def __init__(self, data):\n' | ||
| handle_list_str = \ | ||
| " self.%s = _handle_list(%s, data.get('%s'))\n" | ||
| get_list_str = \ | ||
| " self.%s = _get_list(data, '%s')\n" | ||
| handle_single_str = \ | ||
| " self.%s = _handle_single(%s, data.get('%s'))\n" | ||
| translations_str = \ | ||
| " self.translations = []\n" | ||
| similarity_str = \ | ||
| " self.name_similarity = 0.0\n" | ||
| added_str = \ | ||
| " # additional attributes needed by the mythtv grabber script:\n" | ||
|
|
||
| if defitem.get('properties') is None: | ||
| classstr += " pass\n" | ||
| else: | ||
| needs_translations = False | ||
| for i in defitem['properties'].keys(): | ||
| if ('items') in defitem['properties'][i]: | ||
| # handle arrays and lists of basic types | ||
| if ('type') in defitem['properties'][i]: | ||
| if defitem['properties'][i]['type'] == 'array': | ||
| if ('$ref') in defitem['properties'][i]['items']: | ||
| ref = defitem['properties'][i]['items']['$ref'] | ||
| atype = ref.split("/")[-1] | ||
| classstr += handle_list_str % (i, atype, i) | ||
| else: | ||
| if i == 'nameTranslations': | ||
| needs_translations = True | ||
| classstr += get_list_str % (i, i) | ||
|
|
||
| elif ('$ref') in defitem['properties'][i]: | ||
| # handle special types | ||
| v = defitem['properties'][i]['$ref'] | ||
| stype = v.split("/")[-1] | ||
| classstr += handle_single_str % (i, stype, i) | ||
|
|
||
| elif ('type') in defitem['properties'][i]: | ||
| # handle basic types | ||
| stype = defitem['properties'][i]['type'] | ||
| if setdefaults: | ||
| d = defaults.get(stype) | ||
| s = " self.%s = data.get('%s', %s)" % (i, i, d) | ||
| else: | ||
| s = " self.%s = data.get('%s')" % (i, i) | ||
| alignment = 80 - len(s) + len(stype) | ||
| classstr += s + ("# %s\n" % stype).rjust(alignment) | ||
| if needs_translations: | ||
| # below are additions needed by the mythtv grabber script | ||
| classstr += added_str | ||
| classstr += translations_str | ||
| classstr += similarity_str | ||
|
|
||
| return(classstr) | ||
|
|
||
|
|
||
| if __name__ == '__main__': | ||
| """ | ||
| Download the latest api specification from the TheTVDB official repo: | ||
| https://github.com/thetvdb/v4-api/blob/main/docs/swagger.yml | ||
| """ | ||
| api_spec = sys.argv[1] | ||
| if not os.path.isfile(api_spec): | ||
| print("Error: main() needs to be called with an OAS3 spec file (yaml)") | ||
| sys.exit(1) | ||
| apidata = read_yaml_from_file(api_spec) | ||
| api_title = apidata['info']['title'] | ||
| api_version = apidata['info']['version'] | ||
| apiv4_basepath = apidata['servers'][0]['url'] | ||
|
|
||
| pathlist.append('TTVDBV4_path = "{}"'.format(apiv4_basepath)) | ||
|
|
||
| # get api functions | ||
| api_calls = "" | ||
| for k in apidata['paths'].keys(): | ||
| if apidata['paths'][k].get('get'): | ||
| api_calls += print_api_gets(k, apidata['paths'][k]) | ||
| api_calls += "\n\n\n" | ||
|
|
||
| print_api(api_title, api_version, pathlist, api_calls) | ||
|
|
||
| # get api definitions | ||
| api_defs = "" | ||
| api_defs += ('"""Generated API for thetvdb.com {} v {}"""' | ||
| .format(api_title, api_version)) | ||
| api_defs += "\n\n" | ||
| schemas = apidata['components']['schemas'] # openapi 3.0 | ||
| for k in schemas.keys(): | ||
| api_defs += "\n" | ||
| api_defs += print_definition(k, schemas[k], setdefaults=True) | ||
| api_defs += "\n" | ||
|
|
||
| print(api_defs) | ||
|
|
||
|
|
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| # -*- coding: UTF-8 -*- | ||
|
|
||
| # ---------------------------------------------------- | ||
| # Purpose: MythTV Python Bindings for TheTVDB v4 API | ||
| # Copyright: (c) 2021 Roland Ernst | ||
| # License: GPL v2 or later, see COPYING for details | ||
| # ---------------------------------------------------- | ||
|
|
||
|
|
||
| from datetime import datetime | ||
| import sys | ||
|
|
||
|
|
||
| if sys.version_info[0] == 2: | ||
| from HTMLParser import HTMLParser | ||
| from StringIO import StringIO | ||
|
|
||
|
|
||
| class MLStripper(HTMLParser): | ||
| def __init__(self): | ||
| self.reset() | ||
| self.text = StringIO() | ||
|
|
||
| def handle_data(self, d): | ||
| self.text.write(d) | ||
|
|
||
| def get_data(self): | ||
| return self.text.getvalue() | ||
|
|
||
| else: | ||
| from io import StringIO | ||
| from html.parser import HTMLParser | ||
|
|
||
| class MLStripper(HTMLParser): | ||
| def __init__(self): | ||
| super().__init__() | ||
| self.reset() | ||
| self.strict = False | ||
| self.convert_charrefs= True | ||
| self.text = StringIO() | ||
|
|
||
| def handle_data(self, d): | ||
| self.text.write(d) | ||
|
|
||
| def get_data(self): | ||
| return self.text.getvalue() | ||
|
|
||
|
|
||
| def strip_tags(html): | ||
| if html is not None and html != "": | ||
| s = MLStripper() | ||
| s.feed(html) | ||
| return s.get_data() | ||
| else: | ||
| return "" | ||
|
|
||
|
|
||
| def convert_date(tstring): | ||
| if tstring is None or tstring == '': | ||
| return None | ||
| try: | ||
| return datetime.strptime(tstring, '%Y-%m-%d').date() | ||
| except(TypeError, ValueError): | ||
| return None | ||
|
|
||
|
|
||
| def convert_time(tstring): | ||
| if tstring is None or tstring == '': | ||
| return None | ||
| try: | ||
| return datetime.strptime(tstring, '%Y-%m-%d') | ||
| except(TypeError, ValueError): | ||
| return None |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,6 @@ | ||
| #ifdef _WIN32 | ||
| #include <sys/stat.h> | ||
| #endif | ||
|
|
||
| #include "mythmiscutil.h" | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| test_unzip |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| Lorem ipsum dolor sit amet, consectetur adipiscing elit. Duis lacinia | ||
| sapien tellus, non fringilla turpis efficitur ut. Vivamus in tellus | ||
| nec mauris congue ultricies id non nulla. Praesent massa mauris, | ||
| viverra in justo sed, malesuada lobortis mauris. Suspendisse vulputate | ||
| purus vel consequat tempus. Quisque sed enim sit amet erat placerat | ||
| egestas in et dui. Sed non tellus ac arcu consectetur finibus. Proin | ||
| suscipit, felis ut consectetur ullamcorper, nisl nunc ullamcorper | ||
| elit, eget porttitor neque ipsum eu lacus. Sed dapibus diam purus, | ||
| sollicitudin ornare libero feugiat nec. Donec id enim sit amet libero | ||
| pharetra fringilla. Nulla et neque nec ex posuere sodales. Vivamus | ||
| porttitor ullamcorper porttitor. Fusce et nunc id justo interdum | ||
| laoreet. | ||
|
|
||
| Integer in nisl quis arcu sollicitudin tristique a et leo. Suspendisse | ||
| posuere varius sodales. Nunc rutrum diam nec est accumsan, vitae | ||
| pellentesque ex auctor. Quisque at mauris scelerisque, tempus mauris | ||
| vel, mollis ante. Pellentesque quis malesuada tellus. Etiam non nisl | ||
| mauris. Morbi quis venenatis diam. | ||
|
|
||
| Donec tempor lobortis turpis, ut venenatis massa viverra vel. Class | ||
| aptent taciti sociosqu ad litora torquent per conubia nostra, per | ||
| inceptos himenaeos. Vivamus vel dui tempus, pretium ligula non, | ||
| sagittis neque. Aenean ultricies malesuada enim, id elementum odio | ||
| placerat nec. Nam ut aliquet libero. Praesent quis enim tempor, | ||
| lacinia tellus id, finibus sapien. In elit sem, vehicula hendrerit | ||
| urna et, finibus ultrices risus. Sed quis felis a sem ultrices porta | ||
| vel ac tortor. Nulla a aliquet diam. Curabitur iaculis maximus | ||
| nulla. Morbi ut turpis neque. | ||
|
|
||
| Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nam sagittis | ||
| lectus quis sem accumsan posuere. Pellentesque nec viverra risus, ac | ||
| suscipit odio. Proin eget condimentum eros. In nunc erat, viverra eget | ||
| quam ut, luctus placerat eros. Donec et lacus tristique, varius libero | ||
| vitae, viverra massa. Donec a scelerisque orci, sit amet accumsan | ||
| erat. Mauris sollicitudin ligula vitae mi congue aliquam. Cras vitae | ||
| sagittis nulla, tempus elementum velit. Maecenas eros velit, imperdiet | ||
| sit amet ante sit amet, bibendum consectetur arcu. | ||
|
|
||
| Phasellus et arcu quis ante auctor tempor. Vestibulum quam nibh, | ||
| hendrerit eget consectetur sit amet, pharetra et justo. Quisque nulla | ||
| tortor, convallis ac dignissim id, aliquet nec metus. Maecenas dictum | ||
| ligula quis ligula efficitur feugiat nec vitae ligula. Nulla a | ||
| elementum velit, non efficitur magna. Vivamus euismod interdum | ||
| pulvinar. Donec eu enim commodo dui tincidunt porttitor. Etiam viverra | ||
| consequat nunc, sed mattis libero gravida eu. Aenean vitae ullamcorper | ||
| mi, ut imperdiet enim. Proin vehicula nibh quis libero ullamcorper | ||
| laoreet quis nec lacus. Nam nunc lorem, iaculis quis gravida | ||
| fringilla, ultrices consequat justo. Aliquam tempor ipsum non ultrices | ||
| ultrices. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| Lorem ipsum dolor sit amet, consectetur adipiscing elit. Duis lacinia | ||
| sapien tellus, non fringilla turpis efficitur ut. Vivamus in tellus | ||
| nec mauris congue ultricies id non nulla. Praesent massa mauris, | ||
| viverra in justo sed, malesuada lobortis mauris. Suspendisse vulputate | ||
| purus vel consequat tempus. Quisque sed enim sit amet erat placerat | ||
| egestas in et dui. Sed non tellus ac arcu consectetur finibus. Proin | ||
| suscipit, felis ut consectetur ullamcorper, nisl nunc ullamcorper | ||
| elit, eget porttitor neque ipsum eu lacus. Sed dapibus diam purus, | ||
| sollicitudin ornare libero feugiat nec. Donec id enim sit amet libero | ||
| pharetra fringilla. Nulla et neque nec ex posuere sodales. Vivamus | ||
| porttitor ullamcorper porttitor. Fusce et nunc id justo interdum | ||
| laoreet. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| Integer in nisl quis arcu sollicitudin tristique a et leo. Suspendisse | ||
| posuere varius sodales. Nunc rutrum diam nec est accumsan, vitae | ||
| pellentesque ex auctor. Quisque at mauris scelerisque, tempus mauris | ||
| vel, mollis ante. Pellentesque quis malesuada tellus. Etiam non nisl | ||
| mauris. Morbi quis venenatis diam. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| Donec tempor lobortis turpis, ut venenatis massa viverra vel. Class | ||
| aptent taciti sociosqu ad litora torquent per conubia nostra, per | ||
| inceptos himenaeos. Vivamus vel dui tempus, pretium ligula non, | ||
| sagittis neque. Aenean ultricies malesuada enim, id elementum odio | ||
| placerat nec. Nam ut aliquet libero. Praesent quis enim tempor, | ||
| lacinia tellus id, finibus sapien. In elit sem, vehicula hendrerit | ||
| urna et, finibus ultrices risus. Sed quis felis a sem ultrices porta | ||
| vel ac tortor. Nulla a aliquet diam. Curabitur iaculis maximus | ||
| nulla. Morbi ut turpis neque. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nam sagittis | ||
| lectus quis sem accumsan posuere. Pellentesque nec viverra risus, ac | ||
| suscipit odio. Proin eget condimentum eros. In nunc erat, viverra eget | ||
| quam ut, luctus placerat eros. Donec et lacus tristique, varius libero | ||
| vitae, viverra massa. Donec a scelerisque orci, sit amet accumsan | ||
| erat. Mauris sollicitudin ligula vitae mi congue aliquam. Cras vitae | ||
| sagittis nulla, tempus elementum velit. Maecenas eros velit, imperdiet | ||
| sit amet ante sit amet, bibendum consectetur arcu. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| Phasellus et arcu quis ante auctor tempor. Vestibulum quam nibh, | ||
| hendrerit eget consectetur sit amet, pharetra et justo. Quisque nulla | ||
| tortor, convallis ac dignissim id, aliquet nec metus. Maecenas dictum | ||
| ligula quis ligula efficitur feugiat nec vitae ligula. Nulla a | ||
| elementum velit, non efficitur magna. Vivamus euismod interdum | ||
| pulvinar. Donec eu enim commodo dui tincidunt porttitor. Etiam viverra | ||
| consequat nunc, sed mattis libero gravida eu. Aenean vitae ullamcorper | ||
| mi, ut imperdiet enim. Proin vehicula nibh quis libero ullamcorper | ||
| laoreet quis nec lacus. Nam nunc lorem, iaculis quis gravida | ||
| fringilla, ultrices consequat justo. Aliquam tempor ipsum non ultrices | ||
| ultrices. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| <?xml version="1.0" encoding="utf-8"?> | ||
| <!DOCTYPE mythuitheme SYSTEM "http://www.mythtv.org/schema/mythuitheme.dtd"> | ||
| <themeinfo> | ||
| <name>Willi</name> | ||
| <aspect>16:9</aspect> | ||
| <author> | ||
| <name>Elkin Fricke</name> | ||
| <email>managementboy@gmail.com</email> | ||
| </author> | ||
| <!-- The Theme Types (Required) --> | ||
| <types> | ||
| <!-- Type. Legal Values are one or more of: UI, OSD and Menu --> | ||
| <type>UI</type> | ||
| <type>OSD</type> | ||
| </types> | ||
| <baseres>1920x1080</baseres> | ||
| <version> | ||
| <major>2</major> | ||
| <minor>28</minor> | ||
| </version> | ||
| <detail> | ||
| <thumbnail name="preview">preview.jpg</thumbnail> | ||
| <description>A UI and OSD theme with focus on fanart, banners and cover display.</description> | ||
| <!-- What is not available --> | ||
| <errata>Supported: Recording/ Scheduling/ Video/ Gallery/ Music/ Stream/ Information/ Notification. Not yet suported: everything else.</errata> | ||
| <downloadurl>http://ftp.osuosl.org/pub/mythtv/themes/trunk/Willi-2.28_trunk.zip</downloadurl> | ||
| </detail> | ||
| <downloadinfo> | ||
| <url>http://ftp.osuosl.org/pub/mythtv/themes/trunk/Willi-2.28_trunk.zip</url> | ||
| <size>5252813</size> | ||
| <md5sum>dcfd19d5763b6e833781c19a2e0bb617</md5sum> | ||
| <packagedate>2021-04-28 16:18:58 UTC</packagedate> | ||
| <infourl>http://themes.mythtv.org/themes/repository/trunk/Willi/themeinfo.xml</infourl> | ||
| <sourceurl>git://github.com/MythTV-Themes/Willi.git</sourceurl> | ||
| <installedsize>9612919</installedsize> | ||
| </downloadinfo> | ||
| </themeinfo> |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| ipsum_lorem_p1.txt |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| /* | ||
| * Class TestUnzip | ||
| * | ||
| * Copyright (c) David Hampton 2021 | ||
| * | ||
| * This program is free software; you can redistribute it and/or modify | ||
| * it under the terms of the GNU General Public License as published by | ||
| * the Free Software Foundation; either version 2 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * This program is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License | ||
| * along with this program; if not, write to the Free Software | ||
| * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | ||
| */ | ||
| #include <iostream> | ||
| #include "mythcoreutil.h" | ||
| #include "test_unzip.h" | ||
| #include <QTemporaryDir> | ||
|
|
||
| QTemporaryDir *gTmpDir {nullptr}; | ||
|
|
||
| void TestUnzip::initTestCase() | ||
| { | ||
| QDir::setCurrent("libmythbase/test/test_unzip"); | ||
|
|
||
| gTmpDir = new QTemporaryDir(); | ||
| QVERIFY(gTmpDir != nullptr); | ||
| gTmpDir->setAutoRemove(true); | ||
| } | ||
|
|
||
| // After each test case | ||
| void TestUnzip::cleanup() | ||
| { | ||
| QDir dir { gTmpDir->path() }; | ||
| dir.removeRecursively(); | ||
| dir.mkpath(dir.absolutePath()); | ||
| } | ||
|
|
||
| // After all test cases | ||
| void TestUnzip::cleanupTestCase() | ||
| { | ||
| delete gTmpDir; | ||
| gTmpDir = nullptr; | ||
| } | ||
|
|
||
| void TestUnzip::test_text_file(void) | ||
| { | ||
| QString filename { "zipfiles/ipsum_lorem.zip" }; | ||
| bool result = extractZIP(filename, gTmpDir->path()); | ||
| QCOMPARE(result, true); | ||
|
|
||
| auto fi = QFileInfo(gTmpDir->path() + "/ipsum_lorem_p1.txt"); | ||
| QCOMPARE(fi.exists(), true); | ||
| QCOMPARE(fi.size(), 755); | ||
| #if QT_VERSION >= QT_VERSION_CHECK(5,10,0) | ||
| auto actualDateTime = QDateTime(QDate(2021,6,24),QTime(9,55,16)); | ||
| QCOMPARE(fi.lastModified(), actualDateTime); | ||
| #endif | ||
|
|
||
| auto orig = QFile("data/ipsum_lorem_p1.txt"); | ||
| orig.open(QIODevice::ReadOnly); | ||
| auto origData = orig.readAll(); | ||
|
|
||
| auto unzipped = QFile(gTmpDir->path() + "/ipsum_lorem_p1.txt"); | ||
| unzipped.open(QIODevice::ReadOnly); | ||
| auto unzippedData = unzipped.readAll(); | ||
| QCOMPARE(origData, unzippedData); | ||
| } | ||
|
|
||
| void TestUnzip::test_theme_file(void) | ||
| { | ||
| QString filename { "zipfiles/themes.zip" }; | ||
| bool result = extractZIP(filename, gTmpDir->path()); | ||
| QCOMPARE(result, true); | ||
|
|
||
| auto fi = QFileInfo(gTmpDir->path() + "/trunk/Willi/themeinfo.xml"); | ||
| QCOMPARE(fi.exists(), true); | ||
| QCOMPARE(fi.size(), 1461); | ||
| #if QT_VERSION >= QT_VERSION_CHECK(5,10,0) | ||
| auto actualDateTime = QDateTime(QDate(2013,7,14),QTime(16,00,56)); | ||
| QCOMPARE(fi.lastModified(), actualDateTime); | ||
| #endif | ||
|
|
||
| auto orig = QFile("data/willi_themeinfo.xml"); | ||
| orig.open(QIODevice::ReadOnly); | ||
| auto origData = orig.readAll(); | ||
|
|
||
| auto unzipped = QFile(gTmpDir->path() + "/trunk/Willi/themeinfo.xml"); | ||
| unzipped.open(QIODevice::ReadOnly); | ||
| auto unzippedData = unzipped.readAll(); | ||
| QCOMPARE(origData, unzippedData); | ||
| } | ||
|
|
||
| QTEST_APPLESS_MAIN(TestUnzip) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| /* | ||
| * Class TestUnzip | ||
| * | ||
| * Copyright (c) David Hampton 2021 | ||
| * | ||
| * This program is free software; you can redistribute it and/or modify | ||
| * it under the terms of the GNU General Public License as published by | ||
| * the Free Software Foundation; either version 2 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * This program is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License | ||
| * along with this program; if not, write to the Free Software | ||
| * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | ||
| */ | ||
|
|
||
| #include <QtTest/QtTest> | ||
| #include <iostream> | ||
|
|
||
| class TestUnzip : public QObject | ||
| { | ||
| Q_OBJECT | ||
|
|
||
| private slots: | ||
| static void initTestCase(void); | ||
| static void cleanup(void); | ||
| static void cleanupTestCase(void); | ||
|
|
||
| static void test_text_file(void); | ||
| static void test_theme_file(void); | ||
| }; |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| include ( ../../../../settings.pro ) | ||
| include ( ../../../../test.pro ) | ||
|
|
||
| QT += testlib | ||
|
|
||
| TEMPLATE = app | ||
| TARGET = test_unzip | ||
| DEPENDPATH += . ../.. ../../logging | ||
| INCLUDEPATH += . ../.. ../../logging | ||
| LIBS += -L../.. -lmythbase-$$LIBVERSION | ||
| LIBS += -Wl,$$_RPATH_$${PWD}/../.. | ||
|
|
||
| # Input | ||
| HEADERS += test_unzip.h | ||
| SOURCES += test_unzip.cpp | ||
|
|
||
| QMAKE_CLEAN += $(TARGET) | ||
| QMAKE_CLEAN += ; ( cd $(OBJECTS_DIR) && rm -f *.gcov *.gcda *.gcno ) | ||
|
|
||
| LIBS += $$EXTRA_LIBS $$LATE_LIBS |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,313 @@ | ||
| /* | ||
| * Class UnZip | ||
| * | ||
| * Copyright (c) David Hampton 2021 | ||
| * | ||
| * This program is free software; you can redistribute it and/or modify | ||
| * it under the terms of the GNU General Public License as published by | ||
| * the Free Software Foundation; either version 2 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * This program is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License | ||
| * along with this program; if not, write to the Free Software | ||
| * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | ||
| */ | ||
|
|
||
| #include "unzip2.h" | ||
|
|
||
| // libmythbase headers | ||
| #include "mythdate.h" | ||
| #include "mythlogging.h" | ||
|
|
||
| UnZip::UnZip(QString &zipFileName) | ||
| : m_zipFileName(std::move(zipFileName)) | ||
| { | ||
| int err { ZIP_ER_OK }; | ||
| m_zip = zip_open(qPrintable(m_zipFileName), 0, &err); | ||
| if (m_zip != nullptr) | ||
| return; | ||
|
|
||
| LOG(VB_GENERAL, LOG_ERR, | ||
| QString("UnZip: Unable to open zip file %1, error %2") | ||
| .arg(m_zipFileName).arg(err)); | ||
| } | ||
|
|
||
| UnZip::~UnZip() | ||
| { | ||
| int err = zip_close(m_zip); | ||
| if (err == 0) | ||
| return; | ||
|
|
||
| LOG(VB_GENERAL, LOG_DEBUG, | ||
| QString("UnZip: Error closing zip file %1, error %2") | ||
| .arg(m_zipFileName).arg(err)); | ||
| } | ||
|
|
||
| bool UnZip::getEntryStats(zipEntry& entry) | ||
| { | ||
| zip_stat_init(&entry.m_stats); | ||
| if (-1 == zip_stat_index(m_zip, entry.m_index, 0, &entry.m_stats)) | ||
| { | ||
| LOG(VB_GENERAL, LOG_ERR, | ||
| QString("UnZip: Can't get info for index %1 in %2") | ||
| .arg(entry.m_index).arg(m_zipFileName)); | ||
| return false; | ||
| } | ||
| if ((entry.m_stats.valid & kSTATS_REQUIRED) != kSTATS_REQUIRED) | ||
| { | ||
| LOG(VB_GENERAL, LOG_ERR, | ||
| QString("UnZip: Invalid status for index %1 in %2") | ||
| .arg(entry.m_index).arg(m_zipFileName)); | ||
| return false; | ||
| } | ||
| return true; | ||
| } | ||
|
|
||
| void UnZip::getEntryAttrs(zipEntry& entry) | ||
| { | ||
| zip_uint8_t opsys {ZIP_OPSYS_UNIX};// NOLINT(readability-uppercase-literal-suffix) | ||
|
|
||
| entry.m_attributes = 0; | ||
| zip_file_get_external_attributes(m_zip, entry.m_index, 0, &opsys, | ||
| &entry.m_attributes); | ||
| } | ||
|
|
||
| bool UnZip::zipCreateDirectory(const zipEntry& entry) | ||
| { | ||
| QDir dir = entry.m_fi.absoluteDir(); | ||
| if (dir.exists()) | ||
| return true; | ||
| if (dir.mkpath(dir.absolutePath())) | ||
| return true; | ||
| LOG(VB_GENERAL, LOG_ERR, | ||
| QString("UnZip: Failed to create directory %1") | ||
| .arg(dir.absolutePath())); | ||
| return false; | ||
| } | ||
|
|
||
| // Validate that the filename is beneath the extraction directrory. | ||
| // This prevents a zip from overwriting arbitrary files by using names | ||
| // with a sequence of ".." directories. | ||
| bool UnZip::zipValidateFilename(const QFileInfo& fi) | ||
| { | ||
| if (fi.absoluteFilePath().startsWith(m_outDir.absolutePath())) | ||
| return true; | ||
| LOG(VB_GENERAL, LOG_ERR, | ||
| QString("UnZip: Attempt to write outside destination directory. File: %1") | ||
| .arg(QString(fi.fileName()))); | ||
| return false; | ||
| } | ||
|
|
||
| // Would be nice if Qt provided a unix perm to Qt perm conversion. | ||
| QFileDevice::Permissions UnZip::zipToQtPerms(const zipEntry& entry) | ||
| { | ||
| QFileDevice::Permissions qt_perms; | ||
| zip_uint32_t attrs = entry.m_attributes; | ||
|
|
||
| int32_t user = (attrs & ZIP_ATTR_USER_PERM_MASK) >> ZIP_ATTR_USER_PERM_SHIFT; | ||
| int32_t group = (attrs & ZIP_ATTR_GROUP_PERM_MASK) >> ZIP_ATTR_GROUP_PERM_SHIFT; | ||
| int32_t other = (attrs & ZIP_ATTR_OTHER_PERM_MASK) >> ZIP_ATTR_OTHER_PERM_SHIFT; | ||
|
|
||
| if (user & 4) | ||
| qt_perms |= (QFileDevice::ReadOwner | QFileDevice::ReadUser); | ||
| if (user & 2) | ||
| qt_perms |= (QFileDevice::WriteOwner | QFileDevice::WriteUser); | ||
| if (user & 1) | ||
| qt_perms |= (QFileDevice::ExeOwner | QFileDevice::ExeUser); | ||
| if (group & 4) | ||
| qt_perms |= QFileDevice::ReadGroup; | ||
| if (group & 2) | ||
| qt_perms |= QFileDevice::WriteGroup; | ||
| if (group & 1) | ||
| qt_perms |= QFileDevice::ExeGroup; | ||
| if (other & 4) | ||
| qt_perms |= QFileDevice::ReadOther; | ||
| if (other & 2) | ||
| qt_perms |= QFileDevice::WriteOther; | ||
| if (other & 1) | ||
| qt_perms |= QFileDevice::ExeOther; | ||
| return qt_perms; | ||
| } | ||
|
|
||
| void UnZip::zipSetFileAttributes(const zipEntry& entry, QFile& outfile) | ||
| { | ||
| #if QT_VERSION >= QT_VERSION_CHECK(5,10,0) | ||
| // Set times | ||
| auto dateTime = MythDate::fromSecsSinceEpoch(entry.m_stats.mtime); | ||
|
|
||
| outfile.setFileTime(dateTime, QFileDevice::FileAccessTime); | ||
| outfile.setFileTime(dateTime, QFileDevice::FileBirthTime); | ||
| outfile.setFileTime(dateTime, QFileDevice::FileMetadataChangeTime); | ||
| outfile.setFileTime(dateTime, QFileDevice::FileModificationTime); | ||
| #endif | ||
|
|
||
| if (entry.m_attributes == 0) | ||
| return; | ||
| outfile.setPermissions(zipToQtPerms(entry)); | ||
| } | ||
|
|
||
| bool UnZip::zipCreateSymlink(const zipEntry& entry) | ||
| { | ||
| zip_file_t *infile = zip_fopen_index(m_zip, entry.m_index, 0); | ||
| if (infile == nullptr) | ||
| { | ||
| LOG(VB_GENERAL, LOG_ERR, | ||
| QString("UnZip: Can't open index %1 in %2") | ||
| .arg(entry.m_index).arg(m_zipFileName)); | ||
| return false; | ||
| } | ||
|
|
||
| int64_t readLen {0}; | ||
| static constexpr int BLOCK_SIZE { 4096 }; | ||
| QByteArray data; data.resize(BLOCK_SIZE); | ||
| readLen = zip_fread(infile, data.data(), BLOCK_SIZE); | ||
| if (readLen < 1) | ||
| { | ||
| LOG(VB_GENERAL, LOG_ERR, | ||
| QString("UnZip: Invalid symlink name for index %1 in %2") | ||
| .arg(entry.m_index).arg(m_zipFileName)); | ||
| return false; | ||
| } | ||
| data.resize(readLen); | ||
|
|
||
| auto target = QFileInfo(entry.m_fi.absolutePath() + "/" + data); | ||
| if (!zipValidateFilename(target)) | ||
| return false; | ||
| if (!QFile::link(target.absoluteFilePath(), entry.m_fi.absoluteFilePath())) | ||
| { | ||
| LOG(VB_GENERAL, LOG_ERR, | ||
| QString("UnZip: Failed to create symlink from %1 to %2") | ||
| .arg(entry.m_fi.absoluteFilePath(), | ||
| target.absoluteFilePath())); | ||
| return false; | ||
| } | ||
| return true; | ||
| } | ||
|
|
||
| bool UnZip::zipWriteOneFile(const zipEntry& entry) | ||
| { | ||
| zip_file_t *infile = zip_fopen_index(m_zip, entry.m_index, 0); | ||
| if (infile == nullptr) | ||
| { | ||
| LOG(VB_GENERAL, LOG_ERR, | ||
| QString("UnZip: Can't open file at index %1 in %2") | ||
| .arg(entry.m_index).arg(m_zipFileName)); | ||
| return false; | ||
| } | ||
|
|
||
| auto outfile = QFile(entry.m_fi.absoluteFilePath()); | ||
| if (!outfile.open(QIODevice::Truncate|QIODevice::WriteOnly)) | ||
| { | ||
| LOG(VB_GENERAL, LOG_ERR, | ||
| QString("UnZip: Failed to open output file %1") | ||
| .arg(entry.m_fi.absoluteFilePath())); | ||
| return false; | ||
| } | ||
|
|
||
| int64_t readLen {0}; | ||
| uint64_t bytesRead {0}; | ||
| uint64_t bytesWritten {0}; | ||
| static constexpr int BLOCK_SIZE { 4096 }; | ||
| QByteArray data; data.resize(BLOCK_SIZE); | ||
| while ((readLen = zip_fread(infile, data.data(), BLOCK_SIZE)) > 0) | ||
| { | ||
| bytesRead += readLen; | ||
| int64_t writeLen = outfile.write(data.data(), readLen); | ||
| if (writeLen < 0) | ||
| { | ||
| LOG(VB_GENERAL, LOG_ERR, | ||
| QString("UnZip: Failed to write %1/%2 bytes to output file %3") | ||
| .arg(writeLen).arg(readLen).arg(entry.m_fi.absoluteFilePath())); | ||
| return false; | ||
| } | ||
| bytesWritten += writeLen; | ||
| } | ||
|
|
||
| if ((entry.m_stats.size != bytesRead) || | ||
| (entry.m_stats.size != bytesWritten)) | ||
| { | ||
| LOG(VB_GENERAL, LOG_ERR, | ||
| QString("UnZip: Failed to copy file %1. Read %2 and wrote %3 of %4.") | ||
| .arg(entry.m_fi.fileName()).arg(bytesRead).arg(bytesWritten) | ||
| .arg(entry.m_stats.size)); | ||
| return false; | ||
| } | ||
|
|
||
| outfile.flush(); | ||
| zipSetFileAttributes(entry, outfile); | ||
| outfile.close(); | ||
| if (zip_fclose(infile) == -1) | ||
| { | ||
| LOG(VB_GENERAL, LOG_ERR, | ||
| QString("UnZip: Failed to close file at index %1 in %2") | ||
| .arg(entry.m_index).arg(entry.m_fi.fileName())); | ||
| return false; | ||
| } | ||
|
|
||
| return true; | ||
| } | ||
|
|
||
| bool UnZip::extractFile(const QString &outDirName) | ||
| { | ||
| if (!isValid()) | ||
| return false; | ||
|
|
||
| m_outDir = QDir(outDirName); | ||
| if (!m_outDir.exists()) | ||
| { | ||
| LOG(VB_GENERAL, LOG_ERR, | ||
| QString("UnZip: Target directory %1 doesn't exist") | ||
| .arg(outDirName)); | ||
| return false; | ||
| } | ||
|
|
||
| m_zipFileCount = zip_get_num_entries(m_zip, 0); | ||
| if (m_zipFileCount < 1) | ||
| { | ||
| LOG(VB_GENERAL, LOG_ERR, | ||
| QString("UnZip: Zip archive %1 is empty") | ||
| .arg(m_zipFileName)); | ||
| return false; | ||
| } | ||
|
|
||
| bool ok { true }; | ||
| for (auto index = 0; ok && (index < m_zipFileCount); index++) | ||
| { | ||
| zipEntry entry; | ||
| entry.m_index = index; | ||
| if (!getEntryStats(entry)) | ||
| return false; | ||
| if (entry.m_stats.encryption_method > 0) | ||
| { | ||
| LOG(VB_GENERAL, LOG_WARNING, | ||
| QString("UnZip: Skipping encryped file %1 in %2") | ||
| .arg(entry.m_index).arg(m_zipFileName)); | ||
| continue; | ||
| } | ||
| getEntryAttrs(entry); | ||
|
|
||
| entry.m_fi = QFileInfo(outDirName + '/' + entry.m_stats.name); | ||
| ok = zipValidateFilename(entry.m_fi); | ||
| if (ok) | ||
| ok = zipCreateDirectory(entry); | ||
| if (ok && (entry.m_stats.size > 0)) | ||
| { | ||
| switch (entry.m_attributes & ZIP_ATTR_FILE_TYPE_MASK) | ||
| { | ||
| case ZIP_ATTR_FILE_TYPE_SYMLINK: | ||
| ok = zipCreateSymlink(entry); | ||
| break; | ||
| default: | ||
| ok = zipWriteOneFile(entry); | ||
| break; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return ok; | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| /* | ||
| * Class UnZip | ||
| * | ||
| * Copyright (c) David Hampton 2021 | ||
| * | ||
| * This program is free software; you can redistribute it and/or modify | ||
| * it under the terms of the GNU General Public License as published by | ||
| * the Free Software Foundation; either version 2 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * This program is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License | ||
| * along with this program; if not, write to the Free Software | ||
| * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | ||
| */ | ||
|
|
||
| #include "zlib.h" | ||
| #undef Z_NULL | ||
| #define Z_NULL nullptr | ||
| #include "zip.h" | ||
|
|
||
| // Qt headers | ||
| #include <QDir> | ||
| #include <QString> | ||
| #include <QFileInfo> | ||
|
|
||
| class zipEntry; | ||
| class UnZip | ||
| { | ||
| // NOLINTNEXTLINE(readability-uppercase-literal-suffix) | ||
| static constexpr uint64_t kSTATS_REQUIRED {ZIP_STAT_NAME|ZIP_STAT_INDEX|ZIP_STAT_SIZE|ZIP_STAT_MTIME|ZIP_STAT_ENCRYPTION_METHOD}; | ||
|
|
||
| #define ZIP_ATTR_FILE_TYPE_MASK 0xFE000000 | ||
| #define ZIP_ATTR_FILE_TYPE_SYMLINK 0xA0000000 | ||
| #define ZIP_ATTR_FILE_TYPE_NORMAL 0x80000000 | ||
| #define ZIP_ATTR_USER_PERM_MASK 0x01C00000 | ||
| #define ZIP_ATTR_GROUP_PERM_MASK 0x03800000 | ||
| #define ZIP_ATTR_OTHER_PERM_MASK 0x00700000 | ||
| #define ZIP_ATTR_USER_PERM_SHIFT 22 | ||
| #define ZIP_ATTR_GROUP_PERM_SHIFT 19 | ||
| #define ZIP_ATTR_OTHER_PERM_SHIFT 16 | ||
|
|
||
| public: | ||
| UnZip(QString &zipFile); | ||
| ~UnZip(); | ||
| bool extractFile(const QString &outDir); | ||
|
|
||
| private: | ||
| bool isValid() { return m_zip != nullptr; }; | ||
| bool getEntryStats(zipEntry& entry); | ||
| void getEntryAttrs(zipEntry& entry); | ||
| static QFileDevice::Permissions zipToQtPerms(const zipEntry& entry); | ||
| static bool zipCreateDirectory(const zipEntry& entry); | ||
| bool zipValidateFilename(const QFileInfo& fi); | ||
| static void zipSetFileAttributes(const zipEntry& entry, QFile& outfile); | ||
| bool zipCreateSymlink(const zipEntry& entry); | ||
| bool zipWriteOneFile(const zipEntry& entry); | ||
|
|
||
| QDir m_outDir; | ||
|
|
||
| // Info about zip file itself | ||
| QString m_zipFileName; | ||
| zip_t *m_zip {nullptr}; | ||
| zip_int64_t m_zipFileCount {-1}; | ||
| }; | ||
|
|
||
| class zipEntry | ||
| { | ||
| friend class UnZip; | ||
| int m_index {0}; | ||
| zip_stat_t m_stats {}; | ||
| zip_uint32_t m_attributes {0}; | ||
| QFileInfo m_fi; | ||
| }; |