diff --git a/Monika After Story/game/0imports.rpy b/Monika After Story/game/0imports.rpy index e7d8835610..ea8ecca472 100644 --- a/Monika After Story/game/0imports.rpy +++ b/Monika After Story/game/0imports.rpy @@ -1,470 +1,26 @@ # mas can import paradigm. -# TODO: This will need to be migrated to use the builtin renpy certifi/ssl packages -# Likewise, we should migrate dynamic imports to use our new `require()` function to keep consistent. -# Additionally, let's look into seeing if we can simplify this, I feel like this is a bit overcomplicated for an ssl/certifi import. +# see python-packages/mas/can_import/masimport.py for details init -1500 python in mas_can_import: + + import renpy + import store.mas_utils as mas_utils + import mas.can_import.masimport as masimport + from mas.can_import import MASImport_ssl, MASImport_certifi + # set importables certifi = MASImport_certifi() + certifi._set_log(mas_utils.mas_log) + ssl = MASImport_ssl() + ssl._set_log(mas_utils.mas_log) # run checks - check_imports() - - -init -1505 python in mas_can_import: - import os - import platform - import datetime - import threading - - import store - from store.mas_threading import MASAsyncWrapper - - # initialize known imports. - - class MASImport_certifi(MASImport): - """ - certifi import - - This also can do cert updating: - 1. call `start_cert_update` to begin the cert update. - - this will return the promise, but you can also use - the following cert update functions to check the status - if you don't want to keep the promise. - 2. call `check_cert_update` to determine if the update is still - running or is completed. - - once the update is completed, all appropriate vars will be - set automatically. - 3. call `get_cert_update` to get the returned value from the - cert update promise. - 4. call `reset_cert_update` to cleanup the cert updater thread. - this must be called before doing another cert update. - - Auto (startup) cert updating happens once every 6 months. - """ - - def __init__(self): - """ - Constructor - """ - super(MASImport_certifi, self).__init__("certifi") - - self.__cert_available = False - self.__cert_update_promise = MASAsyncWrapper(self._update_cert) - - self.__cert_available_lock = threading.Lock() - self.__cert_available_cond = threading.Condition( - self.__cert_available_lock - ) - - @property - def cert_available(self): - """ - True if cert is available. - """ - return self.__th_get_cert_avail() - - def import_try(self): - """ - Also checks if a cert is available. - """ - import certifi - certifi.set_parent_dir(renpy.config.gamedir) - self.__th_set_cert_avail(certifi.has_cert()) - - # start the cert update - this will be checked later. - self.start_cert_update() - - return True - - def import_except(self, err): - # kill the update thread and mark unavailble cert - # updater thread probably failed so we just wont bother with it. - self.__th_set_cert_avail(False) - - def load(self): - try: - super(MASImport_certifi, self).load() - except AttributeError as e: - self.__th_set_cert_avail(False) - - def start_cert_update(self, force=False): - """ - Starts the cert update process. - - NOTE: will NOT start the process if the updater is currently - running. The promise is still returned though. - - IN: - force - True to force the cert to update now. - - RETURNS: the cert update promise - """ - if not self.__cert_update_promise.ready: - return self.__cert_update_promise - - # update force arg - self.__cert_update_promise._th_kwargs = {"force": force} - - # begin update - self.__cert_update_promise.start() - return self.__cert_update_promise - - def check_cert_update(self): - """ - checks the status of the cert update. - - RETURNS: True if the cert update is done, False if not - """ - return self.__cert_update_promise.done() - - def get_cert_update(self): - """ - Gets the result from the cert update. - - RETURNS: certifi RV value of the check_update function, or None if - the check_update function could not run. THIS WILL ALSO RETURN - NONE IF THE CERT UPDATE IS NOT FINISHED. - Use `check_cert_update` to check that the update is done - before calling this. - """ - return self.__cert_update_promise.get() - - def reset_cert_update(self): - """ - Resets the cert update thread. This may fail if the cert update is - not done yet. - """ - self.__cert_update_promise.end() - - def is_cert_update_running(self): - """ - Determines if the cert update is running. - - RETURNS: True if the cert update is running. - """ - return not self.check_cert_update() - - def ch30_day_cert_update(self): - """ - Call this during ch30_day to handle cert update checks - """ - if not self.__cert_update_promise.ready: - - if self.is_cert_update_running(): - # cert thread not done - assume thread is running and will - # finish later - return - - # reset cert update so we can start the thread - self.reset_cert_update() - - self.start_cert_update() - - def _update_cert(self, force=False): - """ - Updates the cert and sets appropriate vars. - - Certs will only be updated if the last cert update was at least - 6 months ago. - - IN: - force - True to force the cert to update right now - - RETURNS: certifi RV value of the check_update function, or None if - the check_update function could not run. - """ - last_update = store.persistent._mas_last_cert_update - if last_update is None: - last_update = datetime.datetime.utcnow() - - rv = None - - if self.enabled: - if ( - force - or ( - datetime.datetime.utcnow() - last_update - ) > datetime.timedelta(days=180) - ): - import certifi - - rv, response = certifi.check_update() - if rv in (certifi.RV_SUCCESS, certifi.RV_NO_UPDATE): - self.__th_set_cert_avail(certifi.has_cert()) - - store.persistent._mas_last_cert_update = last_update - - return rv - - def __th_get_cert_avail(self): - """ - thread-safe get cert available - for internal use only - - RETURNS: cert_available value - """ - self.__cert_available_cond.acquire() - cert_avail = self.__cert_available - self.__cert_available_cond.release() - - return cert_avail - - def __th_set_cert_avail(self, value): - """ - Thread-safe set cert available - for internal use only - - IN: - value - value to set to cert_available - """ - self.__cert_available_cond.acquire() - self.__cert_available = value - self.__cert_available_cond.release() - - - class MASImport_ssl(MASImport): - """ - SSL Import - """ - - def __init__(self): - """ - Constructor - """ - super(MASImport_ssl, self).__init__("ssl", set_sys=True) - - def import_try(self): - """ - Also adjusts httplib. - - this is a hack. - """ - import sys - - ssl_pkg = "python-packages/ssl" - bit64 = "x86_64" - bit32 = "i686" - new_ssl = None - - if platform.system() == "Windows": - sys.path.append(os.path.normcase(os.path.join( - renpy.config.gamedir, - ssl_pkg, - "windows/32/", - ))) - - import win32_ssl - new_ssl = win32_ssl - - elif platform.system() == "Linux": - - if platform.machine() == bit64: - sys.path.append(os.path.normcase(os.path.join( - renpy.config.gamedir, - ssl_pkg, - "linux/64/", - ))) - - import linux64_ssl - new_ssl = linux64_ssl - - elif platform.machine() == bit32: - sys.path.append(os.path.normcase(os.path.join( - renpy.config.gamedir, - ssl_pkg, - "linux/32/", - ))) - - import linux32_ssl - new_ssl = linux32_ssl - - elif platform.system() == "Darwin" and platform.machine() == bit64: - sys.path.append(os.path.normcase(os.path.join( - renpy.config.gamedir, - ssl_pkg, - "mac/64/", - ))) - - import mac64_ssl - new_ssl = mac64_ssl - - if new_ssl is None: - return False - - # overwrties httplib's ssl ref so later HTTPS will work. - self._set_sys_module(new_ssl) - import httplib - httplib.ssl = new_ssl - return True + masimport.check_imports(renpy) init -1510 python in mas_can_import: - import store.mas_logging as mas_logging - import store.mas_utils as mas_utils - - # new data pattern - import store.mas_can_import_data as Data - - - class MASImport(object): - """ - Wrapper around import checks for MAS-based imports. - All conditional imports should extend this class and implement all - required functions. - Use this before actually running a third-party import. - All functionality that relies on third-party imports should be capable - of being turned off. - """ - _IMPORT_ERR = "Failed to import `{0}`: {1}" - - def __init__(self, module_name, set_sys=False): - """ - Constructor - - IN: - module_name - the name of the module to import - set_sys - pass True to allow this to overwrite the sys modules. - This should only be used in cases where you need to - override a system (aka built-in) module. - (Default: False) - """ - self.__module_name = module_name - self.__set_sys = set_sys - self.__enabled = False - - # add to imports dict - # crashes if already exists - if module_name in Data.imports: - raise Exception( - "duplicate import object - {0}".format(module_name) - ) - - Data.imports[self.__module_name] = self - - def __call__(self): - """ - Just returns if this is enabled or not - """ - return self.enabled - - @property - def enabled(self): - """ - True if this import is enabled (can be imported) - """ - return self.__enabled - - @property - def module_name(self): - """ - module name for this import - """ - return self.__module_name - - @property - def set_sys(self): - """ - True if sys module can be overwritten by this. - """ - return self.__set_sys - - def log_import_error(self, exp=None): - """ - Logs import error message. - - IN: - exp - current exception if available. - (Default: None) - """ - mas_log = mas_utils.init_mas_log() - mas_log.error(self._IMPORT_ERR.format( - self.module_name, - "" if exp is None else repr(exp) - )) - - def import_try(self): - """ - DERIVED CLASSES MUST IMPLEMENT THIS - - This should check if the import should be enabled. - For more info, see `load`. - - RETURNS: True if the import should be enabled, False otherwise. - """ - raise NotImplementedError - - def import_except(self, err): - """ - Runs if an ImportError is encountered. The import will always - be disabled and an import error is logged after this runs. - - If you need to run additional behavior or set other vars on a - failed import, override this function. - - For more info, see `load`. - - IN: - err - the exception that was raised - """ - pass - - def load(self): - """ - Loads the import and checks that it works. This is called - sometime before init level -1000. - - This will call `import_try` and mark the this import as enabled if - appropriate. - - If an ImportError/AttributeError/NameError is triggered, - disable this import, log an import error, and call `import_except. - - If any other errors occur, the import will be disabled and - an import error will be logged, but the error will percolate up. - If you wish to catch those errors, override this function and wrap - a try-except block around the base call. - """ - try: - self.__enabled = self.import_try() - - except (ImportError, AttributeError, NameError) as e: - self.__enabled = False - self.log_import_error(e) - self.import_except(e) - - except Exception as e: - self.__enabled = False - self.log_import_error(e) - raise e - - def _set_sys_module(self, module): - """ - Sets the system module so this import can be used elsewhere. - Not called by the load system - you must call this manually - if desired. - - This is also guardrailed on construction so no injection. - - IN: - module - the imported module - """ - if self.set_sys: - import sys - sys.modules[self.module_name] = module - - - def check_imports(): - """ - checks import availability - """ - for module_name in Data.imports: - Data.imports[module_name].load() - -init -1511 python in mas_can_import_data: - # data store + from mas.can_import import MASImport - imports = {} - # key: module name - # value: MASImport object diff --git a/Monika After Story/game/python-packages/mas/__init__.py b/Monika After Story/game/python-packages/mas/__init__.py new file mode 100644 index 0000000000..139597f9cb --- /dev/null +++ b/Monika After Story/game/python-packages/mas/__init__.py @@ -0,0 +1,2 @@ + + diff --git a/Monika After Story/game/python-packages/mas/can_import/__init__.py b/Monika After Story/game/python-packages/mas/can_import/__init__.py new file mode 100644 index 0000000000..c301f0d005 --- /dev/null +++ b/Monika After Story/game/python-packages/mas/can_import/__init__.py @@ -0,0 +1,3 @@ + +from .masimport import MASImport +from .exports import MASImport_ssl, MASImport_certifi \ No newline at end of file diff --git a/Monika After Story/game/python-packages/mas/can_import/exports.py b/Monika After Story/game/python-packages/mas/can_import/exports.py new file mode 100644 index 0000000000..d204a01a50 --- /dev/null +++ b/Monika After Story/game/python-packages/mas/can_import/exports.py @@ -0,0 +1,36 @@ + + +from .masimport import MASImport +from ..renpyspec import Renpy + + +class MASImport_certifi(MASImport): + """ + certifi import + """ + + def __init__(self): + """ + Constructor + """ + super().__init__("certifi") + + def import_try(self, renpy: Renpy = None): + import certifi + return True + + +class MASImport_ssl(MASImport): + """ + SSL Import + """ + + def __init__(self): + """ + Constructor + """ + super().__init__("ssl") + + def import_try(self, renpy: Renpy = None): + import ssl + return True \ No newline at end of file diff --git a/Monika After Story/game/python-packages/mas/can_import/masimport.py b/Monika After Story/game/python-packages/mas/can_import/masimport.py new file mode 100644 index 0000000000..4512d40bf8 --- /dev/null +++ b/Monika After Story/game/python-packages/mas/can_import/masimport.py @@ -0,0 +1,174 @@ + +from typing import Dict + +from .. renpyspec import Renpy + +_imports: Dict[str, 'MASImport'] = {} +# key: module name +# value: MASImport object + + +def check_imports(renpy: Renpy): + """ + checks import availability + + :param renpy: renpy object for import usage + """ + for module_name in _imports: + _imports[module_name].load(renpy) + + +class MASImport(): + """ + Wrapper around import checks for MAS-based imports. + All conditional imports should extend this class and implement all + required functions. + Use this before actually running a third-party import. + All functionality that relies on third-party imports should be capable + of being turned off. + """ + _IMPORT_ERR = "Failed to import `{0}`: {1}" + + def __init__(self, module_name: str, set_sys: bool = False): + """ + Constructor + + :param module_name: module being imported + :param set_sys: pass True to allow this to overwrite the sys modules. + This should only be used in cases wher eyou need to override a system module. + (Default: False) + """ + + self.__module_name = module_name + self.__set_sys = set_sys + self.__enabled = False + self.mas_log = None + + # add to imports dict + # crashes if already exists + if module_name in _imports: + raise Exception( + "duplicate import object - {0}".format(module_name) + ) + + _imports[self.__module_name] = self + + def _set_log(self, log): + """ + set the mas log (temp use only) + :param log: log to set mas log to + """ + self.mas_log = log + + def __call__(self): + """ + Just returns if this is enabled or not + """ + return self.enabled + + @property + def enabled(self): + """ + True if this import is enabled (can be imported) + """ + return self.__enabled + + @property + def module_name(self): + """ + module name for this import + """ + return self.__module_name + + @property + def set_sys(self): + """ + True if sys module can be overwritten by this. + """ + return self.__set_sys + + def log_import_error(self, exp=None): + """ + Logs import error message. + + IN: + exp - current exception if available. + (Default: None) + """ + self.mas_log.error(self._IMPORT_ERR.format( + self.module_name, + "" if exp is None else repr(exp) + )) + + def import_try(self, renpy: Renpy = None): + """ + DERIVED CLASSES MUST IMPLEMENT THIS + + This should check if the import should be enabled. + For more info, see `load`. + + :param renpy: renpy object for usage by derived classes + :returns: True if the import should be enabled, False otherwise. + """ + raise NotImplementedError + + def import_except(self, err, renpy: Renpy = None): + """ + Runs if an ImportError is encountered. The import will always + be disabled and an import error is logged after this runs. + + If you need to run additional behavior or set other vars on a + failed import, override this function. + + For more info, see `load`. + + :param err: the exception that was raised + :param renpy: renpy object for usage by derived classes + """ + pass + + def load(self, renpy: Renpy = None): + """ + Loads the import and checks that it works. This is called + sometime before init level -1000. + + This will call `import_try` and mark the this import as enabled if + appropriate. + + If an ImportError/AttributeError/NameError is triggered, + disable this import, log an import error, and call `import_except. + + If any other errors occur, the import will be disabled and + an import error will be logged, but the error will percolate up. + If you wish to catch those errors, override this function and wrap + a try-except block around the base call. + + :param renpy: renpy object for usage by dervied classes + """ + try: + self.__enabled = self.import_try(renpy=renpy) + + except (ImportError, AttributeError, NameError) as e: + self.__enabled = False + self.log_import_error(e) + self.import_except(e, renpy=renpy) + + except Exception as e: + self.__enabled = False + self.log_import_error(e) + raise e + + def _set_sys_module(self, module): + """ + Sets the system module so this import can be used elsewhere. + Not called by the load system - you must call this manually + if desired. + + This is also guardrailed on construction so no injection. + + IN: + module - the imported module + """ + if self.set_sys: + import sys + sys.modules[self.module_name] = module \ No newline at end of file diff --git a/Monika After Story/game/python-packages/mas/threading/__init__.py b/Monika After Story/game/python-packages/mas/threading/__init__.py new file mode 100644 index 0000000000..d99e192154 --- /dev/null +++ b/Monika After Story/game/python-packages/mas/threading/__init__.py @@ -0,0 +1,2 @@ + +from .asyncwrapper import MASAsyncWrapper \ No newline at end of file diff --git a/Monika After Story/game/python-packages/mas/threading/asyncwrapper.py b/Monika After Story/game/python-packages/mas/threading/asyncwrapper.py new file mode 100644 index 0000000000..1bbfabea22 --- /dev/null +++ b/Monika After Story/game/python-packages/mas/threading/asyncwrapper.py @@ -0,0 +1,170 @@ + +import threading + +from typing import Callable + +class MASAsyncWrapper(object): + """ + Class designed for calling a function asynchronously and returning + the result. + + This uses threading. + + NOTE: we are not using context managers because we might have an + issue with them on macs + + TODO: you cannot use this to spawm multiple calls of the same function + if we need that, then we make extension of this class + + Main things are: + 1. storing the function to call when threading + 2. not spawning a new thread if the previous one isn't done yet + (adjustable) + 3. check function that checks status of thread's return value + 4. closing the spawn'ed thread + + PROPERTIES: + ready - True means we are ready to spawn a thread, False means we + are either waiting for the thread or waiting for main thread + to retrieve the value. + + PRIVATE PROPERTIES: + _th_lock - the threading lock we are using for var checking + _th_cond - the threading condition for var checking + _th_result - data returned from the function + _th_function - the function we are calling + _th_args - args to pass into the function + _th_kwargs - kwargs to pass into the function + _th_thread - thread object + _th_done - True means the thread has returned and set the value + False means thread is still running + """ + + def __init__( + self, + async_fun: Callable, + async_args: list = None, + async_kwargs: dict = None + ): + """ + Constructor + + :param async_fun: function to call asyncs + :param async_args: list of arguments to send to the async function + :param async_kwargs: dict of keyword args to send to the async function + """ + + # setup threading stuff + self._th_lock = threading.Lock() + self._th_cond = threading.Condition(self._th_lock) + self._th_result = None + self._th_function = async_fun + self._th_args = async_args if async_args else [] + self._th_kwargs = async_kwargs if async_kwargs else {} + self._th_thread = None + self._th_done = True + self.ready = True + + # check for key things + if ( + self._th_function is None + or self._th_args is None + or self._th_kwargs is None + ): + self.ready = False + + + def done(self): + """ + Returns true if the thread is Done and has returned data, False + otherwise. + """ + if self.ready: + return True + + is_done = False + + # lock and check + self._th_cond.acquire() + is_done = self._th_done + self._th_cond.release() + + return is_done + + + def end(self): + """ + Resets thread status vars and more so we can spawn a new thread. + + Checks if the thread is done before doing any resets + """ + if self.ready or not self.done(): + return + + # otherwise we can reset + self.__end() + + + def get(self): + """ + Retrieves value set by thread and resets everything so we can + spawn a new thread. + + RETURNS: + value returned from async call. + or None if the async call is still returning. (or if your + async call returned None) + """ + # dont need to waste time here if we arent running anything + if self.ready: + return self._th_result + + # dont do anything if we arent done + if not self.done(): + return None + + # otherwise we are DONE and we return the result + ret_val = self._th_result + self.__end() + + return ret_val + + + def start(self): + """ + Starts the threaded function call. + """ + if not self.ready: + return + + # otherwise time to make new thread I guess + self._th_done = False + self._th_result = None + self.ready = False + self._th_thread = threading.Thread(target=self._th_start) + self._th_thread.daemon = True + self._th_thread.start() + + + def _th_start(self): + """ + Actually runs the async function and sets the result var + appropriately. + """ + temp_result = self._th_function(*self._th_args, **self._th_kwargs) + + # acquire lock and set the result var + self._th_cond.acquire() + self._th_result = temp_result + self._th_done = True + self._th_cond.release() + + + def __end(self): + """ + Resets vars so we can spawn a new thred. Does NOT check if the + thread is done. + """ + self._th_result = None + self._th_done = True + self.ready = True diff --git a/Monika After Story/game/zz_threading.rpy b/Monika After Story/game/zz_threading.rpy index 56ab560834..500ce60f44 100644 --- a/Monika After Story/game/zz_threading.rpy +++ b/Monika After Story/game/zz_threading.rpy @@ -1,170 +1,3 @@ init -2000 python in mas_threading: - # threading related vars - import threading - - class MASAsyncWrapper(object): - """ - Class designed for calling a function asynchronously and returning - the result. - - This uses threading. - - NOTE: we are not using context managers because we might have an - issue with them on macs - - TODO: you cannot use this to spawm multiple calls of the same function - if we need that, then we make extension of this class - - Main things are: - 1. storing the function to call when threading - 2. not spawning a new thread if the previous one isn't done yet - (adjustable) - 3. check function that checks status of thread's return value - 4. closing the spawn'ed thread - - PROPERTIES: - ready - True means we are ready to spawn a thread, False means we - are either waiting for the thread or waiting for main thread - to retrieve the value. - - PRIVATE PROPERTIES: - _th_lock - the threading lock we are using for var checking - _th_cond - the threading condition for var checking - _th_result - data returned from the function - _th_function - the function we are calling - _th_args - args to pass into the function - _th_kwargs - kwargs to pass into the function - _th_thread - thread object - _th_done - True means the thread has returned and set the value - False means thread is still running - """ - - def __init__(self, - async_fun, - async_args=[],# FIXME: mutable obj as default - async_kwargs={} - ): - """ - IN: - async_fun - function to call asynchronously - async_args - list of arguments to send to the async function - (Default: []) - async_kwargs - dict of keyword args to send to the async - function - (Default: {}) - """ - # setup threading stuff - self._th_lock = threading.Lock() - self._th_cond = threading.Condition(self._th_lock) - self._th_result = None - self._th_function = async_fun - self._th_args = async_args - self._th_kwargs = async_kwargs - self._th_thread = None - self._th_done = True - self.ready = True - - # check for key things - if ( - self._th_function is None - or self._th_args is None - or self._th_kwargs is None - ): - self.ready = False - - - def done(self): - """ - Returns true if the thread is Done and has returned data, False - otherwise. - """ - if self.ready: - return True - - is_done = False - - # lock and check - self._th_cond.acquire() - is_done = self._th_done - self._th_cond.release() - - return is_done - - - def end(self): - """ - Resets thread status vars and more so we can spawn a new thread. - - Checks if the thread is done before doing any resets - """ - if self.ready or not self.done(): - return - - # otherwise we can reset - self.__end() - - - def get(self): - """ - Retrieves value set by thread and resets everything so we can - spawn a new thread. - - RETURNS: - value returned from async call. - or None if the async call is still returning. (or if your - async call returned None) - """ - # dont need to waste time here if we arent running anything - if self.ready: - return self._th_result - - # dont do anything if we arent done - if not self.done(): - return None - - # otherwise we are DONE and we return the result - ret_val = self._th_result - self.__end() - - return ret_val - - - def start(self): - """ - Starts the threaded function call. - """ - if not self.ready: - return - - # otherwise time to make new thread I guess - self._th_done = False - self._th_result = None - self.ready = False - self._th_thread = threading.Thread(target=self._th_start) - self._th_thread.daemon = True - self._th_thread.start() - - - def _th_start(self): - """ - Actually runs the async function and sets the result var - appropriately. - """ - temp_result = self._th_function(*self._th_args, **self._th_kwargs) - - # acquire lock and set the result var - self._th_cond.acquire() - self._th_result = temp_result - self._th_done = True - self._th_cond.release() - - - def __end(self): - """ - Resets vars so we can spawn a new thred. Does NOT check if the - thread is done. - """ - self._th_result = None - self._th_done = True - self.ready = True + from mas.threading import MASAsyncWrapper \ No newline at end of file