diff --git a/common/util/reload.py b/common/util/reload.py index 1633cd1af..953664acd 100644 --- a/common/util/reload.py +++ b/common/util/reload.py @@ -1,122 +1,233 @@ +import sublime +import sublime_plugin +import os +import posixpath +import threading import builtins import functools import importlib import sys -import types +from inspect import ismodule from contextlib import contextmanager +from .debug import StackMeter -import sublime_plugin -from .debug import StackMeter, trace +try: + from package_control.package_manager import PackageManager + def is_dependency(pkg_name): + return PackageManager()._is_dependency(pkg_name) -dprint = trace.for_tag("reload") +except ImportError: + def is_dependency(pkg_name): + return False def reload_plugin(): - """Reload the GitSavvy plugin among with all its modules.""" - from GitSavvy import git_savvy - dprint("begin", fill='═') + threading.Thread(target=functools.partial(reload_package, 'GitSavvy')).start() + + +def dprint(*args, fill=None, fill_width=60, **kwargs): + if fill is not None: + sep = str(kwargs.get('sep', ' ')) + caption = sep.join(args) + args = "{0:{fill}<{width}}".format(caption and caption + sep, + fill=fill, width=fill_width), + print("[Package Reloader]", *args, **kwargs) + + +def path_contains(a, b): + return a == b or b.startswith(a + os.sep) + + +def get_package_modules(pkg_name): + in_installed_path = functools.partial( + path_contains, + os.path.join( + sublime.installed_packages_path(), + pkg_name + '.sublime-package' + ) + ) + + in_package_path = functools.partial( + path_contains, + os.path.join(sublime.packages_path(), pkg_name) + ) + + def module_in_package(module): + file = getattr(module, '__file__', '') + paths = getattr(module, '__path__', ()) + return ( + in_installed_path(file) or any(map(in_installed_path, paths)) or + in_package_path(file) or any(map(in_package_path, paths)) + ) + + return { + name: module + for name, module in sys.modules.items() + if module_in_package(module) + } + + +def package_plugins(pkg_name): + return [ + pkg_name + '.' + posixpath.basename(posixpath.splitext(path)[0]) + for path in sublime.find_resources("*.py") + if posixpath.dirname(path) == 'Packages/' + pkg_name + ] + + +def reload_package(pkg_name, dummy=True, verbose=True): + if pkg_name not in sys.modules: + dprint("error:", pkg_name, "is not loaded.") + return + + if is_dependency(pkg_name): + dependencies, packages = resolve_dependencies(pkg_name) + else: + dependencies = set() + packages = {pkg_name} - modules = {name: module for name, module in sys.modules.items() - if name.startswith("GitSavvy.")} + if verbose: + dprint("begin", fill='=') + + all_modules = { + module_name: module + for pkg_name in dependencies | packages + for module_name, module in get_package_modules(pkg_name).items() + } + + # Tell Sublime to unload plugins + for pkg_name in packages: + for plugin in package_plugins(pkg_name): + module = sys.modules.get(plugin) + if module: + sublime_plugin.unload_module(module) + + # Unload modules + for module_name in all_modules: + sys.modules.pop(module_name) + + # Reload packages try: - reload_modules(git_savvy, modules) + with intercepting_imports(all_modules, verbose), importing_fromlist_aggresively(all_modules): + for pkg_name in packages: + for plugin in package_plugins(pkg_name): + sublime_plugin.reload_plugin(plugin) except Exception: - dprint("ERROR", fill='─') - reload_modules(git_savvy, modules, perform_reload=False) + dprint("reload failed.", fill='-') + reload_missing(all_modules, verbose) raise - finally: - ensure_loaded(git_savvy, modules) - dprint("end", fill='━') + if dummy: + load_dummy(verbose) + + if verbose: + dprint("end", fill='-') + + +def resolve_dependencies(root_name): + """Given the name of a dependency, return all dependencies and packages + that require that dependency, directly or indirectly. + """ + manager = PackageManager() + + all_packages = manager.list_packages() + all_dependencies = manager.list_dependencies() + + recursive_dependencies = set() + dependent_packages = set() + + dependency_relationships = { + name: manager.get_dependencies(name) + for name in all_packages + all_dependencies + } + + def rec(name): + if name in recursive_dependencies: + return + + recursive_dependencies.add(name) + + for dep_name in all_dependencies: + if name in dependency_relationships[dep_name]: + rec(dep_name) + for pkg_name in all_packages: + if name in dependency_relationships[pkg_name]: + dependent_packages.add(pkg_name) -def ensure_loaded(main, modules): - # More simple (comparing to reload_modules(perform_reload=False)) and dumb - # approach to ensure all modules are back. Quite useful when debugging the - # "reload" module itself, i.e. for cases when reloading might fail due to - # bugs in reload_modules(). + rec(root_name) + return (recursive_dependencies, dependent_packages) + + +def load_dummy(verbose): + """ + Hack to trigger automatic "reloading plugins". + + This is needed to ensure TextCommand's and WindowCommand's are ready. + """ + if verbose: + dprint("installing dummy package") + dummy = "_dummy_package" + dummy_py = os.path.join(sublime.packages_path(), "%s.py" % dummy) + with open(dummy_py, "w"): + pass + + def remove_dummy(trial=0): + if dummy in sys.modules: + if verbose: + dprint("removing dummy package") + try: + os.unlink(dummy_py) + except FileNotFoundError: + pass + after_remove_dummy() + elif trial < 300: + threading.Timer(0.1, lambda: remove_dummy(trial + 1)).start() + else: + try: + os.unlink(dummy_py) + except FileNotFoundError: + pass + + condition = threading.Condition() + + def after_remove_dummy(trial=0): + if dummy not in sys.modules: + condition.acquire() + condition.notify() + condition.release() + elif trial < 300: + threading.Timer(0.1, lambda: after_remove_dummy(trial + 1)).start() + + threading.Timer(0.1, remove_dummy).start() + condition.acquire() + condition.wait(30) # 30 seconds should be enough for all regular usages + condition.release() + + +def reload_missing(modules, verbose): missing_modules = {name: module for name, module in modules.items() if name not in sys.modules} if missing_modules: - for name, module in missing_modules: - sys.modules[name] = modules - print("GS [reload] BUG!", "restored", name) - sublime_plugin.reload_plugin(git_savvy.__name__) - - -def reload_modules(main, modules, perform_reload=True): - """Implements the machinery for reloading a given plugin module.""" - # - # Here's the approach in general: - # - # - Hide GitSavvy modules from the sys.modules temporarily; - # - # - Install a special import hook onto sys.meta_path; - # - # - Call sublime_plugin.reload_plugin(), which imports the main - # "git_savvy" module under the hood, triggering the hook; - # - # - The hook, instead of creating a new module object, peeks the saved - # one and reloads it. Once the module encounters an import statement - # requesting another module, not yet reloaded, the hook reenters and - # processes that new module recursively, then get back to the previous - # one, and so on. - # - # This makes the modules reload in the very same order as they were loaded - # initially, as if they were imported from scratch. - # - if perform_reload: - sublime_plugin.unload_module(main) - - # Insert the main "git_savvy" module at the beginning to make the reload - # order be as close to the order of the "natural" import as possible. - module_names = [main.__name__] + sorted(name for name in modules - if name != main.__name__) - - # First, remove all the loaded modules from the sys.modules cache, - # otherwise the reloading hook won't be called. - loaded_modules = dict(sys.modules) - for name in loaded_modules: - if name in modules: - del sys.modules[name] - - stack_meter = StackMeter() - - @FilteringImportHook.when(condition=lambda name: name in modules) - def module_reloader(name): - module = modules[name] - sys.modules[name] = module # restore the module back + if verbose: + dprint("reload missing modules") + for name in missing_modules: + if verbose: + dprint("reloading missing module", name) + sys.modules[name] = modules[name] - if perform_reload: - with stack_meter as depth: - dprint("reloading", ('╿ ' * depth) + '┡━─', name) - try: - return module.__loader__.load_module(name) - except Exception: - if name in sys.modules: - del sys.modules[name] # to indicate an error - raise - else: - if name not in loaded_modules: - dprint("NO RELOAD", '╺━─', name) - return module - - with intercepting_imports(module_reloader), \ - importing_fromlist_aggresively(modules): - # Now, import all the modules back, in order, starting with the main - # module. This will reload all the modules directly or indirectly - # referenced by the main one, i.e. usually most of our modules. - sublime_plugin.reload_plugin(main.__name__) - - # Be sure to bring back *all* the modules that used to be loaded, not - # only these imported through the main one. Otherwise, some of them - # might end up being created from scratch as new module objects in - # case of being imported after detaching the hook. In general, most of - # the imports below (if not all) are no-ops though. - for name in module_names: - importlib.import_module(name) + +@contextmanager +def intercepting_imports(modules, verbose): + finder = FilterFinder(modules, verbose) + sys.meta_path.insert(0, finder) + try: + yield + finally: + if finder in sys.meta_path: + sys.meta_path.remove(finder) @contextmanager @@ -125,59 +236,14 @@ def importing_fromlist_aggresively(modules): @functools.wraps(orig___import__) def __import__(name, globals=None, locals=None, fromlist=(), level=0): - # Given an import statement like this: - # - # from .some.module import something - # - # The original __import__ performs roughly the following steps: - # - # - Import ".some.module", just like the importlib.import_module() - # function would do, i.e. resolve packages, calculate the absolute - # name, check sys.modules for that module, invoke import hooks and - # so on... - # - # - For each name specified in the "fromlist" (a "something" in our - # case), ensure the module have that name in its namespace. This - # could be: - # - # - a regular name defined within that module, like a function - # named "something", and in this case we're done; - # - # - or, in case the module is missing that attribute, there's a - # chance that the requested name refers to a submodule of that - # module, ".some.module.something", and we need to import it. - # Once imported it will take care to register itself within - # the parent's namespace. - # - # This looks natural and it is indeed in case of loading a module for - # the first time. But things start to behave slightly different once - # you try to reload a module. - # - # The main difference is that during the reload the module code is - # executed with its dictionary retained. And this has an undesired - # effect on handling the "fromlist" as described above: the second - # part (involving import of a submodule) is only executed when the - # module dictionary is missing the submodule name, which is not the - # case during the reload. - # - # This is generally not a problem: the name refers to the submodule - # imported earlier anyway. But we need to import it in order to force - # the necessary hook to reload that submodule too. - module = orig___import__(name, globals, locals, fromlist, level) if fromlist and module.__name__ in modules: - # Refer to _handle_fromlist() from "importlib/_bootstrap.py" if '*' in fromlist: fromlist = list(fromlist) fromlist.remove('*') fromlist.extend(getattr(module, '__all__', [])) for x in fromlist: - # Here's an altered part of logic. - # - # The original __import__ doesn't even try to import a - # submodule if its name is already in the module namespace, - # but we do that for certain set of the known submodule. - if isinstance(getattr(module, x, None), types.ModuleType): + if ismodule(getattr(module, x, None)): from_name = '{}.{}'.format(module.__name__, x) if from_name in modules: importlib.import_module(from_name) @@ -190,31 +256,25 @@ def __import__(name, globals=None, locals=None, fromlist=(), level=0): builtins.__import__ = orig___import__ -@contextmanager -def intercepting_imports(hook): - sys.meta_path.insert(0, hook) - try: - yield hook - finally: - if hook in sys.meta_path: - sys.meta_path.remove(hook) - - -class FilteringImportHook: - """ - PEP-302 importer that delegates loading of given modules to a function. - """ - - def __init__(self, condition, load_module): - super().__init__() - self.condition = condition - self.load_module = load_module - - @classmethod - def when(cls, condition): - """A handy loader function decorator.""" - return lambda load_module: cls(condition, load_module) +class FilterFinder: + def __init__(self, modules, verbose): + self._modules = modules + self._stack_meter = StackMeter() + self._verbose = verbose def find_module(self, name, path=None): - if self.condition(name): + if name in self._modules: return self + + def load_module(self, name): + module = self._modules[name] + sys.modules[name] = module # restore the module back + with self._stack_meter as depth: + if self._verbose: + dprint("reloading", ('| ' * depth) + '|--', name) + try: + return module.__loader__.load_module(name) + except Exception: + if name in sys.modules: + del sys.modules[name] # to indicate an error + raise