diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py index 1f4d8fc6..9729b8a1 100644 --- a/ansible_mitogen/loaders.py +++ b/ansible_mitogen/loaders.py @@ -49,7 +49,7 @@ __all__ = [ ANSIBLE_VERSION_MIN = (2, 10) -ANSIBLE_VERSION_MAX = (2, 13) +ANSIBLE_VERSION_MAX = (2, 16) NEW_VERSION_MSG = ( "Your Ansible version (%s) is too recent. The most recent version\n" diff --git a/ansible_mitogen/plugins/connection/mitogen_ssh.py b/ansible_mitogen/plugins/connection/mitogen_ssh.py index 75f2d42f..2cfbee2f 100644 --- a/ansible_mitogen/plugins/connection/mitogen_ssh.py +++ b/ansible_mitogen/plugins/connection/mitogen_ssh.py @@ -75,6 +75,7 @@ import ansible_mitogen.loaders class Connection(ansible_mitogen.connection.Connection): transport = 'ssh' + # It actually returns a namedtuple 'ansible.plugins.loader.get_with_context_result' vanilla_class = ansible_mitogen.loaders.connection_loader__get( 'ssh', class_only=True, @@ -84,4 +85,5 @@ class Connection(ansible_mitogen.connection.Connection): def _create_control_path(*args, **kwargs): """Forward _create_control_path() to the implementation in ssh.py.""" # https://github.com/dw/mitogen/issues/342 + # print("mitogen_ssh.Connection._create_control_path", type(vanilla_class)) return Connection.vanilla_class._create_control_path(*args, **kwargs) diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index 63caa88a..453302f2 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -185,8 +185,8 @@ def _setup_responder(responder): Configure :class:`mitogen.master.ModuleResponder` to only permit certain packages, and to generate custom responses for certain modules. """ - responder.whitelist_prefix('ansible') - responder.whitelist_prefix('ansible_mitogen') + #responder.whitelist_prefix('ansible') + #responder.whitelist_prefix('ansible_mitogen') # Ansible 2.3 is compatible with Python 2.4 targets, however # ansible/__init__.py is not. Instead, executor/module_common.py writes @@ -285,7 +285,13 @@ class Broker(mitogen.master.Broker): the exuberant syscall expense of EpollPoller, so override it and restore the poll() poller. """ - poller_class = mitogen.core.Poller + # poller_class = mitogen.core.Poller + pass + + if mitogen.parent.PollPoller.SUPPORTED: + poller_class = mitogen.parent.PollPoller + else: + poller_class = mitogen.core.Poller class Binding(object): diff --git a/ansible_mitogen/strategy.py b/ansible_mitogen/strategy.py index 0a98e316..257a4e31 100644 --- a/ansible_mitogen/strategy.py +++ b/ansible_mitogen/strategy.py @@ -88,7 +88,9 @@ def wrap_action_loader__get(name, *args, **kwargs): get_kwargs = {'class_only': True} if name in ('fetch',): name = 'mitogen_' + name - get_kwargs['collection_list'] = kwargs.pop('collection_list', None) + get_kwargs['collection_list'] = None + else: + get_kwargs['collection_list'] = kwargs.pop('collection_list', None) klass = ansible_mitogen.loaders.action_loader__get(name, **get_kwargs) if klass: diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 5fc78185..e52a1ce8 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -475,12 +475,13 @@ class PlayContextSpec(Spec): ) def ssh_args(self): + local_vars = self._task_vars.get("hostvars", {}).get(self._inventory_name, {}) return [ mitogen.core.to_text(term) for s in ( - C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})), - C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})), - C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})) + C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=local_vars), + C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=local_vars), + C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=local_vars) ) for term in ansible.utils.shlex.shlex_split(s or '') ] @@ -707,12 +708,13 @@ class MitogenViaSpec(Spec): ) def ssh_args(self): + local_vars = self._task_vars.get("hostvars", {}).get(self._inventory_name, {}) return [ mitogen.core.to_text(term) for s in ( - C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})), - C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})), - C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})) + C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=local_vars), + C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=local_vars), + C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=local_vars) ) for term in ansible.utils.shlex.shlex_split(s) if s diff --git a/mitogen/core.py b/mitogen/core.py index bee722e6..efa2a977 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -58,9 +58,17 @@ import warnings import weakref import zlib -# Python >3.7 deprecated the imp module. -warnings.filterwarnings('ignore', message='the imp module is deprecated') -import imp +has_importlib = False +try: + import importlib + # importlib.util is available in importlib package, but sometimes + # it is not. This was observed to vary, despite exactly same Python + # version installed on different machines. + import importlib.util + from importlib.machinery import ModuleSpec + has_importlib = True +except: + has_importlib = False # Absolute imports for <2.5. select = __import__('select') @@ -95,6 +103,15 @@ try: except NameError: ModuleNotFoundError = ImportError +has_imp = False +try: + import imp + has_imp = True +except ModuleNotFoundError: + has_imp = False + +assert has_imp or has_importlib + # TODO: usage of 'import' after setting __name__, but before fixing up # sys.modules generates a warning. This happens when profiling = True. warnings.filterwarnings('ignore', @@ -300,6 +317,19 @@ class Kwargs(dict): return (Kwargs, (dict(self),)) +AnsibleUnsafeText = None + +def lazy_AnsibleUnsafeText(): + global AnsibleUnsafeText + if AnsibleUnsafeText is not None: + return AnsibleUnsafeText + mod = __import__("ansible.utils.unsafe_proxy", fromlist=("AnsibleUnsafeText",)) + AnsibleUnsafeText = getattr(mod, "AnsibleUnsafeText") + assert type(AnsibleUnsafeText) is type, f"AnsibleUnsafeText {AnsibleUnsafeText} is not a type" + assert callable(AnsibleUnsafeText), f"AnsibleUnsafeText {AnsibleUnsafeText} is not callable" + return AnsibleUnsafeText + + class CallError(Error): """ Serializable :class:`Error` subclass raised when :meth:`Context.call() @@ -767,6 +797,9 @@ else: _Unpickler = pickle.Unpickler +PICKLE_PROTOCOL = 2 + + class Message(object): """ Messages are the fundamental unit of communication, comprising fields from @@ -860,6 +893,8 @@ class Message(object): return Secret elif func == 'Kwargs': return Kwargs + elif module == 'ansible.utils.unsafe_proxy' and func == 'AnsibleUnsafeText': + return lazy_AnsibleUnsafeText() elif module == '_codecs' and func == 'encode': return self._unpickle_bytes elif module == '__builtin__' and func == 'bytes': @@ -895,10 +930,10 @@ class Message(object): """ self = cls(**kwargs) try: - self.data = pickle__dumps(obj, protocol=2) + self.data = pickle__dumps(obj, protocol=PICKLE_PROTOCOL) except pickle.PicklingError: e = sys.exc_info()[1] - self.data = pickle__dumps(CallError(e), protocol=2) + self.data = pickle__dumps(CallError(e), protocol=PICKLE_PROTOCOL) return self def reply(self, msg, router=None, **kwargs): @@ -965,6 +1000,7 @@ class Message(object): try: obj = unpickler.load() except: + LOG.exception("unpickler.load exception") LOG.error('raw pickle was: %r', self.data) raise self._unpickled = obj @@ -1360,17 +1396,44 @@ class Importer(object): if fullname == '__main__': raise ModuleNotFoundError() + assert hasattr(_tls, 'running') + parent, _, modname = str_rpartition(fullname, '.') if parent: path = sys.modules[parent].__path__ else: path = None + if has_importlib: + # Python 3.4+ + if hasattr(importlib.util, 'find_spec'): + # spec = importlib.util.find_spec(modname, package=parent or None) + spec = importlib.util.find_spec(fullname) + if spec: + return # Good + else: + # raise ImportError() + raise ModuleNotFoundError() + + # Python 3.3 and earlier + loader = importlib.find_loader(modname, path) + if loader: + return # Good + else: + # raise ImportError() + raise ModuleNotFoundError() + + assert has_imp fp, pathname, description = imp.find_module(modname, path) if fp: fp.close() + return # Good + def find_module(self, fullname, path=None): + _vv and self._log.debug('') + _vv and self._log.debug('%r.find_module %r %r called', self, fullname, path) + """ Return a loader (ourself) or None, for the module with fullname. @@ -1378,9 +1441,12 @@ class Importer(object): Deprecrated in Python 3.4+, replaced by find_spec(). Raises ImportWarning in Python 3.10+. - fullname A (fully qualified?) module name, e.g. "os.path". + fullname A fully qualified module name, e.g. "os.path". path __path__ of parent packge. None for a top level module. """ + + # This is to break recursion to itself due to the use of + # module import attempt via self.builtin_find_module if hasattr(_tls, 'running'): return None @@ -1406,15 +1472,28 @@ class Importer(object): if any(fullname.startswith(s) for s in self.whitelist): return self + if fullname.startswith("ansible") or fullname.startswith("ansible_mitogen"): + return self + + _vv and self._log.debug('checking if %r is available locally', fullname) + try: self.builtin_find_module(fullname) _vv and self._log.debug('%r is available locally', fullname) + return None except ImportError: _vv and self._log.debug('we will try to load %r', fullname) return self finally: del _tls.running + def find_spec(self, fullname, path, target=None): + loader = self.find_module(fullname, path=path) + if loader is None: + return None + + return ModuleSpec(fullname, self, loader_state={}) + blacklisted_msg = ( '%r is present in the Mitogen importer blacklist, therefore this ' 'context will not attempt to request it from the master, as the ' @@ -1460,16 +1539,18 @@ class Importer(object): return tup = msg.unpickle() - fullname = tup[0] + fullname, pkg_present, filename, payload = tup[0], tup[1], tup[2], tup[3] _v and self._log.debug('received %s', fullname) self._lock.acquire() try: + # Note: We cache not only succesfull loads, but also failed loads + # (i.e. no such module), in which case tup[2] (filename) will be None. self._cache[fullname] = tup - if tup[2] is not None and PY24: + if filename is not None and PY24: self._update_linecache( - path='master:' + tup[2], - data=zlib.decompress(tup[3]) + path='master:' + filename, + data=zlib.decompress(payload) ) callbacks = self._callbacks.pop(fullname, []) finally: @@ -1508,6 +1589,27 @@ class Importer(object): Implements importlib.abc.Loader.load_module(). Deprecated in Python 3.4+, replaced by create_module() & exec_module(). """ + + _vv and self._log.debug('%r load_module %r called', self, fullname) + + # spec = ModuleSpec(fullname, self) + # return importlib.util.module_from_spec(spec) + + mod = spec.loader.create_module(fullname) + assert spec.loader is self + spec.loader.exec_module(mod) + return mod + + def create_module(self, spec): + # Note: Import machinery only uses this with importlib faciilties with + # Python 3.3+, but to avoid code duplication, we also use that in older + # Python version as we call it in `self.load_module`. So one still + # needs to check for `has_importlib` in few places. + + _vv and self._log.debug('%r create_module %r called', self, spec) + + fullname = spec.name + fullname = to_text(fullname) _v and self._log.debug('requesting %s', fullname) self._refuse_imports(fullname) @@ -1521,7 +1623,15 @@ class Importer(object): raise ModuleNotFoundError(self.absent_msg % (fullname,)) pkg_present = ret[1] - mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) + + if has_importlib: + import types + mod = types.ModuleType(fullname) + else: + assert has_imp + mod = imp.new_module(fullname) + + mod = sys.modules.setdefault(fullname, mod) mod.__file__ = self.get_filename(fullname) mod.__loader__ = self if pkg_present is not None: # it's a package. @@ -1535,7 +1645,23 @@ class Importer(object): # 2.x requires __package__ to be exactly a string. mod.__package__, _ = encodings.utf_8.encode(mod.__package__) - source = self.get_source(fullname) + mod.__spec__ = spec + + # This can be arbitrary (i.e. to pass from find_spec, or to exec_module) + spec.loader_state = {"source": self.get_source(fullname)} + + return mod + + def exec_module(self, mod): + # Note: Similarly this is only used in Python 3.3, but due our + # implementation of self.load_module this can also be called in older + # Pythons, including possibly Python 2. + + _vv and self._log.debug('%r exec_module %r called', self, mod) + + fullname = mod.__name__ + # source = self.get_source(fullname) + source = mod.__spec__.loader_state["source"] try: code = compile(source, mod.__file__, 'exec', 0, 1) except SyntaxError: @@ -1543,6 +1669,7 @@ class Importer(object): raise if PY3: + _vv and self._log.debug('%r exec_module exec code %r vars %r', self, code, vars(mod)) exec(code, vars(mod)) else: exec('exec code in vars(mod)') @@ -3682,6 +3809,9 @@ class Dispatcher(object): _v and LOG.debug('%r: dispatching %r', self, data) chain_id, modname, klass, func, args, kwargs = data + + _v and LOG.debug('%r: before import_module our sys.modules = %r', self, sys.modules) + obj = import_module(modname) if klass: obj = getattr(obj, klass) @@ -3921,7 +4051,13 @@ class ExternalContext(object): def _setup_package(self): global mitogen - mitogen = imp.new_module('mitogen') + + if not has_imp: + import types + mitogen = types.ModuleType('mitogen') + else: + mitogen = imp.new_module('mitogen') + mitogen.__package__ = 'mitogen' mitogen.__path__ = [] mitogen.__loader__ = self.importer diff --git a/mitogen/master.py b/mitogen/master.py index 4fb535f0..53b1459f 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -239,8 +239,12 @@ if mitogen.is_master: mitogen.parent._get_core_source = _get_core_source +BUILD_TUPLE = dis.opname.index('BUILD_TUPLE') LOAD_CONST = dis.opname.index('LOAD_CONST') +LOAD_NAME = dis.opname.index('LOAD_NAME') IMPORT_NAME = dis.opname.index('IMPORT_NAME') +if sys.version_info >= (3, 0): + LOAD_BUILD_CLASS = dis.opname.index('LOAD_BUILD_CLASS') def _getarg(nextb, c): @@ -296,14 +300,21 @@ def scan_code_imports(co): return if sys.version_info >= (2, 5): - for oparg1, oparg2, (op3, arg3) in izip(opit, opit2, opit3): - if op3 == IMPORT_NAME: - op2, arg2 = oparg2 - op1, arg1 = oparg1 - if op1 == op2 == LOAD_CONST: + for (op1, arg1), (op2, arg2), (op3, arg3) in izip(opit, opit2, opit3): + if op3 == IMPORT_NAME and op1 == op2 == LOAD_CONST: yield (co.co_consts[arg1], co.co_names[arg3], co.co_consts[arg2] or ()) + + # Scan defined classes for imports + if sys.version_info < (3, 0): + if op1 == LOAD_NAME and op2 == BUILD_TUPLE and op3 == LOAD_CONST and isinstance(co.co_consts[arg3],types.CodeType): + for level, modname, namelist in scan_code_imports(co.co_consts[arg3]): + yield (level, modname, namelist) + if sys.version_info >= (3, 0): + if op1 == LOAD_BUILD_CLASS and op2 == LOAD_CONST and isinstance(co.co_consts[arg2],types.CodeType): + for level, modname, namelist in scan_code_imports(co.co_consts[arg2]): + yield (level, modname, namelist) else: # Python 2.4 did not yet have 'level', so stack format differs. for oparg1, (op2, arg2) in izip(opit, opit2): @@ -854,15 +865,19 @@ class ModuleFinder(object): if tup: return tup + # LOG.debug('Searching for %s', fullname) + for method in self.get_module_methods: tup = method.find(fullname) if tup: - #LOG.debug('%r returned %r', method, tup) + # LOG.debug('method=%r returned path=%r source=%r...%d is_pkg=%r', method, tup[0], tup[1][0:30], len(tup[1]), tup[2]) break else: tup = None, None, None LOG.debug('get_module_source(%r): cannot find source', fullname) + # LOG.debug(' ') + self._found_cache[fullname] = tup return tup