diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py index 1f4d8fc6..9729b8a1 100644 --- a/ansible_mitogen/loaders.py +++ b/ansible_mitogen/loaders.py @@ -49,7 +49,7 @@ __all__ = [ ANSIBLE_VERSION_MIN = (2, 10) -ANSIBLE_VERSION_MAX = (2, 13) +ANSIBLE_VERSION_MAX = (2, 16) NEW_VERSION_MSG = ( "Your Ansible version (%s) is too recent. The most recent version\n" diff --git a/ansible_mitogen/plugins/connection/mitogen_ssh.py b/ansible_mitogen/plugins/connection/mitogen_ssh.py index 75f2d42f..2cfbee2f 100644 --- a/ansible_mitogen/plugins/connection/mitogen_ssh.py +++ b/ansible_mitogen/plugins/connection/mitogen_ssh.py @@ -75,6 +75,7 @@ import ansible_mitogen.loaders class Connection(ansible_mitogen.connection.Connection): transport = 'ssh' + # It actually returns a namedtuple 'ansible.plugins.loader.get_with_context_result' vanilla_class = ansible_mitogen.loaders.connection_loader__get( 'ssh', class_only=True, @@ -84,4 +85,5 @@ class Connection(ansible_mitogen.connection.Connection): def _create_control_path(*args, **kwargs): """Forward _create_control_path() to the implementation in ssh.py.""" # https://github.com/dw/mitogen/issues/342 + # print("mitogen_ssh.Connection._create_control_path", type(vanilla_class)) return Connection.vanilla_class._create_control_path(*args, **kwargs) diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index 63caa88a..453302f2 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -185,8 +185,8 @@ def _setup_responder(responder): Configure :class:`mitogen.master.ModuleResponder` to only permit certain packages, and to generate custom responses for certain modules. """ - responder.whitelist_prefix('ansible') - responder.whitelist_prefix('ansible_mitogen') + #responder.whitelist_prefix('ansible') + #responder.whitelist_prefix('ansible_mitogen') # Ansible 2.3 is compatible with Python 2.4 targets, however # ansible/__init__.py is not. Instead, executor/module_common.py writes @@ -285,7 +285,13 @@ class Broker(mitogen.master.Broker): the exuberant syscall expense of EpollPoller, so override it and restore the poll() poller. """ - poller_class = mitogen.core.Poller + # poller_class = mitogen.core.Poller + pass + + if mitogen.parent.PollPoller.SUPPORTED: + poller_class = mitogen.parent.PollPoller + else: + poller_class = mitogen.core.Poller class Binding(object): diff --git a/ansible_mitogen/strategy.py b/ansible_mitogen/strategy.py index 0a98e316..257a4e31 100644 --- a/ansible_mitogen/strategy.py +++ b/ansible_mitogen/strategy.py @@ -88,7 +88,9 @@ def wrap_action_loader__get(name, *args, **kwargs): get_kwargs = {'class_only': True} if name in ('fetch',): name = 'mitogen_' + name - get_kwargs['collection_list'] = kwargs.pop('collection_list', None) + get_kwargs['collection_list'] = None + else: + get_kwargs['collection_list'] = kwargs.pop('collection_list', None) klass = ansible_mitogen.loaders.action_loader__get(name, **get_kwargs) if klass: diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 5fc78185..e52a1ce8 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -475,12 +475,13 @@ class PlayContextSpec(Spec): ) def ssh_args(self): + local_vars = self._task_vars.get("hostvars", {}).get(self._inventory_name, {}) return [ mitogen.core.to_text(term) for s in ( - C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})), - C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})), - C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})) + C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=local_vars), + C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=local_vars), + C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=local_vars) ) for term in ansible.utils.shlex.shlex_split(s or '') ] @@ -707,12 +708,13 @@ class MitogenViaSpec(Spec): ) def ssh_args(self): + local_vars = self._task_vars.get("hostvars", {}).get(self._inventory_name, {}) return [ mitogen.core.to_text(term) for s in ( - C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})), - C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})), - C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})) + C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=local_vars), + C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=local_vars), + C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=local_vars) ) for term in ansible.utils.shlex.shlex_split(s) if s diff --git a/mitogen/core.py b/mitogen/core.py index bee722e6..9f879fc0 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -58,9 +58,9 @@ import warnings import weakref import zlib -# Python >3.7 deprecated the imp module. -warnings.filterwarnings('ignore', message='the imp module is deprecated') import imp +import importlib +from importlib.machinery import ModuleSpec # Absolute imports for <2.5. select = __import__('select') @@ -300,6 +300,19 @@ class Kwargs(dict): return (Kwargs, (dict(self),)) +AnsibleUnsafeText = None + +def lazy_AnsibleUnsafeText(): + global AnsibleUnsafeText + if AnsibleUnsafeText is not None: + return AnsibleUnsafeText + mod = __import__("ansible.utils.unsafe_proxy", fromlist=("AnsibleUnsafeText",)) + AnsibleUnsafeText = getattr(mod, "AnsibleUnsafeText") + assert type(AnsibleUnsafeText) is type, f"AnsibleUnsafeText {AnsibleUnsafeText} is not a type" + assert callable(AnsibleUnsafeText), f"AnsibleUnsafeText {AnsibleUnsafeText} is not callable" + return AnsibleUnsafeText + + class CallError(Error): """ Serializable :class:`Error` subclass raised when :meth:`Context.call() @@ -767,6 +780,9 @@ else: _Unpickler = pickle.Unpickler +PICKLE_PROTOCOL = 2 + + class Message(object): """ Messages are the fundamental unit of communication, comprising fields from @@ -860,6 +876,8 @@ class Message(object): return Secret elif func == 'Kwargs': return Kwargs + elif module == 'ansible.utils.unsafe_proxy' and func == 'AnsibleUnsafeText': + return lazy_AnsibleUnsafeText() elif module == '_codecs' and func == 'encode': return self._unpickle_bytes elif module == '__builtin__' and func == 'bytes': @@ -895,10 +913,10 @@ class Message(object): """ self = cls(**kwargs) try: - self.data = pickle__dumps(obj, protocol=2) + self.data = pickle__dumps(obj, protocol=PICKLE_PROTOCOL) except pickle.PicklingError: e = sys.exc_info()[1] - self.data = pickle__dumps(CallError(e), protocol=2) + self.data = pickle__dumps(CallError(e), protocol=PICKLE_PROTOCOL) return self def reply(self, msg, router=None, **kwargs): @@ -965,6 +983,7 @@ class Message(object): try: obj = unpickler.load() except: + LOG.exception("unpickler.load exception") LOG.error('raw pickle was: %r', self.data) raise self._unpickled = obj @@ -1366,11 +1385,35 @@ class Importer(object): else: path = None + # + + # Python 3.4+ + # spec = importlib.util.find_spec(modname, package=parent or None) + spec = importlib.util.find_spec(fullname) + if spec: + return # Good + else: + # raise ImportError() + raise ModuleNotFoundError() + + # Python 3.3 and earlier: + loader = importlib.find_loader(modname, path) + if loader: + return # Good + else: + # raise ImportError() + raise ModuleNotFoundError() + fp, pathname, description = imp.find_module(modname, path) if fp: fp.close() + return # Good + def find_module(self, fullname, path=None): + _vv and self._log.debug('') + _vv and self._log.debug('%r.find_module %r %r called', self, fullname, path) + """ Return a loader (ourself) or None, for the module with fullname. @@ -1378,9 +1421,12 @@ class Importer(object): Deprecrated in Python 3.4+, replaced by find_spec(). Raises ImportWarning in Python 3.10+. - fullname A (fully qualified?) module name, e.g. "os.path". + fullname A fully qualified module name, e.g. "os.path". path __path__ of parent packge. None for a top level module. """ + + # This is to break recursion to itself due to the use of + # module import attempt via self.builtin_find_module if hasattr(_tls, 'running'): return None @@ -1405,6 +1451,10 @@ class Importer(object): if self.whitelist != ['']: if any(fullname.startswith(s) for s in self.whitelist): return self + if fullname.startswith("ansible") or fullname.startswith("ansible_mitogen"): + return self + + _vv and self._log.debug('checking if %r is available locally', fullname) try: self.builtin_find_module(fullname) @@ -1415,6 +1465,51 @@ class Importer(object): finally: del _tls.running + def find_spec(self, fullname, path, target=None): + _vv and self._log.debug('') + + _vv and self._log.debug('%r.find_spec %r %r %r called', self, fullname, path, target) + + if hasattr(_tls, 'running'): + self._log.debug('%r.find_spec %r %r %r recursion prevention', self, fullname, path, target) + return None + + _tls.running = True + try: + fullname = to_text(fullname) + pkgname, dot, _ = str_rpartition(fullname, '.') + pkg = sys.modules.get(pkgname) + if pkgname and getattr(pkg, '__loader__', None) is not self: + self._log.debug('%s is submodule of a locally loaded package', + fullname) + return None + + suffix = fullname[len(pkgname+dot):] + if pkgname and suffix not in self._present.get(pkgname, ()): + self._log.debug('%s has no submodule %s', pkgname, suffix) + return None + + # #114: explicitly whitelisted prefixes override any + # system-installed package. + if self.whitelist != ['']: + if any(fullname.startswith(s) for s in self.whitelist): + return ModuleSpec(fullname, self) + + if fullname.startswith("ansible") or fullname.startswith("ansible_mitogen"): + return ModuleSpec(fullname, self) + + _vv and self._log.debug('checking if %r is available locally', fullname) + + try: + self.builtin_find_module(fullname) + _vv and self._log.debug('%r is available locally', fullname) + return None + except ImportError: + _vv and self._log.debug('we will try to load %r', fullname) + return ModuleSpec(fullname, self) + finally: + del _tls.running + blacklisted_msg = ( '%r is present in the Mitogen importer blacklist, therefore this ' 'context will not attempt to request it from the master, as the ' @@ -1502,12 +1597,27 @@ class Importer(object): callback() def load_module(self, fullname): + _vv and self._log.debug('%r load_module %r called', self, fullname) + """ Return the loaded module specified by fullname. Implements importlib.abc.Loader.load_module(). Deprecated in Python 3.4+, replaced by create_module() & exec_module(). """ + + # spec = ModuleSpec(fullname, self) + # return importlib.util.module_from_spec(spec) + + mod = spec.loader.create_module(fullname) + spec.loader.exec_module(mod) + return mod + + def create_module(self, spec): + _vv and self._log.debug('%r create_module %r called', self, spec) + + fullname = spec.name + fullname = to_text(fullname) _v and self._log.debug('requesting %s', fullname) self._refuse_imports(fullname) @@ -1535,7 +1645,18 @@ class Importer(object): # 2.x requires __package__ to be exactly a string. mod.__package__, _ = encodings.utf_8.encode(mod.__package__) - source = self.get_source(fullname) + mod.__spec__ = spec + # This can be arbitrary (i.e. to pass from find_spec, or to exec_module) + spec.loader_state = {"source": self.get_source(fullname)} + + return mod + + def exec_module(self, mod): + _vv and self._log.debug('%r exec_module %r called', self, mod) + + fullname = mod.__name__ + # source = self.get_source(fullname) + source = mod.__spec__.loader_state["source"] try: code = compile(source, mod.__file__, 'exec', 0, 1) except SyntaxError: @@ -1543,6 +1664,7 @@ class Importer(object): raise if PY3: + _vv and self._log.debug('%r exec_module exec code %r vars %r', self, code, vars(mod)) exec(code, vars(mod)) else: exec('exec code in vars(mod)') @@ -1551,6 +1673,7 @@ class Importer(object): # is necessary. This matches PyImport_ExecCodeModuleEx() return sys.modules.get(fullname, mod) + def get_filename(self, fullname): if fullname in self._cache: path = self._cache[fullname][2] @@ -3682,6 +3805,9 @@ class Dispatcher(object): _v and LOG.debug('%r: dispatching %r', self, data) chain_id, modname, klass, func, args, kwargs = data + + _v and LOG.debug('%r: before import_module our sys.modules = %r', self, sys.modules) + obj = import_module(modname) if klass: obj = getattr(obj, klass) @@ -3921,7 +4047,11 @@ class ExternalContext(object): def _setup_package(self): global mitogen - mitogen = imp.new_module('mitogen') + + # mitogen = imp.new_module('mitogen') + import types + mitogen = types.ModuleType('mitogen') + mitogen.__package__ = 'mitogen' mitogen.__path__ = [] mitogen.__loader__ = self.importer diff --git a/mitogen/master.py b/mitogen/master.py index 4fb535f0..53b1459f 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -239,8 +239,12 @@ if mitogen.is_master: mitogen.parent._get_core_source = _get_core_source +BUILD_TUPLE = dis.opname.index('BUILD_TUPLE') LOAD_CONST = dis.opname.index('LOAD_CONST') +LOAD_NAME = dis.opname.index('LOAD_NAME') IMPORT_NAME = dis.opname.index('IMPORT_NAME') +if sys.version_info >= (3, 0): + LOAD_BUILD_CLASS = dis.opname.index('LOAD_BUILD_CLASS') def _getarg(nextb, c): @@ -296,14 +300,21 @@ def scan_code_imports(co): return if sys.version_info >= (2, 5): - for oparg1, oparg2, (op3, arg3) in izip(opit, opit2, opit3): - if op3 == IMPORT_NAME: - op2, arg2 = oparg2 - op1, arg1 = oparg1 - if op1 == op2 == LOAD_CONST: + for (op1, arg1), (op2, arg2), (op3, arg3) in izip(opit, opit2, opit3): + if op3 == IMPORT_NAME and op1 == op2 == LOAD_CONST: yield (co.co_consts[arg1], co.co_names[arg3], co.co_consts[arg2] or ()) + + # Scan defined classes for imports + if sys.version_info < (3, 0): + if op1 == LOAD_NAME and op2 == BUILD_TUPLE and op3 == LOAD_CONST and isinstance(co.co_consts[arg3],types.CodeType): + for level, modname, namelist in scan_code_imports(co.co_consts[arg3]): + yield (level, modname, namelist) + if sys.version_info >= (3, 0): + if op1 == LOAD_BUILD_CLASS and op2 == LOAD_CONST and isinstance(co.co_consts[arg2],types.CodeType): + for level, modname, namelist in scan_code_imports(co.co_consts[arg2]): + yield (level, modname, namelist) else: # Python 2.4 did not yet have 'level', so stack format differs. for oparg1, (op2, arg2) in izip(opit, opit2): @@ -854,15 +865,19 @@ class ModuleFinder(object): if tup: return tup + # LOG.debug('Searching for %s', fullname) + for method in self.get_module_methods: tup = method.find(fullname) if tup: - #LOG.debug('%r returned %r', method, tup) + # LOG.debug('method=%r returned path=%r source=%r...%d is_pkg=%r', method, tup[0], tup[1][0:30], len(tup[1]), tup[2]) break else: tup = None, None, None LOG.debug('get_module_source(%r): cannot find source', fullname) + # LOG.debug(' ') + self._found_cache[fullname] = tup return tup