Temporary (#31677)

* allow shells to have per host options, remote_tmp

added language to shell
removed module lang setting from general as  plugins have it now
use get to avoid bad powershell plugin
more resilient tmp discovery, fall back to `pwd`
add shell to docs
fixed options for when frags are only options
added shell set ops in t_e and fixed option frags
normalize tmp dir usag4e

- pass tmpdir/tmp/temp options as env var to commands, making it default for tempfile
- adjusted ansiballz tmpdir
- default local tempfile usage to the configured local tmp
- set env temp in action

add options to powershell
shift temporary to internal envvar/params
ensure tempdir is set if we pass var
ensure basic and url use expected tempdir
ensure localhost uses local tmp
give /var/tmp priority, less perms issues
more consistent tempfile mgmt for ansiballz
made async_dir configurable
better action handling, allow for finally rm tmp
fixed tmp issue and no more tempdir in ballz
hostvarize world readable and admin users
always set shell tempdir
added comment to discourage use of exception/flow control

* Mostly revert expand_user as it's not quite working.

This was an additional feature anyhow.

Kept the use of pwd as a fallback but moved it to a second ssh
connection.  This is not optimal but getting that to work in a single
ssh connection was part of the problem holding this up.

(cherry picked from commit 395b714120522f15e4c90a346f5e8e8d79213aca)

* fixed script and other action plugins

ensure tmpdir deletion
allow for connections that don't support new options (legacy, 3rd party)
fixed tests

docs/bin/plugin_formatter.py

@@ -49,6 +49,7 @@ def html_escape(text, quote=True):
 
 from ansible.errors import AnsibleError
 from ansible.module_utils._text import to_bytes
+from ansible.plugins.loader import fragment_loader
 from ansible.utils import plugin_docs
 from ansible.utils.display import Display
 
@@ -235,7 +236,7 @@ def get_plugin_info(module_dir, limit_to=None, verbose=False):
             primary_category = module_categories[0]
 
         # use ansible core library to parse out doc metadata YAML and plaintext examples
-        doc, examples, returndocs, metadata = plugin_docs.get_docstring(module_path, verbose=verbose)
+        doc, examples, returndocs, metadata = plugin_docs.get_docstring(module_path, fragment_loader, verbose=verbose)
 
         # save all the information
         module_info[module] = {'path': module_path,

docs/docsite/rst/dev_guide/developing_modules_best_practices.rst

@@ -22,7 +22,7 @@ and guidelines:
 
 * In the event of failure, a key of 'failed' should be included, along with a string explanation in 'msg'.  Modules that raise tracebacks (stacktraces) are generally considered 'poor' modules, though Ansible can deal with these returns and will automatically convert anything unparseable into a failed result.  If you are using the AnsibleModule common Python code, the 'failed' element will be included for you automatically when you call 'fail_json'.
 
-* Return codes from modules are actually not significant, but continue on with 0=success and non-zero=failure for reasons of future proofing.
+* Return codes from modules are used if 'failed' is missing, 0=success and non-zero=failure.
 
 * As results from many hosts will be aggregated at once, modules should return only relevant output.  Returning the entire contents of a log file is generally bad form.
 
@@ -194,5 +194,4 @@ Avoid creating a module that does the work of other modules; this leads to code
 
 Avoid creating 'caches'. Ansible is designed without a central server or authority, so you cannot guarantee it will not run with different permissions, options or locations. If you need a central authority, have it on top of Ansible (for example, using bastion/cm/ci server or tower); do not try to build it into modules.
 
-Always use the hacking/test-module script when developing modules and it will warn
-you about these kind of things.
+Always use the hacking/test-module script when developing modules and it will warn you about these kind of things.

lib/ansible/cli/console.py

@@ -44,7 +44,7 @@
 from ansible.module_utils.parsing.convert_bool import boolean
 from ansible.parsing.splitter import parse_kv
 from ansible.playbook.play import Play
-from ansible.plugins.loader import module_loader
+from ansible.plugins.loader import module_loader, fragment_loader
 from ansible.utils import plugin_docs
 from ansible.utils.color import stringc
 
@@ -356,7 +356,7 @@ def helpdefault(self, module_name):
         if module_name in self.modules:
             in_path = module_loader.find_plugin(module_name)
             if in_path:
-                oc, a, _, _ = plugin_docs.get_docstring(in_path)
+                oc, a, _, _ = plugin_docs.get_docstring(in_path, fragment_loader)
                 if oc:
                     display.display(oc['short_description'])
                     display.display('Parameters:')
@@ -388,7 +388,7 @@ def completedefault(self, text, line, begidx, endidx):
 
     def module_args(self, module_name):
         in_path = module_loader.find_plugin(module_name)
-        oc, a, _, _ = plugin_docs.get_docstring(in_path)
+        oc, a, _, _ = plugin_docs.get_docstring(in_path, fragment_loader)
         return list(oc['options'].keys())
 
     def run(self):

lib/ansible/cli/doc.py

@@ -29,8 +29,9 @@
 from ansible.module_utils.six import string_types
 from ansible.parsing.yaml.dumper import AnsibleDumper
 from ansible.plugins.loader import module_loader, action_loader, lookup_loader, callback_loader, cache_loader, \
-    vars_loader, connection_loader, strategy_loader, inventory_loader
-from ansible.utils import plugin_docs
+    vars_loader, connection_loader, strategy_loader, inventory_loader, shell_loader, fragment_loader
+from ansible.utils.plugin_docs import BLACKLIST, get_docstring
+
 try:
     from __main__ import display
 except ImportError:
@@ -71,7 +72,7 @@ def parse(self):
                                help='**For internal testing only** Show documentation for all plugins.')
         self.parser.add_option("-t", "--type", action="store", default='module', dest='type', type='choice',
                                help='Choose which plugin type (defaults to "module")',
-                               choices=['cache', 'callback', 'connection', 'inventory', 'lookup', 'module', 'strategy', 'vars'])
+                               choices=['cache', 'callback', 'connection', 'inventory', 'lookup', 'module', 'shell', 'strategy', 'vars'])
 
         super(DocCLI, self).parse()
 
@@ -101,6 +102,8 @@ def run(self):
             loader = vars_loader
         elif plugin_type == 'inventory':
             loader = inventory_loader
+        elif plugin_type == 'shell':
+            loader = shell_loader
         else:
             loader = module_loader
 
@@ -146,7 +149,6 @@ def run(self):
         # process command line list
         text = ''
         for plugin in self.args:
-
             try:
                 # if the plugin lives in a non-python file (eg, win_X.ps1), require the corresponding python file for docs
                 filename = loader.find_plugin(plugin, mod_type='.py', ignore_deprecated=True, check_aliases=True)
@@ -158,7 +160,7 @@ def run(self):
                     continue
 
                 try:
-                    doc, plainexamples, returndocs, metadata = plugin_docs.get_docstring(filename, verbose=(self.options.verbosity > 0))
+                    doc, plainexamples, returndocs, metadata = get_docstring(filename, fragment_loader, verbose=(self.options.verbosity > 0))
                 except:
                     display.vvv(traceback.format_exc())
                     display.error("%s %s has a documentation error formatting or is missing documentation." % (plugin_type, plugin))
@@ -229,7 +231,7 @@ def find_plugins(self, path, ptype):
             plugin = os.path.splitext(plugin)[0]  # removes the extension
             plugin = plugin.lstrip('_')  # remove underscore from deprecated plugins
 
-            if plugin not in plugin_docs.BLACKLIST.get(bkey, ()):
+            if plugin not in BLACKLIST.get(bkey, ()):
                 self.plugin_list.add(plugin)
                 display.vvvv("Added %s" % plugin)
 
@@ -254,7 +256,7 @@ def get_plugin_list_text(self, loader):
 
                 doc = None
                 try:
-                    doc, plainexamples, returndocs, metadata = plugin_docs.get_docstring(filename)
+                    doc, plainexamples, returndocs, metadata = get_docstring(filename, fragment_loader)
                 except:
                     display.warning("%s has a documentation formatting error" % plugin)
 

lib/ansible/config/base.yml

@@ -1,18 +1,6 @@
 # Copyright (c) 2017 Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 ---
-ALLOW_WORLD_READABLE_TMPFILES:
-  name: Allow world readable temporary files
-  default: False
-  description:
-    - This makes the temporary files created on the machine to be world readable and will issue a warning instead of failing the task.
-    - It is useful when becoming an unprivileged user.
-  env: []
-  ini:
-  - {key: allow_world_readable_tmpfiles, section: defaults}
-  type: boolean
-  yaml: {key: defaults.allow_world_readable_tmpfiles}
-  version_added: "2.1"
 ANSIBLE_COW_SELECTION:
   name: Cowsay filter selection
   default: default
@@ -744,15 +732,6 @@ DEFAULT_MODULE_COMPRESSION:
   - {key: module_compression, section: defaults}
 # vars:
 #   - name: ansible_module_compression
-DEFAULT_MODULE_LANG:
-  name: Target language environment
-  default: "{{CONTROLER_LANG}}"
-  description: "Language locale setting to use for modules when they execute on the target, if empty it defaults to 'en_US.UTF-8'"
-  env: [{name: ANSIBLE_MODULE_LANG}]
-  ini:
-  - {key: module_lang, section: defaults}
-# vars:
-#   - name: ansible_module_lang
 DEFAULT_MODULE_NAME:
   name: Default adhoc module
   default: command
@@ -768,16 +747,6 @@ DEFAULT_MODULE_PATH:
   ini:
   - {key: library, section: defaults}
   type: pathspec
-DEFAULT_MODULE_SET_LOCALE:
-  name: Target locale
-  default: False
-  description: Controls if we set locale for modules when executing on the target.
-  env: [{name: ANSIBLE_MODULE_SET_LOCALE}]
-  ini:
-  - {key: module_set_locale, section: defaults}
-  type: boolean
-# vars:
-#   - name: ansible_module_locale
 DEFAULT_MODULE_UTILS_PATH:
   name: Module Utils Path
   description: Colon separated paths in which Ansible will search for Module utils files, which are shared by modules.
@@ -851,17 +820,6 @@ DEFAULT_REMOTE_PORT:
   - {key: remote_port, section: defaults}
   type: integer
   yaml: {key: defaults.remote_port}
-DEFAULT_REMOTE_TMP:
-  name: Target temporary directory
-  default: ~/.ansible/tmp
-  description:
-    - Temporary directory to use on targets when executing tasks.
-    - In some cases Ansible may still choose to use a system temporary dir to avoid permission issues.
-  env: [{name: ANSIBLE_REMOTE_TEMP}]
-  ini:
-  - {key: remote_tmp, section: defaults}
-  vars:
-    - name: ansible_remote_tmp
 DEFAULT_REMOTE_USER:
   name: Login/Remote User
   default:

lib/ansible/constants.py

@@ -5,7 +5,7 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import os  # used to set lang and for backwards compat get_config
+import os
 
 from ast import literal_eval
 from jinja2 import Template
@@ -114,7 +114,6 @@ def set_constant(name, value, export=vars()):
     module_compression=('ansible_module_compression', ),
     shell=('ansible_shell_type', ),
     executable=('ansible_shell_executable', ),
-    remote_tmp_dir=('ansible_remote_tmp', ),
 
     # connection common
     remote_addr=('ansible_ssh_host', 'ansible_host'),

lib/ansible/errors/__init__.py

@@ -252,11 +252,38 @@ def __init__(self, message="", obj=None, show_content=True, suppress_extended_er
                                                   suppress_extended_error=suppress_extended_error, orig_exc=orig_exc)
 
 
-class AnsibleActionSkip(AnsibleRuntimeError):
+# These Exceptions are temporary, using them as flow control until we can get a better solution.
+# DO NOT USE as they will probably be removed soon.
+class AnsibleAction(AnsibleRuntimeError):
+    ''' Base Exception for Action plugin flow control '''
+
+    def __init__(self, message="", obj=None, show_content=True, suppress_extended_error=False, orig_exc=None, result=None):
+
+        super(AnsibleAction, self).__init__(message=message, obj=obj, show_content=show_content,
+                                            suppress_extended_error=suppress_extended_error, orig_exc=orig_exc)
+        if result is None:
+            self.result = {}
+        else:
+            self.result = result
+
+
+class AnsibleActionSkip(AnsibleAction):
     ''' an action runtime skip'''
-    pass
+
+    def __init__(self, message="", obj=None, show_content=True, suppress_extended_error=False, orig_exc=None, result=None):
+        super(AnsibleActionSkip, self).__init__(message=message, obj=obj, show_content=show_content,
+                                                suppress_extended_error=suppress_extended_error, orig_exc=orig_exc, result=result)
+        self.result.update({'skipped': True, 'msg': message})
 
 
-class AnsibleActionFail(AnsibleRuntimeError):
+class AnsibleActionFail(AnsibleAction):
     ''' an action runtime failure'''
+    def __init__(self, message="", obj=None, show_content=True, suppress_extended_error=False, orig_exc=None, result=None):
+        super(AnsibleActionFail, self).__init__(message=message, obj=obj, show_content=show_content,
+                                                suppress_extended_error=suppress_extended_error, orig_exc=orig_exc, result=result)
+        self.result.update({'failed': True, 'msg': message})
+
+
+class AnsibleActionDone(AnsibleAction):
+    ''' an action runtime early exit'''
     pass

lib/ansible/executor/task_executor.py

@@ -486,6 +486,7 @@ def _execute(self, variables=None):
             self._connection._play_context = self._play_context
 
         self._set_connection_options(variables, templar)
+        self._set_shell_options(variables, templar)
 
         # get handler
         self._handler = self._get_action_handler(connection=self._connection, templar=templar)
@@ -774,6 +775,15 @@ def _set_connection_options(self, variables, templar):
 
         # set options with 'templated vars' specific to this plugin
         self._connection.set_options(var_options=options)
+        self._set_shell_options(final_vars, templar)
+
+    def _set_shell_options(self, variables, templar):
+        option_vars = C.config.get_plugin_vars('shell', self._connection._shell._load_name)
+        options = {}
+        for k in option_vars:
+            if k in variables:
+                options[k] = templar.template(variables[k])
+        self._connection._shell.set_options(var_options=options)
 
     def _get_action_handler(self, connection, templar):
         '''

lib/ansible/inventory/data.py

@@ -97,6 +97,7 @@ def _create_implicit_localhost(self, pattern):
                                 'You can correct this by setting ansible_python_interpreter for localhost')
             new_host.set_variable("ansible_python_interpreter", py_interp)
             new_host.set_variable("ansible_connection", 'local')
+            new_host.set_variable("ansible_remote_tmp", C.DEFAULT_LOCAL_TMP)
 
             self.localhost = new_host