Lookups: use Ansible's config manager whenever possible (#5440)

* Start using Ansible's config manager to handle options.

* Docs improvements.

* Fix documentation, make options actual lookup options.

* The cyberarkpassword lookup does too strange things.

* The onepassword lookups are converted in #4728, let's not interfere.

* Improve docs.

* Skip shelvefile as well.

* Convert lmdb_kv.

* Convert and fix credstash.

* Convert manifold.

* Drop chef_databag.

* Convert dig.

* Update examples.

* Forgot the most important part.

* Fix lmdb_kv docs.

* Python 2.6 compatibility.

* Convert AnsibleUnicode to str.

* Load lookup with lookup loader.

* Fix environment handling and error message checking.

* Improve docs formatting.

.github/workflows/docs-pr.yml

@@ -26,6 +26,7 @@ jobs:
       init-fail-on-error: true
       provide-link-targets: |
         ansible_collections.ansible.builtin.dict2items_filter
+        ansible_collections.ansible.builtin.items_lookup
         ansible_collections.ansible.builtin.path_join_filter
         ansible_collections.community.kubevirt.kubevirt_cdi_upload_module
         ansible_collections.community.kubevirt.kubevirt_inventory

changelogs/fragments/lookup-options.yml

@@ -0,0 +1,14 @@
+minor_changes:
+  - "cartesian lookup plugin - start using Ansible's configuration manager to parse options (https://github.com/ansible-collections/community.general/pull/5440)."
+  - "credstash lookup plugin - start using Ansible's configuration manager to parse options (https://github.com/ansible-collections/community.general/pull/5440)."
+  - "dependent lookup plugin - start using Ansible's configuration manager to parse options (https://github.com/ansible-collections/community.general/pull/5440)."
+  - "dig lookup plugin - start using Ansible's configuration manager to parse options. All documented options can now also be passed as lookup parameters (https://github.com/ansible-collections/community.general/pull/5440)."
+  - "dnstxt lookup plugin - start using Ansible's configuration manager to parse options (https://github.com/ansible-collections/community.general/pull/5440)."
+  - "filetree lookup plugin - start using Ansible's configuration manager to parse options (https://github.com/ansible-collections/community.general/pull/5440)."
+  - "flattened lookup plugin - start using Ansible's configuration manager to parse options (https://github.com/ansible-collections/community.general/pull/5440)."
+  - "hiera lookup plugin - start using Ansible's configuration manager to parse options. The Hiera executable and config file can now also be passed as lookup parameters (https://github.com/ansible-collections/community.general/pull/5440)."
+  - "keyring lookup plugin - start using Ansible's configuration manager to parse options (https://github.com/ansible-collections/community.general/pull/5440)."
+  - "lmdb_kv lookup plugin - start using Ansible's configuration manager to parse options (https://github.com/ansible-collections/community.general/pull/5440)."
+  - "manifold lookup plugin - start using Ansible's configuration manager to parse options (https://github.com/ansible-collections/community.general/pull/5440)."
+bugfixes:
+  - "credstash lookup plugin - pass plugin options to credstash for all terms, not just for the first (https://github.com/ansible-collections/community.general/pull/5440)."

plugins/lookup/cartesian.py

@@ -15,9 +15,11 @@
         - It is clearer with an example, it turns [1, 2, 3], [a, b] into [1, a], [1, b], [2, a], [2, b], [3, a], [3, b].
          You can see the exact syntax in the examples section.
     options:
-      _raw:
+      _terms:
         description:
           - a set of lists
+        type: list
+        elements: list
         required: true
 '''
 
@@ -69,6 +71,7 @@ def _lookup_variables(self, terms):
         return results
 
     def run(self, terms, variables=None, **kwargs):
+        self.set_options(var_options=variables, direct=kwargs)
 
         terms = self._lookup_variables(terms)
 

plugins/lookup/credstash.py

@@ -22,25 +22,33 @@
         required: true
       table:
         description: name of the credstash table to query
+        type: str
         default: 'credential-store'
       version:
         description: Credstash version
+        type: str
+        default: ''
       region:
         description: AWS region
+        type: str
       profile_name:
         description: AWS profile to use for authentication
+        type: str
         env:
           - name: AWS_PROFILE
       aws_access_key_id:
         description: AWS access key ID
+        type: str
         env:
           - name: AWS_ACCESS_KEY_ID
       aws_secret_access_key:
         description: AWS access key
+        type: str
         env:
           - name: AWS_SECRET_ACCESS_KEY
       aws_session_token:
         description: AWS session token
+        type: str
         env:
           - name: AWS_SESSION_TOKEN
 '''
@@ -100,28 +108,39 @@
 
 
 class LookupModule(LookupBase):
-    def run(self, terms, variables, **kwargs):
-
+    def run(self, terms, variables=None, **kwargs):
         if not CREDSTASH_INSTALLED:
             raise AnsibleError('The credstash lookup plugin requires credstash to be installed.')
 
+        self.set_options(var_options=variables, direct=kwargs)
+
+        version = self.get_option('version')
+        region = self.get_option('region')
+        table = self.get_option('table')
+        profile_name = self.get_option('profile_name')
+        aws_access_key_id = self.get_option('aws_access_key_id')
+        aws_secret_access_key = self.get_option('aws_secret_access_key')
+        aws_session_token = self.get_option('aws_session_token')
+
+        context = dict(
+            (k, v) for k, v in kwargs.items()
+            if k not in ('version', 'region', 'table', 'profile_name', 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token')
+        )
+
+        kwargs_pass = {
+            'profile_name': profile_name,
+            'aws_access_key_id': aws_access_key_id,
+            'aws_secret_access_key': aws_secret_access_key,
+            'aws_session_token': aws_session_token,
+        }
+
         ret = []
         for term in terms:
             try:
-                version = kwargs.pop('version', '')
-                region = kwargs.pop('region', None)
-                table = kwargs.pop('table', 'credential-store')
-                profile_name = kwargs.pop('profile_name', os.getenv('AWS_PROFILE', None))
-                aws_access_key_id = kwargs.pop('aws_access_key_id', os.getenv('AWS_ACCESS_KEY_ID', None))
-                aws_secret_access_key = kwargs.pop('aws_secret_access_key', os.getenv('AWS_SECRET_ACCESS_KEY', None))
-                aws_session_token = kwargs.pop('aws_session_token', os.getenv('AWS_SESSION_TOKEN', None))
-                kwargs_pass = {'profile_name': profile_name, 'aws_access_key_id': aws_access_key_id,
-                               'aws_secret_access_key': aws_secret_access_key, 'aws_session_token': aws_session_token}
-                val = credstash.getSecret(term, version, region, table, context=kwargs, **kwargs_pass)
+                ret.append(credstash.getSecret(term, version, region, table, context=context, **kwargs_pass))
             except credstash.ItemNotFound:
                 raise AnsibleError('Key {0} not found'.format(term))
             except Exception as e:
                 raise AnsibleError('Encountered exception while fetching {0}: {1}'.format(term, e))
-            ret.append(val)
 
         return ret

plugins/lookup/cyberarkpassword.py

@@ -174,7 +174,6 @@ class LookupModule(LookupBase):
     """
 
     def run(self, terms, variables=None, **kwargs):
-
         display.vvvv("%s" % terms)
         if isinstance(terms, list):
             return_values = []

plugins/lookup/dependent.py

@@ -16,7 +16,7 @@
      or template expressions which evaluate to lists or dicts, composed of the elements of
      the input evaluated lists and dictionaries."
 options:
-  _raw:
+  _terms:
     description:
       - A list where the elements are one-element dictionaries, mapping a name to a string, list, or dictionary.
         The name is the index that is used in the result object. The value is iterated over as described below.
@@ -180,6 +180,8 @@ def __process(self, result, terms, index, current, templar, variables):
 
     def run(self, terms, variables=None, **kwargs):
         """Generate list."""
+        self.set_options(var_options=variables, direct=kwargs)
+
         result = []
         if len(terms) > 0:
             templar = Templar(loader=self._templar._loader)

plugins/lookup/dig.py

@@ -21,22 +21,26 @@
       - In addition to (default) A record, it is also possible to specify a different record type that should be queried.
         This can be done by either passing-in additional parameter of format qtype=TYPE to the dig lookup, or by appending /TYPE to the FQDN being queried.
       - If multiple values are associated with the requested record, the results will be returned as a comma-separated list.
-        In such cases you may want to pass option wantlist=True to the plugin, which will result in the record values being returned as a list
-        over which you can iterate later on.
+        In such cases you may want to pass option I(wantlist=true) to the lookup call, or alternatively use C(query) instead of C(lookup),
+        which will result in the record values being returned as a list over which you can iterate later on.
       - By default, the lookup will rely on system-wide configured DNS servers for performing the query.
         It is also possible to explicitly specify DNS servers to query using the @DNS_SERVER_1,DNS_SERVER_2,...,DNS_SERVER_N notation.
         This needs to be passed-in as an additional parameter to the lookup
     options:
       _terms:
         description: Domain(s) to query.
+        type: list
+        elements: str
       qtype:
         description:
             - Record type to query.
             - C(DLV) has been removed in community.general 6.0.0.
+        type: str
         default: 'A'
         choices: [A, ALL, AAAA, CNAME, DNAME, DNSKEY, DS, HINFO, LOC, MX, NAPTR, NS, NSEC3PARAM, PTR, RP, RRSIG, SOA, SPF, SRV, SSHFP, TLSA, TXT]
       flat:
         description: If 0 each record is returned as a dictionary, otherwise a string.
+        type: int
         default: 1
       retry_servfail:
         description: Retry a nameserver if it returns SERVFAIL.
@@ -59,6 +63,11 @@
         default: false
         type: bool
         version_added: 6.0.0
+      class:
+        description:
+          - "Class."
+        type: str
+        default: 'IN'
     notes:
       - ALL is not a record per-se, merely the listed fields are available for any record results you retrieve in the form of a dictionary.
       - While the 'dig' lookup plugin supports anything which dnspython supports out of the box, only a subset can be converted into a dictionary.
@@ -74,7 +83,7 @@
 
 - name: "The TXT record for example.org."
   ansible.builtin.debug:
-    msg: "{{ lookup('community.general.dig', 'example.org.', 'qtype=TXT') }}"
+    msg: "{{ lookup('community.general.dig', 'example.org.', qtype='TXT') }}"
 
 - name: "The TXT record for example.org, alternative syntax."
   ansible.builtin.debug:
@@ -83,24 +92,24 @@
 - name: use in a loop
   ansible.builtin.debug:
     msg: "MX record for gmail.com {{ item }}"
-  with_items: "{{ lookup('community.general.dig', 'gmail.com./MX', wantlist=True) }}"
+  with_items: "{{ lookup('community.general.dig', 'gmail.com./MX', wantlist=true) }}"
 
 - ansible.builtin.debug:
     msg: "Reverse DNS for 192.0.2.5 is {{ lookup('community.general.dig', '192.0.2.5/PTR') }}"
 - ansible.builtin.debug:
     msg: "Reverse DNS for 192.0.2.5 is {{ lookup('community.general.dig', '5.2.0.192.in-addr.arpa./PTR') }}"
 - ansible.builtin.debug:
-    msg: "Reverse DNS for 192.0.2.5 is {{ lookup('community.general.dig', '5.2.0.192.in-addr.arpa.', 'qtype=PTR') }}"
+    msg: "Reverse DNS for 192.0.2.5 is {{ lookup('community.general.dig', '5.2.0.192.in-addr.arpa.', qtype='PTR') }}"
 - ansible.builtin.debug:
     msg: "Querying 198.51.100.23 for IPv4 address for example.com. produces {{ lookup('dig', 'example.com', '@198.51.100.23') }}"
 
 - ansible.builtin.debug:
     msg: "XMPP service for gmail.com. is available at {{ item.target }} on port {{ item.port }}"
-  with_items: "{{ lookup('community.general.dig', '_xmpp-server._tcp.gmail.com./SRV', 'flat=0', wantlist=True) }}"
+  with_items: "{{ lookup('community.general.dig', '_xmpp-server._tcp.gmail.com./SRV', flat=0, wantlist=true) }}"
 
 - name: Retry nameservers that return SERVFAIL
   ansible.builtin.debug:
-    msg: "{{ lookup('community.general.dig', 'example.org./A', 'retry_servfail=True') }}"
+    msg: "{{ lookup('community.general.dig', 'example.org./A', retry_servfail=true) }}"
 """
 
 RETURN = """
@@ -279,21 +288,26 @@ def run(self, terms, variables=None, **kwargs):
 
             ... flat=0                                      # returns a dict; default is 1 == string
         '''
-
         if HAVE_DNS is False:
             raise AnsibleError("The dig lookup requires the python 'dnspython' library and it is not installed")
 
+        self.set_options(var_options=variables, direct=kwargs)
+
         # Create Resolver object so that we can set NS if necessary
         myres = dns.resolver.Resolver(configure=True)
         edns_size = 4096
         myres.use_edns(0, ednsflags=dns.flags.DO, payload=edns_size)
 
         domain = None
-        qtype = 'A'
-        flat = True
-        fail_on_error = False
-        real_empty = False
-        rdclass = dns.rdataclass.from_text('IN')
+        qtype = self.get_option('qtype')
+        flat = self.get_option('flat')
+        fail_on_error = self.get_option('fail_on_error')
+        real_empty = self.get_option('real_empty')
+        try:
+            rdclass = dns.rdataclass.from_text(self.get_option('class'))
+        except Exception as e:
+            raise AnsibleError("dns lookup illegal CLASS: %s" % to_native(e))
+        myres.retry_servfail = self.get_option('retry_servfail')
 
         for t in terms:
             if t.startswith('@'):       # e.g. "@10.0.1.2,192.0.2.1" is ok.
@@ -316,7 +330,7 @@ def run(self, terms, variables=None, **kwargs):
                 continue
             if '=' in t:
                 try:
-                    opt, arg = t.split('=')
+                    opt, arg = t.split('=', 1)
                 except Exception:
                     pass
 

plugins/lookup/dnstxt.py

@@ -71,6 +71,7 @@
 class LookupModule(LookupBase):
 
     def run(self, terms, variables=None, **kwargs):
+        self.set_options(var_options=variables, direct=kwargs)
 
         if HAVE_DNS is False:
             raise AnsibleError("Can't LOOKUP(dnstxt): module dns.resolver is not installed")

plugins/lookup/filetree.py

@@ -201,6 +201,8 @@ def file_props(root, path):
 class LookupModule(LookupBase):
 
     def run(self, terms, variables=None, **kwargs):
+        self.set_options(var_options=variables, direct=kwargs)
+
         basedir = self.get_basedir(variables)
 
         ret = []

plugins/lookup/flattened.py

@@ -11,14 +11,17 @@
     author: Serge van Ginderachter (!UNKNOWN) <serge@vanginderachter.be>
     short_description: return single list completely flattened
     description:
-      - given one or more lists, this lookup will flatten any list elements found recursively until only 1 list is left.
+      - Given one or more lists, this lookup will flatten any list elements found recursively until only 1 list is left.
     options:
       _terms:
         description: lists to flatten
+        type: list
+        elements: raw
         required: true
     notes:
-      - unlike 'items' which only flattens 1 level, this plugin will continue to flatten until it cannot find lists anymore.
-      - aka highlander plugin, there can only be one (list).
+      - Unlike the R(items lookup,ansible_collections.ansible.builtin.items_lookup) which only flattens 1 level,
+        this plugin will continue to flatten until it cannot find lists anymore.
+      - Aka highlander plugin, there can only be one (list).
 '''
 
 EXAMPLES = """
@@ -78,9 +81,10 @@ def _do_flatten(self, terms, variables):
 
         return ret
 
-    def run(self, terms, variables, **kwargs):
-
+    def run(self, terms, variables=None, **kwargs):
         if not isinstance(terms, list):
             raise AnsibleError("with_flattened expects a list")
 
+        self.set_options(var_options=variables, direct=kwargs)
+
         return self._do_flatten(terms, variables)

plugins/lookup/hiera.py

@@ -14,23 +14,23 @@
     requirements:
       - hiera (command line utility)
     description:
-        - Retrieves data from an Puppetmaster node using Hiera as ENC
+        - Retrieves data from an Puppetmaster node using Hiera as ENC.
     options:
-      _hiera_key:
+      _terms:
             description:
-                - The list of keys to lookup on the Puppetmaster
+                - The list of keys to lookup on the Puppetmaster.
             type: list
             elements: string
             required: true
-      _bin_file:
+      executable:
             description:
-                - Binary file to execute Hiera
+                - Binary file to execute Hiera.
             default: '/usr/bin/hiera'
             env:
                 - name: ANSIBLE_HIERA_BIN
-      _hierarchy_file:
+      config_file:
             description:
-                - File that describes the hierarchy of Hiera
+                - File that describes the hierarchy of Hiera.
             default: '/etc/hiera.yaml'
             env:
                 - name: ANSIBLE_HIERA_CFG
@@ -67,25 +67,28 @@
 from ansible.utils.cmd_functions import run_cmd
 from ansible.module_utils.common.text.converters import to_text
 
-ANSIBLE_HIERA_CFG = os.getenv('ANSIBLE_HIERA_CFG', '/etc/hiera.yaml')
-ANSIBLE_HIERA_BIN = os.getenv('ANSIBLE_HIERA_BIN', '/usr/bin/hiera')
-
 
 class Hiera(object):
+    def __init__(self, hiera_cfg, hiera_bin):
+        self.hiera_cfg = hiera_cfg
+        self.hiera_bin = hiera_bin
+
     def get(self, hiera_key):
-        pargs = [ANSIBLE_HIERA_BIN]
-        pargs.extend(['-c', ANSIBLE_HIERA_CFG])
+        pargs = [self.hiera_bin]
+        pargs.extend(['-c', self.hiera_cfg])
 
         pargs.extend(hiera_key)
 
         rc, output, err = run_cmd("{0} -c {1} {2}".format(
-            ANSIBLE_HIERA_BIN, ANSIBLE_HIERA_CFG, hiera_key[0]))
+            self.hiera_bin, self.hiera_cfg, hiera_key[0]))
 
         return to_text(output.strip())
 
 
 class LookupModule(LookupBase):
-    def run(self, terms, variables=''):
-        hiera = Hiera()
+    def run(self, terms, variables=None, **kwargs):
+        self.set_options(var_options=variables, direct=kwargs)
+
+        hiera = Hiera(self.get_option('config_file'), self.get_option('executable'))
         ret = [hiera.get(terms)]
         return ret