Merge pull request #138 from otakup0pe/misc

misc changes

README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.org/Autodesk/aomi.svg?branch=master)](https://travis-ci.org/Autodesk/aomi)[![PyPI](https://img.shields.io/pypi/v/aomi.svg)](https://pypi.python.org/pypi/aomi)[![Maintenance](https://img.shields.io/maintenance/yes/2017.svg)]()
+[![Build Status](https://travis-ci.org/Autodesk/aomi.svg?branch=master)](https://travis-ci.org/Autodesk/aomi)[![PyPI](https://img.shields.io/pypi/v/aomi.svg)](https://pypi.python.org/pypi/aomi)[![Coverage Status](https://coveralls.io/repos/github/Autodesk/aomi/badge.svg?branch=master)](https://coveralls.io/github/Autodesk/aomi?branch=master)[![Maintenance](https://img.shields.io/maintenance/yes/2017.svg)]()
 
 # Aomi: Opinionlessly Express Opinions on Vault
 

aomi/helpers.py

@@ -258,3 +258,15 @@ def normalize_vault_path(path):
     a variety of user specified formats and what HCV
     itself will return in API calls"""
     return '/'.join([x for x in path.split('/') if x])
+
+
+def map_val(dest, src, key, default=None, src_key=None):
+    """Will ensure a dict has values sourced from either
+        another dict or based on the provided default"""
+    if not src_key:
+        src_key = key
+
+    if src_key in src:
+        dest[key] = src[src_key]
+    else:
+        dest[key] = default

aomi/model/auth.py

@@ -11,8 +11,8 @@
 import hvac
 import aomi.exceptions
 from aomi.vault import wrap_hvac as wrap_vault
-from aomi.helpers import hard_path, merge_dicts, cli_hash
-from aomi.template import load_var_files, render
+from aomi.helpers import hard_path, merge_dicts, map_val
+from aomi.template import load_vars, render
 from aomi.model.resource import Auth, Resource, NOOP, ADD
 from aomi.validation import secret_file, sanitize_mount
 LOG = logging.getLogger(__name__)
@@ -43,7 +43,13 @@ def diff(self, obj=None):
         return Resource.diff_write_only(self)
 
     def __init__(self, duo, secret, opt):
-        super(DUOAccess, self).__init__({}, opt)
+        s_obj = {
+            'state': 'present'
+        }
+        if not duo.present:
+            s_obj['state'] = 'absent'
+
+        super(DUOAccess, self).__init__(s_obj, opt)
         self.backend = duo.backend
         self.path = "auth/%s/duo/access" % self.backend
         self.filename = secret
@@ -107,7 +113,7 @@ def __init__(self, obj, opt):
         super(AppRoleSecret, self).__init__(obj, opt)
 
     def diff(self, obj=None):
-        if 'secret_id_accessor' in self.existing:
+        if self.existing and 'secret_id_accessor' in self.existing:
             return NOOP
 
         return ADD
@@ -160,18 +166,6 @@ class AppRole(Auth):
     required_fields = ['name', 'policies']
     config_key = 'approles'
 
-    @staticmethod
-    def map_val(dest, src, key, default, src_key=None):
-        """Will ensure a dict has values sourced from either
-        another dict or based on the provided default"""
-        if not src_key:
-            src_key = key
-
-        if src_key in src:
-            dest[key] = src[src_key]
-        else:
-            dest[key] = default
-
     def resources(self):
         return [self] + self.secret_ids
 
@@ -190,14 +184,14 @@ def __init__(self, obj, opt):
         role_obj = {
             'policies': ','.join(policies)
         }
-        AppRole.map_val(role_obj, obj, 'bound_cidr_list', '', 'cidr_list')
-        AppRole.map_val(role_obj, obj, 'secret_id_num_uses', 0, 'secret_uses')
-        AppRole.map_val(role_obj, obj, 'secret_id_ttl', 0, 'secret_ttl')
-        AppRole.map_val(role_obj, obj, 'period', 0)
-        AppRole.map_val(role_obj, obj, 'token_max_ttl', 0)
-        AppRole.map_val(role_obj, obj, 'token_ttl', 0)
-        AppRole.map_val(role_obj, obj, 'bind_secret_id', True)
-        AppRole.map_val(role_obj, obj, 'token_num_uses', 0)
+        map_val(role_obj, obj, 'bound_cidr_list', '', 'cidr_list')
+        map_val(role_obj, obj, 'secret_id_num_uses', 0, 'secret_uses')
+        map_val(role_obj, obj, 'secret_id_ttl', 0, 'secret_ttl')
+        map_val(role_obj, obj, 'period', 0)
+        map_val(role_obj, obj, 'token_max_ttl', 0)
+        map_val(role_obj, obj, 'token_ttl', 0)
+        map_val(role_obj, obj, 'bind_secret_id', True)
+        map_val(role_obj, obj, 'token_num_uses', 0)
         self._obj = role_obj
         if 'preset' in obj:
             self.presets(obj['preset'], opt)
@@ -232,6 +226,84 @@ def delete(self, client):
         client.delete_role(self.app_name)
 
 
+class LDAP(Auth):
+    """LDAP Authentication"""
+    required_fields = ['url']
+    config_key = 'ldap_auth'
+
+    def __init__(self, obj, opt):
+        super(LDAP, self).__init__('ldap', obj, opt)
+        auth_obj = {
+            'url': obj['url']
+        }
+        self.mount = 'ldap'
+        self.path = sanitize_mount("auth/ldap/config")
+        if 'secrets' in obj:
+            self.secret = obj['secrets']
+        else:
+            self.secret = None
+
+        map_val(auth_obj, obj, 'starttls', False)
+        map_val(auth_obj, obj, 'insecure_tls', False)
+        map_val(auth_obj, obj, 'discoverdn')
+        map_val(auth_obj, obj, 'userdn')
+        map_val(auth_obj, obj, 'userattr')
+        map_val(auth_obj, obj, 'deny_null_bind', True)
+        map_val(auth_obj, obj, 'upndomain')
+        map_val(auth_obj, obj, 'groupfilter')
+        map_val(auth_obj, obj, 'groupdn')
+        map_val(auth_obj, obj, 'groupattr')
+        map_val(auth_obj, obj, 'binddn')
+        self._obj = auth_obj
+
+    def obj(self):
+        ldap_obj = self._obj
+        if self.secret:
+            filename = hard_path(self.secret, self.opt.secrets)
+            secret_file(filename)
+
+        return ldap_obj
+
+
+class LDAPGroup(Resource):
+    """LDAP Group Policy Mapping"""
+    required_fields = ['policies', 'group']
+    config_key = 'ldap_groups'
+
+    def __init__(self, obj, opt):
+        super(LDAPGroup, self).__init__(obj, opt)
+        self.path = sanitize_mount("auth/%s/groups/%s" %
+                                   (obj.get('mount', 'ldap'), obj['group']))
+        self._obj = {
+            "policies": obj['policies']
+        }
+
+    def obj(self):
+        return {
+            'policies': ','.join(self._obj.get('policies', []))
+        }
+
+
+class LDAPUser(Resource):
+    """LDAP User Membership"""
+    required_fields = ['user']
+    config_key = 'ldap_users'
+
+    def __init__(self, obj, opt):
+        super(LDAPUser, self).__init__(obj, opt)
+        self.path = sanitize_mount("auth/%s/users/%s" %
+                                   (obj.get('mount', 'ldap'), obj['user']))
+        self._obj = {}
+        map_val(self._obj, obj, 'groups', [])
+        map_val(self._obj, obj, 'policies', [])
+
+    def obj(self):
+        return {
+            'groups': ','.join(self._obj.get('groups', [])),
+            'policies': ','.join(self._obj.get('policies', []))
+        }
+
+
 class UserPass(Auth):
     """UserPass"""
     required_fields = ['username', 'password_file', 'policies']
@@ -275,9 +347,8 @@ def __init__(self, obj, opt):
         self.path = obj['name']
         if self.present:
             self.filename = obj['file']
-            cli_obj = merge_dicts(load_var_files(opt),
-                                  cli_hash(opt.extra_vars))
-            self._obj = merge_dicts(cli_obj, obj.get('vars', {}))
+            base_obj = load_vars(opt)
+            self._obj = merge_dicts(base_obj, obj.get('vars', {}))
 
     def validate(self, obj):
         super(Policy, self).validate(obj)

aomi/model/aws.py

@@ -5,8 +5,8 @@
 import aomi.model.resource
 from aomi.vault import is_mounted
 from aomi.model.resource import Secret, Resource
-from aomi.helpers import hard_path, merge_dicts, cli_hash
-from aomi.template import load_var_files, render
+from aomi.helpers import hard_path, merge_dicts
+from aomi.template import load_vars, render
 from aomi.validation import sanitize_mount, secret_file, check_obj
 LOG = logging.getLogger(__name__)
 
@@ -58,9 +58,8 @@ def obj(self):
         s_obj = {}
         if 'policy' in self._obj:
             role_template_obj = self._obj.get('vars', {})
-            cli_obj = merge_dicts(load_var_files(self.opt),
-                                  cli_hash(self.opt.extra_vars))
-            template_obj = merge_dicts(role_template_obj, cli_obj)
+            base_obj = load_vars(self.opt)
+            template_obj = merge_dicts(role_template_obj, base_obj)
             aws_role = render(self._obj['policy'], template_obj)
             aws_role = aws_role.replace(" ", "").replace("\n", "")
             s_obj = {'policy': aws_role}

aomi/render.py

@@ -7,9 +7,9 @@
 from pkg_resources import resource_filename
 import hvac
 from cryptorito import portable_b64decode, is_base64
-from aomi.helpers import cli_hash, merge_dicts, \
+from aomi.helpers import merge_dicts, cli_hash, \
     path_pieces, abspath
-from aomi.template import render, load_var_files
+from aomi.template import render, load_vars
 from aomi.error import output as error_output
 import aomi.exceptions
 LOG = logging.getLogger(__name__)
@@ -80,9 +80,8 @@ def grok_template_file(src):
 
 def blend_vars(secrets, opt):
     """Blends secret and static variables together"""
-    extra_obj = merge_dicts(load_var_files(opt),
-                            cli_hash(opt.extra_vars))
-    merged = merge_dicts(extra_obj, secrets)
+    base_obj = load_vars(opt)
+    merged = merge_dicts(base_obj, secrets)
     template_obj = dict((k, v) for k, v in iteritems(merged) if v)
     # give templates something to iterate over
     template_obj['aomi_items'] = template_obj.copy()

aomi/seed_action.py

@@ -129,6 +129,7 @@ def details_dict(resource, opt):
 def maybe_details(resource, opt):
     """At the first level of verbosity this will print out detailed
     change information on for the specified Vault resource"""
+
     if opt.verbose == 0:
         return
 

aomi/template.py

@@ -78,11 +78,21 @@ def render(filename, obj):
                                     ' '.join(template_traces))
 
 
-def load_var_files(opt):
+def load_vars(opt):
+    """Loads variable from cli and var files, passing in cli options
+    as a seed (although they can be overwritten!)"""
+    cli_opts = cli_hash(opt.extra_vars)
+    return merge_dicts(load_var_files(opt, cli_opts), cli_opts)
+
+
+def load_var_files(opt, p_obj=None):
     """Load variable files, merge, return contents"""
     obj = {}
+    if p_obj:
+        obj = p_obj
+
     for var_file in opt.extra_vars_file:
-        yamlz = yaml.safe_load(open(abspath(var_file)).read())
+        yamlz = yaml.safe_load(render(var_file, obj))
         obj = merge_dicts(obj.copy(), yamlz)
 
     return obj
@@ -144,6 +154,5 @@ def get_secretfile(opt):
 def render_secretfile(opt):
     """Renders and returns the Secretfile construct"""
     secretfile_path = abspath(opt.secretfile)
-    obj = merge_dicts(load_var_files(opt),
-                      cli_hash(opt.extra_vars))
+    obj = load_vars(opt)
     return render(secretfile_path, obj)

aomi/vault.py

@@ -106,6 +106,8 @@ def __init__(self, _url=None, token=None, _cert=None, _verify=True,
         ssl_verify = True
         if 'VAULT_SKIP_VERIFY' in os.environ:
             if os.environ['VAULT_SKIP_VERIFY'] == '1':
+                import urllib3
+                urllib3.disable_warnings()
                 ssl_verify = False
 
         self.initial_token = None

docs/auth-resources.md

@@ -69,3 +69,63 @@ duo:
 key: "foo"
 secret: "bar"
 ```
+
+# LDAP
+
+The aomi tool supports [LDAP authentication](https://www.vaultproject.io/docs/auth/ldap.html), along with mapping users to policies/groups, and groups to policies. It should work with all the ways that Vault supports LDAP.
+
+The basic configuration will setup an auth endpoint. You can specify all the config variables listed in the Vault documentation itself. Note that the `bindpass` and `certificate` options _must_ be specified in a "secret" file indicated by the `secrets` option.
+
+----
+
+`Secretfile`
+
+```
+ldap_auth:
+  - url: "ldap://example.com"
+    binddn: "cn=vault,dc=example,dc=com"
+    userattr: "uid"
+    userdn: "dc=example,dc=com"
+    groupdn: "dc=example,dc=com"
+    secrets: "ldap.yml"
+```
+
+`.secrets/ldap.yml`
+
+```
+bindpass: "password"
+```
+
+## LDAP Users
+
+Vault provides the ability to map users to policies and LDAP groups. Note that depending on your particular Vault/LDAP configuration, you may not be able to set overrides for users.
+
+----
+
+`Secretfile`
+
+```
+ldap_users:
+  - user: "test"
+    groups:
+      - "some-group"
+      - "another-group"
+    policies:
+      - "another-policy"
+```
+
+## LDAP Groups
+
+Vault provides the ability to map LDAP groups to policies.
+
+----
+
+`Secretfile`
+
+```
+ldap_groups:
+   - name: "some-group"
+     policies:
+       - "some-policy"
+```
+

docs/data.md

@@ -52,7 +52,7 @@ The seed command will go through the [`Secretfile`]({{site.baseurl}}/secretfile)
 
 It is possible to have aomi clean up unrecognzied Vault mount points. Note that by doing this, _any_ mount point that is not defined in the fully rendered `Secretfile` will be unmounted. This causes non-recoverable data from the perspective of Vault. Care should be taken to back up your data using [`freeze`]({{site.baseurl}}/data#freeze) and [`thaw`]({{site.baseurl}}/data#thaw) prior to using this option. It may be enabled by specifying `--remove-unknown`.
 
-The `Secretfile` is interpreted as a Jinja2 template, and you can pass in `--extra-vars` and `--extra-vars-file` to `seed`. This opens up some possibilities for bulk-creating sets of credentials based on integrations with other systems, while still preserving various paths and structures.
+The `Secretfile` is interpreted as a Jinja2 template, and you can pass in `--extra-vars` and `--extra-vars-file` to `seed`. This opens up some possibilities for bulk-creating sets of credentials based on integrations with other systems, while still preserving various paths and structures. The files passed to `--extra-vars-file` will be interpreted in order, with each being merged subsequently. They are also treated as templates prior to being interpreted as YAML.
 
 The `seed` command will make some sanity checks as it goes. One of these is to check for the presence of the secrets directory within your `.gitignore`. As this directory can contain plaintext secrets, it should never be committed. A recommended alternative is to specify an icefile with the `--thaw-from` option. When doing this, plain text secrets are only accessible in the clear during the seed operation and are removed immediately after.
 

scripts/integration

@@ -21,7 +21,7 @@ VAULTBIN="${VAULTDIR}/vault"
 # Define our test groups. Currently not every test is executed
 # in every single execution context.
 GPG_TESTS="cold_storage"
-TESTS="a_smoke environment exec_context seed template cubbyhole output ux diff"
+TESTS="a_smoke environment exec_context seed template cubbyhole output ux diff auth"
 VAULT_TESTS="aws"
 
 # These are the HCV paths where we can find the secrets used