Initial release.

README.md

@@ -1 +1,38 @@
-# Vaultutils
+# LSST Vault Utilities
+
+This package is a set of Vault utilities useful for the LSST use case.
+
+## Classes
+
+The package name is `lsstvaultutils`.  Its functional classes are:
+
+1. `SecretCopier` -- this copies secrets between the current Kubernetes
+   context and a Vault instance.
+
+2. `TokenAdmin` -- this highly LSST-specific class allows you to specify a
+   path under the Vault secret store, and it will generate three tokens
+   (read, write, and admin) for manipulating secrets under the path.  It
+   stores those under secret/delegated, so that an admin can find (and,
+   if need be, revoke) them later.  It also manages revoking those
+   tokens and removing them from the secret/delegated path.
+
+3. `RecursiveDeleter` -- this adds a recursive deletion feature to Vault
+   for removing a whole secret tree at a time.
+
+There is also a TimeFormatter class that exists only to add milliseconds
+to the debugging logs.
+
+## Programs
+
+The major functionality of these classes is also exposed as standalone
+programs.
+
+1. `copyk2v` -- copy a Kubernetes secret to a Vault secret path.
+
+2. `copyv2k` -- copy a set of Vault secrets at a specified path to a
+   Kubernetes secret.
+
+3. `tokenadmin` -- Creating or revoke token sets for a given Vault secret
+   path.
+
+4. `vaultrmrf` -- Remove a Vault secret path and everything underneath it.

lsstvaultutils/recursivedeleter.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+"""RecursiveDeleter removes an entire secret tree from Vault.
+"""
+
+import logging
+import click
+import hvac
+from .timeformatter import TimeFormatter
+
+
+@click.command()
+@click.argument('vault_secret_path')
+@click.option('--url', envvar='VAULT_ADDR',
+              help="URL of Vault endpoint.")
+@click.option('--token', envvar='VAULT_TOKEN',
+              help="Vault token to use.")
+@click.option('--cacert', envvar='VAULT_CAPATH',
+              help="Path to Vault CA certificate.")
+@click.option('--debug', envvar='DEBUG', is_flag=True,
+              help="Enable debugging.")
+def standalone(vault_secret_path, url, token, cacert, debug):
+    client = RecursiveDeleter(url, token, cacert, debug)
+    client.recursive_delete(vault_secret_path)
+
+
+class RecursiveDeleter(object):
+    """Class to remove a whole secret tree from Vault.
+    """
+
+    def __init__(self, url, token, cacert, debug):
+        logger = logging.getLogger(__name__)
+        if debug:
+            logger.setLevel(logging.DEBUG)
+        ch = logging.StreamHandler()
+        if debug:
+            ch.setLevel(logging.DEBUG)
+        formatter = TimeFormatter(
+            '%(asctime)s [%(levelname)s] %(name)s | %(message)s',
+            datefmt='%Y-%m-%d %H:%M:%S.%F %Z(%z)')
+        ch.setFormatter(formatter)
+        logger.addHandler(ch)
+        self.logger = logger
+        if debug:
+            self.logger.debug("Debug logging started.")
+        if not url and token and cacert:
+            raise ValueError("All of Vault URL, Vault Token, and Vault CA " +
+                             "path must be present, either in the " +
+                             "or as options.")
+        self.vault_client = self.get_vault_client(url, token, cacert)
+
+    def get_vault_client(self, url, token, cacert):
+        """Acquire a Vault client.
+        """
+        self.logger.debug("Acquiring Vault client for '%s'." % url)
+        client = hvac.Client(url=url, token=token, verify=cacert)
+        assert client.is_authenticated()
+        return client
+
+    def recursive_delete(self, path):
+        """Delete path and everything under it.
+        """
+        self.logger.debug("Removing '%s' recursively." % path)
+        pkeys = []
+        resp = self.vault_client.list(path)
+        if resp:
+            self.logger.debug("Removing tree rooted at '%s'" % path)
+            pkeys = resp["data"]["keys"]
+            for item in [(path + "/" + x) for x in pkeys]:
+                self.recursive_delete(item)
+        else:
+            self.logger.debug("Removing '%s' as leaf node." % path)
+            self.vault_client.delete(path)
+
+
+if __name__ == '__main__':
+    standalone()

lsstvaultutils/secretcopier.py

@@ -3,12 +3,12 @@
 """
 
 import logging
-import os
 import click
 import hvac
 from base64 import b64decode, b64encode
 from kubernetes import client, config
 from kubernetes.client.rest import ApiException
+from .timeformatter import TimeFormatter
 
 
 @click.command()
@@ -20,8 +20,13 @@
               help="Vault token to use.")
 @click.option('--cacert', envvar='VAULT_CAPATH',
               help="Path to Vault CA certificate.")
-def standalonek2v(url, token, cacert, k8s_secret_name, vault_secret_path):
-    client = SecretCopier(url, token, cacert)
+@click.option('--debug', envvar='DEBUG', is_flag=True,
+              help="Enable debugging.")
+def standalonek2v(url, token, cacert, k8s_secret_name, vault_secret_path,
+                  debug):
+    """Copy from Kubernetes to Vault.
+    """
+    client = SecretCopier(url, token, cacert, debug)
     client.copy_k8s_to_vault(k8s_secret_name, vault_secret_path)
 
 
@@ -34,22 +39,30 @@ def standalonek2v(url, token, cacert, k8s_secret_name, vault_secret_path):
               help="Vault token to use.")
 @click.option('--cacert', envvar='VAULT_CAPATH',
               help="Path to Vault CA certificate.")
-def standalonev2k(url, token, cacert, k8s_secret_name, vault_secret_path):
-    client = SecretCopier(url, token, cacert)
+@click.option('--debug', envvar='DEBUG', is_flag=True,
+              help="Enable debugging.")
+def standalonev2k(url, token, cacert, k8s_secret_name, vault_secret_path,
+                  debug):
+    """Copy from Vault to Kubernetes.
+    """
+    client = SecretCopier(url, token, cacert, debug)
     client.copy_vault_to_k8s(vault_secret_path, k8s_secret_name)
 
 
 class SecretCopier(object):
-    def __init__(self, url, token, cacert):
-        debug = os.getenv('DEBUG')
+    """Class to copy secrets between Kubernetes and Vault.
+    """
+
+    def __init__(self, url, token, cacert, debug=False):
         logger = logging.getLogger(__name__)
         if debug:
             logger.setLevel(logging.DEBUG)
         ch = logging.StreamHandler()
         if debug:
             ch.setLevel(logging.DEBUG)
-        formatter = logging.Formatter(
-            '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        formatter = TimeFormatter(
+            '%(asctime)s [%(levelname)s] %(name)s | %(message)s',
+            datefmt='%Y-%m-%d %H:%M:%S.%F %Z(%z)')
         ch.setFormatter(formatter)
         logger.addHandler(ch)
         self.logger = logger
@@ -74,19 +87,25 @@ def get_vault_client(self, url, token, cacert):
         return client
 
     def get_k8s_client(self):
+        """Acquire a Kubernetes client from currently-selected config.
+        """
         self.logger.debug("Acquiring k8s client.")
         config.load_kube_config()
         k8s_client = client.CoreV1Api()
         return k8s_client
 
     def encode_secret_data(self):
+        """Base64-encode secret data for Kubernetes storage.
+        """
         self.logger.debug("Base64-encoding secret data")
         self.encoded_secret = {}
         for k in self.secret:
             self.encoded_secret[k] = b64encode(
                 self.secret[k].encode('utf-8')).decode('utf-8')
 
     def read_k8s_secret(self, k8s_secret_name):
+        """Read a secret from Kubernetes.
+        """
         self.logger.debug("Reading secret from '%s' " % k8s_secret_name +
                           " in namespace '%s'." % self.namespace)
         secret = self.k8s_client.read_namespaced_secret(
@@ -96,19 +115,29 @@ def read_k8s_secret(self, k8s_secret_name):
         self.secret = {}
         for k in data:
             v = data[k]
-            self.secret[k] = b64decode(v).decode('utf-8')
+            if v:
+                self.secret[k] = b64decode(v).decode('utf-8')
+            else:
+                self.secret[k] = v
 
     def write_vault_secret(self, vault_secret_path):
+        """Write a secret to Vault.
+        """
         self.logger.debug("Writing secret to '%s'." % vault_secret_path)
         for k in self.secret:
-            self.vault_client.write(vault_secret_path + "/" + k,
-                                    value=self.secret[k])
+            spath = vault_secret_path + "/" + k
+            self.logger.debug("Writing secret to '%s'." % spath)
+            self.vault_client.write(spath, value=self.secret[k])
 
     def copy_k8s_to_vault(self, k8s_secret_name, vault_secret_path):
+        """Copy secret from Kubernetes to Vault.
+        """
         self.read_k8s_secret(k8s_secret_name)
         self.write_vault_secret(vault_secret_path)
 
     def read_vault_secret(self, vault_secret_path):
+        """Read a secret from Vault.
+        """
         self.logger.debug("Reading secret from '%s'." % vault_secret_path)
         path = vault_secret_path
         pathcomps = path.split('/')
@@ -131,6 +160,8 @@ def read_vault_secret(self, vault_secret_path):
         self.secret = valdata
 
     def write_k8s_secret(self, k8s_secret_name):
+        """Write a secret to Kubernetes.
+        """
         oldsecret = None
         self.logger.debug("Determining whether secret '%s'" % k8s_secret_name +
                           "exists in namespace '%s'." % self.namespace)
@@ -165,6 +196,8 @@ def write_k8s_secret(self, k8s_secret_name):
                                                      secret)
 
     def copy_vault_to_k8s(self, vault_secret_path, k8s_secret_name):
+        """Copy a secret from Vault to Kubernetes.
+        """
         self.read_vault_secret(vault_secret_path)
         self.write_k8s_secret(k8s_secret_name)
 

lsstvaultutils/timeformatter.py

@@ -0,0 +1,21 @@
+import logging
+import time
+
+
+class TimeFormatter(logging.Formatter):
+    """Time formatter that does milliseconds.
+    https://stackoverflow.com/questions/6290739/\
+     python-logging-use-milliseconds-in-time-format
+    """
+
+    def formatTime(self, record, datefmt=None):
+        ct = self.converter(record.created)
+        if datefmt:
+            if "%F" in datefmt:
+                msec = "%03d" % record.msecs
+                datefmt = datefmt.replace("%F", msec)
+            s = time.strftime(datefmt, ct)
+        else:
+            t = time.strftime("%Y-%m-%d %H:%M:%S", ct)
+            s = "%s,%03d" % (t, record.msecs)
+        return s

lsstvaultutils/tokenadmin.py

@@ -6,7 +6,7 @@
 import click
 import hvac
 import logging
-import os
+from .timeformatter import TimeFormatter
 
 POLICY_ROOT = "policy/delegated"
 SECRET_ROOT = "secret/delegated"
@@ -23,10 +23,12 @@
               help="Path to Vault CA certificate.")
 @click.option('--ttl', envvar="VAULT_TTL", default="8766h",
               help="TTL for tokens [ 1 year = \"8776h\" ]")
-def standalone(verb, vault_secret_path, url, token, cacert, ttl):
+@click.option('--debug', envvar='DEBUG', is_flag=True,
+              help="Enable debugging.")
+def standalone(verb, vault_secret_path, url, token, cacert, ttl, debug):
     """Run as standalone program.
     """
-    client = AdminTool(url, token, cacert, ttl)
+    client = AdminTool(url, token, cacert, ttl, debug)
     client.execute(verb, vault_secret_path)
 
 
@@ -44,16 +46,16 @@ class AdminTool(object):
     """Class to build and destroy token hierarchy in LSST taxonomy.
     """
 
-    def __init__(self, url, token, cacert, ttl='8766h'):
-        debug = os.getenv('DEBUG')
+    def __init__(self, url, token, cacert, ttl='8766h', debug=False):
         logger = logging.getLogger(__name__)
         if debug:
             logger.setLevel(logging.DEBUG)
         ch = logging.StreamHandler()
         if debug:
             ch.setLevel(logging.DEBUG)
-        formatter = logging.Formatter(
-            '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        formatter = TimeFormatter(
+            '%(asctime)s [%(levelname)s] %(name)s | %(message)s',
+            datefmt='%Y-%m-%d %H:%M:%S.%F %Z(%z)')
         ch.setFormatter(formatter)
         logger.addHandler(ch)
         self.logger = logger

setup.py

@@ -61,7 +61,8 @@ def local_read(filename):
         'console_scripts': [
             'copyk2v = lsstvaultutils.secretcopier:standalonek2v',
             'copyv2k = lsstvaultutils.secretcopier:standalonev2k',
-            'tokenadmin = lsstvaultutils.tokenadmin:standalone'
+            'tokenadmin = lsstvaultutils.tokenadmin:standalone',
+            'vaultrmrf = lsstvaultutils.recursivedeleter:standalone'
         ]
     },
 )