From 97dac65b362f9f0699ada0a4d46b96bd4eb401dd Mon Sep 17 00:00:00 2001 From: adam Date: Tue, 19 Feb 2019 16:31:50 -0700 Subject: [PATCH] Initial release. --- README.md | 39 ++++++++++++++- lsstvaultutils/recursivedeleter.py | 76 ++++++++++++++++++++++++++++++ lsstvaultutils/secretcopier.py | 57 +++++++++++++++++----- lsstvaultutils/timeformatter.py | 21 +++++++++ lsstvaultutils/tokenadmin.py | 16 ++++--- setup.py | 3 +- 6 files changed, 191 insertions(+), 21 deletions(-) create mode 100644 lsstvaultutils/recursivedeleter.py create mode 100644 lsstvaultutils/timeformatter.py diff --git a/README.md b/README.md index d576523..ea8a80c 100644 --- a/README.md +++ b/README.md @@ -1 +1,38 @@ -# Vaultutils +# LSST Vault Utilities + +This package is a set of Vault utilities useful for the LSST use case. + +## Classes + +The package name is `lsstvaultutils`. Its functional classes are: + +1. `SecretCopier` -- this copies secrets between the current Kubernetes + context and a Vault instance. + +2. `TokenAdmin` -- this highly LSST-specific class allows you to specify a + path under the Vault secret store, and it will generate three tokens + (read, write, and admin) for manipulating secrets under the path. It + stores those under secret/delegated, so that an admin can find (and, + if need be, revoke) them later. It also manages revoking those + tokens and removing them from the secret/delegated path. + +3. `RecursiveDeleter` -- this adds a recursive deletion feature to Vault + for removing a whole secret tree at a time. + +There is also a TimeFormatter class that exists only to add milliseconds +to the debugging logs. + +## Programs + +The major functionality of these classes is also exposed as standalone +programs. + +1. `copyk2v` -- copy a Kubernetes secret to a Vault secret path. + +2. `copyv2k` -- copy a set of Vault secrets at a specified path to a + Kubernetes secret. + +3. `tokenadmin` -- Creating or revoke token sets for a given Vault secret + path. + +4. `vaultrmrf` -- Remove a Vault secret path and everything underneath it. diff --git a/lsstvaultutils/recursivedeleter.py b/lsstvaultutils/recursivedeleter.py new file mode 100644 index 0000000..c1607d3 --- /dev/null +++ b/lsstvaultutils/recursivedeleter.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +"""RecursiveDeleter removes an entire secret tree from Vault. +""" + +import logging +import click +import hvac +from .timeformatter import TimeFormatter + + +@click.command() +@click.argument('vault_secret_path') +@click.option('--url', envvar='VAULT_ADDR', + help="URL of Vault endpoint.") +@click.option('--token', envvar='VAULT_TOKEN', + help="Vault token to use.") +@click.option('--cacert', envvar='VAULT_CAPATH', + help="Path to Vault CA certificate.") +@click.option('--debug', envvar='DEBUG', is_flag=True, + help="Enable debugging.") +def standalone(vault_secret_path, url, token, cacert, debug): + client = RecursiveDeleter(url, token, cacert, debug) + client.recursive_delete(vault_secret_path) + + +class RecursiveDeleter(object): + """Class to remove a whole secret tree from Vault. + """ + + def __init__(self, url, token, cacert, debug): + logger = logging.getLogger(__name__) + if debug: + logger.setLevel(logging.DEBUG) + ch = logging.StreamHandler() + if debug: + ch.setLevel(logging.DEBUG) + formatter = TimeFormatter( + '%(asctime)s [%(levelname)s] %(name)s | %(message)s', + datefmt='%Y-%m-%d %H:%M:%S.%F %Z(%z)') + ch.setFormatter(formatter) + logger.addHandler(ch) + self.logger = logger + if debug: + self.logger.debug("Debug logging started.") + if not url and token and cacert: + raise ValueError("All of Vault URL, Vault Token, and Vault CA " + + "path must be present, either in the " + + "or as options.") + self.vault_client = self.get_vault_client(url, token, cacert) + + def get_vault_client(self, url, token, cacert): + """Acquire a Vault client. + """ + self.logger.debug("Acquiring Vault client for '%s'." % url) + client = hvac.Client(url=url, token=token, verify=cacert) + assert client.is_authenticated() + return client + + def recursive_delete(self, path): + """Delete path and everything under it. + """ + self.logger.debug("Removing '%s' recursively." % path) + pkeys = [] + resp = self.vault_client.list(path) + if resp: + self.logger.debug("Removing tree rooted at '%s'" % path) + pkeys = resp["data"]["keys"] + for item in [(path + "/" + x) for x in pkeys]: + self.recursive_delete(item) + else: + self.logger.debug("Removing '%s' as leaf node." % path) + self.vault_client.delete(path) + + +if __name__ == '__main__': + standalone() diff --git a/lsstvaultutils/secretcopier.py b/lsstvaultutils/secretcopier.py index 879adf5..8352844 100644 --- a/lsstvaultutils/secretcopier.py +++ b/lsstvaultutils/secretcopier.py @@ -3,12 +3,12 @@ """ import logging -import os import click import hvac from base64 import b64decode, b64encode from kubernetes import client, config from kubernetes.client.rest import ApiException +from .timeformatter import TimeFormatter @click.command() @@ -20,8 +20,13 @@ help="Vault token to use.") @click.option('--cacert', envvar='VAULT_CAPATH', help="Path to Vault CA certificate.") -def standalonek2v(url, token, cacert, k8s_secret_name, vault_secret_path): - client = SecretCopier(url, token, cacert) +@click.option('--debug', envvar='DEBUG', is_flag=True, + help="Enable debugging.") +def standalonek2v(url, token, cacert, k8s_secret_name, vault_secret_path, + debug): + """Copy from Kubernetes to Vault. + """ + client = SecretCopier(url, token, cacert, debug) client.copy_k8s_to_vault(k8s_secret_name, vault_secret_path) @@ -34,22 +39,30 @@ def standalonek2v(url, token, cacert, k8s_secret_name, vault_secret_path): help="Vault token to use.") @click.option('--cacert', envvar='VAULT_CAPATH', help="Path to Vault CA certificate.") -def standalonev2k(url, token, cacert, k8s_secret_name, vault_secret_path): - client = SecretCopier(url, token, cacert) +@click.option('--debug', envvar='DEBUG', is_flag=True, + help="Enable debugging.") +def standalonev2k(url, token, cacert, k8s_secret_name, vault_secret_path, + debug): + """Copy from Vault to Kubernetes. + """ + client = SecretCopier(url, token, cacert, debug) client.copy_vault_to_k8s(vault_secret_path, k8s_secret_name) class SecretCopier(object): - def __init__(self, url, token, cacert): - debug = os.getenv('DEBUG') + """Class to copy secrets between Kubernetes and Vault. + """ + + def __init__(self, url, token, cacert, debug=False): logger = logging.getLogger(__name__) if debug: logger.setLevel(logging.DEBUG) ch = logging.StreamHandler() if debug: ch.setLevel(logging.DEBUG) - formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + formatter = TimeFormatter( + '%(asctime)s [%(levelname)s] %(name)s | %(message)s', + datefmt='%Y-%m-%d %H:%M:%S.%F %Z(%z)') ch.setFormatter(formatter) logger.addHandler(ch) self.logger = logger @@ -74,12 +87,16 @@ def get_vault_client(self, url, token, cacert): return client def get_k8s_client(self): + """Acquire a Kubernetes client from currently-selected config. + """ self.logger.debug("Acquiring k8s client.") config.load_kube_config() k8s_client = client.CoreV1Api() return k8s_client def encode_secret_data(self): + """Base64-encode secret data for Kubernetes storage. + """ self.logger.debug("Base64-encoding secret data") self.encoded_secret = {} for k in self.secret: @@ -87,6 +104,8 @@ def encode_secret_data(self): self.secret[k].encode('utf-8')).decode('utf-8') def read_k8s_secret(self, k8s_secret_name): + """Read a secret from Kubernetes. + """ self.logger.debug("Reading secret from '%s' " % k8s_secret_name + " in namespace '%s'." % self.namespace) secret = self.k8s_client.read_namespaced_secret( @@ -96,19 +115,29 @@ def read_k8s_secret(self, k8s_secret_name): self.secret = {} for k in data: v = data[k] - self.secret[k] = b64decode(v).decode('utf-8') + if v: + self.secret[k] = b64decode(v).decode('utf-8') + else: + self.secret[k] = v def write_vault_secret(self, vault_secret_path): + """Write a secret to Vault. + """ self.logger.debug("Writing secret to '%s'." % vault_secret_path) for k in self.secret: - self.vault_client.write(vault_secret_path + "/" + k, - value=self.secret[k]) + spath = vault_secret_path + "/" + k + self.logger.debug("Writing secret to '%s'." % spath) + self.vault_client.write(spath, value=self.secret[k]) def copy_k8s_to_vault(self, k8s_secret_name, vault_secret_path): + """Copy secret from Kubernetes to Vault. + """ self.read_k8s_secret(k8s_secret_name) self.write_vault_secret(vault_secret_path) def read_vault_secret(self, vault_secret_path): + """Read a secret from Vault. + """ self.logger.debug("Reading secret from '%s'." % vault_secret_path) path = vault_secret_path pathcomps = path.split('/') @@ -131,6 +160,8 @@ def read_vault_secret(self, vault_secret_path): self.secret = valdata def write_k8s_secret(self, k8s_secret_name): + """Write a secret to Kubernetes. + """ oldsecret = None self.logger.debug("Determining whether secret '%s'" % k8s_secret_name + "exists in namespace '%s'." % self.namespace) @@ -165,6 +196,8 @@ def write_k8s_secret(self, k8s_secret_name): secret) def copy_vault_to_k8s(self, vault_secret_path, k8s_secret_name): + """Copy a secret from Vault to Kubernetes. + """ self.read_vault_secret(vault_secret_path) self.write_k8s_secret(k8s_secret_name) diff --git a/lsstvaultutils/timeformatter.py b/lsstvaultutils/timeformatter.py new file mode 100644 index 0000000..46008ba --- /dev/null +++ b/lsstvaultutils/timeformatter.py @@ -0,0 +1,21 @@ +import logging +import time + + +class TimeFormatter(logging.Formatter): + """Time formatter that does milliseconds. + https://stackoverflow.com/questions/6290739/\ + python-logging-use-milliseconds-in-time-format + """ + + def formatTime(self, record, datefmt=None): + ct = self.converter(record.created) + if datefmt: + if "%F" in datefmt: + msec = "%03d" % record.msecs + datefmt = datefmt.replace("%F", msec) + s = time.strftime(datefmt, ct) + else: + t = time.strftime("%Y-%m-%d %H:%M:%S", ct) + s = "%s,%03d" % (t, record.msecs) + return s diff --git a/lsstvaultutils/tokenadmin.py b/lsstvaultutils/tokenadmin.py index 16e1271..d4f071c 100644 --- a/lsstvaultutils/tokenadmin.py +++ b/lsstvaultutils/tokenadmin.py @@ -6,7 +6,7 @@ import click import hvac import logging -import os +from .timeformatter import TimeFormatter POLICY_ROOT = "policy/delegated" SECRET_ROOT = "secret/delegated" @@ -23,10 +23,12 @@ help="Path to Vault CA certificate.") @click.option('--ttl', envvar="VAULT_TTL", default="8766h", help="TTL for tokens [ 1 year = \"8776h\" ]") -def standalone(verb, vault_secret_path, url, token, cacert, ttl): +@click.option('--debug', envvar='DEBUG', is_flag=True, + help="Enable debugging.") +def standalone(verb, vault_secret_path, url, token, cacert, ttl, debug): """Run as standalone program. """ - client = AdminTool(url, token, cacert, ttl) + client = AdminTool(url, token, cacert, ttl, debug) client.execute(verb, vault_secret_path) @@ -44,16 +46,16 @@ class AdminTool(object): """Class to build and destroy token hierarchy in LSST taxonomy. """ - def __init__(self, url, token, cacert, ttl='8766h'): - debug = os.getenv('DEBUG') + def __init__(self, url, token, cacert, ttl='8766h', debug=False): logger = logging.getLogger(__name__) if debug: logger.setLevel(logging.DEBUG) ch = logging.StreamHandler() if debug: ch.setLevel(logging.DEBUG) - formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + formatter = TimeFormatter( + '%(asctime)s [%(levelname)s] %(name)s | %(message)s', + datefmt='%Y-%m-%d %H:%M:%S.%F %Z(%z)') ch.setFormatter(formatter) logger.addHandler(ch) self.logger = logger diff --git a/setup.py b/setup.py index 7c8ecae..ad787eb 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,8 @@ def local_read(filename): 'console_scripts': [ 'copyk2v = lsstvaultutils.secretcopier:standalonek2v', 'copyv2k = lsstvaultutils.secretcopier:standalonev2k', - 'tokenadmin = lsstvaultutils.tokenadmin:standalone' + 'tokenadmin = lsstvaultutils.tokenadmin:standalone', + 'vaultrmrf = lsstvaultutils.recursivedeleter:standalone' ] }, )