From bf9b23156e6f9cc79e2889c74e5852ec863d1219 Mon Sep 17 00:00:00 2001 From: Constantin Muraru Date: Mon, 11 Feb 2019 18:52:06 +0200 Subject: [PATCH] Retrieve secrets from AWS SSM --- README.md | 50 +++++++++++++------ requirements.txt | 2 +- .../ansible/filter_plugins/commonfilters.py | 14 ++++-- src/ops/simplessm.py | 41 +++++++++++++++ src/ops/{simplesecrets.py => simplevault.py} | 0 5 files changed, 86 insertions(+), 21 deletions(-) create mode 100644 src/ops/simplessm.py rename src/ops/{simplesecrets.py => simplevault.py} (100%) diff --git a/README.md b/README.md index 7c7feea9..05c9eb5e 100644 --- a/README.md +++ b/README.md @@ -138,18 +138,11 @@ $ aws configure --profile aws_account_name ## Azure TBD -## SKMS -Create a file in `~/.skms/credentials.yaml` which looks like the following: -```yaml -endpoint: "api.skms.mycompany.com" -username: -password: -``` +## Examples -## Example - -See `examples/` folder: +See [examples/](https://github.com/adobe/ops-cli/tree/master/examples) folder: - cassandra-stress - n-node cassandra cluster used for stress-testing; a basic stress profile is included +- spin up a Kubernetes clsuter - distinct `ops` features ## Usage help @@ -526,14 +519,11 @@ optional arguments: ops clusters/centos7.yaml packer build ``` -## Development - -### Running tests +## Secrets Management -- docker: `buildrunner -f buildrunner.yaml` -- on your machine: `py.test tests` +There are cases where you need to reference sensitive data in your `cluster.yaml` file (credentials, passwords, tokens etc). Given that the cluster configuration file can be stored in a version control system (such as Git), the best practice is to not put sensitive data in the file itself. Instead, we can use `ops-cli` to fetch the desired credentials from a secrets manager such as Vault or Amazon SSM, at runtime. -## Secrets Management +### Vault Ops can manage the automatic generation of secrets and their push in Vault, without actually persisting the secrets in the cluster file. A cluster file will only need to use a construct like the following: @@ -548,6 +538,20 @@ Which will translate behind the scenes in : This allows us to just refer in cluster files a secret that actually exists in vault and make sure we only generate it once - if it was already created by os or any other system, we will just use what is already there. The reference is by means of fixed form jinja call added to the cluster file, which ends up interpretted later during the templating phase. +### Amazon Secrets Manager (SSM) + +Amazon offers the possibility to use their [Secrets Manager](https://docs.aws.amazon.com/systems-manager/latest/userguide/what-is-systems-manager.html) in order to manage configuration data such as credentials, passwords and license keys. + +We can use `ops-cli` to fetch the sensitive data from SSM, at runtime. Just define this in your cluster configuration file (eg. `mycluster.yaml`). + +``` +db_password: "{{ '/my/ssm/path' | read_ssm(aws_profile='myprofile') }}" +``` + +`ops-cli` will read the SSM value by running a command similar to: `AWS_PROFILE=aam-npe aws ssm get-parameter --name "/my/ssm/path" --region us-east-1 --with-decryption`. +Note that you can specify the AWS region via `read_ssm(aws_profile='myprofile', region_name='us-west-2')`. + + ## Using jinja2 filters in playbooks and terraform templates You can register your own jinja2 filters that you can use in the cluster config file, terraform templates and ansible playbooks @@ -575,6 +579,20 @@ class FilterModule(object): # test_custom_filters: "{{ 'value' | my_filter }}" ``` +## SKMS +Create a file in `~/.skms/credentials.yaml` which looks like the following: +```yaml +endpoint: "api.skms.mycompany.com" +username: +password: +``` + +## Development + +### Running tests + +- on your machine: `py.test tests` + ## Troubleshooting - Permission issues when installing: you should install the tool in a python virtualenv diff --git a/requirements.txt b/requirements.txt index d6e40720..89305525 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ simpledi>=0.2 awscli==1.16.97 ansible==2.3.1.0 -boto3==1.9.87 +boto3==1.9.91 boto==2.49.0 azure-common==1.1.4 azure==2.0.0rc5 diff --git a/src/ops/ansible/filter_plugins/commonfilters.py b/src/ops/ansible/filter_plugins/commonfilters.py index af3a36c5..feb61199 100644 --- a/src/ops/ansible/filter_plugins/commonfilters.py +++ b/src/ops/ansible/filter_plugins/commonfilters.py @@ -65,12 +65,12 @@ def flatten_tree(d, parent_key='', sep='/'): return dict(items) def read_vault(secret_path, key='value', fetch_all=False, vault_user=None,vault_url=None, token=None, auto_prompt=True): - from ops.simplesecrets import SimpleVault + from ops.simplevault import SimpleVault sv = SimpleVault(vault_user=vault_user, vault_addr=vault_url, vault_token=token,auto_prompt=auto_prompt) return sv.get(path=secret_path, key=key, fetch_all=fetch_all) def write_vault(secret_path, key='value', data="", vault_user=None, vault_url=None, token=None, auto_prompt=True): - from ops.simplesecrets import SimpleVault + from ops.simplevault import SimpleVault sv = SimpleVault(vault_user=vault_user, vault_addr=vault_url, vault_token=token, auto_prompt=auto_prompt) new_data = {} if isinstance(data, dict): @@ -82,13 +82,18 @@ def write_vault(secret_path, key='value', data="", vault_user=None, vault_url=No return False return sv.put(path=secret_path, value=new_data ) +def read_ssm(key, aws_profile, region_name='us-east-1'): + from ops.simplessm import SimpleSSM + ssm = SimpleSSM(aws_profile, region_name) + return ssm.get(key) + def managed_vault_secret(secret_path,key='value', policy={}, vault_user=None, vault_addr=None, vault_token=None, auto_prompt=True): - from ops.simplesecrets import ManagedVaultSecret + from ops.simplevault import ManagedVaultSecret ms = ManagedVaultSecret(path=secret_path, key=key, policy=policy, @@ -111,5 +116,6 @@ def filters(self): 'read_yaml': read_yaml, 'read_vault': read_vault, 'write_vault': write_vault, - 'managed_vault_secret': managed_vault_secret + 'managed_vault_secret': managed_vault_secret, + 'read_ssm': read_ssm } diff --git a/src/ops/simplessm.py b/src/ops/simplessm.py new file mode 100644 index 00000000..534521ea --- /dev/null +++ b/src/ops/simplessm.py @@ -0,0 +1,41 @@ +#Copyright 2019 Adobe. All rights reserved. +#This file is licensed to you under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. You may obtain a copy +#of the License at http://www.apache.org/licenses/LICENSE-2.0 + +#Unless required by applicable law or agreed to in writing, software distributed under +#the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS +#OF ANY KIND, either express or implied. See the License for the specific language +#governing permissions and limitations under the License. + +#!/usr/bin/env python + +from ops.cli import display +from botocore.exceptions import ClientError +import boto3 +import os + +class SimpleSSM(object): + def __init__(self, aws_profile, region_name): + self.initial_aws_profile = os.getenv('AWS_PROFILE', None) + self.aws_profile = aws_profile + self.region_name = region_name + + def get(self, key): + client = self.get_ssm_client() + try: + return client.get_parameter(Name=key, WithDecryption=True).get("Parameter").get("Value") + except ClientError as e: + raise Exception('Error while trying to read SSM value for key: %s - %s' % (key, e.response['Error']['Code'])) + finally: + self.release_ssm_client() + + def get_ssm_client(self): + os.environ['AWS_PROFILE'] = self.aws_profile + return boto3.client('ssm', region_name=self.region_name) + + def release_ssm_client(self): + if self.initial_aws_profile is None: + del os.environ['AWS_PROFILE'] + else: + os.environ['AWS_PROFILE'] = self.initial_aws_profile diff --git a/src/ops/simplesecrets.py b/src/ops/simplevault.py similarity index 100% rename from src/ops/simplesecrets.py rename to src/ops/simplevault.py