Merge pull request #159 from feikesteenbergen/feature/postgres-encryp…

…t-values

Feature/postgres encrypt values

senza/aws.py

@@ -3,6 +3,7 @@
 import functools
 import time
 import boto3
+import base64
 from botocore.exceptions import ClientError
 
 
@@ -24,6 +25,28 @@ def get_security_group(region: str, sg_name: str):
             raise
 
 
+def encrypt(region: str, KeyId: str, Plaintext: str, b64encode=False):
+    kms = boto3.client('kms', region)
+    encrypted = kms.encrypt(KeyId=KeyId, Plaintext=Plaintext)['CiphertextBlob']
+    if b64encode:
+        return base64.b64encode(encrypted).decode('utf-8')
+
+    return encrypted
+
+
+def list_kms_keys(region: str, details=True):
+    kms = boto3.client('kms', region)
+    keys = list(kms.list_keys()['Keys'])
+    if details:
+        aliases = kms.list_aliases()['Aliases']
+
+        for key in keys:
+            key['aliases'] = [a['AliasName'] for a in aliases if a.get('TargetKeyId') == key['KeyId']]
+            key.update(kms.describe_key(KeyId=key['KeyId'])['KeyMetadata'])
+
+    return keys
+
+
 def resolve_security_groups(security_groups: list, region: str):
     result = []
     for security_group in security_groups:

senza/templates/postgresapp.py

@@ -3,10 +3,12 @@
 '''
 
 import click
-from clickclick import warning, error
-from senza.aws import get_security_group
+from clickclick import warning, error, choice
+from senza.aws import get_security_group, encrypt, list_kms_keys
 from senza.utils import pystache_render
 import requests
+import random
+import string
 
 from ._helper import prompt, check_security_group, check_s3_bucket, get_account_alias
 
@@ -85,6 +87,9 @@
           SCOPE: "{{=<% %>=}}{{Arguments.version}}<%={{ }}=%>"
           ETCD_DISCOVERY_DOMAIN: "{{discovery_domain}}"
           WAL_S3_BUCKET: "{{wal_s3_bucket}}"
+          PGPASSWORD_SUPERUSER: "{{pgpassword_superuser}}"
+          PGPASSWORD_ADMIN: "{{pgpassword_admin}}"
+          PGPASSWORD_STANDBY: "{{pgpassword_standby}}"
         root: True
         mounts:
           /home/postgres/pgdata:
@@ -180,7 +185,7 @@
           Action: sts:AssumeRole
       Path: /
       Policies:
-      - PolicyName: SpiloEC2S3Access
+      - PolicyName: SpiloEC2S3KMSAccess
         PolicyDocument:
           Version: "2012-10-17"
           Statement:
@@ -195,11 +200,25 @@
           - Effect: Allow
             Action: ec2:Describe*
             Resource: "*"
+          {{#kms_arn}}
+          - Effect: Allow
+            Action:
+              - "kms:Decrypt"
+              - "kms:Encrypt"
+            Resource:
+              - {{kms_arn}}
+          {{/kms_arn}}
 '''
 
 
 def ebs_optimized_supported(instance_type):
     # per http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSOptimized.html
+    """
+    >>> ebs_optimized_supported('c3.xlarge')
+    True
+    >>> ebs_optimized_supported('t2.micro')
+    False
+    """
     return instance_type in ('c1.large', 'c3.xlarge', 'c3.2xlarge', 'c3.4xlarge',
                              'c4.large', 'c4.xlarge', 'c4.2xlarge', 'c4.4xlarge', 'c4.8xlarge',
                              'd2.xlarge', 'd2.2xlarge', 'd2.4xlarge', 'd2.8xlarge',
@@ -209,44 +228,96 @@ def ebs_optimized_supported(instance_type):
                              'r3.4xlarge')
 
 
+def set_default_variables(variables):
+    variables.setdefault('discovery_domain', 'postgres.example.com')
+    variables.setdefault('docker_image', None)
+    variables.setdefault('ebs_optimized', None)
+    variables.setdefault('fsoptions', 'noatime,nodiratime,nobarrier')
+    variables.setdefault('fstype', 'ext4')
+    variables.setdefault('healthcheck_port', HEALTHCHECK_PORT)
+    variables.setdefault('hosted_zone', 'example.com')
+    variables.setdefault('instance_type', 't2.micro')
+    variables.setdefault('kms_arn', None)
+    variables.setdefault('pgpassword_admin', 'admin')
+    variables.setdefault('pgpassword_standby', 'standby')
+    variables.setdefault('pgpassword_superuser', 'zalando')
+    variables.setdefault('postgres_port', POSTGRES_PORT)
+    variables.setdefault('scalyr_account_key', None)
+    variables.setdefault('snapshot_id', None)
+    variables.setdefault('spilo_sg_id', None)
+    variables.setdefault('use_ebs', True)
+    variables.setdefault('volume_iops', 300)
+    variables.setdefault('volume_size', 10)
+    variables.setdefault('volume_type', 'gp2')
+    variables.setdefault('wal_s3_bucket', None)
+
+    return variables
+
+
 def gather_user_variables(variables, region, account_info):
+    defaults = set_default_variables(dict())
+
     if click.confirm('Do you want to set the docker image now? [No]'):
         prompt(variables, "docker_image", "Docker Image Version", default=get_latest_spilo_image())
-    else:
-        variables['docker_image'] = None
+
     prompt(variables, 'wal_s3_bucket', 'Postgres WAL S3 bucket to use',
            default='{}-{}-spilo-app'.format(get_account_alias(), region))
+
     prompt(variables, 'instance_type', 'EC2 instance type', default='t2.micro')
-    variables['hosted_zone'] = account_info.Domain or 'example.com'
+
+    variables['hosted_zone'] = account_info.Domain or defaults['hosted_zone']
     if (variables['hosted_zone'][-1:] != '.'):
         variables['hosted_zone'] += '.'
     prompt(variables, 'discovery_domain', 'ETCD Discovery Domain',
            default='postgres.' + variables['hosted_zone'][:-1])
+
     if variables['instance_type'].lower().split('.')[0] in ('c3', 'g2', 'hi1', 'i2', 'm3', 'r3'):
         variables['use_ebs'] = click.confirm('Do you want database data directory on external (EBS) storage? [Yes]',
-                                             default=True)
+                                             default=defaults['use_ebs'])
     else:
         variables['use_ebs'] = True
-    variables['ebs_optimized'] = None
-    variables['volume_iops'] = None
-    variables['snapshot_id'] = None
+
     if variables['use_ebs']:
-        prompt(variables, 'volume_size', 'Database volume size (GB, 10 or more)', default=10)
-        prompt(variables, 'volume_type', 'Database volume type (gp2, io1 or standard)', default='gp2')
+        prompt(variables, 'volume_size', 'Database volume size (GB, 10 or more)', default=defaults['volume_size'])
+        prompt(variables, 'volume_type', 'Database volume type (gp2, io1 or standard)',
+               default=defaults['volume_type'])
         if variables['volume_type'] == 'io1':
             pio_max = variables['volume_size'] * 30
             prompt(variables, "volume_iops", 'Provisioned I/O operations per second (100 - {0})'.
                    format(pio_max), default=str(pio_max))
         prompt(variables, "snapshot_id", "ID of the snapshot to populate EBS volume from", default="")
         if ebs_optimized_supported(variables['instance_type']):
             variables['ebs_optimized'] = True
-    prompt(variables, "fstype", "Filesystem for the data partition", default="ext4")
+    prompt(variables, "fstype", "Filesystem for the data partition", default=defaults['fstype'])
     prompt(variables, "fsoptions", "Filesystem mount options (comma-separated)",
-           default="noatime,nodiratime,nobarrier")
+           default=defaults['fsoptions'])
     prompt(variables, "scalyr_account_key", "Account key for your scalyr account", "")
 
-    variables['postgres_port'] = POSTGRES_PORT
-    variables['healthcheck_port'] = HEALTHCHECK_PORT
+    prompt(variables, 'pgpassword_superuser', "Password for PostgreSQL superuser [random]", show_default=False,
+           default=generate_random_password, hide_input=True, confirmation_prompt=True)
+    prompt(variables, 'pgpassword_standby', "Password for PostgreSQL user standby [random]", show_default=False,
+           default=generate_random_password, hide_input=True, confirmation_prompt=True)
+    prompt(variables, 'pgpassword_admin', "Password for PostgreSQL user admin", show_default=True,
+           default=defaults['pgpassword_admin'], hide_input=True, confirmation_prompt=True)
+
+    if click.confirm('Do you wish to encrypt these passwords using KMS?', default=False):
+        kms_keys = [k for k in list_kms_keys(region) if 'alias/aws/ebs' not in k['aliases']]
+
+        if len(kms_keys) == 0:
+            raise click.UsageError('No KMS key is available for encrypting and decrypting. '
+                                   'Ensure you have at least 1 key available.')
+
+        options = ['{}: {}'.format(k['KeyId'], k['Description']) for k in kms_keys]
+        kms_key = choice(prompt='Please select the encryption key', options=options)
+        kms_keyid = kms_key.split(':')[0]
+
+        variables['kms_arn'] = [k['Arn'] for k in kms_keys if k['KeyId'] == kms_keyid][0]
+
+        for key in [k for k in variables if k.startswith('pgpassword_')]:
+            encrypted = encrypt(region=region, KeyId=kms_keyid, Plaintext=variables[key], b64encode=True)
+            variables[key] = 'aws:kms:{}'.format(encrypted)
+
+    set_default_variables(variables)
 
     sg_name = 'app-spilo'
     rules_missing = check_security_group(sg_name,
@@ -272,7 +343,20 @@ def gather_user_variables(variables, region, account_info):
     return variables
 
 
+def generate_random_password(length=64):
+    """
+    >>> len(generate_random_password(61))
+    61
+    """
+    return ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(length))
+
+
 def generate_definition(variables):
+    """
+    >>> variables = set_default_variables(dict())
+    >>> len(generate_definition(variables)) > 300
+    True
+    """
     definition_yaml = pystache_render(TEMPLATE, variables)
     return definition_yaml
 

tests/test_aws.py

@@ -1,6 +1,6 @@
 from unittest.mock import MagicMock
 from senza.aws import resolve_topic_arn
-from senza.aws import get_security_group, resolve_security_groups, get_account_id, get_account_alias
+from senza.aws import get_security_group, resolve_security_groups, get_account_id, get_account_alias, list_kms_keys, encrypt
 
 
 def test_resolve_security_groups(monkeypatch):
@@ -30,6 +30,25 @@ def test_create(monkeypatch):
     assert 'arn:123:mytopic' == resolve_topic_arn('myregion', 'mytopic')
 
 
+def test_encrypt(monkeypatch):
+    boto3 = MagicMock()
+    boto3.encrypt.return_value = {'CiphertextBlob':b'Hello World'}
+    monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3))
+
+    assert b'Hello World' == encrypt(region=None, KeyId='key_a', Plaintext='Hello World', b64encode=False)
+    assert 'SGVsbG8gV29ybGQ=' == encrypt(region=None, KeyId='key_a', Plaintext='Hello World', b64encode=True)
+
+
+def test_list_kms_keys(monkeypatch):
+    boto3 = MagicMock()
+    boto3.list_keys.return_value = {'Keys': [{'KeyId':'key_a'},{'KeyId':'key_b'}]}
+    boto3.list_aliases.return_value = {'Aliases': [{'AliasName':'a', 'TargetKeyId':'key_a'}]}
+    boto3.describe_key.return_value = {'KeyMetadata':{'Description':'This is key a'}}
+    monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3))
+
+    assert len(list_kms_keys(region=None, details=True)) == 2
+
+
 def test_get_account_id(monkeypatch):
     boto3 = MagicMock()
     boto3.get_user.return_value = {'User': {'Arn': 'arn:aws:iam::0123456789:user/admin'}}