diff --git a/contentcuration/contentcuration/utils/secretmanagement.py b/contentcuration/contentcuration/utils/secretmanagement.py index 912c803681..d94b1b35c7 100644 --- a/contentcuration/contentcuration/utils/secretmanagement.py +++ b/contentcuration/contentcuration/utils/secretmanagement.py @@ -2,7 +2,9 @@ import logging import os -from google.cloud import kms_v1 +import six +from crcmod.predefined import mkPredefinedCrcFun +from google.cloud import kms from google.cloud.storage import Client ENV_VARS = "ENV_VARS" @@ -71,10 +73,22 @@ def decrypt_secret(ciphertext, project_id, loc, env, secret_name): """ Decrypt the ciphertext by using the GCloud KMS keys for that secret. """ - kms_client = kms_v1.KeyManagementServiceClient() - key_path = kms_client.crypto_key_path_path(project_id, loc, env, secret_name) + kms_client = kms.KeyManagementServiceClient() + key_path = kms_client.crypto_key_path(project_id, loc, env, secret_name) + + # Optional, but recommended: compute ciphertext's CRC32C. + # See crc32c() function defined below. + ciphertext_crc32c = crc32c(ciphertext) + + response = kms_client.decrypt( + request={'name': key_path, 'ciphertext': ciphertext, 'ciphertext_crc32c': ciphertext_crc32c}) + + # Optional, but recommended: perform integrity verification on decrypt_response. + # For more details on ensuring E2E in-transit integrity to and from Cloud KMS visit: + # https://cloud.google.com/kms/docs/data-integrity-guidelines + if not response.plaintext_crc32c == crc32c(response.plaintext): + raise Exception('The response received from the server was corrupted in-transit.') - response = kms_client.decrypt(key_path, ciphertext) return response.plaintext @@ -103,3 +117,15 @@ def get_encrypted_secret(secret_name, project_id, env): ) return ret + + +def crc32c(data): + """ + Calculates the CRC32C checksum of the provided data. + Args: + data: the bytes over which the checksum should be calculated. + Returns: + An int representing the CRC32C checksum of the provided bytes. + """ + crc32c_fun = mkPredefinedCrcFun('crc-32c') + return crc32c_fun(six.ensure_binary(data)) diff --git a/deploy/secretmanage b/deploy/secretmanage deleted file mode 100755 index 1dfe4cccba..0000000000 --- a/deploy/secretmanage +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env python2 -""" -./secretmanage -- manage production secrets! - -secretmanage allows you to upload new secrets. These secrets are then available -in production (either in real production, staging or develop) by using the -importsecret module. - - -Usage: - secretmanage upload (--dev|--staging|--prod) - - is the name of the secret. This will be used as a reference when you import -the secret. - - is the content of the secret. These can be the actual DB credentials, -the password to a message broker, or your life's nemesis. -""" -# NOTE to maintainers: -# This encrypts the given plaintext using a keyring with the name of the environment, -# and a key with the same name as the secret. -# We will then upload this key to a bucket called '-secrets/', inside a folder in that bucket -# with the same name as the environment. The file in that bucket is named the same as -# the secret. -# -# This means that you need to ensure that a keyring with the same name as the environment is created. -# You also need to make sure that the -secrets/ bucket is available. -# -# The persons running this script must also have th Encrypt permission for KMS, and has Edit and Get -# permissions for the bucket. -import logging -import os - -from docopt import docopt -from google.cloud import kms_v1 -from google.cloud.storage import Client - -logging.basicConfig(level=logging.INFO) - -# PROJECT_ID = "contentworkshop-159920" -PROJECT_ID = "ops-central" -LOCATION = "global" - -kms_client = kms_v1.KeyManagementServiceClient() - - -def create_key(project_id, location, env, name): - """ - Create the key. The keyring named after the env should already exist! - """ - keyring = kms_client.key_ring_path(project_id, location, env) - key_purpose = kms_v1.enums.CryptoKey.CryptoKeyPurpose.ENCRYPT_DECRYPT - - response = kms_client.create_crypto_key(keyring, name, {"purpose": key_purpose}) - - -def get_key_url(project_id, location, env, secret_name): - """ - Return the URL we can pass to google.cloud.kms to encrypt a string. - """ - return kms_client.crypto_key_path_path(project_id, location, env, secret_name) - - -def encrypt_string(key_url, plaintext): - """ - Encrypt the plaintext string using the given GCloud KMS key. - - Returns the ciphertext. - """ - response = kms_client.encrypt(key_url, plaintext) - return response.ciphertext - - -def upload_ciphertext(project_id, env, name, ciphertext): - """ - Upload the given ciphertext to the secrets bucket. - - The secret is written as a file located in -secrets//. - - Note that a bucket with the name {project_id}-secrets should exist. If it doesn't, - just create one. No special config needed. - """ - - bucket_name = "{id}-secrets".format(id=project_id) - loc = "{env}/{name}".format(env=env, name=name) - - bucket = Client().get_bucket(bucket_name) - - # Write the ciphertext to the bucket! - bucket.blob(loc).upload_from_string(ciphertext) - - -def main(): - args = docopt(__doc__) - - if args["--dev"]: - args["env"] = "dev" - elif args["--staging"]: - args["env"] = "staging" - elif args["--prod"]: - args["env"] = "prod" - - env = args["env"] - name = args[""] - plaintext = args[""] - project_id = PROJECT_ID - location = LOCATION - - logging.info("Creating key...") - create_key(project_id, location, env, name) - - logging.info("Getting KMS key...") - key = get_key_url(project_id, location, env, name) - - logging.info("Encrypting string...") - ciphertext = encrypt_string(key, plaintext) - - # logging.info("Writing encrypted string...") - logging.info("Uploading encrypted secret to storage...") - upload_ciphertext(project_id, env, name, ciphertext) - - logging.info("Done!") - - -main() diff --git a/requirements.in b/requirements.in index 5c1007de08..c72aba150e 100644 --- a/requirements.in +++ b/requirements.in @@ -25,7 +25,7 @@ google-cloud-core django-db-readonly==0.7.0 oauth2client django-mathfilters -google-cloud-kms==1.4.0 +google-cloud-kms==2.0.0 backoff backports-abc==0.5 django-model-utils==4.3.1 @@ -40,3 +40,4 @@ python-dateutil>=2.8.1 jsonschema>=3.2.0 importlib-metadata==1.7.0 django-celery-results +crcmod==1.7 diff --git a/requirements.txt b/requirements.txt index 632ab58f91..48de98f8d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,6 +54,8 @@ click-repl==0.2.0 # via celery confusable-homoglyphs==3.2.0 # via django-registration +crcmod==1.7 + # via -r requirements.in django==3.2.18 # via # -r requirements.in @@ -121,7 +123,7 @@ google-cloud-core==1.7.3 # google-cloud-storage google-cloud-error-reporting==1.4.0 # via -r requirements.in -google-cloud-kms==1.4.0 +google-cloud-kms==2.0.0 # via -r requirements.in google-cloud-logging==2.3.1 # via google-cloud-error-reporting @@ -166,6 +168,10 @@ kombu==5.2.4 # via celery le-utils==0.1.42 # via -r requirements.in +libcst==0.4.9 + # via google-cloud-kms +mypy-extensions==1.0.0 + # via typing-inspect newrelic==6.2.0.156 # via -r requirements.in oauth2client==4.1.3 @@ -185,6 +191,7 @@ prompt-toolkit==3.0.23 proto-plus==1.18.1 # via # google-cloud-error-reporting + # google-cloud-kms # google-cloud-logging protobuf==3.20.3 # via @@ -222,6 +229,8 @@ pytz==2022.1 # django # django-postmark # google-api-core +pyyaml==6.0 + # via libcst redis==4.5.4 # via # -r requirements.in @@ -251,6 +260,12 @@ six==1.16.0 # python-dateutil sqlparse==0.4.1 # via django +typing-extensions==4.5.0 + # via + # libcst + # typing-inspect +typing-inspect==0.8.0 + # via libcst urllib3==1.26.14 # via # botocore