From bbd02c1403bed8c4d5426e09a938b232ab2d2c14 Mon Sep 17 00:00:00 2001 From: Justin Guathier Date: Fri, 15 Mar 2019 21:20:18 -0400 Subject: [PATCH 1/2] Add basic support for HashiCorp Vault --- secrets.sh | 215 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 186 insertions(+), 29 deletions(-) diff --git a/secrets.sh b/secrets.sh index da3981c..119e80d 100755 --- a/secrets.sh +++ b/secrets.sh @@ -1,4 +1,30 @@ #!/usr/bin/env bash +# shellcheck disable=SC1003 + +# Parts of this project are MIT Licensed, they will be denoted below. + +# MIT License + +# Original work Copyright (c) 2017 Jonathan Peres +# Modified work Copyright (c) 2019 Just_Insane + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. # The suffix to use for decrypted files. The default can be overridden using # the HELM_SECRETS_DEC_SUFFIX environment variable. @@ -57,6 +83,18 @@ This allows you to first decrypt the file, edit it, then encrypt it again. You can use plain sops to encrypt - https://github.com/mozilla/sops +Vault secrets + +If VAULT_TOKEN env variable is set, automatically store values in Vault. +Secret values must be entered into the helm chart as `changeme`, for example + + db: + name: nextcloud + user: nextcloud + password: changeme + +would prompt to enter a value for the db password. + Example: $ ${HELM_BIN} secrets enc $ git add @@ -75,6 +113,18 @@ Produces ${DEC_SUFFIX} file. You can use plain sops to decrypt specific files - https://github.com/mozilla/sops +Vault secrets + +If VAULT_TOKEN env variable is set, automatically pull values from Vault in a plaintext file. +Values must be pulled in order to update or install via Helm. + +For example: + + db: + name: nextcloud + user: nextcloud + password: + Example: $ ${HELM_BIN} secrets dec @@ -208,6 +258,100 @@ is_help() { esac } +# Parses yaml document +# Based on https://github.com/jasperes/bash-yaml +parse_yaml() { + local yaml_file=$1 + local s + local w + local fs + + s='[[:space:]]*' + w='[a-zA-Z0-9_.-]*' + fs="$(echo @|tr @ '\034')" + + ( + sed -e '/- [^\“]'"[^\']"'.*: /s|\([ ]*\)- \([[:space:]]*\)|\1-\'$'\n'' \1\2|g' | + + sed -ne '/^--/s|--||g; s|\"|\\\"|g; s/[[:space:]]*$//g;' \ + -e "/#.*[\"\']/!s| #.*||g; /^#/s|#.*||g;" \ + -e "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \ + -e "s|^\($s\)\($w\)${s}[:-]$s\(.*\)$s\$|\1$fs\2$fs\3|p" | + + awk -F"$fs" '{ + indent = length($1)/2; + if (length($2) == 0) { conj[indent]="+";} else {conj[indent]="";} + vname[indent] = $2; + for (i in vname) {if (i > indent) {delete vname[i]}} + if (length($3) > 0) { + vn=""; for (i=0; i "$yml" + echo "Encrypted $ymldec to $yml" + fi else - sops --encrypt --input-type yaml --output-type yaml "$ymldec" > "$yml" - echo "Encrypted $ymldec to $yml" + create_variables $yml + set_secrets fi } @@ -265,27 +415,34 @@ decrypt_helper() { __dec=0 [[ -e "$yml" ]] || { echo "File does not exist: $yml"; exit 1; } - if [[ $(grep -C10000 'sops:' "$yml" | grep -c 'version:') -eq 0 ]] - then - echo "Not encrypted: $yml" - __ymldec="$yml" - else - __ymldec=$(sed -e "s/\\.yaml$/${DEC_SUFFIX}/" <<<"$yml") - if [[ -e $__ymldec && $__ymldec -nt $yml ]] - then - echo "$__ymldec is newer than $yml" - else - sops --decrypt --input-type yaml --output-type yaml "$yml" > "$__ymldec" || { rm "$__ymldec"; exit 1; } - __dec=1 - fi - fi - if [[ ${BASH_VERSINFO[0]} -lt 4 ]] + if [[ -z "${VAULT_TOKEN}" ]] then - [[ $__ymldec_var ]] && eval $__ymldec_var="'$__ymldec'" - [[ $__dec_var ]] && eval $__dec_var="'$__dec'" + if [[ $(grep -C10000 'sops:' "$yml" | grep -c 'version:') -eq 0 ]] + then + echo "Not encrypted: $yml" + __ymldec="$yml" + else + __ymldec=$(sed -e "s/\\.yaml$/${DEC_SUFFIX}/" <<<"$yml") + if [[ -e $__ymldec && $__ymldec -nt $yml ]] + then + echo "$__ymldec is newer than $yml" + else + sops --decrypt --input-type yaml --output-type yaml "$yml" > "$__ymldec" || { rm "$__ymldec"; exit 1; } + __dec=1 + fi + fi + + if [[ ${BASH_VERSINFO[0]} -lt 4 ]] + then + [[ $__ymldec_var ]] && eval $__ymldec_var="'$__ymldec'" + [[ $__dec_var ]] && eval $__dec_var="'$__dec'" + fi + true # just so that decrypt_helper will exit with a true status on no error + else + create_variables $yml + get_secrets fi - true # just so that decrypt_helper will exit with a true status on no error } @@ -495,4 +652,4 @@ case "${1:-help}" in ;; esac -exit 0 +exit 0 \ No newline at end of file From 437379bd992893508dc68171eb49e61d984df526 Mon Sep 17 00:00:00 2001 From: Justin Guathier Date: Fri, 22 Mar 2019 20:25:25 -0400 Subject: [PATCH 2/2] Add customization support --- secrets.sh | 76 +++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 64 insertions(+), 12 deletions(-) diff --git a/secrets.sh b/secrets.sh index 119e80d..9cc5856 100755 --- a/secrets.sh +++ b/secrets.sh @@ -33,6 +33,27 @@ DEC_SUFFIX="${HELM_SECRETS_DEC_SUFFIX:-.yaml.dec}" # Make sure HELM_BIN is set (normally by the helm command) HELM_BIN="${HELM_BIN:-helm}" +# The secret store to use to store values +# Defaults to "secret/helm" +if [[ -z "${VAULT_PATH}" ]] +then + default=secret/helm + read -p "Enter a Vault KV Store path [$default]: " VAULT_PATH + VAULT_PATH=${VAULT_PATH:-$default} + unset default +fi + +# The plaintext deliminator used to specify which values need to be stored in Vault +# Defaults to "changme" +if [[ -z "${secret_deliminator}" ]] +then + default=changeme + read -p "Enter a secret deliminator [$default]: " secret_deliminator + secret_deliminator=${secret_deliminator:-$default} + echo "Secret Deliminator is $secret_deliminator" + unset default +fi + getopt --test > /dev/null if [[ $? -ne 4 ]] then @@ -86,7 +107,7 @@ You can use plain sops to encrypt - https://github.com/mozilla/sops Vault secrets If VAULT_TOKEN env variable is set, automatically store values in Vault. -Secret values must be entered into the helm chart as `changeme`, for example +Secret values must be entered into the helm chart as a specific vaule, in this case "changme", for example db: name: nextcloud @@ -306,8 +327,31 @@ parse_yaml() { # Based on https://github.com/jasperes/bash-yaml create_variables() { local yaml_file="$1" - eval "$(parse_yaml "$yaml_file" | awk '/changeme/{print $0}')" - eval "$(parse_yaml "$yaml_file" | awk '/image_repository/{print $0}')" + eval "$(parse_yaml "$yaml_file" | awk -v secret_deliminator="$secret_deliminator" '$0 ~ secret_deliminator {print}')" +} + +# Get file path from root of Git repo to provide a well defined location to store secrets +get_path() { + repository_path=`git rev-parse --show-prefix` + repository_path=`echo $repository_path | sed 's/.$//'` +} + +# Cleans the environment variable array to remove the "secret_deliminator" variable from being returned +clean_array() { + delete=(secret_deliminator) + for target in "${delete[@]}"; do + for i in "${!envsarray[@]}"; do + if [[ ${envsarray[i]} = "${delete[0]}" ]]; then + unset 'envsarray[i]' + fi + done + done + + for i in "${!envsarray[@]}"; do + new_envsarray+=( "${envsarray[i]}" ) + done + envsarray=("${new_envsarray[@]}") + unset new_envsarray } # Prompts user for secret material and uploads to vault K/V Store @@ -317,13 +361,18 @@ set_secrets() { envsarray=() while IFS= read -r line; do envsarray+=( "$line" ) - done < <( set -o posix +o allexport; set | grep "changeme" | awk 'match($0, "\.=") {print substr($0, 1, RSTART)}' ) + done < <( set -o posix +o allexport; set | grep "$secret_deliminator" | awk 'match($0, "\.=") {print substr($0, 1, RSTART)}' ) + + clean_array for env in "${envsarray[@]}"; do - echo "Enter a secret value for $image_repository/$env" - read -r usersecret - vault kv put secret/helm/$image_repository/$env value=$usersecret + echo "Enter a secret value for $env" + echo "Stored at $VAULT_PATH/$repository_path/$yml/$env" + stty -echo + read -r usersecret; + stty echo + vault kv put $VAULT_PATH/$repository_path/$yml/$env value=$usersecret done } @@ -334,19 +383,20 @@ get_secrets() { envsarray=() while IFS= read -r line; do envsarray+=( "$line" ) - done < <( set -o posix +o allexport; set | grep "changeme" | awk 'match($0, "\.=") {print substr($0, 1, RSTART)}' ) + done < <( set -o posix +o allexport; set | grep "$secret_deliminator" | awk 'match($0, "\.=") {print substr($0, 1, RSTART)}' ) + + clean_array yml_dec="$yml.dec" cp $yml $yml_dec for env in "${envsarray[@]}"; do - sec_values=`vault kv get secret/helm/$image_repository/$env | grep "value" | awk '/value/{print $2}'` - echo "Secret Values = $sec_values" + sec_values=`vault kv get $VAULT_PATH/$repository_path/$yml/$env | grep "value" | awk '/value/{print $2}'` for sec in "${sec_values[@]}"; do - #this will fail if "changeme" is on the first line of the file, but is required for GNU sed - sed -i.dec "1,// s/changeme/$sec/" $yml_dec + #this will fail if "$secret_delminator" is on the first line of the file, but is required for GNU sed + sed -i.dec "1,// s/$secret_deliminator/$sec/" $yml_dec rm "$yml_dec.dec" done done @@ -376,6 +426,7 @@ encrypt_helper() { echo "Encrypted $ymldec to $yml" fi else + get_path create_variables $yml set_secrets fi @@ -440,6 +491,7 @@ decrypt_helper() { fi true # just so that decrypt_helper will exit with a true status on no error else + get_path create_variables $yml get_secrets fi