diff --git a/dataeng/jobs/analytics/SnowflakeRefreshSnowpipe.groovy b/dataeng/jobs/analytics/SnowflakeRefreshSnowpipe.groovy index 64a8236c7..b4e11afad 100644 --- a/dataeng/jobs/analytics/SnowflakeRefreshSnowpipe.groovy +++ b/dataeng/jobs/analytics/SnowflakeRefreshSnowpipe.groovy @@ -82,7 +82,7 @@ class SnowflakeRefreshSnowpipe { } publishers common_publishers(allVars) steps { - shell(dslFactory.readFileFromWorkspace('dataeng/resources/secrets-manager-setup.sh')) + shell(dslFactory.readFileFromWorkspace('dataeng/resources/secrets-manager.sh')) shell(dslFactory.readFileFromWorkspace('dataeng/resources/snowflake-refresh-snowpipe.sh')) } } diff --git a/dataeng/resources/secrets-manager.sh b/dataeng/resources/secrets-manager.sh index 8251ccb55..1a8b8458f 100755 --- a/dataeng/resources/secrets-manager.sh +++ b/dataeng/resources/secrets-manager.sh @@ -1,38 +1,66 @@ #!/bin/bash +# Define the location of the script in the Jenkins workspace +SCRIPT_PATH="$WORKSPACE/secrets-manager.sh" + +echo "running setup" + +# Write the script content to the specified location +cat < "$SCRIPT_PATH" + +#!/usr/bin/env bash + extract_value_from_json() { - local json="$1" - local key="$2" - local value=$(echo "$json" | jq -r ".$key") + local secret_json="\$1" + local secret_key="\$2" + + local secret_value=\$(echo "$secret_json" | jq -r ".$secret_key") } fetch_whole_secret() { - local secret_name="$1" - local variable_name="$2" - local secret_value=$(aws secretsmanager get-secret-value --secret-id "$secret_name" --query "SecretString" --output text) - #set whole file as env var - declare "${secret_name%=*}=${secret_value}" + local secret_name="\$1" + local variable_name="\$2" + echo "\$secret_name" + echo "\$variable_name" + SECRET_JSON=\$(aws secretsmanager get-secret-value --secret-id "\$secret_name" --region "us-east-1" --output json) + echo "\$SECRET_JSON" + value=\$(echo "\$SECRET_JSON" | jq -r ".SecretString" 2>/dev/null) + echo "\$value" + echo "\$value" > "\$WORKSPACE/\$variable_name" + # Output the contents of the file to verify + cat "\$WORKSPACE/\$variable_name" + declare "\${variable_name%=*}=\${value}" + + #declare "$variable_name=$secret_value" + #declare "$variable_name=\"$secret_value\"" + #what brian said to do + #declare "\${variable_name%=*}=\${value}" } fetch_specific_key() { - local secret_name="$1" - local key="$2" + local secret_name="\$1" + local key="\$2" local secret_value=$(aws secretsmanager get-secret-value --secret-id "$secret_name" --query "SecretString" --output text) local extracted_value=$(extract_value_from_json "$secret_value" "$key") declare "${key%=*}=${extracted_value}" } -# Main script -if [[ "$1" == "-w" ]]; then - if [ $# -ne 3 ]; then - echo "Usage: $0 -w " - exit 1 +secret_script() { + echo "\$1" + echo "\$2" + echo "\$3" + if [[ "\$1" == "-w" ]]; then + if [ \$# -ne 3 ]; then + echo "Usage: $0 -w " + exit 1 + fi + fetch_whole_secret "\$2" "\$3" + else + if [ $# -ne 2 ]; then + echo "Usage: $0 " + exit 1 + fi + fetch_specific_key "\$1" "\$2" fi - fetch_whole_secret "$2" "$3" -else - if [ $# -ne 2 ]; then - echo "Usage: $0 " - exit 1 - fi - fetch_specific_key "$1" "$2" -fi +} +EOF diff --git a/dataeng/resources/snowflake-refresh-snowpipe.sh b/dataeng/resources/snowflake-refresh-snowpipe.sh index 3cf265aa3..480200197 100644 --- a/dataeng/resources/snowflake-refresh-snowpipe.sh +++ b/dataeng/resources/snowflake-refresh-snowpipe.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -ex + # Creating Python virtual env PYTHON_VENV="python_venv" virtualenv --python=$PYTHON_VENV_VERSION --clear "${PYTHON_VENV}" @@ -10,16 +11,30 @@ source "${PYTHON_VENV}/bin/activate" cd $WORKSPACE/analytics-tools/snowflake make requirements +# Define the location of the script in the Jenkins workspace +SCRIPT_PATH="$WORKSPACE/secrets-manager.sh" + + # Source the secrets-manager.sh script to make the function available source $WORKSPACE/secrets-manager.sh # Fetch the secrets from AWS set +x +echo "Running secrets manager script" +python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_snowpipe_user.p8 -v rsa_key_snowpipe_user +python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_passphrase_snowpipe_user -v rsa_key_passphrase_snowpipe_user +#secret_script -w analytics-secure/snowflake/rsa_key_snowpipe_user.p8 rsa_key_snowpipe_user +#echo "Running secrets manager script x2" +#secret_script -w analytics-secure/snowflake/rsa_key_passphrase_snowpipe_user rsa_key_passphrase_snowpipe_user +echo "catting files" +#cat $WORKSPACE/rsa_key_snowpipe_user +#cat $WORKSPACE/rsa_key_passphrase_snowpipe_user +cat rsa_key_snowpipe_user +cat rsa_key_passphrase_snowpipe_user +#set -x -secrets-manager.sh -w analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS snowflake/rsa_key_snowpipe_user.p8 -secrets-manager.sh -w analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS snowflake/rsa_key_passphrase_snowpipe_user - -set -x +unset KEY_PATH +unset PASSPHRASE_PATH python refresh_snowpipe.py \ --user 'SNOWPIPE' \ @@ -28,6 +43,10 @@ python refresh_snowpipe.py \ --pipe_name $PIPE_NAME \ --table_name $TABLE_NAME \ --delay $DELAY \ - --limit $LIMIT - --key_file $KEY_PATH \ - --passphrase_file $PASSPHRASE_PATH + --limit $LIMIT \ + --key_file "$(cat "rsa_key_snowpipe_user")" \ + --passphrase_file "$(cat "rsa_key_passphrase_snowpipe_user")" + +rm rsa_key_snowpipe_user +rm rsa_key_passphrase_snowpipe_user +