Merge pull request #1758 from edx/katebygrace/fix-refresh-job

chore: fix refresh job
edx · May 17, 2024 · 94f9613 · 94f9613
2 parents 83d7195 + 8c2d2fe
commit 94f9613
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 12 deletions.
diff --git a/dataeng/jobs/analytics/SnowflakeRefreshSnowpipe.groovy b/dataeng/jobs/analytics/SnowflakeRefreshSnowpipe.groovy
@@ -82,7 +82,6 @@ class SnowflakeRefreshSnowpipe {
                 }
                 publishers common_publishers(allVars)
                 steps {
-                    shell(dslFactory.readFileFromWorkspace('dataeng/resources/secrets-manager-setup.sh'))
                     shell(dslFactory.readFileFromWorkspace('dataeng/resources/snowflake-refresh-snowpipe.sh'))
                 }
             }

diff --git a/dataeng/resources/secrets-manager.sh b/dataeng/resources/secrets-manager.sh
@@ -35,4 +35,4 @@ else
         exit 1
     fi
     fetch_specific_key "$1" "$2"
-fi
+fi
diff --git a/dataeng/resources/snowflake-refresh-snowpipe.sh b/dataeng/resources/snowflake-refresh-snowpipe.sh
@@ -10,16 +10,14 @@ source "${PYTHON_VENV}/bin/activate"
 cd $WORKSPACE/analytics-tools/snowflake
 make requirements
 
-# Source the secrets-manager.sh script to make the function available
-source $WORKSPACE/secrets-manager.sh
-# Fetch the secrets from AWS
-set +x
 
 
-secrets-manager.sh -w analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS snowflake/rsa_key_snowpipe_user.p8
-secrets-manager.sh -w analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS snowflake/rsa_key_passphrase_snowpipe_user
+python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_snowpipe_user.p8 -v rsa_key_snowpipe_user
+python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_passphrase_snowpipe_user -v rsa_key_passphrase_snowpipe_user
+#set -x
 
-set -x
+unset KEY_PATH
+unset PASSPHRASE_PATH
 
 python refresh_snowpipe.py \
     --user 'SNOWPIPE' \
@@ -28,6 +26,10 @@ python refresh_snowpipe.py \
     --pipe_name $PIPE_NAME \
     --table_name $TABLE_NAME \
     --delay $DELAY \
-    --limit $LIMIT
-    --key_file $KEY_PATH \
-    --passphrase_file $PASSPHRASE_PATH
+    --limit $LIMIT \
+    --key_file "$(cat "rsa_key_snowpipe_user")" \
+    --passphrase_file "$(cat "rsa_key_passphrase_snowpipe_user")"
+
+rm rsa_key_snowpipe_user
+rm rsa_key_passphrase_snowpipe_user
+