From 9789b34338489a0f2c0fa413a6a551cb544ea6eb Mon Sep 17 00:00:00 2001 From: Justin Reese Date: Mon, 11 May 2020 14:19:51 -0700 Subject: [PATCH 01/11] Remove awkward, repeated 'cd config' cmds from Jenkinfile --- Jenkinsfile | 75 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b581a8eb..0136c74b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,9 +3,9 @@ pipeline { options { timestamps() } - stages { + stages { // Very first: pause for a minute to give a chance to - // cancel and clean the workspace before use. + // cancel and clean the workspace before use. stage('Ready and clean') { steps { // Give us a minute to cancel if we want. @@ -13,6 +13,7 @@ pipeline { cleanWs() } } + stage('Initialize') { steps { // Start preparing environment. @@ -28,9 +29,10 @@ pipeline { }) } } + stage('Build kg_covid_19') { steps { - dir('./config') { + dir('./gitrepo') { git( url: 'https://github.com/Knowledge-Graph-Hub/kg-covid-19', branch: 'master' @@ -43,57 +45,72 @@ pipeline { } } } + stage('Download') { steps { - script { - def run_py_dl = sh( - script: 'cd config;. venv/bin/activate; python3.7 run.py download', returnStatus: true - ) - withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { - if (run_py_dl == 0) { // upload raw to s3 - sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put -r data/raw s3://kg-hub-public-data/' - } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/ - sh 'cd config; rm -fr data/raw; mkdir -p data/raw' - sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate get -r s3://kg-hub-public-data/raw/ data/raw/' + dir('./gitrepo') { + script { + def run_py_dl = sh( + script: '. venv/bin/activate; python3.7 run.py download', returnStatus: true + ) + withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { + if (run_py_dl == 0) { // upload raw to s3 + sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put -r data/raw s3://kg-hub-public-data/' + } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/ + sh 'rm -fr data/raw; mkdir -p data/raw' + sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate get -r s3://kg-hub-public-data/raw/ data/raw/' + } } } } } } + stage('Transform') { steps { - sh 'cd config;. venv/bin/activate; python3.7 run.py transform' + dir('./gitrepo') { + sh '. venv/bin/activate; python3.7 run.py transform' + } } } + stage('Load') { steps { - sh 'cd config;. venv/bin/activate; python3.7 run.py load' - sh 'cd config;. venv/bin/activate; pigz merged-kg.tar' + dir('./gitrepo') { + sh '. venv/bin/activate; python3.7 run.py load' + sh '. venv/bin/activate; pigz merged-kg.tar' + } } } + stage('Convert to RDF') { steps { - sh 'cd config;. venv/bin/activate; kgx transform --input-type tsv --output-type nt -o ./merged-kg.nt merged-kg.tar.gz' - sh 'cd config;. venv/bin/activate; pigz merged-kg.nt' + dir('./gitrepo') { + sh '. venv/bin/activate; kgx transform --input-type tsv --output-type nt -o ./merged-kg.nt merged-kg.tar.gz' + sh '. venv/bin/activate; pigz merged-kg.nt' + } } - } + } + stage('Publish') { steps { + dir('./gitrepo') { + script { + if (env.BRANCH_NAME != 'master') { + echo "Will not push if not on correct branch." + } else { + withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { + sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.nt.gz s3://kg-hub-public-data/kg-covid-19.nt.gz' + sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar.gz s3://kg-hub-public-data/kg-covid-19.tar.gz' + // Should now appear at: + // https://kg-hub.berkeleybop.io/[artifact name] + } - script { - if (env.BRANCH_NAME != 'master') { - echo "Will not push if not on correct branch." - } else { - withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { - sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.nt.gz s3://kg-hub-public-data/kg-covid-19.nt.gz' - sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar.gz s3://kg-hub-public-data/kg-covid-19.tar.gz' - // Should now appear at: - // https://kg-hub.berkeleybop.io/[artifact name] } - } } } } + } } From b7acc5c79adb8d7613d9aad46a40ddeed0bc03e4 Mon Sep 17 00:00:00 2001 From: Justin Reese Date: Mon, 11 May 2020 15:57:39 -0700 Subject: [PATCH 02/11] Some tidying up based on Seth's suggestions --- Jenkinsfile | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0136c74b..d64dd828 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -51,13 +51,14 @@ pipeline { dir('./gitrepo') { script { def run_py_dl = sh( - script: '. venv/bin/activate; python3.7 run.py download', returnStatus: true + script: '. venv/bin/activate && python3.7 run.py download', returnStatus: true ) withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { if (run_py_dl == 0) { // upload raw to s3 sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put -r data/raw s3://kg-hub-public-data/' } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/ - sh 'rm -fr data/raw; mkdir -p data/raw' + sh 'rm -fr data/raw || true;' + sh 'mkdir -p data/raw || true' sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate get -r s3://kg-hub-public-data/raw/ data/raw/' } } @@ -69,7 +70,7 @@ pipeline { stage('Transform') { steps { dir('./gitrepo') { - sh '. venv/bin/activate; python3.7 run.py transform' + sh '. venv/bin/activate && python3.7 run.py transform' } } } @@ -77,8 +78,8 @@ pipeline { stage('Load') { steps { dir('./gitrepo') { - sh '. venv/bin/activate; python3.7 run.py load' - sh '. venv/bin/activate; pigz merged-kg.tar' + sh '. venv/bin/activate && python3.7 run.py load' + sh 'pigz merged-kg.tar' } } } @@ -86,8 +87,8 @@ pipeline { stage('Convert to RDF') { steps { dir('./gitrepo') { - sh '. venv/bin/activate; kgx transform --input-type tsv --output-type nt -o ./merged-kg.nt merged-kg.tar.gz' - sh '. venv/bin/activate; pigz merged-kg.nt' + sh '. venv/bin/activate && kgx transform --input-type tsv --output-type nt -o ./merged-kg.nt merged-kg.tar.gz' + sh 'pigz merged-kg.nt' } } } From ec78532379a92aeae08007e46f1bcbb161f351b6 Mon Sep 17 00:00:00 2001 From: Justin Reese Date: Mon, 11 May 2020 16:27:04 -0700 Subject: [PATCH 03/11] Make variable for long s3cmd with arguments --- Jenkinsfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d64dd828..3df0fed3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,3 +1,5 @@ +def s3cmd_with_args = "s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate" + pipeline { agent any options { @@ -55,11 +57,11 @@ pipeline { ) withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { if (run_py_dl == 0) { // upload raw to s3 - sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put -r data/raw s3://kg-hub-public-data/' + sh '${s3cmd_with_args} put -r data/raw s3://kg-hub-public-data/' } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/ sh 'rm -fr data/raw || true;' sh 'mkdir -p data/raw || true' - sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate get -r s3://kg-hub-public-data/raw/ data/raw/' + sh '${s3cmd_with_args} get -r s3://kg-hub-public-data/raw/ data/raw/' } } } @@ -101,8 +103,8 @@ pipeline { echo "Will not push if not on correct branch." } else { withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { - sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.nt.gz s3://kg-hub-public-data/kg-covid-19.nt.gz' - sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar.gz s3://kg-hub-public-data/kg-covid-19.tar.gz' + sh '${s3cmd_with_args} put merged-kg.nt.gz s3://kg-hub-public-data/kg-covid-19.nt.gz' + sh '${s3cmd_with_args} put merged-kg.tar.gz s3://kg-hub-public-data/kg-covid-19.tar.gz' // Should now appear at: // https://kg-hub.berkeleybop.io/[artifact name] } From 946b5c89cf39e8651c0541f388aa01a87685536e Mon Sep 17 00:00:00 2001 From: Justin Reese Date: Mon, 11 May 2020 16:49:09 -0700 Subject: [PATCH 04/11] Move s3cmd variable to within WithCredentials() --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3df0fed3..dfab2f1f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,3 @@ -def s3cmd_with_args = "s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate" - pipeline { agent any options { @@ -56,6 +54,7 @@ pipeline { script: '. venv/bin/activate && python3.7 run.py download', returnStatus: true ) withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { + def s3cmd_with_args = 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate' if (run_py_dl == 0) { // upload raw to s3 sh '${s3cmd_with_args} put -r data/raw s3://kg-hub-public-data/' } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/ @@ -103,6 +102,7 @@ pipeline { echo "Will not push if not on correct branch." } else { withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { + def s3cmd_with_args = 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate' sh '${s3cmd_with_args} put merged-kg.nt.gz s3://kg-hub-public-data/kg-covid-19.nt.gz' sh '${s3cmd_with_args} put merged-kg.tar.gz s3://kg-hub-public-data/kg-covid-19.tar.gz' // Should now appear at: From 12c8ab941dd2012b8d9a46ea5f5fa266dda2a7c4 Mon Sep 17 00:00:00 2001 From: Justin Reese Date: Mon, 11 May 2020 17:16:49 -0700 Subject: [PATCH 05/11] Branch check when pushing downloaded files to s3 --- Jenkinsfile | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index dfab2f1f..c361be9d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -50,17 +50,21 @@ pipeline { steps { dir('./gitrepo') { script { - def run_py_dl = sh( - script: '. venv/bin/activate && python3.7 run.py download', returnStatus: true - ) - withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { - def s3cmd_with_args = 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate' - if (run_py_dl == 0) { // upload raw to s3 - sh '${s3cmd_with_args} put -r data/raw s3://kg-hub-public-data/' - } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/ - sh 'rm -fr data/raw || true;' - sh 'mkdir -p data/raw || true' - sh '${s3cmd_with_args} get -r s3://kg-hub-public-data/raw/ data/raw/' + if (env.BRANCH_NAME != 'master') { + echo "Will not push if not on correct branch." + } else { + def run_py_dl = sh( + script: '. venv/bin/activate && python3.7 run.py download', returnStatus: true + ) + withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { + def s3cmd_with_args = 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate' + if (run_py_dl == 0) { // upload raw to s3 + sh '${s3cmd_with_args} put -r data/raw s3://kg-hub-public-data/' + } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/ + sh 'rm -fr data/raw || true;' + sh 'mkdir -p data/raw || true' + sh '${s3cmd_with_args} get -r s3://kg-hub-public-data/raw/ data/raw/' + } } } } @@ -114,6 +118,5 @@ pipeline { } } } - } } From 2802ee03bc4c6ccf38d9923b42f8025ddba11225 Mon Sep 17 00:00:00 2001 From: Justin Reese Date: Mon, 11 May 2020 17:20:19 -0700 Subject: [PATCH 06/11] move cf-invalidate to only put s3cmd commands --- Jenkinsfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c361be9d..ecf87c81 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -57,9 +57,9 @@ pipeline { script: '. venv/bin/activate && python3.7 run.py download', returnStatus: true ) withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { - def s3cmd_with_args = 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate' + def s3cmd_with_args = 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text' if (run_py_dl == 0) { // upload raw to s3 - sh '${s3cmd_with_args} put -r data/raw s3://kg-hub-public-data/' + sh '${s3cmd_with_args} --cf-invalidate put -r data/raw s3://kg-hub-public-data/' } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/ sh 'rm -fr data/raw || true;' sh 'mkdir -p data/raw || true' @@ -106,9 +106,9 @@ pipeline { echo "Will not push if not on correct branch." } else { withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { - def s3cmd_with_args = 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate' - sh '${s3cmd_with_args} put merged-kg.nt.gz s3://kg-hub-public-data/kg-covid-19.nt.gz' - sh '${s3cmd_with_args} put merged-kg.tar.gz s3://kg-hub-public-data/kg-covid-19.tar.gz' + def s3cmd_with_args = 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text' + sh '${s3cmd_with_args} --cf-invalidate put merged-kg.nt.gz s3://kg-hub-public-data/kg-covid-19.nt.gz' + sh '${s3cmd_with_args} --cf-invalidate put merged-kg.tar.gz s3://kg-hub-public-data/kg-covid-19.tar.gz' // Should now appear at: // https://kg-hub.berkeleybop.io/[artifact name] } From 08d449ed1d89a7b0ed14fa606860b550382e4600 Mon Sep 17 00:00:00 2001 From: Justin Reese Date: Tue, 12 May 2020 10:21:55 -0700 Subject: [PATCH 07/11] Fix logical error in download stage (download was only running on master branch) --- Jenkinsfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ecf87c81..b90cedea 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -50,12 +50,14 @@ pipeline { steps { dir('./gitrepo') { script { + def run_py_dl = sh( + script: '. venv/bin/activate && python3.7 run.py download', returnStatus: true + ) if (env.BRANCH_NAME != 'master') { - echo "Will not push if not on correct branch." + if (run_py_dl == 0) { // upload raw to s3 + echo "Will not push if not on correct branch." + } } else { - def run_py_dl = sh( - script: '. venv/bin/activate && python3.7 run.py download', returnStatus: true - ) withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { def s3cmd_with_args = 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text' if (run_py_dl == 0) { // upload raw to s3 From 94e7db91439a50b3c82a18c1b618bedabe5e9852 Mon Sep 17 00:00:00 2001 From: Justin Reese Date: Tue, 12 May 2020 10:37:42 -0700 Subject: [PATCH 08/11] Add put/get to transform step, and refactor download stage to be more readable --- Jenkinsfile | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b90cedea..aeeee9d6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -53,20 +53,20 @@ pipeline { def run_py_dl = sh( script: '. venv/bin/activate && python3.7 run.py download', returnStatus: true ) - if (env.BRANCH_NAME != 'master') { - if (run_py_dl == 0) { // upload raw to s3 + if (run_py_dl == 0) { + if (env.BRANCH_NAME == 'master') { // upload raw to s3 if we're on correct branch echo "Will not push if not on correct branch." + } else { + withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { + def s3cmd_with_args = 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text' + sh '${s3cmd_with_args} --cf-invalidate put -r data/raw s3://kg-hub-public-data/' } - } else { + } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/ withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { def s3cmd_with_args = 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text' - if (run_py_dl == 0) { // upload raw to s3 - sh '${s3cmd_with_args} --cf-invalidate put -r data/raw s3://kg-hub-public-data/' - } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/ - sh 'rm -fr data/raw || true;' - sh 'mkdir -p data/raw || true' - sh '${s3cmd_with_args} get -r s3://kg-hub-public-data/raw/ data/raw/' - } + sh 'rm -fr data/raw || true;' + sh 'mkdir -p data/raw || true' + sh '${s3cmd_with_args} get -r s3://kg-hub-public-data/raw/ data/raw/' } } } @@ -77,7 +77,27 @@ pipeline { stage('Transform') { steps { dir('./gitrepo') { - sh '. venv/bin/activate && python3.7 run.py transform' + script { + def run_py_transform = sh( + script: '. venv/bin/activate && python3.7 run.py transform', returnStatus: true + ) + if (run_py_transform == 0) { // upload transformed to s3 if we're on correct branch + if (env.BRANCH_NAME == 'master') { + echo "Will not push if not on correct branch." + } else { + withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { + def s3cmd_with_args = 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text' + sh '${s3cmd_with_args} --cf-invalidate put -r data/transformed s3://kg-hub-public-data/' + } + } else { // 'run.py transform' failed - let's try to download last good copy of transformed/ from s3 to data/ + withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { + def s3cmd_with_args = 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text' + sh 'rm -fr data/transformed || true;' + sh 'mkdir -p data/transformed || true' + sh '${s3cmd_with_args} get -r s3://kg-hub-public-data/transformed/ data/transformed/' + } + } + } } } } From c58cb85d7af8ee04d9187c6c67f2e04a2cff0f18 Mon Sep 17 00:00:00 2001 From: Justin Reese Date: Tue, 12 May 2020 15:24:40 -0700 Subject: [PATCH 09/11] Fix missing }s in Jenkinsfile --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index aeeee9d6..63203335 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -60,6 +60,7 @@ pipeline { withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { def s3cmd_with_args = 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text' sh '${s3cmd_with_args} --cf-invalidate put -r data/raw s3://kg-hub-public-data/' + } } } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/ withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { @@ -88,6 +89,7 @@ pipeline { withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { def s3cmd_with_args = 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text' sh '${s3cmd_with_args} --cf-invalidate put -r data/transformed s3://kg-hub-public-data/' + } } } else { // 'run.py transform' failed - let's try to download last good copy of transformed/ from s3 to data/ withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { From 7fcd41b16c0e559379ff0f9d0a47f64b667d8f45 Mon Sep 17 00:00:00 2001 From: Justin Reese Date: Tue, 12 May 2020 17:25:49 -0700 Subject: [PATCH 10/11] Remove s3cmd variable --- Jenkinsfile | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 63203335..95075b6f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -58,16 +58,14 @@ pipeline { echo "Will not push if not on correct branch." } else { withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { - def s3cmd_with_args = 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text' - sh '${s3cmd_with_args} --cf-invalidate put -r data/raw s3://kg-hub-public-data/' + sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put -r data/raw s3://kg-hub-public-data/' } } } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/ withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { - def s3cmd_with_args = 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text' sh 'rm -fr data/raw || true;' sh 'mkdir -p data/raw || true' - sh '${s3cmd_with_args} get -r s3://kg-hub-public-data/raw/ data/raw/' + sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text' get -r s3://kg-hub-public-data/raw/ data/raw/' } } } @@ -87,16 +85,14 @@ pipeline { echo "Will not push if not on correct branch." } else { withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { - def s3cmd_with_args = 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text' - sh '${s3cmd_with_args} --cf-invalidate put -r data/transformed s3://kg-hub-public-data/' + sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put -r data/transformed s3://kg-hub-public-data/' } } } else { // 'run.py transform' failed - let's try to download last good copy of transformed/ from s3 to data/ withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { - def s3cmd_with_args = 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text' sh 'rm -fr data/transformed || true;' sh 'mkdir -p data/transformed || true' - sh '${s3cmd_with_args} get -r s3://kg-hub-public-data/transformed/ data/transformed/' + sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text get -r s3://kg-hub-public-data/transformed/ data/transformed/' } } } @@ -130,9 +126,8 @@ pipeline { echo "Will not push if not on correct branch." } else { withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { - def s3cmd_with_args = 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text' - sh '${s3cmd_with_args} --cf-invalidate put merged-kg.nt.gz s3://kg-hub-public-data/kg-covid-19.nt.gz' - sh '${s3cmd_with_args} --cf-invalidate put merged-kg.tar.gz s3://kg-hub-public-data/kg-covid-19.tar.gz' + sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.nt.gz s3://kg-hub-public-data/kg-covid-19.nt.gz' + sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar.gz s3://kg-hub-public-data/kg-covid-19.tar.gz' // Should now appear at: // https://kg-hub.berkeleybop.io/[artifact name] } From 9334c0684eb43ff092e0b69a45f2f7c8b306cdd4 Mon Sep 17 00:00:00 2001 From: Justin Reese Date: Tue, 12 May 2020 17:36:21 -0700 Subject: [PATCH 11/11] Remove stray apostrophe --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 95075b6f..2a9dbdea 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -65,7 +65,7 @@ pipeline { withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { sh 'rm -fr data/raw || true;' sh 'mkdir -p data/raw || true' - sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text' get -r s3://kg-hub-public-data/raw/ data/raw/' + sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text get -r s3://kg-hub-public-data/raw/ data/raw/' } } }