Add step in download to upload/dl raw to s3

Knowledge-Graph-Hub · May 6, 2020 · 6b9fdd6 · 6b9fdd6
1 parent 5862e93
commit 6b9fdd6
Showing 1 changed file with 9 additions and 2 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -45,8 +45,15 @@ pipeline {
             }
         }
         stage('Download') {
-            steps {
-                sh 'cd config;. venv/bin/activate; python3.7 run.py download'
+        steps {
+            def run_py_dl = sh(
+                script: 'cd config;. venv/bin/activate; python3.7 run.py download', returnStatus: true
+            )
+            if (run_py_dl == 0) { // upload raw to s3
+                sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put -r data/raw s3://kg-hub-public-data/raw'
+            } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/
+                sh 'cd config; rm -fr data/raw'
+                sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate get -r s3://kg-hub-public-data/raw data/'
             }
         }
         stage('Transform') {