Add back blazegraph journal step

Knowledge-Graph-Hub · Sep 23, 2020 · 7c0b9bb · 7c0b9bb
1 parent e8af5ce
commit 7c0b9bb
Showing 1 changed file with 20 additions and 17 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -111,27 +111,30 @@ pipeline {
                     sh 'touch TEST_stats.yaml'
                     sh 'touch merged_graph_stats_$BUILDSTARTDATE.yaml'
                     sh 'mkdir -p data/merged/'
-                    sh 'touch data/merged/merged-kg.nt.gz'
+                    withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_CFG')]) {
+                        sh 'mkdir -p data/merged/'
+                        sh 's3cmd -c $S3CMD_CFG --acl-public --mime-type=plain/text --cf-invalidate get s3://kg-hub-public-data/kg-covid-19.jnl.gz data/merged/merged-kg.nt.gz'
+                    }
                     sh 'touch data/merged/merged-kg.tar.gz'
                 }
             }
         }
 
-//         stage('Make blazegraph journal'){
-//             steps {
-//                 dir('./gitrepo/blazegraph') {
-//                         git(
-//                                 url: 'https://github.com/balhoff/blazegraph-runner.git',
-//                                 branch: 'master'
-//                         )
-//                         sh 'sbt stage'
-//                         sh 'pigz -d ../data/merged/merged-kg.nt.gz'
-//                         sh 'export JAVA_OPTS=-Xmx128G && ./target/universal/stage/bin/blazegraph-runner load --informat=ntriples --journal=../merged-kg.jnl --use-ontology-graph=true ../data/merged/merged-kg.nt'
-//                         sh 'pigz ../merged-kg.jnl'
-//                         sh 'pigz ../data/merged/merged-kg.nt'
-//                 }
-//             }
-//         }
+        stage('Make blazegraph journal'){
+            steps {
+                dir('./gitrepo/blazegraph') {
+                        git(
+                                url: 'https://github.com/balhoff/blazegraph-runner.git',
+                                branch: 'master'
+                        )
+                        sh 'sbt stage'
+                        sh 'pigz -d ../data/merged/merged-kg.nt.gz'
+                        sh 'export JAVA_OPTS=-Xmx128G && ./target/universal/stage/bin/blazegraph-runner load --informat=ntriples --journal=../merged-kg.jnl --use-ontology-graph=true ../data/merged/merged-kg.nt'
+                        sh 'pigz ../merged-kg.jnl'
+                        sh 'pigz ../data/merged/merged-kg.nt'
+                }
+            }
+        }
 
         stage('Publish') {
             steps {
@@ -188,7 +191,7 @@ pipeline {
 
                                 //
                                 // make $BUILDSTARTDATE the new current/
-                                // 	    
+                                //
 				// The following cp always times out:
                                 // sh 's3cmd -c $S3CMD_CFG --acl-public --mime-type=text/html --cf-invalidate cp -v -pr s3://kg-hub-public-data/$BUILDSTARTDATE/ s3://kg-hub-public-data/new_current/'
                                 // sh 's3cmd -c $S3CMD_CFG --acl-public --mime-type=text/html --cf-invalidate rm -fr s3://kg-hub-public-data/current'