From 4785dd9a3675b203e6cd38063501808872e83249 Mon Sep 17 00:00:00 2001 From: Justin Reese Date: Tue, 12 May 2020 15:03:21 -0700 Subject: [PATCH 1/8] Added stage to make blazegraph journal --- Jenkinsfile | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b581a8eb..baf097a0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,9 +3,9 @@ pipeline { options { timestamps() } - stages { + stages { // Very first: pause for a minute to give a chance to - // cancel and clean the workspace before use. + // cancel and clean the workspace before use. stage('Ready and clean') { steps { // Give us a minute to cancel if we want. @@ -76,7 +76,20 @@ pipeline { sh 'cd config;. venv/bin/activate; kgx transform --input-type tsv --output-type nt -o ./merged-kg.nt merged-kg.tar.gz' sh 'cd config;. venv/bin/activate; pigz merged-kg.nt' } - } + } + stage('Make blazegraph journal'){ + dir('./config/blazegraph') { + git( + url: 'https://github.com/balhoff/blazegraph-runner.git', + branch: 'master' + ) + sh 'sbt stage' + sh 'cd ..' + sh 'pigz -d ../merged-kg.nt.gz' + sh './blazegraph-runner/target/universal/stage/bin/blazegraph-runner/blazegraph-runner load --informat=ntriples --journal=merged-kg.jnl --use-ontology-graph=true ./merged-kg.nt' + sh 'pigz merged-kg.jnl' + } + } stage('Publish') { steps { @@ -87,6 +100,7 @@ pipeline { withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.nt.gz s3://kg-hub-public-data/kg-covid-19.nt.gz' sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar.gz s3://kg-hub-public-data/kg-covid-19.tar.gz' + sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.jnl.gz s3://kg-hub-public-data/kg-covid-19.jnl.gz' // Should now appear at: // https://kg-hub.berkeleybop.io/[artifact name] } From 0341d82415eced9cb17cf2c22109d8032be0bf0d Mon Sep 17 00:00:00 2001 From: Justin Reese Date: Tue, 12 May 2020 15:20:43 -0700 Subject: [PATCH 2/8] Added missing steps {} for blazegraph step --- Jenkinsfile | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index baf097a0..3a4a5c1e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -78,16 +78,18 @@ pipeline { } } stage('Make blazegraph journal'){ - dir('./config/blazegraph') { - git( - url: 'https://github.com/balhoff/blazegraph-runner.git', - branch: 'master' - ) - sh 'sbt stage' - sh 'cd ..' - sh 'pigz -d ../merged-kg.nt.gz' - sh './blazegraph-runner/target/universal/stage/bin/blazegraph-runner/blazegraph-runner load --informat=ntriples --journal=merged-kg.jnl --use-ontology-graph=true ./merged-kg.nt' - sh 'pigz merged-kg.jnl' + steps { + dir('./config/blazegraph') { + git( + url: 'https://github.com/balhoff/blazegraph-runner.git', + branch: 'master' + ) + sh 'sbt stage' + sh 'cd ..' + sh 'pigz -d ../merged-kg.nt.gz' + sh './blazegraph-runner/target/universal/stage/bin/blazegraph-runner/blazegraph-runner load --informat=ntriples --journal=merged-kg.jnl --use-ontology-graph=true ./merged-kg.nt' + sh 'pigz merged-kg.jnl' + } } } stage('Publish') { From b53edd6d7aacb9cfe70cb79b3e1985ed4c93e752 Mon Sep 17 00:00:00 2001 From: Justin Reese Date: Tue, 12 May 2020 16:12:36 -0700 Subject: [PATCH 3/8] Fix incorrect path to blazegraph-runner --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3a4a5c1e..39bb53d1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -87,7 +87,7 @@ pipeline { sh 'sbt stage' sh 'cd ..' sh 'pigz -d ../merged-kg.nt.gz' - sh './blazegraph-runner/target/universal/stage/bin/blazegraph-runner/blazegraph-runner load --informat=ntriples --journal=merged-kg.jnl --use-ontology-graph=true ./merged-kg.nt' + sh './blazegraph-runner/target/universal/stage/bin/blazegraph-runner load --informat=ntriples --journal=merged-kg.jnl --use-ontology-graph=true ./merged-kg.nt' sh 'pigz merged-kg.jnl' } } From 7a4db9b60a3ce4fe893ceb7c11b390c111fbf08e Mon Sep 17 00:00:00 2001 From: Justin Reese Date: Wed, 13 May 2020 08:35:07 -0700 Subject: [PATCH 4/8] Fixed (hopefully) path to blazegraph-runner --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 39bb53d1..818a90d3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -87,7 +87,7 @@ pipeline { sh 'sbt stage' sh 'cd ..' sh 'pigz -d ../merged-kg.nt.gz' - sh './blazegraph-runner/target/universal/stage/bin/blazegraph-runner load --informat=ntriples --journal=merged-kg.jnl --use-ontology-graph=true ./merged-kg.nt' + sh './blazegraph/target/universal/stage/bin/blazegraph-runner load --informat=ntriples --journal=merged-kg.jnl --use-ontology-graph=true ./merged-kg.nt' sh 'pigz merged-kg.jnl' } } From c80bcfd1b2ec1b58cc91a62639cd8bbea7234d4d Mon Sep 17 00:00:00 2001 From: Justin Reese Date: Thu, 14 May 2020 06:59:48 -0700 Subject: [PATCH 5/8] Find correct path to blazegraph-runner --- Jenkinsfile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 818a90d3..d2a621a7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -28,6 +28,22 @@ pipeline { }) } } + + stage('Build blazegraph and die'){ + steps { + dir('./config/blazegraph') { + git( + url: 'https://github.com/balhoff/blazegraph-runner.git', + branch: 'master' + ) + sh 'sbt stage' + sh 'cd ..' + sh 'pwd;find . -name "blazegraph-runner"' + sh 'false' + } + } + } + stage('Build kg_covid_19') { steps { dir('./config') { From 8fbe2018110babff559512600db811b8701fe1be Mon Sep 17 00:00:00 2001 From: Justin Reese Date: Thu, 14 May 2020 07:15:46 -0700 Subject: [PATCH 6/8] Fix blazegraph-runner path --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d2a621a7..6157a9d5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,6 +39,7 @@ pipeline { sh 'sbt stage' sh 'cd ..' sh 'pwd;find . -name "blazegraph-runner"' + sh './target/universal/stage/bin/blazegraph-runner' sh 'false' } } @@ -103,7 +104,7 @@ pipeline { sh 'sbt stage' sh 'cd ..' sh 'pigz -d ../merged-kg.nt.gz' - sh './blazegraph/target/universal/stage/bin/blazegraph-runner load --informat=ntriples --journal=merged-kg.jnl --use-ontology-graph=true ./merged-kg.nt' + sh './target/universal/stage/bin/blazegraph-runner load --informat=ntriples --journal=merged-kg.jnl --use-ontology-graph=true ./merged-kg.nt' sh 'pigz merged-kg.jnl' } } From 92233441cfbd404ce904625d9b0b29e99f94239f Mon Sep 17 00:00:00 2001 From: Justin Reese Date: Thu, 14 May 2020 07:20:11 -0700 Subject: [PATCH 7/8] Remove test stage --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6157a9d5..0f762441 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -102,9 +102,8 @@ pipeline { branch: 'master' ) sh 'sbt stage' - sh 'cd ..' sh 'pigz -d ../merged-kg.nt.gz' - sh './target/universal/stage/bin/blazegraph-runner load --informat=ntriples --journal=merged-kg.jnl --use-ontology-graph=true ./merged-kg.nt' + sh './target/universal/stage/bin/blazegraph-runner load --informat=ntriples --journal=merged-kg.jnl --use-ontology-graph=true ../merged-kg.nt' sh 'pigz merged-kg.jnl' } } From 96bd0fca073464f8f54a4eef461c85c853ee2f8a Mon Sep 17 00:00:00 2001 From: Justin Reese Date: Thu, 14 May 2020 07:30:31 -0700 Subject: [PATCH 8/8] Remove test stage --- Jenkinsfile | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0f762441..af13030b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -28,23 +28,7 @@ pipeline { }) } } - - stage('Build blazegraph and die'){ - steps { - dir('./config/blazegraph') { - git( - url: 'https://github.com/balhoff/blazegraph-runner.git', - branch: 'master' - ) - sh 'sbt stage' - sh 'cd ..' - sh 'pwd;find . -name "blazegraph-runner"' - sh './target/universal/stage/bin/blazegraph-runner' - sh 'false' - } - } - } - + stage('Build kg_covid_19') { steps { dir('./config') {