Skip to content

Commit

Permalink
first draft for deployment; will have to wait on blazegraph journal a…
Browse files Browse the repository at this point in the history 
…vailability; work on #134
  • Loading branch information
@kltm
kltm committed May 20, 2020
1 parent f8c51e0 commit 7def065
Showing 1 changed file with 16 additions and 121 deletions.
137 changes: 16 additions & 121 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,127 +30,22 @@ pipeline {
}
}

stage('Build kg_covid_19') {
steps {
dir('./gitrepo') {
git(
url: 'https://github.com/Knowledge-Graph-Hub/kg-covid-19',
branch: 'master'
)
sh '/usr/bin/python3.7 -m venv venv'
sh '. venv/bin/activate'
sh './venv/bin/pip install bmt'
sh './venv/bin/pip install -r requirements.txt'
sh './venv/bin/python setup.py install'
}
}
}

stage('Download') {
steps {
dir('./gitrepo') {
script {
def run_py_dl = sh(
script: '. venv/bin/activate && python3.7 run.py download', returnStatus: true
)
if (run_py_dl == 0) {
if (env.BRANCH_NAME == 'master') { // upload raw to s3 if we're on correct branch
echo "Will not push if not on correct branch."
} else {
withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) {
sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put -r data/raw s3://kg-hub-public-data/'
}
}
} else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/
withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) {
sh 'rm -fr data/raw || true;'
sh 'mkdir -p data/raw || true'
sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text get -r s3://kg-hub-public-data/raw/ data/raw/'
}
}
}
}
}
}

stage('Transform') {
steps {
dir('./gitrepo') {
script {
def run_py_transform = sh(
script: '. venv/bin/activate && python3.7 run.py transform', returnStatus: true
)
if (run_py_transform == 0) { // upload transformed to s3 if we're on correct branch
if (env.BRANCH_NAME != 'master') {
echo "Will not push if not on correct branch."
} else {
withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) {
sh 'rm -fr data/transformed/.gitkeep'
sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put -r data/transformed s3://kg-hub-public-data/'
}
}
} else { // 'run.py transform' failed - let's try to download last good copy of transformed/ from s3 to data/
withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) {
sh 'rm -fr data/transformed || true;'
sh 'mkdir -p data/transformed || true'
sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text get -r s3://kg-hub-public-data/transformed/ data/transformed/'
}
}
}
}
}
}

stage('Load') {
steps {
dir('./gitrepo') {
sh '. venv/bin/activate && python3.7 run.py load'
sh 'pigz merged-kg.tar'
}
}
}

stage('Convert to RDF') {
steps {
dir('./gitrepo') {
sh '. venv/bin/activate && kgx transform --input-type tsv --output-type nt -o ./merged-kg.nt merged-kg.tar.gz'
sh 'pigz merged-kg.nt'
}
}
}

stage('Make blazegraph journal'){
steps {
dir('./gitrepo/blazegraph') {
git(
url: 'https://github.com/balhoff/blazegraph-runner.git',
branch: 'master'
)
sh 'sbt stage'
sh 'pigz -d ../merged-kg.nt.gz'
sh './target/universal/stage/bin/blazegraph-runner load --informat=ntriples --journal=merged-kg.jnl --use-ontology-graph=true ../merged-kg.nt'
sh 'pigz merged-kg.jnl'
}
}
}

stage('Publish') {
steps {
dir('./gitrepo') {
script {
if (env.BRANCH_NAME != 'master') {
echo "Will not push if not on correct branch."
} else {
withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) {
sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.nt.gz s3://kg-hub-public-data/kg-covid-19.nt.gz'
sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar.gz s3://kg-hub-public-data/kg-covid-19.tar.gz'
sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.jnl.gz s3://kg-hub-public-data/kg-covid-19.jnl.gz'
// Should now appear at:
// https://kg-hub.berkeleybop.io/[artifact name]
}

}
}
stage('Deploy') {
when { anyOf { branch 'master' } }
steps {

git([branch: 'master',
credentialsId: 'justaddcoffee_github_api_token_username_pw',
url: 'https://github.com/geneontology/operations.git'])
dir('./ansible') {

withCredentials([file(credentialsId: 'ansible-bbop-local-slave', variable: 'DEPLOY_LOCAL_IDENTITY')]) {

echo 'Push master out to public Blazegraph'
retry(3){
sh 'ansible-playbook update-kg-hub-endpoint.yaml --inventory=hosts.local-rdf-endpoint --private-key="$DEPLOY_LOCAL_IDENTITY" -e target_user=bbop --extra-vars="endpoint=internal"'
}
}
}
}
}
Expand Down

0 comments on commit 7def065

Please sign in to comment.