From 7def065ede114448effd9bc8b13dcdf9c5b2916c Mon Sep 17 00:00:00 2001 From: Seth Carbon Date: Tue, 19 May 2020 17:20:44 -0700 Subject: [PATCH] first draft for deployment; will have to wait on blazegraph journal availability; work on #134 --- Jenkinsfile | 137 ++++++---------------------------------------------- 1 file changed, 16 insertions(+), 121 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6cb320a8..56f17b5b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -30,127 +30,22 @@ pipeline { } } - stage('Build kg_covid_19') { - steps { - dir('./gitrepo') { - git( - url: 'https://github.com/Knowledge-Graph-Hub/kg-covid-19', - branch: 'master' - ) - sh '/usr/bin/python3.7 -m venv venv' - sh '. venv/bin/activate' - sh './venv/bin/pip install bmt' - sh './venv/bin/pip install -r requirements.txt' - sh './venv/bin/python setup.py install' - } - } - } - - stage('Download') { - steps { - dir('./gitrepo') { - script { - def run_py_dl = sh( - script: '. venv/bin/activate && python3.7 run.py download', returnStatus: true - ) - if (run_py_dl == 0) { - if (env.BRANCH_NAME == 'master') { // upload raw to s3 if we're on correct branch - echo "Will not push if not on correct branch." - } else { - withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { - sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put -r data/raw s3://kg-hub-public-data/' - } - } - } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/ - withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { - sh 'rm -fr data/raw || true;' - sh 'mkdir -p data/raw || true' - sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text get -r s3://kg-hub-public-data/raw/ data/raw/' - } - } - } - } - } - } - - stage('Transform') { - steps { - dir('./gitrepo') { - script { - def run_py_transform = sh( - script: '. venv/bin/activate && python3.7 run.py transform', returnStatus: true - ) - if (run_py_transform == 0) { // upload transformed to s3 if we're on correct branch - if (env.BRANCH_NAME != 'master') { - echo "Will not push if not on correct branch." - } else { - withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { - sh 'rm -fr data/transformed/.gitkeep' - sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put -r data/transformed s3://kg-hub-public-data/' - } - } - } else { // 'run.py transform' failed - let's try to download last good copy of transformed/ from s3 to data/ - withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { - sh 'rm -fr data/transformed || true;' - sh 'mkdir -p data/transformed || true' - sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text get -r s3://kg-hub-public-data/transformed/ data/transformed/' - } - } - } - } - } - } - - stage('Load') { - steps { - dir('./gitrepo') { - sh '. venv/bin/activate && python3.7 run.py load' - sh 'pigz merged-kg.tar' - } - } - } - - stage('Convert to RDF') { - steps { - dir('./gitrepo') { - sh '. venv/bin/activate && kgx transform --input-type tsv --output-type nt -o ./merged-kg.nt merged-kg.tar.gz' - sh 'pigz merged-kg.nt' - } - } - } - - stage('Make blazegraph journal'){ - steps { - dir('./gitrepo/blazegraph') { - git( - url: 'https://github.com/balhoff/blazegraph-runner.git', - branch: 'master' - ) - sh 'sbt stage' - sh 'pigz -d ../merged-kg.nt.gz' - sh './target/universal/stage/bin/blazegraph-runner load --informat=ntriples --journal=merged-kg.jnl --use-ontology-graph=true ../merged-kg.nt' - sh 'pigz merged-kg.jnl' - } - } - } - - stage('Publish') { - steps { - dir('./gitrepo') { - script { - if (env.BRANCH_NAME != 'master') { - echo "Will not push if not on correct branch." - } else { - withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) { - sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.nt.gz s3://kg-hub-public-data/kg-covid-19.nt.gz' - sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar.gz s3://kg-hub-public-data/kg-covid-19.tar.gz' - sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.jnl.gz s3://kg-hub-public-data/kg-covid-19.jnl.gz' - // Should now appear at: - // https://kg-hub.berkeleybop.io/[artifact name] - } - - } - } + stage('Deploy') { + when { anyOf { branch 'master' } } + steps { + + git([branch: 'master', + credentialsId: 'justaddcoffee_github_api_token_username_pw', + url: 'https://github.com/geneontology/operations.git']) + dir('./ansible') { + + withCredentials([file(credentialsId: 'ansible-bbop-local-slave', variable: 'DEPLOY_LOCAL_IDENTITY')]) { + + echo 'Push master out to public Blazegraph' + retry(3){ + sh 'ansible-playbook update-kg-hub-endpoint.yaml --inventory=hosts.local-rdf-endpoint --private-key="$DEPLOY_LOCAL_IDENTITY" -e target_user=bbop --extra-vars="endpoint=internal"' + } + } } } }