Skip to content

Commit

Permalink
Comment out stages to get to the good (i.e. bad) part in Publish
Browse files Browse the repository at this point in the history
  • Loading branch information
@justaddcoffee
justaddcoffee committed Oct 1, 2020
1 parent a3d608b commit 71d6d10
Showing 1 changed file with 82 additions and 82 deletions.
164 changes: 82 additions & 82 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,68 +59,68 @@ pipeline {
}
}

stage('Download') {
steps {
dir('./gitrepo') {
script {
def run_py_dl = sh(
script: '. venv/bin/activate && python3.7 run.py download', returnStatus: true
)
if (run_py_dl == 0) {
if (env.BRANCH_NAME != 'master') { // upload raw to s3 if we're on correct branch
echo "Will not push if not on correct branch."
} else {
withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_CFG')]) {
sh 's3cmd -c $S3CMD_CFG --acl-public --mime-type=plain/text --cf-invalidate put -r data/raw s3://kg-hub-public-data/$S3PROJECTDIR/'
}
}
} else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/
withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_CFG')]) {
sh 'rm -fr data/raw || true;'
sh 'mkdir -p data/raw || true'
sh 's3cmd -c $S3CMD_CFG --acl-public --mime-type=plain/text get -r s3://kg-hub-public-data/$S3PROJECTDIR/raw/ data/raw/'
}
}
}
}
}
}

stage('Transform') {
steps {
dir('./gitrepo') {
sh 'env'
sh '. venv/bin/activate && env && python3.7 run.py transform'
}
}
}

stage('Merge') {
steps {
dir('./gitrepo') {
sh '. venv/bin/activate && python3.7 run.py merge -y merge_jenkins.yaml'
sh 'env'
sh 'cp merged_graph_stats.yaml merged_graph_stats_$BUILDSTARTDATE.yaml'
sh 'tar -rvf data/merged/merged-kg.tar merged_graph_stats_$BUILDSTARTDATE.yaml'
}
}
}

stage('Make blazegraph journal'){
steps {
dir('./gitrepo/blazegraph') {
git(
url: 'https://github.com/balhoff/blazegraph-runner.git',
branch: 'master'
)
sh 'sbt stage'
sh 'pigz -d ../data/merged/merged-kg.nt.gz'
sh 'export JAVA_OPTS=-Xmx128G && ./target/universal/stage/bin/blazegraph-runner load --informat=ntriples --journal=../merged-kg.jnl --use-ontology-graph=true ../data/merged/merged-kg.nt'
sh 'pigz ../merged-kg.jnl'
sh 'pigz ../data/merged/merged-kg.nt'
}
}
}
// stage('Download') {
// steps {
// dir('./gitrepo') {
// script {
// def run_py_dl = sh(
// script: '. venv/bin/activate && python3.7 run.py download', returnStatus: true
// )
// if (run_py_dl == 0) {
// if (env.BRANCH_NAME != 'master') { // upload raw to s3 if we're on correct branch
// echo "Will not push if not on correct branch."
// } else {
// withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_CFG')]) {
// sh 's3cmd -c $S3CMD_CFG --acl-public --mime-type=plain/text --cf-invalidate put -r data/raw s3://kg-hub-public-data/$S3PROJECTDIR/'
// }
// }
// } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/
// withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_CFG')]) {
// sh 'rm -fr data/raw || true;'
// sh 'mkdir -p data/raw || true'
// sh 's3cmd -c $S3CMD_CFG --acl-public --mime-type=plain/text get -r s3://kg-hub-public-data/$S3PROJECTDIR/raw/ data/raw/'
// }
// }
// }
// }
// }
// }
//
// stage('Transform') {
// steps {
// dir('./gitrepo') {
// sh 'env'
// sh '. venv/bin/activate && env && python3.7 run.py transform'
// }
// }
// }
//
// stage('Merge') {
// steps {
// dir('./gitrepo') {
// sh '. venv/bin/activate && python3.7 run.py merge -y merge_jenkins.yaml'
// sh 'env'
// sh 'cp merged_graph_stats.yaml merged_graph_stats_$BUILDSTARTDATE.yaml'
// sh 'tar -rvf data/merged/merged-kg.tar merged_graph_stats_$BUILDSTARTDATE.yaml'
// }
// }
// }
//
// stage('Make blazegraph journal'){
// steps {
// dir('./gitrepo/blazegraph') {
// git(
// url: 'https://github.com/balhoff/blazegraph-runner.git',
// branch: 'master'
// )
// sh 'sbt stage'
// sh 'pigz -d ../data/merged/merged-kg.nt.gz'
// sh 'export JAVA_OPTS=-Xmx128G && ./target/universal/stage/bin/blazegraph-runner load --informat=ntriples --journal=../merged-kg.jnl --use-ontology-graph=true ../data/merged/merged-kg.nt'
// sh 'pigz ../merged-kg.jnl'
// sh 'pigz ../data/merged/merged-kg.nt'
// }
// }
// }

stage('Publish') {
steps {
Expand Down Expand Up @@ -212,26 +212,26 @@ pipeline {
}
}

stage('Deploy blazegraph') {
when { anyOf { branch 'master' } }
steps {

git([branch: 'master',
credentialsId: 'justaddcoffee_github_api_token_username_pw',
url: 'https://github.com/geneontology/operations.git'])

dir('./ansible') {

withCredentials([file(credentialsId: 'ansible-bbop-local-slave', variable: 'DEPLOY_LOCAL_IDENTITY')]) {
echo 'Push master out to public Blazegraph'
retry(3){
sh 'ansible-playbook update-kg-hub-endpoint.yaml --inventory=hosts.local-rdf-endpoint --private-key="$DEPLOY_LOCAL_IDENTITY" -e target_user=bbop --extra-vars="endpoint=internal"'
}
}
}

}
}
// stage('Deploy blazegraph') {
// when { anyOf { branch 'master' } }
// steps {
//
// git([branch: 'master',
// credentialsId: 'justaddcoffee_github_api_token_username_pw',
// url: 'https://github.com/geneontology/operations.git'])
//
// dir('./ansible') {
//
// withCredentials([file(credentialsId: 'ansible-bbop-local-slave', variable: 'DEPLOY_LOCAL_IDENTITY')]) {
// echo 'Push master out to public Blazegraph'
// retry(3){
// sh 'ansible-playbook update-kg-hub-endpoint.yaml --inventory=hosts.local-rdf-endpoint --private-key="$DEPLOY_LOCAL_IDENTITY" -e target_user=bbop --extra-vars="endpoint=internal"'
// }
// }
// }
//
// }
// }

}
}

0 comments on commit 71d6d10

Please sign in to comment.