From a4ea8bab4bc067082e8a0d61c827054af0c01a44 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Fri, 3 Apr 2020 15:57:10 -0700
Subject: [PATCH 01/63] Cribbed Jenkinsfile from GO

---
 Jenkinsfile | 395 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 395 insertions(+)
 create mode 100644 Jenkinsfile

diff --git a/Jenkinsfile b/Jenkinsfile
new file mode 100644
index 00000000..2cafedba
--- /dev/null
+++ b/Jenkinsfile
@@ -0,0 +1,395 @@
+pipeline {
+    agent any
+    // In additional to manual runs, trigger somewhere at midnight to
+    // give us the max time in a day to get things right.
+    triggers {
+	// Master never runs--Feb 31st.
+	//cron('0 0 31 2 *')
+	// Nightly @12am, for "snapshot", skip "release" night.
+	//cron('0 0 2-31 * *')
+	// First of the month @12am, for "release" (also "current").
+	//cron('0 0 1 * *')
+	// Every week on Friday, noon.
+	cron('0 12 * * 5')
+    }
+    environment {
+	// Pin dates and day to beginning of run.
+	START_DATE = sh (
+	    script: 'date +%Y-%m-%d',
+	    returnStdout: true
+	).trim()
+
+	START_DAY = sh (
+	    script: 'date +%A',
+	    returnStdout: true
+	).trim()
+	// The branch of geneontology/go-site to use.
+	TARGET_GO_SITE_BRANCH = 'master'
+	// The people to call when things go bad. It is a comma-space
+	// "separated" string.
+	TARGET_ADMIN_EMAILS = 'sjcarbon@lbl.gov'
+	TARGET_SUCCESS_EMAILS = 'sjcarbon@lbl.gov'
+	// The file bucket(/folder) combination to use.
+	TARGET_BUCKET = 'no'
+	// The URL prefix to use when creating site indices.
+	TARGET_INDEXER_PREFIX = 'no'
+	// This variable should typically be 'TRUE', which will cause
+	// some additional basic checks to be made. There are some
+	// very exotic cases where these check may need to be skipped
+	// for a run, in that case this variable is set to 'FALSE'.
+	WE_ARE_BEING_SAFE_P = 'TRUE'
+	// The Zenodo concept ID to use for releases (and occasionally
+	// master testing).
+	ZENODO_REFERENCE_CONCEPT = '0'
+	ZENODO_ARCHIVE_CONCEPT = '0'
+	// Control make to get through our loads faster if
+	// possible. Assuming we're cpu bound for some of these...
+	// wok has 48 "processors" over 12 "cores", so I have no idea;
+	// let's go with conservative and see if we get an
+	// improvement.
+	//MAKECMD = 'make --jobs 3 --max-load 10.0'
+	MAKECMD = 'make'
+	// GOlr load profile.
+	GOLR_SOLR_MEMORY = "128G"
+	GOLR_LOADER_MEMORY = "192G"
+	GOLR_INPUT_ONTOLOGIES = [
+	    //"http://purl.obolibrary.org/obo/go/snapshot/extensions/go-lego.owl",
+	    //"http://purl.obolibrary.org/obo/eco.owl",
+	    //"http://purl.obolibrary.org/obo/ncbitaxon/subsets/taxslim.owl",
+	    //"http://purl.obolibrary.org/obo/cl/cl-basic.owl",
+	    //"http://purl.obolibrary.org/obo/go/extensions/gorel.owl",
+	    //"http://purl.obolibrary.org/obo/pato.owl",
+	    //"http://purl.obolibrary.org/obo/po.owl",
+	    //"http://purl.obolibrary.org/obo/chebi.owl",
+	    //"http://purl.obolibrary.org/obo/uberon/basic.owl",
+	    //"http://purl.obolibrary.org/obo/wbbt.owl",
+	    //"http://purl.obolibrary.org/obo/go/extensions/go-modules-annotations.owl",
+	    //"http://purl.obolibrary.org/obo/go/extensions/go-taxon-subsets.owl"
+            //"http://skyhook.berkeleybop.org/issue-35-neo-test/ontology/neo.owl"
+            "http://skyhook.berkeleybop.org/issue-35-neo-test/ontology/extensions/go-lego.owl"
+	].join(" ")
+    }
+    options{
+	timestamps()
+    }
+    stages {
+	// Very first: pause for a few minutes to give a chance to
+	// cancel and clean the workspace before use.
+	stage('Ready and clean') {
+	    steps {
+
+		// Check that we do not affect public targets on
+		// non-mainline runs.
+		script {
+		    if( BRANCH_NAME != 'master' && TARGET_BUCKET == 'go-data-product-experimental'){
+			echo 'Only master can touch that target.'
+			sh '`exit -1`'
+		    }else if( BRANCH_NAME != 'snapshot' && TARGET_BUCKET == 'go-data-product-snapshot'){
+			echo 'Only master can touch that target.'
+			sh '`exit -1`'
+		    }else if( BRANCH_NAME != 'release' && TARGET_BUCKET == 'go-data-product-release'){
+			echo 'Only master can touch that target.'
+			sh '`exit -1`'
+		    }
+		}
+
+		// Give us a minute to cancel if we want.
+		sleep time: 1, unit: 'MINUTES'
+		cleanWs()
+	    }
+	}
+	stage('Initialize') {
+	    steps {
+		// Start preparing environment.
+		parallel(
+		    "Report": {
+			sh 'env > env.txt'
+			sh 'echo $BRANCH_NAME > branch.txt'
+			sh 'echo "$BRANCH_NAME"'
+			sh 'cat env.txt'
+			sh 'cat branch.txt'
+			sh 'echo $START_DAY > dow.txt'
+			sh 'echo "$START_DAY"'
+		    },
+		    "Reset base": {
+			// Get a mount point ready
+			sh 'mkdir -p $WORKSPACE/mnt || true'
+			// Ninja in our file credentials from Jenkins.
+			withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) {
+			    // Try and ssh fuse skyhook onto our local system.
+			    sh 'sshfs -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY -o idmap=user skyhook@skyhook.berkeleybop.org:/home/skyhook $WORKSPACE/mnt/'
+			}
+			// Remove anything we might have left around from
+			// times past.
+			sh 'rm -r -f $WORKSPACE/mnt/$BRANCH_NAME || true'
+			// Rebuild directory structure.
+			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/bin || true'
+			// WARNING/BUG: needed for arachne to run at
+			// this point.
+			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/lib || true'
+			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products || true'
+			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/ttl || true'
+			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/blazegraph || true'
+			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/annotations || true'
+			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/pages || true'
+			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/solr || true'
+			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/panther || true'
+			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/metadata || true'
+			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/annotations || true'
+			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/ontology || true'
+			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/reports || true'
+			// Tag the top to let the world know I was at least
+			// here.
+			sh 'echo "Runtime summary." > $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
+			sh 'echo "" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
+			sh 'date >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
+			sh 'echo "" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
+			sh 'echo "Release notes: https://github.com/geneontology/go-site/tree/master/releases" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
+			sh 'echo "Branch: $BRANCH_NAME" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
+			sh 'echo "Start day: $START_DAY" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
+			sh 'echo "Start date: $START_DATE" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
+
+			sh 'echo "Official release date: metadata/release-date.json" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
+			sh 'echo "Official Zenodo archive DOI: metadata/release-archive-doi.json" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
+			sh 'echo "Official Zenodo archive DOI: metadata/release-reference-doi.json" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
+			sh 'echo "TODO: Note software versions." >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
+			// TODO: This should be wrapped in exception
+			// handling. In fact, this whole thing should be.
+			sh 'fusermount -u $WORKSPACE/mnt/ || true'
+		    }
+		)
+	    }
+	}
+	// Build owltools and get it into the shared filesystem.
+	stage('Ready production software') {
+	    steps {
+		// Legacy: build 'owltools-build'
+		dir('./owltools') {
+		    // Remember that git lays out into CWD.
+		    git 'https://github.com/owlcollab/owltools.git'
+		    sh 'mvn -f OWLTools-Parent/pom.xml -U clean install -DskipTests -Dmaven.javadoc.skip=true -Dsource.skip=true'
+		    // Attempt to rsync produced into bin/.
+		    withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) {
+			sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" OWLTools-Runner/target/owltools skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/'
+			sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" OWLTools-Oort/bin/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/'
+			sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" OWLTools-NCBI/bin/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/'
+			sh 'rsync -vhac -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" --exclude ".git" OWLTools-Oort/reporting/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/'
+			sh 'rsync -vhac -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" --exclude ".git" OWLTools-Runner/contrib/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/'
+		    }
+		}
+	    }
+	}
+	// See https://github.com/geneontology/go-ontology for details
+	// on the ontology release pipeline. This ticket runs
+	// daily(TODO?) and creates all the files normally included in
+	// a release, and deploys to S3.
+	stage('Produce NEO') {
+	    steps {
+		// Create a relative working directory and setup our
+		// data environment.
+		dir('./neo') {
+		    git 'https://github.com/geneontology/neo.git'
+
+		    // Default namespace.
+		    sh 'OBO=http://purl.obolibrary.org/obo'
+
+		    // Make all software products available in bin/.
+		    sh 'mkdir -p bin/'
+		    withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) {
+			sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/* ./bin/'
+		    }
+		    sh 'chmod +x bin/*'
+
+		    //
+		    withEnv(['PATH+EXTRA=:bin:./bin', 'JAVA_OPTS=-Xmx128G', 'OWLTOOLS_MEMORY=128G', 'BGMEM=128G']){
+			retry(3){
+			    sh 'make clean all'
+			}
+		    }
+
+		    // Make sure that we copy any files there,
+		    // including the core dump of produced.
+		    withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) {
+			sh 'scp -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY neo.obo neo.owl skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology/'
+		    }
+
+		    // WARNING/BUG: This occurs "early" as we need NEO
+		    // in the proper location before building GO (for
+		    // testing, solr loading, etc.). Once we have
+		    // ubiquitous ontology catalogs, publishing can
+		    // occur properly at the end and after testing.
+		    // See commented out section below.
+		    //
+		    // Deploy to S3 location for pickup by PURL via CF.
+		    withCredentials([file(credentialsId: 'aws_go_push_json', variable: 'S3_PUSH_JSON'), file(credentialsId: 's3cmd_go_push_configuration', variable: 'S3CMD_JSON'), string(credentialsId: 'aws_go_access_key', variable: 'AWS_ACCESS_KEY_ID'), string(credentialsId: 'aws_go_secret_key', variable: 'AWS_SECRET_ACCESS_KEY')]) {
+			sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=application/rdf+xml --cf-invalidate put neo.owl s3://go-build/build-noctua-entity-ontology/latest/'
+			sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=application/rdf+xml --cf-invalidate put neo.obo s3://go-build/build-noctua-entity-ontology/latest/'
+		    }
+		}
+	    }
+	}
+	// Produce the standard GO package.
+	stage('Produce GO') {
+	    agent {
+		docker {
+		    image 'obolibrary/odkfull:v1.2.22'
+		    // Reset Jenkins Docker agent default to original
+		    // root.
+		    args '-u root:root'
+		}
+	    }
+	    steps {
+		// Create a relative working directory and setup our
+		// data environment.
+		dir('./go-ontology') {
+		    git 'https://github.com/geneontology/go-ontology.git'
+
+		    // Default namespace.
+		    sh 'env'
+
+		    dir('./src/ontology') {
+			retry(3){
+			    sh 'make RELEASEDATE=$START_DATE OBO=http://purl.obolibrary.org/obo ROBOT_ENV="ROBOT_JAVA_ARGS=-Xmx48G" all'
+			}
+			retry(3){
+			    sh 'make prepare_release'
+			}
+		    }
+
+		    // Make sure that we copy any files there,
+		    // including the core dump of produced.
+		    withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) {
+			//sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" target/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology'
+			sh 'scp -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY -r target/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology/'
+		    }
+
+		    // Now that the files are safely away onto skyhook for
+		    // debugging, test for the core dump.
+		    script {
+			if( WE_ARE_BEING_SAFE_P == 'TRUE' ){
+
+			    def found_core_dump_p = fileExists 'target/core_dump.owl'
+			    if( found_core_dump_p ){
+				error 'ROBOT core dump detected--bailing out.'
+			    }
+			}
+		    }
+		}
+	    }
+	}
+	//..
+	stage('Produce derivatives') {
+            agent {
+                docker {
+		    image 'geneontology/golr-autoindex-ontology:0aeeb57b6e20a4b41d677a8ae934fdf9ecd4b0cd_2019-01-24T124316'
+		    // Reset Jenkins Docker agent default to original
+		    // root.
+		    args '-u root:root --mount type=tmpfs,destination=/srv/solr/data'
+		}
+            }
+            steps {
+		///
+		/// Produce Solr index.
+		///
+
+                // sh 'ls /srv'
+                // sh 'ls /tmp'
+
+		// Build index into tmpfs.
+		sh 'bash /tmp/run-indexer.sh'
+
+		// Copy tmpfs Solr contents onto skyhook.
+		sh 'tar --use-compress-program=pigz -cvf /tmp/golr-index-contents.tgz -C /srv/solr/data/index .'
+		withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) {
+		    // Copy over index.
+		    sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" /tmp/golr-index-contents.tgz skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/products/solr/'
+		    // Copy over log.
+		    sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" /tmp/golr_timestamp.log skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/products/solr/'
+		}
+
+		///
+		/// Produce go-lego (w/NEO) blazegraph.
+		///
+
+		// An awkward download and protective cleanup dance.
+		sh 'rm blazegraph.jnl || true'
+		sh 'curl -L -o /tmp/blazegraph.jar https://github.com/blazegraph/database/releases/download/BLAZEGRAPH_2_1_6_RC/blazegraph.jar'
+		sh 'curl -L -o /tmp/blazegraph.properties https://raw.githubusercontent.com/geneontology/minerva/master/minerva-core/src/main/resources/org/geneontology/minerva/blazegraph.properties'
+		sh 'curl -L -o /tmp/go-lego.owl http://skyhook.berkeleybop.org/$BRANCH_NAME/ontology/extensions/go-lego.owl'
+		// WARNING: Having trouble getting the journal to the
+		// right location. Theoretically, if the pipeline
+		// choked at the wrong time, a hard-to-erase file
+		// could be left on the system. See "rm" above.
+		sh 'java -cp /tmp/blazegraph.jar com.bigdata.rdf.store.DataLoader -defaultGraph http://example.org /tmp/blazegraph.properties /tmp/go-lego.owl'
+		sh 'mv blazegraph.jnl /tmp/blazegraph.jnl'
+		sh 'pigz /tmp/blazegraph.jnl'
+		withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) {
+		    // Copy over journal.
+		    sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" /tmp/blazegraph.jnl.gz skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/products/blazegraph/blazegraph-go-lego.jnl.gz'
+		}
+	    }
+	}
+	//...
+	stage('Sanity 0') {
+	    agent {
+		docker {
+		    image 'obolibrary/odkfull:v1.2.22'
+		    // Reset Jenkins Docker agent default to original
+		    // root.
+		    args '-u root:root'
+		}
+	    }
+	    steps {
+		// Get files back to local.
+		withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) {
+		    sh 'scp -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology/neo.owl /tmp/neo.owl'
+		    sh 'scp -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology/neo.obo /tmp/neo.obo'
+		    sh 'scp -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology/extensions/go-lego.owl /tmp/go-lego.owl'
+		}
+
+		git 'https://github.com/geneontology/go-ontology.git'
+		sh 'ROBOT_JAVA_ARGS=-Xmx48G robot report --input /tmp/go-lego.owl --tdb true --tdb-directory /tmp/tdb/ -k true -p go-ontology/src/sparql/neo/profile.txt -o neo-violations.report.txt --print 50 -v'
+
+		withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) {
+		    sh 'scp -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY neo-violations.report.txt skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/reports/'
+		}
+
+		// // TODO: here. Files available /tmp/go-lego.owl /tmp/neo.owl /tmp/go-lego.obo.
+		// sh 'make RELEASEDATE=$START_DATE OBO=http://purl.obolibrary.org/obo ROBOT_ENV="ROBOT_JAVA_ARGS=-Xmx48G" all'
+	    }
+	}
+	// // WARNING/BUG: This can only occur in its proper location
+	// // once ontology catalogs are produced in the proper
+	// // locations.
+	// // Currently only the NEO files.
+	// stage('Publish') {
+	//     // Get files back to local.
+	//     withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) {
+	// 	sh 'scp -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology/neo.owl neo.owl'
+	// 	sh 'scp -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology/neo.obo neo.obo'
+	//     }
+	//     // Deploy to S3 location for pickup by PURL via CF.
+	//     withCredentials([file(credentialsId: 'aws_go_push_json', variable: 'S3_PUSH_JSON'), file(credentialsId: 's3cmd_go_push_configuration', variable: 'S3CMD_JSON'), string(credentialsId: 'aws_go_access_key', variable: 'AWS_ACCESS_KEY_ID'), string(credentialsId: 'aws_go_secret_key', variable: 'AWS_SECRET_ACCESS_KEY')]) {
+	// 	sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=application/rdf+xml --cf-invalidate put neo.owl s3://go-build/build-noctua-entity-ontology/latest/'
+	// 	sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=application/rdf+xml --cf-invalidate put neo.obo s3://go-build/build-noctua-entity-ontology/latest/'
+	//     }
+	// }
+    }
+    post {
+	// Let's let our people know if things go well.
+	success {
+	    echo "There has been a successful run of the ${env.BRANCH_NAME} pipeline."
+	    mail bcc: '', body: "There has been successful run of the ${env.BRANCH_NAME} pipeline. Please see: https://build.geneontology.org/job/geneontology/job/pipeline/job/${env.BRANCH_NAME}", cc: '', from: '', replyTo: '', subject: "GO Pipeline success for ${env.BRANCH_NAME}", to: "${TARGET_SUCCESS_EMAILS}"
+	}
+	// Let's let our internal people know if things change.
+        changed {
+            echo "There has been a change in the ${env.BRANCH_NAME} pipeline."
+	    mail bcc: '', body: "There has been a pipeline status change in ${env.BRANCH_NAME}. Please see: https://build.geneontology.org/job/geneontology/job/pipeline/job/${env.BRANCH_NAME}", cc: '', from: '', replyTo: '', subject: "GO Pipeline change for ${env.BRANCH_NAME}", to: "${TARGET_ADMIN_EMAILS}"
+	}
+	// Let's let our internal people know if things go badly.
+	failure {
+            echo "There has been a failure in the ${env.BRANCH_NAME} pipeline."
+	    mail bcc: '', body: "There has been a pipeline failure in ${env.BRANCH_NAME}. Please see: https://build.geneontology.org/job/geneontology/job/pipeline/job/${env.BRANCH_NAME}", cc: '', from: '', replyTo: '', subject: "GO Pipeline FAIL for ${env.BRANCH_NAME}", to: "${TARGET_ADMIN_EMAILS}"
+        }
+    }
+}

From b795b327e85ec7c1e8be0954e4157cea7832f2a3 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Fri, 3 Apr 2020 16:01:37 -0700
Subject: [PATCH 02/63] Change some env variables

---
 Jenkinsfile | 38 ++++++--------------------------------
 1 file changed, 6 insertions(+), 32 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 2cafedba..69273abd 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -3,14 +3,8 @@ pipeline {
     // In additional to manual runs, trigger somewhere at midnight to
     // give us the max time in a day to get things right.
     triggers {
-	// Master never runs--Feb 31st.
-	//cron('0 0 31 2 *')
-	// Nightly @12am, for "snapshot", skip "release" night.
-	//cron('0 0 2-31 * *')
-	// First of the month @12am, for "release" (also "current").
-	//cron('0 0 1 * *')
-	// Every week on Friday, noon.
-	cron('0 12 * * 5')
+	// Every two days at 11 pm
+	cron('0      23     */2       *       *')
     }
     environment {
 	// Pin dates and day to beginning of run.
@@ -23,12 +17,12 @@ pipeline {
 	    script: 'date +%A',
 	    returnStdout: true
 	).trim()
-	// The branch of geneontology/go-site to use.
-	TARGET_GO_SITE_BRANCH = 'master'
+	// The branch of to use.
+	TARGET_GO_SITE_BRANCH = 'jenkins'
 	// The people to call when things go bad. It is a comma-space
 	// "separated" string.
-	TARGET_ADMIN_EMAILS = 'sjcarbon@lbl.gov'
-	TARGET_SUCCESS_EMAILS = 'sjcarbon@lbl.gov'
+	TARGET_ADMIN_EMAILS = 'justaddcoffee@gmail.com'
+	TARGET_SUCCESS_EMAILS = 'justaddcoffee@gmail.com'
 	// The file bucket(/folder) combination to use.
 	TARGET_BUCKET = 'no'
 	// The URL prefix to use when creating site indices.
@@ -47,26 +41,6 @@ pipeline {
 	// wok has 48 "processors" over 12 "cores", so I have no idea;
 	// let's go with conservative and see if we get an
 	// improvement.
-	//MAKECMD = 'make --jobs 3 --max-load 10.0'
-	MAKECMD = 'make'
-	// GOlr load profile.
-	GOLR_SOLR_MEMORY = "128G"
-	GOLR_LOADER_MEMORY = "192G"
-	GOLR_INPUT_ONTOLOGIES = [
-	    //"http://purl.obolibrary.org/obo/go/snapshot/extensions/go-lego.owl",
-	    //"http://purl.obolibrary.org/obo/eco.owl",
-	    //"http://purl.obolibrary.org/obo/ncbitaxon/subsets/taxslim.owl",
-	    //"http://purl.obolibrary.org/obo/cl/cl-basic.owl",
-	    //"http://purl.obolibrary.org/obo/go/extensions/gorel.owl",
-	    //"http://purl.obolibrary.org/obo/pato.owl",
-	    //"http://purl.obolibrary.org/obo/po.owl",
-	    //"http://purl.obolibrary.org/obo/chebi.owl",
-	    //"http://purl.obolibrary.org/obo/uberon/basic.owl",
-	    //"http://purl.obolibrary.org/obo/wbbt.owl",
-	    //"http://purl.obolibrary.org/obo/go/extensions/go-modules-annotations.owl",
-	    //"http://purl.obolibrary.org/obo/go/extensions/go-taxon-subsets.owl"
-            //"http://skyhook.berkeleybop.org/issue-35-neo-test/ontology/neo.owl"
-            "http://skyhook.berkeleybop.org/issue-35-neo-test/ontology/extensions/go-lego.owl"
 	].join(" ")
     }
     options{

From 8c7659e463efe0ba097a6046e6d8fb76e84bbda3 Mon Sep 17 00:00:00 2001
From: Seth Carbon <github.com@genkisugi.net>
Date: Fri, 3 Apr 2020 16:15:18 -0700
Subject: [PATCH 03/63] work on #30

---
 Jenkinsfile | 364 +++++-----------------------------------------------
 1 file changed, 32 insertions(+), 332 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 69273abd..f7bd1f32 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,72 +1,17 @@
 pipeline {
     agent any
-    // In additional to manual runs, trigger somewhere at midnight to
-    // give us the max time in a day to get things right.
     triggers {
 	// Every two days at 11 pm
-	cron('0      23     */2       *       *')
-    }
-    environment {
-	// Pin dates and day to beginning of run.
-	START_DATE = sh (
-	    script: 'date +%Y-%m-%d',
-	    returnStdout: true
-	).trim()
-
-	START_DAY = sh (
-	    script: 'date +%A',
-	    returnStdout: true
-	).trim()
-	// The branch of to use.
-	TARGET_GO_SITE_BRANCH = 'jenkins'
-	// The people to call when things go bad. It is a comma-space
-	// "separated" string.
-	TARGET_ADMIN_EMAILS = 'justaddcoffee@gmail.com'
-	TARGET_SUCCESS_EMAILS = 'justaddcoffee@gmail.com'
-	// The file bucket(/folder) combination to use.
-	TARGET_BUCKET = 'no'
-	// The URL prefix to use when creating site indices.
-	TARGET_INDEXER_PREFIX = 'no'
-	// This variable should typically be 'TRUE', which will cause
-	// some additional basic checks to be made. There are some
-	// very exotic cases where these check may need to be skipped
-	// for a run, in that case this variable is set to 'FALSE'.
-	WE_ARE_BEING_SAFE_P = 'TRUE'
-	// The Zenodo concept ID to use for releases (and occasionally
-	// master testing).
-	ZENODO_REFERENCE_CONCEPT = '0'
-	ZENODO_ARCHIVE_CONCEPT = '0'
-	// Control make to get through our loads faster if
-	// possible. Assuming we're cpu bound for some of these...
-	// wok has 48 "processors" over 12 "cores", so I have no idea;
-	// let's go with conservative and see if we get an
-	// improvement.
-	].join(" ")
+	cron('0 23 */2 * *')
     }
     options{
 	timestamps()
     }
     stages {
-	// Very first: pause for a few minutes to give a chance to
+	// Very first: pause for a minute to give a chance to
 	// cancel and clean the workspace before use.
 	stage('Ready and clean') {
 	    steps {
-
-		// Check that we do not affect public targets on
-		// non-mainline runs.
-		script {
-		    if( BRANCH_NAME != 'master' && TARGET_BUCKET == 'go-data-product-experimental'){
-			echo 'Only master can touch that target.'
-			sh '`exit -1`'
-		    }else if( BRANCH_NAME != 'snapshot' && TARGET_BUCKET == 'go-data-product-snapshot'){
-			echo 'Only master can touch that target.'
-			sh '`exit -1`'
-		    }else if( BRANCH_NAME != 'release' && TARGET_BUCKET == 'go-data-product-release'){
-			echo 'Only master can touch that target.'
-			sh '`exit -1`'
-		    }
-		}
-
 		// Give us a minute to cancel if we want.
 		sleep time: 1, unit: 'MINUTES'
 		cleanWs()
@@ -84,286 +29,41 @@ pipeline {
 			sh 'cat branch.txt'
 			sh 'echo $START_DAY > dow.txt'
 			sh 'echo "$START_DAY"'
-		    },
-		    "Reset base": {
-			// Get a mount point ready
-			sh 'mkdir -p $WORKSPACE/mnt || true'
-			// Ninja in our file credentials from Jenkins.
-			withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) {
-			    // Try and ssh fuse skyhook onto our local system.
-			    sh 'sshfs -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY -o idmap=user skyhook@skyhook.berkeleybop.org:/home/skyhook $WORKSPACE/mnt/'
-			}
-			// Remove anything we might have left around from
-			// times past.
-			sh 'rm -r -f $WORKSPACE/mnt/$BRANCH_NAME || true'
-			// Rebuild directory structure.
-			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/bin || true'
-			// WARNING/BUG: needed for arachne to run at
-			// this point.
-			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/lib || true'
-			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products || true'
-			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/ttl || true'
-			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/blazegraph || true'
-			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/annotations || true'
-			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/pages || true'
-			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/solr || true'
-			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/products/panther || true'
-			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/metadata || true'
-			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/annotations || true'
-			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/ontology || true'
-			sh 'mkdir -p $WORKSPACE/mnt/$BRANCH_NAME/reports || true'
-			// Tag the top to let the world know I was at least
-			// here.
-			sh 'echo "Runtime summary." > $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
-			sh 'echo "" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
-			sh 'date >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
-			sh 'echo "" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
-			sh 'echo "Release notes: https://github.com/geneontology/go-site/tree/master/releases" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
-			sh 'echo "Branch: $BRANCH_NAME" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
-			sh 'echo "Start day: $START_DAY" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
-			sh 'echo "Start date: $START_DATE" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
-
-			sh 'echo "Official release date: metadata/release-date.json" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
-			sh 'echo "Official Zenodo archive DOI: metadata/release-archive-doi.json" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
-			sh 'echo "Official Zenodo archive DOI: metadata/release-reference-doi.json" >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
-			sh 'echo "TODO: Note software versions." >> $WORKSPACE/mnt/$BRANCH_NAME/summary.txt'
-			// TODO: This should be wrapped in exception
-			// handling. In fact, this whole thing should be.
-			sh 'fusermount -u $WORKSPACE/mnt/ || true'
-		    }
-		)
-	    }
-	}
-	// Build owltools and get it into the shared filesystem.
-	stage('Ready production software') {
-	    steps {
-		// Legacy: build 'owltools-build'
-		dir('./owltools') {
-		    // Remember that git lays out into CWD.
-		    git 'https://github.com/owlcollab/owltools.git'
-		    sh 'mvn -f OWLTools-Parent/pom.xml -U clean install -DskipTests -Dmaven.javadoc.skip=true -Dsource.skip=true'
-		    // Attempt to rsync produced into bin/.
-		    withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) {
-			sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" OWLTools-Runner/target/owltools skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/'
-			sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" OWLTools-Oort/bin/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/'
-			sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" OWLTools-NCBI/bin/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/'
-			sh 'rsync -vhac -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" --exclude ".git" OWLTools-Oort/reporting/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/'
-			sh 'rsync -vhac -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" --exclude ".git" OWLTools-Runner/contrib/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/'
-		    }
-		}
+		    })
 	    }
 	}
-	// See https://github.com/geneontology/go-ontology for details
-	// on the ontology release pipeline. This ticket runs
-	// daily(TODO?) and creates all the files normally included in
-	// a release, and deploys to S3.
-	stage('Produce NEO') {
-	    steps {
-		// Create a relative working directory and setup our
-		// data environment.
-		dir('./neo') {
-		    git 'https://github.com/geneontology/neo.git'
-
-		    // Default namespace.
-		    sh 'OBO=http://purl.obolibrary.org/obo'
-
-		    // Make all software products available in bin/.
-		    sh 'mkdir -p bin/'
-		    withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) {
-			sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/bin/* ./bin/'
-		    }
-		    sh 'chmod +x bin/*'
 
-		    //
-		    withEnv(['PATH+EXTRA=:bin:./bin', 'JAVA_OPTS=-Xmx128G', 'OWLTOOLS_MEMORY=128G', 'BGMEM=128G']){
-			retry(3){
-			    sh 'make clean all'
-			}
-		    }
-
-		    // Make sure that we copy any files there,
-		    // including the core dump of produced.
-		    withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) {
-			sh 'scp -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY neo.obo neo.owl skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology/'
-		    }
-
-		    // WARNING/BUG: This occurs "early" as we need NEO
-		    // in the proper location before building GO (for
-		    // testing, solr loading, etc.). Once we have
-		    // ubiquitous ontology catalogs, publishing can
-		    // occur properly at the end and after testing.
-		    // See commented out section below.
-		    //
-		    // Deploy to S3 location for pickup by PURL via CF.
-		    withCredentials([file(credentialsId: 'aws_go_push_json', variable: 'S3_PUSH_JSON'), file(credentialsId: 's3cmd_go_push_configuration', variable: 'S3CMD_JSON'), string(credentialsId: 'aws_go_access_key', variable: 'AWS_ACCESS_KEY_ID'), string(credentialsId: 'aws_go_secret_key', variable: 'AWS_SECRET_ACCESS_KEY')]) {
-			sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=application/rdf+xml --cf-invalidate put neo.owl s3://go-build/build-noctua-entity-ontology/latest/'
-			sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=application/rdf+xml --cf-invalidate put neo.obo s3://go-build/build-noctua-entity-ontology/latest/'
-		    }
-		}
-	    }
-	}
-	// Produce the standard GO package.
-	stage('Produce GO') {
-	    agent {
-		docker {
-		    image 'obolibrary/odkfull:v1.2.22'
-		    // Reset Jenkins Docker agent default to original
-		    // root.
-		    args '-u root:root'
-		}
-	    }
-	    steps {
-		// Create a relative working directory and setup our
-		// data environment.
-		dir('./go-ontology') {
-		    git 'https://github.com/geneontology/go-ontology.git'
-
-		    // Default namespace.
-		    sh 'env'
-
-		    dir('./src/ontology') {
-			retry(3){
-			    sh 'make RELEASEDATE=$START_DATE OBO=http://purl.obolibrary.org/obo ROBOT_ENV="ROBOT_JAVA_ARGS=-Xmx48G" all'
-			}
-			retry(3){
-			    sh 'make prepare_release'
-			}
-		    }
-
-		    // Make sure that we copy any files there,
-		    // including the core dump of produced.
-		    withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) {
-			//sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" target/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology'
-			sh 'scp -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY -r target/* skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology/'
-		    }
-
-		    // Now that the files are safely away onto skyhook for
-		    // debugging, test for the core dump.
-		    script {
-			if( WE_ARE_BEING_SAFE_P == 'TRUE' ){
-
-			    def found_core_dump_p = fileExists 'target/core_dump.owl'
-			    if( found_core_dump_p ){
-				error 'ROBOT core dump detected--bailing out.'
-			    }
-			}
-		    }
-		}
-	    }
-	}
-	//..
-	stage('Produce derivatives') {
-            agent {
-                docker {
-		    image 'geneontology/golr-autoindex-ontology:0aeeb57b6e20a4b41d677a8ae934fdf9ecd4b0cd_2019-01-24T124316'
-		    // Reset Jenkins Docker agent default to original
-		    // root.
-		    args '-u root:root --mount type=tmpfs,destination=/srv/solr/data'
-		}
+        stage('Build kg_covid_19') {
+            steps {
+                dir('./config') {
+                    git(
+                        url: 'https://github.com/Knowledge-Graph-Hub/kg-covid-19',
+                        branch: 'master'
+                    )
+		    // TODO: Also consider Docker image?
+                    echo "# venv and pip stuff"
+                }
             }
+        }
+        stage('Download') {
             steps {
-		///
-		/// Produce Solr index.
-		///
-
-                // sh 'ls /srv'
-                // sh 'ls /tmp'
-
-		// Build index into tmpfs.
-		sh 'bash /tmp/run-indexer.sh'
-
-		// Copy tmpfs Solr contents onto skyhook.
-		sh 'tar --use-compress-program=pigz -cvf /tmp/golr-index-contents.tgz -C /srv/solr/data/index .'
-		withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) {
-		    // Copy over index.
-		    sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" /tmp/golr-index-contents.tgz skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/products/solr/'
-		    // Copy over log.
-		    sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" /tmp/golr_timestamp.log skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/products/solr/'
-		}
-
-		///
-		/// Produce go-lego (w/NEO) blazegraph.
-		///
-
-		// An awkward download and protective cleanup dance.
-		sh 'rm blazegraph.jnl || true'
-		sh 'curl -L -o /tmp/blazegraph.jar https://github.com/blazegraph/database/releases/download/BLAZEGRAPH_2_1_6_RC/blazegraph.jar'
-		sh 'curl -L -o /tmp/blazegraph.properties https://raw.githubusercontent.com/geneontology/minerva/master/minerva-core/src/main/resources/org/geneontology/minerva/blazegraph.properties'
-		sh 'curl -L -o /tmp/go-lego.owl http://skyhook.berkeleybop.org/$BRANCH_NAME/ontology/extensions/go-lego.owl'
-		// WARNING: Having trouble getting the journal to the
-		// right location. Theoretically, if the pipeline
-		// choked at the wrong time, a hard-to-erase file
-		// could be left on the system. See "rm" above.
-		sh 'java -cp /tmp/blazegraph.jar com.bigdata.rdf.store.DataLoader -defaultGraph http://example.org /tmp/blazegraph.properties /tmp/go-lego.owl'
-		sh 'mv blazegraph.jnl /tmp/blazegraph.jnl'
-		sh 'pigz /tmp/blazegraph.jnl'
-		withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) {
-		    // Copy over journal.
-		    sh 'rsync -avz -e "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY" /tmp/blazegraph.jnl.gz skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/products/blazegraph/blazegraph-go-lego.jnl.gz'
-		}
-	    }
-	}
-	//...
-	stage('Sanity 0') {
-	    agent {
-		docker {
-		    image 'obolibrary/odkfull:v1.2.22'
-		    // Reset Jenkins Docker agent default to original
-		    // root.
-		    args '-u root:root'
-		}
-	    }
-	    steps {
-		// Get files back to local.
-		withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) {
-		    sh 'scp -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology/neo.owl /tmp/neo.owl'
-		    sh 'scp -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology/neo.obo /tmp/neo.obo'
-		    sh 'scp -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology/extensions/go-lego.owl /tmp/go-lego.owl'
-		}
-
-		git 'https://github.com/geneontology/go-ontology.git'
-		sh 'ROBOT_JAVA_ARGS=-Xmx48G robot report --input /tmp/go-lego.owl --tdb true --tdb-directory /tmp/tdb/ -k true -p go-ontology/src/sparql/neo/profile.txt -o neo-violations.report.txt --print 50 -v'
-
-		withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) {
-		    sh 'scp -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY neo-violations.report.txt skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/reports/'
-		}
-
-		// // TODO: here. Files available /tmp/go-lego.owl /tmp/neo.owl /tmp/go-lego.obo.
-		// sh 'make RELEASEDATE=$START_DATE OBO=http://purl.obolibrary.org/obo ROBOT_ENV="ROBOT_JAVA_ARGS=-Xmx48G" all'
-	    }
-	}
-	// // WARNING/BUG: This can only occur in its proper location
-	// // once ontology catalogs are produced in the proper
-	// // locations.
-	// // Currently only the NEO files.
-	// stage('Publish') {
-	//     // Get files back to local.
-	//     withCredentials([file(credentialsId: 'skyhook-private-key', variable: 'SKYHOOK_IDENTITY')]) {
-	// 	sh 'scp -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology/neo.owl neo.owl'
-	// 	sh 'scp -o StrictHostKeyChecking=no -o IdentitiesOnly=true -o IdentityFile=$SKYHOOK_IDENTITY skyhook@skyhook.berkeleybop.org:/home/skyhook/$BRANCH_NAME/ontology/neo.obo neo.obo'
-	//     }
-	//     // Deploy to S3 location for pickup by PURL via CF.
-	//     withCredentials([file(credentialsId: 'aws_go_push_json', variable: 'S3_PUSH_JSON'), file(credentialsId: 's3cmd_go_push_configuration', variable: 'S3CMD_JSON'), string(credentialsId: 'aws_go_access_key', variable: 'AWS_ACCESS_KEY_ID'), string(credentialsId: 'aws_go_secret_key', variable: 'AWS_SECRET_ACCESS_KEY')]) {
-	// 	sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=application/rdf+xml --cf-invalidate put neo.owl s3://go-build/build-noctua-entity-ontology/latest/'
-	// 	sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=application/rdf+xml --cf-invalidate put neo.obo s3://go-build/build-noctua-entity-ontology/latest/'
-	//     }
-	// }
-    }
-    post {
-	// Let's let our people know if things go well.
-	success {
-	    echo "There has been a successful run of the ${env.BRANCH_NAME} pipeline."
-	    mail bcc: '', body: "There has been successful run of the ${env.BRANCH_NAME} pipeline. Please see: https://build.geneontology.org/job/geneontology/job/pipeline/job/${env.BRANCH_NAME}", cc: '', from: '', replyTo: '', subject: "GO Pipeline success for ${env.BRANCH_NAME}", to: "${TARGET_SUCCESS_EMAILS}"
-	}
-	// Let's let our internal people know if things change.
-        changed {
-            echo "There has been a change in the ${env.BRANCH_NAME} pipeline."
-	    mail bcc: '', body: "There has been a pipeline status change in ${env.BRANCH_NAME}. Please see: https://build.geneontology.org/job/geneontology/job/pipeline/job/${env.BRANCH_NAME}", cc: '', from: '', replyTo: '', subject: "GO Pipeline change for ${env.BRANCH_NAME}", to: "${TARGET_ADMIN_EMAILS}"
-	}
-	// Let's let our internal people know if things go badly.
-	failure {
-            echo "There has been a failure in the ${env.BRANCH_NAME} pipeline."
-	    mail bcc: '', body: "There has been a pipeline failure in ${env.BRANCH_NAME}. Please see: https://build.geneontology.org/job/geneontology/job/pipeline/job/${env.BRANCH_NAME}", cc: '', from: '', replyTo: '', subject: "GO Pipeline FAIL for ${env.BRANCH_NAME}", to: "${TARGET_ADMIN_EMAILS}"
+                  python run.py download
+            }
+        }
+        stage('Transform') {
+            steps {
+                echo "python run.py transform"
+            }
         }
+        stage('Load') {
+            steps {
+                echo "python run.py load"
+            }
+        }
+        stage('Push to s3 bucket') {
+            steps {
+                echo "???? # Need some details for this item"
+            }
+	}
     }
 }

From 2b7bc9b52ca690949c95df6bbb34ec57fa55d2e9 Mon Sep 17 00:00:00 2001
From: kltm <github.com@genkisugi.net>
Date: Fri, 3 Apr 2020 17:11:21 -0700
Subject: [PATCH 04/63] Update Jenkinsfile

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f7bd1f32..f7ffd321 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -47,7 +47,7 @@ pipeline {
         }
         stage('Download') {
             steps {
-                  python run.py download
+                  echo "python run.py download"
             }
         }
         stage('Transform') {

From ab3de8557da3b76f83e9cc4b829c9dcfa037eb11 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 10:47:48 -0700
Subject: [PATCH 05/63] Update Jenkins with install commands

---
 Jenkinsfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f7ffd321..83e68936 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -40,8 +40,9 @@ pipeline {
                         url: 'https://github.com/Knowledge-Graph-Hub/kg-covid-19',
                         branch: 'master'
                     )
-		    // TODO: Also consider Docker image?
-                    echo "# venv and pip stuff"
+			sh 'pip install bmt'
+			sh 'pip install -r requirements.txt'
+			sh 'python setup.py install'
                 }
             }
         }

From 7f50f7293dba615ad1e940c793440976b9aca089 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 10:48:49 -0700
Subject: [PATCH 06/63] Add cd command

---
 Jenkinsfile | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 83e68936..83add119 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -40,9 +40,10 @@ pipeline {
                         url: 'https://github.com/Knowledge-Graph-Hub/kg-covid-19',
                         branch: 'master'
                     )
-			sh 'pip install bmt'
-			sh 'pip install -r requirements.txt'
-			sh 'python setup.py install'
+		    sh 'cd kg_covid_19'
+		    sh 'pip install bmt'
+		    sh 'pip install -r requirements.txt'
+		    sh 'python setup.py install'
                 }
             }
         }

From d4d6305eddf7c3bb9c0f68d87879a39deb085f1e Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 11:16:47 -0700
Subject: [PATCH 07/63] Add venv

---
 Jenkinsfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 83add119..2f24cc9f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -41,6 +41,8 @@ pipeline {
                         branch: 'master'
                     )
 		    sh 'cd kg_covid_19'
+		    sh 'virtualenv venv --distribute'
+		    sh 'source venv/bin/activate'
 		    sh 'pip install bmt'
 		    sh 'pip install -r requirements.txt'
 		    sh 'python setup.py install'

From bc90e3105bc69d170a2b1ba7f175edde6f4fa1e9 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 11:19:43 -0700
Subject: [PATCH 08/63] Activate venv

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 2f24cc9f..1573063e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -42,7 +42,7 @@ pipeline {
                     )
 		    sh 'cd kg_covid_19'
 		    sh 'virtualenv venv --distribute'
-		    sh 'source venv/bin/activate'
+		    sh '. venv/bin/activate'
 		    sh 'pip install bmt'
 		    sh 'pip install -r requirements.txt'
 		    sh 'python setup.py install'

From 5c16d100d146c17a8becc8d07e1cdbb2155368fe Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 11:27:36 -0700
Subject: [PATCH 09/63] Update Jenkinsfile

---
 Jenkinsfile | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 1573063e..80e6ec88 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -43,25 +43,25 @@ pipeline {
 		    sh 'cd kg_covid_19'
 		    sh 'virtualenv venv --distribute'
 		    sh '. venv/bin/activate'
-		    sh 'pip install bmt'
-		    sh 'pip install -r requirements.txt'
-		    sh 'python setup.py install'
+		    sh './venv/bin/pip install bmt'
+		    sh './venv/bin/pip  install -r requirements.txt'
+		    sh './venv/bin/python setup.py install'
                 }
             }
         }
         stage('Download') {
             steps {
-                  echo "python run.py download"
+                  echo "./venv/bin/python run.py download"
             }
         }
         stage('Transform') {
             steps {
-                echo "python run.py transform"
+                echo "./venv/bin/python run.py transform"
             }
         }
         stage('Load') {
             steps {
-                echo "python run.py load"
+                echo "./venv/bin/python run.py load"
             }
         }
         stage('Push to s3 bucket') {

From 33ab9efc968ea21e4c1e1ca6632959304b12c43f Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 11:33:12 -0700
Subject: [PATCH 10/63] Fix python version in venv

---
 Jenkinsfile | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 80e6ec88..ca0b843b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -13,7 +13,7 @@ pipeline {
 	stage('Ready and clean') {
 	    steps {
 		// Give us a minute to cancel if we want.
-		sleep time: 1, unit: 'MINUTES'
+		// sleep time: 1, unit: 'MINUTES'
 		cleanWs()
 	    }
 	}
@@ -40,12 +40,14 @@ pipeline {
                         url: 'https://github.com/Knowledge-Graph-Hub/kg-covid-19',
                         branch: 'master'
                     )
-		    sh 'cd kg_covid_19'
-		    sh 'virtualenv venv --distribute'
-		    sh '. venv/bin/activate'
-		    sh './venv/bin/pip install bmt'
-		    sh './venv/bin/pip  install -r requirements.txt'
-		    sh './venv/bin/python setup.py install'
+                    sh '''
+		        cd kg_covid_19
+		        virtualenv -p python3.7 venv --distribute
+		        . venv/bin/activate
+		        ./venv/bin/pip install bmt
+		        ./venv/bin/pip install -r requirements.txt
+		        ./venv/bin/python setup.py install
+		    '''
                 }
             }
         }

From cdc18e4252157d8cabc87ab2517aee161270066d Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 11:34:40 -0700
Subject: [PATCH 11/63] Guess at where python3.7 lives

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index ca0b843b..6a003534 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -42,7 +42,7 @@ pipeline {
                     )
                     sh '''
 		        cd kg_covid_19
-		        virtualenv -p python3.7 venv --distribute
+		        virtualenv -p /usr/bin/python3.7 venv --distribute
 		        . venv/bin/activate
 		        ./venv/bin/pip install bmt
 		        ./venv/bin/pip install -r requirements.txt

From a4160bc5966250a3c716acd4e49316178be3054a Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 11:37:29 -0700
Subject: [PATCH 12/63] Okay, see if python3 works (we need 3.7 though)

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 6a003534..7c5d49f7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -42,7 +42,7 @@ pipeline {
                     )
                     sh '''
 		        cd kg_covid_19
-		        virtualenv -p /usr/bin/python3.7 venv --distribute
+		        virtualenv -p python3 venv --distribute
 		        . venv/bin/activate
 		        ./venv/bin/pip install bmt
 		        ./venv/bin/pip install -r requirements.txt

From 9545132b491ace68ae19492810b08de98304660f Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 11:39:54 -0700
Subject: [PATCH 13/63] Leave out bmt during install

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7c5d49f7..d1ecf792 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -44,7 +44,7 @@ pipeline {
 		        cd kg_covid_19
 		        virtualenv -p python3 venv --distribute
 		        . venv/bin/activate
-		        ./venv/bin/pip install bmt
+		        # ./venv/bin/pip install bmt
 		        ./venv/bin/pip install -r requirements.txt
 		        ./venv/bin/python setup.py install
 		    '''

From de439619ce9feb84cf5fba09c00bbb31133ad18f Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 11:48:05 -0700
Subject: [PATCH 14/63] List directory contents

---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index d1ecf792..f215b8c2 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -45,6 +45,7 @@ pipeline {
 		        virtualenv -p python3 venv --distribute
 		        . venv/bin/activate
 		        # ./venv/bin/pip install bmt
+			ls .
 		        ./venv/bin/pip install -r requirements.txt
 		        ./venv/bin/python setup.py install
 		    '''

From 66600abc5e6a0485061c4d16af01eeab6c71f923 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 11:50:32 -0700
Subject: [PATCH 15/63] Possibly in wrong directory

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f215b8c2..1c0711b0 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -41,7 +41,7 @@ pipeline {
                         branch: 'master'
                     )
                     sh '''
-		        cd kg_covid_19
+		        # cd kg_covid_19
 		        virtualenv -p python3 venv --distribute
 		        . venv/bin/activate
 		        # ./venv/bin/pip install bmt

From 57b7df8a0f8b8ea16636343ab6d7e1027e56d288 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 11:53:25 -0700
Subject: [PATCH 16/63] Python version issues

---
 Jenkinsfile | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 1c0711b0..215245da 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -40,15 +40,11 @@ pipeline {
                         url: 'https://github.com/Knowledge-Graph-Hub/kg-covid-19',
                         branch: 'master'
                     )
-                    sh '''
-		        # cd kg_covid_19
-		        virtualenv -p python3 venv --distribute
-		        . venv/bin/activate
-		        # ./venv/bin/pip install bmt
-			ls .
-		        ./venv/bin/pip install -r requirements.txt
-		        ./venv/bin/python setup.py install
-		    '''
+		    sh 'virtualenv -p python3.5 venv --distribute'
+		    sh '. venv/bin/activate'
+		    sh '# ./venv/bin/pip install bmt'
+		    sh './venv/bin/pip install -r requirements.txt'
+		    sh './venv/bin/python setup.py install'
                 }
             }
         }

From 1a3b2ff0dfcf3de3ebfde99f8634b37f49c055f8 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 11:56:32 -0700
Subject: [PATCH 17/63] Change KGX github repo from Luca's to the main one

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 2a4100c5..2c7e277f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1 @@
-git+git://github.com/LucaCappelletti94/kgx.git
+git+git://github.com/NCATS-Tangerine/kgx.git

From 1b2bcf3f374f9182ab0a1f851f8aee2f8c48e6a4 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 12:11:30 -0700
Subject: [PATCH 18/63] venv

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 215245da..8c95c0a6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -40,7 +40,7 @@ pipeline {
                         url: 'https://github.com/Knowledge-Graph-Hub/kg-covid-19',
                         branch: 'master'
                     )
-		    sh 'virtualenv -p python3.5 venv --distribute'
+		    sh 'python3 -m venv venv'
 		    sh '. venv/bin/activate'
 		    sh '# ./venv/bin/pip install bmt'
 		    sh './venv/bin/pip install -r requirements.txt'

From c72bc04680672c9c0cc7992fcbaa212ead16f5c5 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 12:15:00 -0700
Subject: [PATCH 19/63] BMT

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 8c95c0a6..7aea2adf 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -42,7 +42,7 @@ pipeline {
                     )
 		    sh 'python3 -m venv venv'
 		    sh '. venv/bin/activate'
-		    sh '# ./venv/bin/pip install bmt'
+		    sh './venv/bin/pip install bmt'
 		    sh './venv/bin/pip install -r requirements.txt'
 		    sh './venv/bin/python setup.py install'
                 }

From e6c147b86018f77f630b5134bfce0b325bd80f83 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 12:19:56 -0700
Subject: [PATCH 20/63] python3.7

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7aea2adf..521a71de 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -40,7 +40,7 @@ pipeline {
                         url: 'https://github.com/Knowledge-Graph-Hub/kg-covid-19',
                         branch: 'master'
                     )
-		    sh 'python3 -m venv venv'
+		    sh 'python3.7 -m venv venv'
 		    sh '. venv/bin/activate'
 		    sh './venv/bin/pip install bmt'
 		    sh './venv/bin/pip install -r requirements.txt'

From 1fe20839edd0186f2e257a02001c18de8b3808fa Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 12:21:47 -0700
Subject: [PATCH 21/63] echo $PATH

---
 Jenkinsfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 521a71de..063ba6f0 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -40,8 +40,9 @@ pipeline {
                         url: 'https://github.com/Knowledge-Graph-Hub/kg-covid-19',
                         branch: 'master'
                     )
-		    sh 'python3.7 -m venv venv'
-		    sh '. venv/bin/activate'
+		    sh 'echo $PATH'
+		    sh '/usr/bin/python3.7 -m venv venv'
+		    sh './venv/bin/activate'
 		    sh './venv/bin/pip install bmt'
 		    sh './venv/bin/pip install -r requirements.txt'
 		    sh './venv/bin/python setup.py install'

From e9d12ebe6ae49051dbc4e1581de3eb4f31d52bf5 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 12:24:12 -0700
Subject: [PATCH 22/63] ?

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 063ba6f0..7c9edf51 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -42,7 +42,7 @@ pipeline {
                     )
 		    sh 'echo $PATH'
 		    sh '/usr/bin/python3.7 -m venv venv'
-		    sh './venv/bin/activate'
+		    sh '. venv/bin/activate'
 		    sh './venv/bin/pip install bmt'
 		    sh './venv/bin/pip install -r requirements.txt'
 		    sh './venv/bin/python setup.py install'

From 6a68dd63257e52a945a1cf452afb13b1fd7cc5c2 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 12:27:17 -0700
Subject: [PATCH 23/63] uncomment run.py commands

---
 Jenkinsfile | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7c9edf51..a9c2c3b8 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -40,7 +40,6 @@ pipeline {
                         url: 'https://github.com/Knowledge-Graph-Hub/kg-covid-19',
                         branch: 'master'
                     )
-		    sh 'echo $PATH'
 		    sh '/usr/bin/python3.7 -m venv venv'
 		    sh '. venv/bin/activate'
 		    sh './venv/bin/pip install bmt'
@@ -51,17 +50,17 @@ pipeline {
         }
         stage('Download') {
             steps {
-                  echo "./venv/bin/python run.py download"
+                  sh './venv/bin/python run.py download'
             }
         }
         stage('Transform') {
             steps {
-                echo "./venv/bin/python run.py transform"
+                sh './venv/bin/python run.py transform'
             }
         }
         stage('Load') {
             steps {
-                echo "./venv/bin/python run.py load"
+                sh './venv/bin/python run.py load'
             }
         }
         stage('Push to s3 bucket') {

From 11d7f9d27133e5439bd00bdd9c489f9b304001e3 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 12:30:12 -0700
Subject: [PATCH 24/63] Fix calls to python

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index a9c2c3b8..5b83e28b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -50,17 +50,17 @@ pipeline {
         }
         stage('Download') {
             steps {
-                  sh './venv/bin/python run.py download'
+                  sh 'python run.py download'
             }
         }
         stage('Transform') {
             steps {
-                sh './venv/bin/python run.py transform'
+                 sh 'python run.py transform'
             }
         }
         stage('Load') {
             steps {
-                sh './venv/bin/python run.py load'
+                 sh 'python run.py load'
             }
         }
         stage('Push to s3 bucket') {

From a53430749918cc888a19d76bea9da8f150aedf65 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 12:33:36 -0700
Subject: [PATCH 25/63] cd

---
 Jenkinsfile | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5b83e28b..aead0e55 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -50,17 +50,20 @@ pipeline {
         }
         stage('Download') {
             steps {
+		  sh 'cd kg-covid-19'
                   sh 'python run.py download'
             }
         }
         stage('Transform') {
             steps {
-                 sh 'python run.py transform'
+		  sh 'cd kg-covid-19'		  
+                  sh 'python run.py transform'
             }
         }
         stage('Load') {
             steps {
-                 sh 'python run.py load'
+		  sh 'cd kg-covid-19'
+		  sh 'python run.py load'
             }
         }
         stage('Push to s3 bucket') {

From 004d7a8633880700addb664f75c4ec2f2e7ea645 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 12:35:53 -0700
Subject: [PATCH 26/63] directory weirdness

---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index aead0e55..dec3071f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -50,6 +50,7 @@ pipeline {
         }
         stage('Download') {
             steps {
+		  sh 'pwd'
 		  sh 'cd kg-covid-19'
                   sh 'python run.py download'
             }

From f68756d13c023d972b66c3c92ac9d04489f562db Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 12:38:21 -0700
Subject: [PATCH 27/63] Update Jenkinsfile

---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index dec3071f..1cf7f446 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -51,6 +51,7 @@ pipeline {
         stage('Download') {
             steps {
 		  sh 'pwd'
+		  sh 'ls'
 		  sh 'cd kg-covid-19'
                   sh 'python run.py download'
             }

From a99c4ca4fc924165112df15e671de1e858734bbe Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 12:46:53 -0700
Subject: [PATCH 28/63] Update Jenkinsfile

---
 Jenkinsfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 1cf7f446..ad19e1cf 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -38,7 +38,7 @@ pipeline {
                 dir('./config') {
                     git(
                         url: 'https://github.com/Knowledge-Graph-Hub/kg-covid-19',
-                        branch: 'master'
+                        branch: 'jenkins'
                     )
 		    sh '/usr/bin/python3.7 -m venv venv'
 		    sh '. venv/bin/activate'
@@ -50,9 +50,10 @@ pipeline {
         }
         stage('Download') {
             steps {
+		  sh 'echo "${env.WORKSPACE}"'
 		  sh 'pwd'
 		  sh 'ls'
-		  sh 'cd kg-covid-19'
+		  sh 'cd config'
                   sh 'python run.py download'
             }
         }

From d70fca463e4fc72f75e79e2b98ecfea9c37e045c Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 12:49:25 -0700
Subject: [PATCH 29/63] Update Jenkinsfile

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index ad19e1cf..921eaef3 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -50,7 +50,7 @@ pipeline {
         }
         stage('Download') {
             steps {
-		  sh 'echo "${env.WORKSPACE}"'
+		  sh '# echo "${env.WORKSPACE}"'
 		  sh 'pwd'
 		  sh 'ls'
 		  sh 'cd config'

From 1161af04756e23c4d87aebd42b27817a471de449 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 12:51:37 -0700
Subject: [PATCH 30/63] Update Jenkinsfile

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 921eaef3..7e5436cb 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -52,7 +52,7 @@ pipeline {
             steps {
 		  sh '# echo "${env.WORKSPACE}"'
 		  sh 'pwd'
-		  sh 'ls'
+		  sh 'ls; find ../ -name "run.py"'
 		  sh 'cd config'
                   sh 'python run.py download'
             }

From 8bee150865b387f4a9e0316527683ad3757b7bba Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 12:55:50 -0700
Subject: [PATCH 31/63] Update Jenkinsfile

---
 Jenkinsfile | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7e5436cb..f6398b6e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -50,11 +50,9 @@ pipeline {
         }
         stage('Download') {
             steps {
-		  sh '# echo "${env.WORKSPACE}"'
 		  sh 'pwd'
 		  sh 'ls; find ../ -name "run.py"'
-		  sh 'cd config'
-                  sh 'python run.py download'
+                  sh 'cd config; python run.py download'
             }
         }
         stage('Transform') {

From 94fb6a6682f0d12c4ae118e7ea9903651b7690b3 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 12:59:07 -0700
Subject: [PATCH 32/63] Update Jenkinsfile

---
 Jenkinsfile | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f6398b6e..71a62029 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -50,21 +50,17 @@ pipeline {
         }
         stage('Download') {
             steps {
-		  sh 'pwd'
-		  sh 'ls; find ../ -name "run.py"'
-                  sh 'cd config; python run.py download'
+                  sh 'cd config; python3.7 run.py download'
             }
         }
         stage('Transform') {
             steps {
-		  sh 'cd kg-covid-19'		  
-                  sh 'python run.py transform'
+                  sh 'cd config; python run.py transform'
             }
         }
         stage('Load') {
             steps {
-		  sh 'cd kg-covid-19'
-		  sh 'python run.py load'
+		  sh 'cd config; python run.py load'
             }
         }
         stage('Push to s3 bucket') {

From 8d897a400887cbc5a7d3b63bfa79c613a3e0b143 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 13:06:25 -0700
Subject: [PATCH 33/63] Update Jenkinsfile

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 71a62029..505828b8 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -50,7 +50,7 @@ pipeline {
         }
         stage('Download') {
             steps {
-                  sh 'cd config; python3.7 run.py download'
+                  sh '. venv/bin/activate; cd config; python3.7 run.py download'
             }
         }
         stage('Transform') {

From 8a74b8e464f2164c0feb88144eb079527c4f2647 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 13:11:47 -0700
Subject: [PATCH 34/63] Update Jenkinsfile

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 505828b8..2f8a76e6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -50,7 +50,7 @@ pipeline {
         }
         stage('Download') {
             steps {
-                  sh '. venv/bin/activate; cd config; python3.7 run.py download'
+                  sh 'cd config;. venv/bin/activate; python3.7 run.py download'
             }
         }
         stage('Transform') {

From 4bf9a889e6446da303d9d0ccfc3ec162eb5fb1cc Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 13:14:50 -0700
Subject: [PATCH 35/63] Update Jenkinsfile

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 2f8a76e6..bd5028fa 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -55,12 +55,12 @@ pipeline {
         }
         stage('Transform') {
             steps {
-                  sh 'cd config; python run.py transform'
+                  sh 'cd config;. venv/bin/activate; python3.7 run.py transform'
             }
         }
         stage('Load') {
             steps {
-		  sh 'cd config; python run.py load'
+		  sh 'cd config;. venv/bin/activate; python3.7 run.py load'
             }
         }
         stage('Push to s3 bucket') {

From d2c30ad0e0f6e4b1acc47379a8912f98c420fafa Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 13:45:05 -0700
Subject: [PATCH 36/63] Point Jenkins to master branch

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index bd5028fa..6375a34c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -38,7 +38,7 @@ pipeline {
                 dir('./config') {
                     git(
                         url: 'https://github.com/Knowledge-Graph-Hub/kg-covid-19',
-                        branch: 'jenkins'
+                        branch: 'master'
                     )
 		    sh '/usr/bin/python3.7 -m venv venv'
 		    sh '. venv/bin/activate'

From 1b128623b0f6b94ffaebaa594a78d61f7b295bfe Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 19:43:50 -0700
Subject: [PATCH 37/63] Add Publish stage (trivial for now)

---
 Jenkinsfile | 74 +++++++++++++++++++++++++++++++++--------------------
 1 file changed, 46 insertions(+), 28 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 6375a34c..89d1b6d6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -8,30 +8,30 @@ pipeline {
 	timestamps()
     }
     stages {
-	// Very first: pause for a minute to give a chance to
-	// cancel and clean the workspace before use.
-	stage('Ready and clean') {
-	    steps {
-		// Give us a minute to cancel if we want.
-		// sleep time: 1, unit: 'MINUTES'
-		cleanWs()
-	    }
-	}
-	stage('Initialize') {
-	    steps {
-		// Start preparing environment.
-		parallel(
-		    "Report": {
-			sh 'env > env.txt'
-			sh 'echo $BRANCH_NAME > branch.txt'
-			sh 'echo "$BRANCH_NAME"'
-			sh 'cat env.txt'
-			sh 'cat branch.txt'
-			sh 'echo $START_DAY > dow.txt'
-			sh 'echo "$START_DAY"'
-		    })
-	    }
-	}
+        // Very first: pause for a minute to give a chance to
+        // cancel and clean the workspace before use.
+        stage('Ready and clean') {
+            steps {
+            // Give us a minute to cancel if we want.
+            sleep time: 1, unit: 'MINUTES'
+            cleanWs()
+            }
+        }
+        stage('Initialize') {
+            steps {
+            // Start preparing environment.
+            parallel(
+                "Report": {
+                sh 'env > env.txt'
+                sh 'echo $BRANCH_NAME > branch.txt'
+                sh 'echo "$BRANCH_NAME"'
+                sh 'cat env.txt'
+                sh 'cat branch.txt'
+                sh 'echo $START_DAY > dow.txt'
+                sh 'echo "$START_DAY"'
+                })
+            }
+        }
 
         stage('Build kg_covid_19') {
             steps {
@@ -60,13 +60,31 @@ pipeline {
         }
         stage('Load') {
             steps {
-		  sh 'cd config;. venv/bin/activate; python3.7 run.py load'
+		          sh 'cd config;. venv/bin/activate; python3.7 run.py load'
             }
         }
-        stage('Push to s3 bucket') {
+        stage('Publish') {
             steps {
-                echo "???? # Need some details for this item"
+
+            script {
+                if( env.BRANCH_NAME != 'master' ){
+                   echo "Will not push if not on correct branch."
+                } else{
+                // Get trivial product back in hand.
+                sh 'echo thing > dow.txt'
+
+                // Push out to your S3 bucket.  The given
+                // command is for small to medium files. If
+                // you need something appropriate for large
+                // files, let me know.
+                withCredentials([file(credentialsId: 's3cmd_idg_push_configuration', variable: 'S3CMD_JSON')]) {
+                    sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put dow.txt s3://idg-public-data/dow.txt'
+                    // Should now appear at:
+                    // https://idg.berkeleybop.io/dow.txt
+                    // http://idg.berkeleybop.io/dow.txt
+                }
+
             }
-	}
+		}
     }
 }

From d655d6e67caf73c5d9528da396f7ec3841ae13bd Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 9 Apr 2020 18:55:45 -0700
Subject: [PATCH 38/63] Update URL for Scibite download to last known working
 version, at BBOP

(cherry picked from commit f8be062be81a6abe983bbf516d3f51922be8e80f)
---
 download.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/download.yaml b/download.yaml
index 89ff2534..219589bb 100644
--- a/download.yaml
+++ b/download.yaml
@@ -74,7 +74,7 @@
 # SciBite CORD-19 annotations v1.2
 #
 -
-  url: https://github.com/SciBiteLabs/CORD19/raw/d2fb4998a504bcb06f5d0fd407f58feda2f140b1/annotated-CORD-19/1.2/CORD-19_1_2.zip
+  url: http://idg.berkeleybop.io/CORD-19_1_2.zip
   local_name: CORD-19_1_2.zip
 
 # SciBite CORD-19 entity co-occurrences v1.0

From b1e354821c53d1f8c672dd8453f953ddecb83b6e Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Fri, 10 Apr 2020 08:51:58 -0700
Subject: [PATCH 39/63] Fix Jenkinsfile - closing parens, indenting

---
 Jenkinsfile | 92 +++++++++++++++++++++++++++--------------------------
 1 file changed, 47 insertions(+), 45 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 89d1b6d6..62f97cd0 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,35 +1,35 @@
 pipeline {
     agent any
     triggers {
-	// Every two days at 11 pm
-	cron('0 23 */2 * *')
+        // Every two days at 11 pm
+        cron('0 23 */2 * *')
     }
-    options{
-	timestamps()
+    options {
+        timestamps()
     }
     stages {
         // Very first: pause for a minute to give a chance to
         // cancel and clean the workspace before use.
         stage('Ready and clean') {
             steps {
-            // Give us a minute to cancel if we want.
-            sleep time: 1, unit: 'MINUTES'
-            cleanWs()
+                // Give us a minute to cancel if we want.
+                sleep time: 1, unit: 'MINUTES'
+                cleanWs()
             }
         }
         stage('Initialize') {
             steps {
-            // Start preparing environment.
-            parallel(
-                "Report": {
-                sh 'env > env.txt'
-                sh 'echo $BRANCH_NAME > branch.txt'
-                sh 'echo "$BRANCH_NAME"'
-                sh 'cat env.txt'
-                sh 'cat branch.txt'
-                sh 'echo $START_DAY > dow.txt'
-                sh 'echo "$START_DAY"'
-                })
+                // Start preparing environment.
+                parallel(
+                        "Report": {
+                            sh 'env > env.txt'
+                            sh 'echo $BRANCH_NAME > branch.txt'
+                            sh 'echo "$BRANCH_NAME"'
+                            sh 'cat env.txt'
+                            sh 'cat branch.txt'
+                            sh 'echo $START_DAY > dow.txt'
+                            sh 'echo "$START_DAY"'
+                        })
             }
         }
 
@@ -37,54 +37,56 @@ pipeline {
             steps {
                 dir('./config') {
                     git(
-                        url: 'https://github.com/Knowledge-Graph-Hub/kg-covid-19',
-                        branch: 'master'
+                            url: 'https://github.com/Knowledge-Graph-Hub/kg-covid-19',
+                            branch: 'master'
                     )
-		    sh '/usr/bin/python3.7 -m venv venv'
-		    sh '. venv/bin/activate'
-		    sh './venv/bin/pip install bmt'
-		    sh './venv/bin/pip install -r requirements.txt'
-		    sh './venv/bin/python setup.py install'
+                    sh '/usr/bin/python3.7 -m venv venv'
+                    sh '. venv/bin/activate'
+                    sh './venv/bin/pip install bmt'
+                    sh './venv/bin/pip install -r requirements.txt'
+                    sh './venv/bin/python setup.py install'
                 }
             }
         }
         stage('Download') {
             steps {
-                  sh 'cd config;. venv/bin/activate; python3.7 run.py download'
+                sh 'cd config;. venv/bin/activate; python3.7 run.py download'
             }
         }
         stage('Transform') {
             steps {
-                  sh 'cd config;. venv/bin/activate; python3.7 run.py transform'
+                sh 'cd config;. venv/bin/activate; python3.7 run.py transform'
             }
         }
         stage('Load') {
             steps {
-		          sh 'cd config;. venv/bin/activate; python3.7 run.py load'
+                sh 'cd config;. venv/bin/activate; python3.7 run.py load'
             }
         }
         stage('Publish') {
             steps {
 
-            script {
-                if( env.BRANCH_NAME != 'master' ){
-                   echo "Will not push if not on correct branch."
-                } else{
-                // Get trivial product back in hand.
-                sh 'echo thing > dow.txt'
+                script {
+                    if (env.BRANCH_NAME != 'master') {
+                        echo "Will not push if not on correct branch."
+                    } else {
+                        // Get trivial product back in hand.
+                        sh 'echo thing > dow.txt'
 
-                // Push out to your S3 bucket.  The given
-                // command is for small to medium files. If
-                // you need something appropriate for large
-                // files, let me know.
-                withCredentials([file(credentialsId: 's3cmd_idg_push_configuration', variable: 'S3CMD_JSON')]) {
-                    sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put dow.txt s3://idg-public-data/dow.txt'
-                    // Should now appear at:
-                    // https://idg.berkeleybop.io/dow.txt
-                    // http://idg.berkeleybop.io/dow.txt
-                }
+                        // Push out to your S3 bucket.  The given
+                        // command is for small to medium files. If
+                        // you need something appropriate for large
+                        // files, let me know.
+                        withCredentials([file(credentialsId: 's3cmd_idg_push_configuration', variable: 'S3CMD_JSON')]) {
+                            sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put dow.txt s3://idg-public-data/dow.txt'
+                            // Should now appear at:
+                            // https://idg.berkeleybop.io/dow.txt
+                            // http://idg.berkeleybop.io/dow.txt
+                        }
 
+                    }
+                }
             }
-		}
+        }
     }
 }

From f5fb8e137a856e1d950ea027ce1af14d2195f0ef Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Sun, 12 Apr 2020 13:41:34 -0700
Subject: [PATCH 40/63] Comment out cron trigger

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 62f97cd0..1c640571 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -2,7 +2,7 @@ pipeline {
     agent any
     triggers {
         // Every two days at 11 pm
-        cron('0 23 */2 * *')
+        // cron('0 23 */2 * *')
     }
     options {
         timestamps()

From d6ebe42b8442817692db23649a80c34d33242ca4 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Tue, 14 Apr 2020 08:16:28 -0700
Subject: [PATCH 41/63] Add stage to convert to RDF (call to KGX)

This call will change if/when we output ntriples instead of TTL
---
 Jenkinsfile | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 1c640571..9326b02e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -63,6 +63,11 @@ pipeline {
                 sh 'cd config;. venv/bin/activate; python3.7 run.py load'
             }
         }
+        stage('Convert to RDF') {
+            steps {
+                sh 'cd config;. venv/bin/activate; kgx transform --input-type tsv --output-type ttl -o . merged-kg.tar'
+            }
+        }        
         stage('Publish') {
             steps {
 
@@ -70,18 +75,15 @@ pipeline {
                     if (env.BRANCH_NAME != 'master') {
                         echo "Will not push if not on correct branch."
                     } else {
-                        // Get trivial product back in hand.
-                        sh 'echo thing > dow.txt'
-
                         // Push out to your S3 bucket.  The given
                         // command is for small to medium files. If
                         // you need something appropriate for large
                         // files, let me know.
                         withCredentials([file(credentialsId: 's3cmd_idg_push_configuration', variable: 'S3CMD_JSON')]) {
-                            sh 's3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put dow.txt s3://idg-public-data/dow.txt'
+                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar s3://idg-public-data/dow.txt'
+                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.ttl s3://idg-public-data/dow.txt'
                             // Should now appear at:
-                            // https://idg.berkeleybop.io/dow.txt
-                            // http://idg.berkeleybop.io/dow.txt
+                            // https://idg.berkeleybop.io/[artifact name]
                         }
 
                     }

From ebc30846f2b0bfb893c31a0f5a2ef393efa493bd Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Tue, 14 Apr 2020 08:37:02 -0700
Subject: [PATCH 42/63] Removed cron trigger for now

---
 Jenkinsfile | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9326b02e..5a5b6330 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,9 +1,5 @@
 pipeline {
     agent any
-    triggers {
-        // Every two days at 11 pm
-        // cron('0 23 */2 * *')
-    }
     options {
         timestamps()
     }
@@ -67,7 +63,7 @@ pipeline {
             steps {
                 sh 'cd config;. venv/bin/activate; kgx transform --input-type tsv --output-type ttl -o . merged-kg.tar'
             }
-        }        
+        }
         stage('Publish') {
             steps {
 

From f7740e5a26c2a416653058d261de45429074f48c Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Fri, 17 Apr 2020 12:59:54 -0700
Subject: [PATCH 43/63] Remove RDF conversion step (this is taking >1 day)

---
 Jenkinsfile | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5a5b6330..5cf5b361 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -59,11 +59,6 @@ pipeline {
                 sh 'cd config;. venv/bin/activate; python3.7 run.py load'
             }
         }
-        stage('Convert to RDF') {
-            steps {
-                sh 'cd config;. venv/bin/activate; kgx transform --input-type tsv --output-type ttl -o . merged-kg.tar'
-            }
-        }
         stage('Publish') {
             steps {
 

From a7eb17dab2396486d685fdeebb2bfd27e4947c89 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Fri, 17 Apr 2020 13:03:27 -0700
Subject: [PATCH 44/63] Removed erroneous s3 upload command

---
 Jenkinsfile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5cf5b361..8b45f114 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -71,8 +71,7 @@ pipeline {
                         // you need something appropriate for large
                         // files, let me know.
                         withCredentials([file(credentialsId: 's3cmd_idg_push_configuration', variable: 'S3CMD_JSON')]) {
-                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar s3://idg-public-data/dow.txt'
-                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.ttl s3://idg-public-data/dow.txt'
+                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar s3://idg-public-data/kg_covid_19.txt'
                             // Should now appear at:
                             // https://idg.berkeleybop.io/[artifact name]
                         }

From e301790bc4f5321a2f4789c07e7e8849b33405f5 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Fri, 17 Apr 2020 21:42:36 -0700
Subject: [PATCH 45/63] Change branch permitted to publish

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 8b45f114..9c60ba4c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -63,7 +63,7 @@ pipeline {
             steps {
 
                 script {
-                    if (env.BRANCH_NAME != 'master') {
+                    if (env.BRANCH_NAME != 'jenkins') {
                         echo "Will not push if not on correct branch."
                     } else {
                         // Push out to your S3 bucket.  The given

From f3341f2b9372806b8f50b7583856179c98b76dbf Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Sat, 18 Apr 2020 08:02:15 -0700
Subject: [PATCH 46/63] Typo in artifact name (txt -> tar)

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9c60ba4c..6486b764 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -71,7 +71,7 @@ pipeline {
                         // you need something appropriate for large
                         // files, let me know.
                         withCredentials([file(credentialsId: 's3cmd_idg_push_configuration', variable: 'S3CMD_JSON')]) {
-                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar s3://idg-public-data/kg_covid_19.txt'
+                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar s3://idg-public-data/kg_covid_19.tar'
                             // Should now appear at:
                             // https://idg.berkeleybop.io/[artifact name]
                         }

From c07febeccef1562588272d9e83a94e41e3a8fabb Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Fri, 24 Apr 2020 14:39:24 -0700
Subject: [PATCH 47/63] Update S3 credentials

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 6486b764..fe94cf5a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -70,8 +70,8 @@ pipeline {
                         // command is for small to medium files. If
                         // you need something appropriate for large
                         // files, let me know.
-                        withCredentials([file(credentialsId: 's3cmd_idg_push_configuration', variable: 'S3CMD_JSON')]) {
-                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar s3://idg-public-data/kg_covid_19.tar'
+                        withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) {
+                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar s3://kg-hub-public-data/kg_covid_19.tar'
                             // Should now appear at:
                             // https://idg.berkeleybop.io/[artifact name]
                         }

From 853180ef1124b97c79e860dc8bbfa08fbcddfb46 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Fri, 24 Apr 2020 14:43:47 -0700
Subject: [PATCH 48/63] Gzip the things

---
 Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index fe94cf5a..3295bf56 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -57,6 +57,7 @@ pipeline {
         stage('Load') {
             steps {
                 sh 'cd config;. venv/bin/activate; python3.7 run.py load'
+                sh 'pigz merged-kg.tar'
             }
         }
         stage('Publish') {
@@ -71,7 +72,7 @@ pipeline {
                         // you need something appropriate for large
                         // files, let me know.
                         withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) {
-                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar s3://kg-hub-public-data/kg_covid_19.tar'
+                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar.gz s3://kg-hub-public-data/kg_covid_19.tar.gz'
                             // Should now appear at:
                             // https://idg.berkeleybop.io/[artifact name]
                         }

From 577e1a8547c76e9d0e378ce7804426ac9fa66c21 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Mon, 27 Apr 2020 10:24:42 -0700
Subject: [PATCH 49/63] Stage to convert TSV to ntriples (and push nt too)

---
 Jenkinsfile | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3295bf56..134c46ff 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -60,6 +60,11 @@ pipeline {
                 sh 'pigz merged-kg.tar'
             }
         }
+        stage('Convert to RDF') {
+            steps {
+                sh 'cd config;. venv/bin/activate; kgx transform --input-type tsv --output-type nt -o ./merged-kg.nt merged-kg.tar.gz'
+            }
+        }        
         stage('Publish') {
             steps {
 
@@ -67,12 +72,9 @@ pipeline {
                     if (env.BRANCH_NAME != 'jenkins') {
                         echo "Will not push if not on correct branch."
                     } else {
-                        // Push out to your S3 bucket.  The given
-                        // command is for small to medium files. If
-                        // you need something appropriate for large
-                        // files, let me know.
                         withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) {
-                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar.gz s3://kg-hub-public-data/kg_covid_19.tar.gz'
+                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.nt s3://kg-hub-public-data/kg-covid-19.nt'
+                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar.gz s3://kg-hub-public-data/kg-covid-19.tar.gz'
                             // Should now appear at:
                             // https://idg.berkeleybop.io/[artifact name]
                         }

From 3452e078b3af7f7c4abfe802cb23651fa0b3793f Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Tue, 28 Apr 2020 13:05:36 -0700
Subject: [PATCH 50/63] Add cd before pigz command

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 134c46ff..947758b1 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -57,7 +57,7 @@ pipeline {
         stage('Load') {
             steps {
                 sh 'cd config;. venv/bin/activate; python3.7 run.py load'
-                sh 'pigz merged-kg.tar'
+                sh 'cd config;. venv/bin/activate; pigz merged-kg.tar'
             }
         }
         stage('Convert to RDF') {

From b9d83b0f28935c7b978f96f00996c950274b7596 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Fri, 1 May 2020 12:43:49 -0700
Subject: [PATCH 51/63] zip nt file to make it less huge

---
 Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 947758b1..e31013c6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -63,6 +63,7 @@ pipeline {
         stage('Convert to RDF') {
             steps {
                 sh 'cd config;. venv/bin/activate; kgx transform --input-type tsv --output-type nt -o ./merged-kg.nt merged-kg.tar.gz'
+                sh 'cd config;. venv/bin/activate; pigz merged-kg.nt'
             }
         }        
         stage('Publish') {
@@ -73,7 +74,7 @@ pipeline {
                         echo "Will not push if not on correct branch."
                     } else {
                         withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) {
-                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.nt s3://kg-hub-public-data/kg-covid-19.nt'
+                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.nt s3://kg-hub-public-data/kg-covid-19.nt.gz'
                             sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar.gz s3://kg-hub-public-data/kg-covid-19.tar.gz'
                             // Should now appear at:
                             // https://idg.berkeleybop.io/[artifact name]

From 5862e9329ac55130d4e7aaa9514e833c4accf59d Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Sat, 2 May 2020 09:31:02 -0700
Subject: [PATCH 52/63] Fix arg in s3cmd referring to uncompressed nt file

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index e31013c6..14131ae9 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -74,7 +74,7 @@ pipeline {
                         echo "Will not push if not on correct branch."
                     } else {
                         withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) {
-                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.nt s3://kg-hub-public-data/kg-covid-19.nt.gz'
+                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.nt.gz s3://kg-hub-public-data/kg-covid-19.nt.gz'
                             sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar.gz s3://kg-hub-public-data/kg-covid-19.tar.gz'
                             // Should now appear at:
                             // https://idg.berkeleybop.io/[artifact name]

From 6b9fdd68e5eeeea9dd5899090a5bd944d4b5510e Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Wed, 6 May 2020 14:25:19 -0700
Subject: [PATCH 53/63] Add step in download to upload/dl raw to s3

---
 Jenkinsfile | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 14131ae9..f1886c0a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -45,8 +45,15 @@ pipeline {
             }
         }
         stage('Download') {
-            steps {
-                sh 'cd config;. venv/bin/activate; python3.7 run.py download'
+        steps {
+            def run_py_dl = sh(
+                script: 'cd config;. venv/bin/activate; python3.7 run.py download', returnStatus: true
+            )
+            if (run_py_dl == 0) { // upload raw to s3
+                sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put -r data/raw s3://kg-hub-public-data/raw'
+            } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/
+                sh 'cd config; rm -fr data/raw'
+                sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate get -r s3://kg-hub-public-data/raw data/'
             }
         }
         stage('Transform') {

From 02ac78c1145f0f4eb95b805e9a56206473624b75 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Wed, 6 May 2020 14:26:59 -0700
Subject: [PATCH 54/63] Missing }

---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index f1886c0a..7fb1bfa5 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -56,6 +56,7 @@ pipeline {
                 sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate get -r s3://kg-hub-public-data/raw data/'
             }
         }
+        }
         stage('Transform') {
             steps {
                 sh 'cd config;. venv/bin/activate; python3.7 run.py transform'

From 872d8321efe2ad72c787130384393887c65c96d4 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Wed, 6 May 2020 14:39:23 -0700
Subject: [PATCH 55/63] invalid Jenkins syntax

---
 Jenkinsfile | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7fb1bfa5..d5298978 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -45,18 +45,20 @@ pipeline {
             }
         }
         stage('Download') {
-        steps {
-            def run_py_dl = sh(
-                script: 'cd config;. venv/bin/activate; python3.7 run.py download', returnStatus: true
-            )
-            if (run_py_dl == 0) { // upload raw to s3
-                sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put -r data/raw s3://kg-hub-public-data/raw'
-            } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/
-                sh 'cd config; rm -fr data/raw'
-                sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate get -r s3://kg-hub-public-data/raw data/'
+            steps {
+                script {
+                    def run_py_dl = sh(
+                        script: 'cd config;. venv/bin/activate; python3.7 run.py download', returnStatus: true
+                    )
+                    if (run_py_dl == 0) { // upload raw to s3
+                        sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put -r data/raw s3://kg-hub-public-data/raw'
+                    } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/
+                        sh 'cd config; rm -fr data/raw'
+                        sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate get -r s3://kg-hub-public-data/raw data/'
+                    }
+                }
             }
         }
-        }
         stage('Transform') {
             steps {
                 sh 'cd config;. venv/bin/activate; python3.7 run.py transform'

From 5d2889a7c90f42cfd8fe06243e81d89c32295f15 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Wed, 6 May 2020 16:17:41 -0700
Subject: [PATCH 56/63] Missing withcredentials block

---
 Jenkinsfile | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d5298978..5a8cb36e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -50,11 +50,13 @@ pipeline {
                     def run_py_dl = sh(
                         script: 'cd config;. venv/bin/activate; python3.7 run.py download', returnStatus: true
                     )
-                    if (run_py_dl == 0) { // upload raw to s3
-                        sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put -r data/raw s3://kg-hub-public-data/raw'
-                    } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/
-                        sh 'cd config; rm -fr data/raw'
-                        sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate get -r s3://kg-hub-public-data/raw data/'
+                    withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) {
+                        if (run_py_dl == 0) { // upload raw to s3
+                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put -r data/raw s3://kg-hub-public-data/raw'
+                        } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/
+                            sh 'cd config; rm -fr data/raw'
+                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate get -r s3://kg-hub-public-data/raw data/'
+                        }
                     }
                 }
             }
@@ -87,7 +89,7 @@ pipeline {
                             sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.nt.gz s3://kg-hub-public-data/kg-covid-19.nt.gz'
                             sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put merged-kg.tar.gz s3://kg-hub-public-data/kg-covid-19.tar.gz'
                             // Should now appear at:
-                            // https://idg.berkeleybop.io/[artifact name]
+                            // https://kg-hub.berkeleybop.io/[artifact name]
                         }
 
                     }

From 2bb2639eb37bb28e78c51cdaa70cd33cedb57289 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Wed, 6 May 2020 17:44:01 -0700
Subject: [PATCH 57/63] Add trailing slash to appease s3cmd

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5a8cb36e..9fdae1eb 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -52,10 +52,10 @@ pipeline {
                     )
                     withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) {
                         if (run_py_dl == 0) { // upload raw to s3
-                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put -r data/raw s3://kg-hub-public-data/raw'
+                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate put -r data/raw s3://kg-hub-public-data/'
                         } else { // 'run.py download' failed - let's try to download last good copy of raw/ from s3 to data/
-                            sh 'cd config; rm -fr data/raw'
-                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate get -r s3://kg-hub-public-data/raw data/'
+                            sh 'cd config; rm -fr data/raw; mkdir -p data/raw'
+                            sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate get -r s3://kg-hub-public-data/raw/ data/raw/'
                         }
                     }
                 }

From 4ef01eb963b0e97f9b0f637aefc8b70e9c39dc87 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 7 May 2020 08:28:23 -0700
Subject: [PATCH 58/63] test download

---
 Jenkinsfile | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9fdae1eb..785f8e31 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -3,9 +3,9 @@ pipeline {
     options {
         timestamps()
     }
-    stages {
+    stages {        
         // Very first: pause for a minute to give a chance to
-        // cancel and clean the workspace before use.
+        // cancel and clean the workspace before use.        
         stage('Ready and clean') {
             steps {
                 // Give us a minute to cancel if we want.
@@ -28,7 +28,6 @@ pipeline {
                         })
             }
         }
-
         stage('Build kg_covid_19') {
             steps {
                 dir('./config') {
@@ -44,6 +43,18 @@ pipeline {
                 }
             }
         }
+        stage('Test Download') {
+            steps {
+                script {
+                    withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) {
+                        sh 'cd config; rm -fr data/raw; mkdir -p data/raw'
+                        sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate get -r s3://kg-hub-public-data/raw/ data/raw/'
+                        sh 'cd config; ls data/raw'
+                        }
+                    }
+                }
+            }
+        }
         stage('Download') {
             steps {
                 script {

From a5f20d3ae979836538a769cd42993dc43a348a9b Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 7 May 2020 08:29:54 -0700
Subject: [PATCH 59/63] stray }

---
 Jenkinsfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 785f8e31..aa810ede 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -50,7 +50,6 @@ pipeline {
                         sh 'cd config; rm -fr data/raw; mkdir -p data/raw'
                         sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate get -r s3://kg-hub-public-data/raw/ data/raw/'
                         sh 'cd config; ls data/raw'
-                        }
                     }
                 }
             }

From 4726ab3f56df23c9f659db5fdbce259da2dd730a Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 7 May 2020 08:36:58 -0700
Subject: [PATCH 60/63] Experiment worked, remove test download

---
 Jenkinsfile | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index aa810ede..d80a65cd 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -43,17 +43,6 @@ pipeline {
                 }
             }
         }
-        stage('Test Download') {
-            steps {
-                script {
-                    withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) {
-                        sh 'cd config; rm -fr data/raw; mkdir -p data/raw'
-                        sh 'cd config; s3cmd -c $S3CMD_JSON --acl-public --mime-type=plain/text --cf-invalidate get -r s3://kg-hub-public-data/raw/ data/raw/'
-                        sh 'cd config; ls data/raw'
-                    }
-                }
-            }
-        }
         stage('Download') {
             steps {
                 script {

From a87ddd54d9138d0d1dc763d4e8883ae4a9de0a97 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 7 May 2020 11:09:00 -0700
Subject: [PATCH 61/63] test download from S3 fxn - fail dl on purpose

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d80a65cd..07a6bff9 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -47,7 +47,7 @@ pipeline {
             steps {
                 script {
                     def run_py_dl = sh(
-                        script: 'cd config;. venv/bin/activate; python3.7 run.py download', returnStatus: true
+                        script: 'false # cd config;. venv/bin/activate; python3.7 run.py download', returnStatus: true
                     )
                     withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) {
                         if (run_py_dl == 0) { // upload raw to s3

From 0d476786e27961b837c35a1d7bef455d38dade6d Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 7 May 2020 11:13:39 -0700
Subject: [PATCH 62/63] Remove intentional fail of dl stage

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 07a6bff9..d80a65cd 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -47,7 +47,7 @@ pipeline {
             steps {
                 script {
                     def run_py_dl = sh(
-                        script: 'false # cd config;. venv/bin/activate; python3.7 run.py download', returnStatus: true
+                        script: 'cd config;. venv/bin/activate; python3.7 run.py download', returnStatus: true
                     )
                     withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) {
                         if (run_py_dl == 0) { // upload raw to s3

From c18fd114bb8c24aac0d2e73c7b9d6b8602726d44 Mon Sep 17 00:00:00 2001
From: Justin Reese <justaddcoffee+github@gmail.com>
Date: Thu, 7 May 2020 11:32:20 -0700
Subject: [PATCH 63/63] Change to check for master branch

when pushing to s3 in publish stage
---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d80a65cd..b581a8eb 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -81,7 +81,7 @@ pipeline {
             steps {
 
                 script {
-                    if (env.BRANCH_NAME != 'jenkins') {
+                    if (env.BRANCH_NAME != 'master') {
                         echo "Will not push if not on correct branch."
                     } else {
                         withCredentials([file(credentialsId: 's3cmd_kg_hub_push_configuration', variable: 'S3CMD_JSON')]) {