add ability for jenkins redo job to update dev solr

monarch-initiative · Jun 13, 2024 · 88f9366 · 88f9366
1 parent f1393e1
commit 88f9366
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 7 deletions.
diff --git a/Jenkinsfile-redo-solr b/Jenkinsfile-redo-solr
@@ -57,6 +57,11 @@ pipeline {
                                 description: 'Upload to bucket',
                                 name: 'UPLOAD'
                             ),
+                            booleanParam(
+                                defaultValue: false,
+                                description: 'Update dev deployment to latest release',
+                                name: 'UPDATE_DEV'
+                            )
                         ])
                     ])
                 }
@@ -87,7 +92,7 @@ pipeline {
                 sh '''
                     mkdir output || true
                     mkdir data || true
-                    gsutil -q -m cp -r gs://data-public-monarchinitiative/monarch-kg-dev/latest/* output/
+                    gsutil -q -m cp -r gs://data-public-monarchinitiative/monarch-kg-dev/${env.RELEASE}/* output/
                     gsutil -q -m cp -r gs://monarch-ingest-data-cache/* data/
                     ls -lasdf
                 '''
@@ -173,5 +178,15 @@ pipeline {
                 sh 'scripts/update_latest_release.sh'
             }
         }
+        stage('update dev deployment') {
+            when {
+                expression {
+                   return params.UPDATE_DEV
+                }
+            }
+            steps {
+                sh 'poetry run python scripts/update-dev-solr.py'
+            }
+        }
     }
 }
diff --git a/scripts/update_latest_release.sh b/scripts/update_latest_release.sh
@@ -3,12 +3,13 @@
 # This script will push a local copy of the Solr, Sqlite and denormalized edge artifacts up to all
 # all copies of the bucket for a given release. It needs to be run from the root of the repo
 
-export RELEASE=$(gsutil ls gs://data-public-monarchinitiative/monarch-kg-dev/latest/ | grep -o '[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}')
 echo "Updating Solr, SQLite and denormalized edge files for $RELEASE"
 
-gsutil cp output/monarch-kg.db.gz gs://monarch-archive/monarch-kg-dev/$RELEASE/
-gsutil cp output/monarch-kg-denormalized-edges.tsv.gz gs://monarch-archive/monarch-kg-dev/$RELEASE/
-gsutil cp output/solr.tar.gz gs://monarch-archive/monarch-kg-dev/$RELEASE/
+gsutil -m cp -r output/* gs://monarch-archive/monarch-kg-dev/$RELEASE/
 
-gsutil cp "gs://monarch-archive/monarch-kg-dev/$RELEASE/*.gz" gs://data-public-monarchinitiative/monarch-kg-dev/$RELEASE/
-gsutil cp "gs://monarch-archive/monarch-kg-dev/$RELEASE/*.gz" gs://data-public-monarchinitiative/monarch-kg-dev/latest/
+# if RELEASE == LATEST_RELEASE, copy all of this release to latest
+export LATEST_RELEASE=$(gsutil ls gs://data-public-monarchinitiative/monarch-kg-dev/latest/ | grep -o '[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}')
+if [ "$RELEASE" == "$LATEST_RELEASE" ]; then
+    gsutil -m
+    gsutil -m cp -r "gs://monarch-archive/monarch-kg-dev/$RELEASE/*" gs://data-public-monarchinitiative/monarch-kg-dev/latest/
+fi