From 5b5e946917204c7f76893780ed74f1da59122e23 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 27 Oct 2020 17:02:16 -0400 Subject: [PATCH 1/3] add script for rebuilding dev environment #7256 --- .../source/developers/troubleshooting.rst | 22 +++--- scripts/dev/dev-rebuild.sh | 71 +++++++++++++++++++ 2 files changed, 83 insertions(+), 10 deletions(-) create mode 100755 scripts/dev/dev-rebuild.sh diff --git a/doc/sphinx-guides/source/developers/troubleshooting.rst b/doc/sphinx-guides/source/developers/troubleshooting.rst index 63f00ce02f6..83d2ab747b2 100755 --- a/doc/sphinx-guides/source/developers/troubleshooting.rst +++ b/doc/sphinx-guides/source/developers/troubleshooting.rst @@ -89,16 +89,18 @@ As another example, here is how to create a Mail Host via command line for Amazo Rebuilding Your Dev Environment ------------------------------- -If you have an old copy of the database and old Solr data and want to start fresh, here are the recommended steps: - -- drop your old database -- clear out your existing Solr index: ``scripts/search/clear`` -- run the installer script above - it will create the db, deploy the app, populate the db with reference data and run all the scripts that create the domain metadata fields. You no longer need to perform these steps separately. -- confirm you are using the latest Dataverse-specific Solr schema.xml and included XML files (schema_dv_cmb_[copies|fields].xml) -- confirm http://localhost:8080 is up -- If you want to set some dataset-specific facets, go to the root dataverse (or any dataverse; the selections can be inherited) and click "General Information" and make choices under "Select Facets". There is a ticket to automate this: https://github.com/IQSS/dataverse/issues/619 - -You may also find https://github.com/IQSS/dataverse/blob/develop/scripts/deploy/phoenix.dataverse.org/deploy and related scripts interesting because they demonstrate how we have at least partially automated the process of tearing down a Dataverse installation and having it rise again, hence the name "phoenix." See :ref:`fresh-reinstall` section of the Installation Guide. +A script called :download:`dev-rebuild.sh <../../../../scripts/dev/dev-rebuild.sh>` is available that does the following: + +- Drops the database. +- Clears our Solr. +- Deletes all data files uploaded by users. +- Deploys the war file located in the ``target`` directory. +- Runs ``setup-all.sh`` in insecure mode so tests will pass. +- Runs post-install SQL statements. +- Publishes the root dataverse. +- Adjusts permissions on on the root dataverse so tests will pass. + +To execute the script, make sure you have built a war file already and then ``cd`` to the root of the source tree and run ``scripts/dev/dev-rebuild.sh``. Feedback on this script is welcome! DataCite -------- diff --git a/scripts/dev/dev-rebuild.sh b/scripts/dev/dev-rebuild.sh new file mode 100755 index 00000000000..7b0b496d5bf --- /dev/null +++ b/scripts/dev/dev-rebuild.sh @@ -0,0 +1,71 @@ +#!/bin/sh +PAYARA_DIR=/usr/local/payara5 +ASADMIN=$PAYARA_DIR/glassfish/bin/asadmin +DB_NAME=dvndb +DB_USER=dvnapp + +echo "Checking if there is a war file to undeploy..." +LIST_APP=$($ASADMIN list-applications -t) +OLD_WAR=$(echo $LIST_APP | awk '{print $1}') +NEW_WAR=target/dataverse*.war +if [ ! -z $OLD_WAR ]; then + $ASADMIN undeploy $OLD_WAR +fi + +echo "Stopping app server..." +$ASADMIN stop-domain + +echo "Deleting \"generated\" directory..." +rm -rf $PAYARA_DIR/glassfish/domains/domain1/generated + +echo "Deleting ALL DATA FILES uploaded to Dataverse..." +rm -rf $PAYARA_DIR/glassfish/domains/domain1/files + +echo "Terminating database settings so we can drop the database..." +psql -U postgres -c " +SELECT pg_terminate_backend(pg_stat_activity.pid) +FROM pg_stat_activity +WHERE pg_stat_activity.datname = '$DB_NAME' + AND pid <> pg_backend_pid(); +" template1 + +echo "Dropping the database..." +psql -U $DB_USER -c "DROP DATABASE \"$DB_NAME\"" template1 +echo $? + +echo "Clearing out data from Solr..." +curl http://localhost:8983/solr/collection1/update/json?commit=true -H "Content-type: application/json" -X POST -d "{\"delete\": { \"query\":\"*:*\"}}" + +echo "Creating a new database..." +psql -U $DB_USER -c "CREATE DATABASE \"$DB_NAME\" WITH OWNER = \"$DB_USER\"" template1 +echo $? + +echo "Starting app server..." +$PAYARA_DIR/glassfish/bin/asadmin start-domain + +echo "Deploying war file..." +$PAYARA_DIR/glassfish/bin/asadmin deploy $NEW_WAR + +echo "Running setup-all.sh (INSECURE MODE)..." +cd scripts/api +./setup-all.sh --insecure -p=admin1 | tee /tmp/setup-all.sh.out +cd ../.. + +echo "Loading SQL reference data..." +psql -U $DB_USER $DB_NAME -f scripts/database/reference_data.sql + +echo "Creating SQL sequence..." +psql -U $DB_USER $DB_NAME -f doc/sphinx-guides/source/_static/util/createsequence.sql + +echo "Setting DOI provider to \"FAKE\"..." +curl http://localhost:8080/api/admin/settings/:DoiProvider -X PUT -d FAKE +export API_TOKEN=`cat /tmp/setup-all.sh.out | grep apiToken| jq .data.apiToken | tr -d \"` + +echo "Publishing root dataverse..." +curl -H X-Dataverse-key:$API_TOKEN -X POST http://localhost:8080/api/dataverses/:root/actions/:publish + +echo "Allowing users to create dataverses and datasets in root..." +curl -H X-Dataverse-key:$API_TOKEN -X POST -H "Content-type:application/json" -d "{\"assignee\": \":authenticated-users\",\"role\": \"fullContributor\"}" "http://localhost:8080/api/dataverses/:root/assignments" + +echo "Checking Dataverse version..." +curl http://localhost:8080/api/info/version From ecde956db1e822b510d730d19fb9b815956621a7 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 29 Oct 2020 16:58:43 -0400 Subject: [PATCH 2/3] note that files dir must be the default #7256 --- doc/sphinx-guides/source/developers/troubleshooting.rst | 2 +- scripts/dev/dev-rebuild.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/troubleshooting.rst b/doc/sphinx-guides/source/developers/troubleshooting.rst index 83d2ab747b2..2964684f7a9 100755 --- a/doc/sphinx-guides/source/developers/troubleshooting.rst +++ b/doc/sphinx-guides/source/developers/troubleshooting.rst @@ -93,7 +93,7 @@ A script called :download:`dev-rebuild.sh <../../../../scripts/dev/dev-rebuild.s - Drops the database. - Clears our Solr. -- Deletes all data files uploaded by users. +- Deletes all data files uploaded by users (assuming you are using the default directory). - Deploys the war file located in the ``target`` directory. - Runs ``setup-all.sh`` in insecure mode so tests will pass. - Runs post-install SQL statements. diff --git a/scripts/dev/dev-rebuild.sh b/scripts/dev/dev-rebuild.sh index 7b0b496d5bf..2bda9c4d9a3 100755 --- a/scripts/dev/dev-rebuild.sh +++ b/scripts/dev/dev-rebuild.sh @@ -19,6 +19,7 @@ echo "Deleting \"generated\" directory..." rm -rf $PAYARA_DIR/glassfish/domains/domain1/generated echo "Deleting ALL DATA FILES uploaded to Dataverse..." +# TODO: Make this configurable. rm -rf $PAYARA_DIR/glassfish/domains/domain1/files echo "Terminating database settings so we can drop the database..." From d3778f3600e8284df6950c19f4fc4c3b38c5c7a2 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 29 Oct 2020 17:01:09 -0400 Subject: [PATCH 3/3] typo #7256 --- scripts/dev/dev-rebuild.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dev/dev-rebuild.sh b/scripts/dev/dev-rebuild.sh index 2bda9c4d9a3..0a053c1d91d 100755 --- a/scripts/dev/dev-rebuild.sh +++ b/scripts/dev/dev-rebuild.sh @@ -22,7 +22,7 @@ echo "Deleting ALL DATA FILES uploaded to Dataverse..." # TODO: Make this configurable. rm -rf $PAYARA_DIR/glassfish/domains/domain1/files -echo "Terminating database settings so we can drop the database..." +echo "Terminating database sessions so we can drop the database..." psql -U postgres -c " SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity