7373 solr upgrade

conf/docker-aio/0prep_deps.sh

@@ -10,12 +10,11 @@ if [ ! -e dv/deps/payara-5.2020.6.zip ]; then
 	wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2020.6/payara-5.2020.6.zip  -O dv/deps/payara-5.2020.6.zip
 fi
 
-if [ ! -e dv/deps/solr-7.7.2dv.tgz ]; then
+if [ ! -e dv/deps/solr-8.8.1dv.tgz ]; then
 	echo "solr dependency prep"
 	# schema changes *should* be the only ones...
-	cd dv/deps/
-	#wget https://archive.apache.org/dist/lucene/solr/7.3.0/solr-7.3.0.tgz -O solr-7.3.0dv.tgz
-	wget https://archive.apache.org/dist/lucene/solr/7.7.2/solr-7.7.2.tgz -O solr-7.7.2dv.tgz
+	cd dv/deps/	
+	wget https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz -O solr-8.8.1dv.tgz
 	cd ../../
 fi
 

conf/docker-aio/1prep.sh

@@ -4,9 +4,9 @@
 # this was based off the phoenix deployment; and is likely uglier and bulkier than necessary in a perfect world
 
 mkdir -p testdata/doc/sphinx-guides/source/_static/util/
-cp ../solr/7.7.2/schema*.xml testdata/
-cp ../solr/7.7.2/solrconfig.xml testdata/
-cp ../solr/7.7.2/updateSchemaMDB.sh testdata/
+cp ../solr/8.8.1/schema*.xml testdata/
+cp ../solr/8.8.1/solrconfig.xml testdata/
+cp ../solr/8.8.1/updateSchemaMDB.sh testdata/
 cp ../jhove/jhove.conf testdata/
 cp ../jhove/jhoveConfig.xsd testdata/
 cd ../../

conf/docker-aio/c8.dockerfile

@@ -17,7 +17,7 @@ COPY testdata/sushi_sample_logs.json /tmp/
 COPY disableipv6.conf /etc/sysctl.d/
 RUN rm /etc/httpd/conf/*
 COPY httpd.conf /etc/httpd/conf 
-RUN cd /opt ; tar zxf /tmp/dv/deps/solr-7.7.2dv.tgz 
+RUN cd /opt ; tar zxf /tmp/dv/deps/solr-8.8.1dv.tgz 
 RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2020.6.zip ; ln -s /opt/payara5 /opt/glassfish4
 
 # this copy of domain.xml is the result of running `asadmin set server.monitoring-service.module-monitoring-levels.jvm=LOW` on a default glassfish installation (aka - enable the glassfish REST monitir endpoint for the jvm`
@@ -28,9 +28,9 @@ RUN sudo -u postgres /usr/bin/initdb /var/lib/pgsql/data
 
 # copy configuration related files
 RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/data/
-RUN cp -r /opt/solr-7.7.2/server/solr/configsets/_default /opt/solr-7.7.2/server/solr/collection1
-RUN cp /tmp/dv/schema*.xml /opt/solr-7.7.2/server/solr/collection1/conf/
-RUN cp /tmp/dv/solrconfig.xml /opt/solr-7.7.2/server/solr/collection1/conf/solrconfig.xml
+RUN cp -r /opt/solr-8.8.1/server/solr/configsets/_default /opt/solr-8.8.1/server/solr/collection1
+RUN cp /tmp/dv/schema*.xml /opt/solr-8.8.1/server/solr/collection1/conf/
+RUN cp /tmp/dv/solrconfig.xml /opt/solr-8.8.1/server/solr/collection1/conf/solrconfig.xml
 
 # skipping payara user and solr user (run both as root)
 

conf/docker-aio/entrypoint.bash

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 export LANG=en_US.UTF-8
 sudo -u postgres /usr/bin/pg_ctl start -D /var/lib/pgsql/data &
-cd /opt/solr-7.7.2/
+cd /opt/solr-8.8.1/
 # TODO: Run Solr as non-root and remove "-force".
 bin/solr start -force
 bin/solr create_core -c collection1 -d server/solr/collection1/conf -force

doc/release-notes/7373-solr-upgrade.md

@@ -0,0 +1,22 @@
+### Solr Update
+
+With this release we upgrade to the latest available stable release in the Solr 8.x branch. We recommend a fresh installation of Solr 8.8.1 (the index will be empty)
+followed by an "index all".
+
+Before you start the "index all", Dataverse will appear to be empty because
+the search results come from Solr. As indexing progresses, partial results will 
+appear until indexing is complete.
+
+
+See http://guides.dataverse.org/installation/prerequisites.html#installing-solr
+
+[for the additional upgrade steps section]
+
+Run the script updateSchemaMDB.sh to generate updated solr schema files and preserve any other custom fields in your Solr configuration.
+For example: (modify the path names as needed)
+cd /usr/local/solr-8.8.1/server/solr/collection1/conf
+wget https://github.com/IQSS/dataverse/releases/download/v5.4/updateSchemaMDB.sh
+chmod +x updateSchemaMDB.sh
+./updateSchemaMDB.sh -t .
+
+See http://guides.dataverse.org/en/5.4/admin/metadatacustomization.html?highlight=updateschemamdb for more information.

doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr

@@ -5,7 +5,7 @@
 # chkconfig: 35 92 08
 # description: Starts and stops Apache Solr
 
-SOLR_DIR="/usr/local/solr/solr-7.7.2"
+SOLR_DIR="/usr/local/solr/solr-8.8.1"
 SOLR_COMMAND="bin/solr"
 SOLR_ARGS="-m 1g -j jetty.host=127.0.0.1"
 SOLR_USER=solr

doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service

@@ -5,9 +5,9 @@ After = syslog.target network.target remote-fs.target nss-lookup.target
 [Service]
 User = solr
 Type = forking
-WorkingDirectory = /usr/local/solr/solr-7.7.2
-ExecStart = /usr/local/solr/solr-7.7.2/bin/solr start -m 1g -j "jetty.host=127.0.0.1"
-ExecStop = /usr/local/solr/solr-7.7.2/bin/solr stop
+WorkingDirectory = /usr/local/solr/solr-8.8.1
+ExecStart = /usr/local/solr/solr-8.8.1/bin/solr start -m 1g -j "jetty.host=127.0.0.1"
+ExecStop = /usr/local/solr/solr-8.8.1/bin/solr stop
 LimitNOFILE=65000
 LimitNPROC=65000
 Restart=on-failure

doc/sphinx-guides/source/admin/metadatacustomization.rst

@@ -644,7 +644,7 @@ configuration, including any enabled metadata schemas:
 
 ``curl http://localhost:8080/api/admin/index/solr/schema``
 
-For convenience and automation you can download and consider running :download:`updateSchemaMDB.sh <../../../../conf/solr/7.7.2/updateSchemaMDB.sh>`. It uses the API endpoint above and writes schema files to the filesystem (so be sure to run it on the Solr server itself as the Unix user who owns the Solr files) and then triggers a Solr reload.
+For convenience and automation you can download and consider running :download:`updateSchemaMDB.sh <../../../../conf/solr/8.8.1/updateSchemaMDB.sh>`. It uses the API endpoint above and writes schema files to the filesystem (so be sure to run it on the Solr server itself as the Unix user who owns the Solr files) and then triggers a Solr reload.
 
 By default, it will download from your Dataverse installation at `http://localhost:8080` and reload Solr at `http://localhost:8983`.
 You may use the following environment variables with this script or mix'n'match with options:
@@ -657,13 +657,13 @@ Environment variable  Option  Description                                      E
 `UNBLOCK_KEY`         `-u`    If your installation has a blocked admin API     *xyz* or */secrets/unblock.key*
                               endpoint, you can provide either the key itself
                               or a path to a keyfile
-`TARGET`              `-t`    Provide the config directory of your Solr core   */usr/local/solr/solr-7.7.2/server/solr/collection1/conf*
+`TARGET`              `-t`    Provide the config directory of your Solr core   */usr/local/solr/solr-8.8.1/server/solr/collection1/conf*
                               "collection1"
 ====================  ======  ===============================================  =========================================================
 
 See the :doc:`/installation/prerequisites/` section of the Installation Guide for a suggested location on disk for the Solr schema file.
 
-Please note that if you are going to make a pull request updating ``conf/solr/7.7.2/schema.xml`` with fields you have added, you should first load all the custom metadata blocks in ``scripts/api/data/metadatablocks`` (including ones you don't care about) to create a complete list of fields.
+Please note that if you are going to make a pull request updating ``conf/solr/8.8.1/schema.xml`` with fields you have added, you should first load all the custom metadata blocks in ``scripts/api/data/metadatablocks`` (including ones you don't care about) to create a complete list of fields.
 
 Reloading a Metadata Block
 --------------------------

doc/sphinx-guides/source/developers/dev-environment.rst

@@ -117,7 +117,7 @@ On Linux, you should just install PostgreSQL from your package manager without w
 Install Solr
 ~~~~~~~~~~~~
 
-`Solr <http://lucene.apache.org/solr/>`_ 7.7.2 is required.
+`Solr <http://lucene.apache.org/solr/>`_ 8.8.1 is required.
 
 To install Solr, execute the following commands:
 
@@ -127,27 +127,27 @@ To install Solr, execute the following commands:
 
 ``cd /usr/local/solr``
 
-``curl -O http://archive.apache.org/dist/lucene/solr/7.7.2/solr-7.7.2.tgz``
+``curl -O http://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz``
 
-``tar xvfz solr-7.7.2.tgz``
+``tar xvfz solr-8.8.1.tgz``
 
-``cd solr-7.7.2/server/solr``
+``cd solr-8.8.1/server/solr``
 
 ``cp -r configsets/_default collection1``
 
-``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.7.2/schema.xml``
+``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.8.1/schema.xml``
 
-``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.7.2/schema_dv_mdb_fields.xml``
+``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.8.1/schema_dv_mdb_fields.xml``
 
-``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.7.2/schema_dv_mdb_copies.xml``
+``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.8.1/schema_dv_mdb_copies.xml``
 
 ``mv schema*.xml collection1/conf``
 
-``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.7.2/solrconfig.xml``
+``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.8.1/solrconfig.xml``
 
 ``mv solrconfig.xml collection1/conf/solrconfig.xml``
 
-``cd /usr/local/solr/solr-7.7.2``
+``cd /usr/local/solr/solr-8.8.1``
 
 (Please note that the extra jetty argument below is a security measure to limit connections to Solr to only your computer. For extra security, run a firewall.)
 

doc/sphinx-guides/source/installation/prerequisites.rst

@@ -166,7 +166,7 @@ The Dataverse Software search index is powered by Solr.
 Supported Versions
 ==================
 
-The Dataverse Software has been tested with Solr version 7.7.2. Future releases in the 7.x series are likely to be compatible; however, this cannot be confirmed until they are officially tested. Major releases above 7.x (e.g. 8.x) are not supported.
+The Dataverse Software has been tested with Solr version 8.8.1. Future releases in the 8.x series are likely to be compatible; however, this cannot be confirmed until they are officially tested. Major releases above 8.x (e.g. 9.x) are not supported.
 
 Installing Solr
 ===============
@@ -181,19 +181,19 @@ Become the ``solr`` user and then download and configure Solr::
 
         su - solr
         cd /usr/local/solr
-        wget https://archive.apache.org/dist/lucene/solr/7.7.2/solr-7.7.2.tgz
-        tar xvzf solr-7.7.2.tgz
-        cd solr-7.7.2
+        wget https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz
+        tar xvzf solr-8.8.1.tgz
+        cd solr-8.8.1
         cp -r server/solr/configsets/_default server/solr/collection1
 
 You should already have a "dvinstall.zip" file that you downloaded from https://github.com/IQSS/dataverse/releases . Unzip it into ``/tmp``. Then copy the files into place::
 
-        cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-7.7.2/server/solr/collection1/conf
-        cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-7.7.2/server/solr/collection1/conf
+        cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-8.8.1/server/solr/collection1/conf
+        cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-8.8.1/server/solr/collection1/conf
 
 Note: The Dataverse Project team has customized Solr to boost results that come from certain indexed elements inside the Dataverse installation, for example prioritizing results from Dataverse collections over Datasets. If you would like to remove this, edit your ``solrconfig.xml`` and remove the ``<str name="qf">`` element and its contents. If you have ideas about how this boosting could be improved, feel free to contact us through our Google Group https://groups.google.com/forum/#!forum/dataverse-dev .
 
-A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-7.7.2/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400``
+A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-8.8.1/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400``
 
 Solr will warn about needing to increase the number of file descriptors and max processes in a production environment but will still run with defaults. We have increased these values to the recommended levels by adding ulimit -n 65000 to the init script, and the following to ``/etc/security/limits.conf``::
 
@@ -212,7 +212,7 @@ Solr launches asynchronously and attempts to use the ``lsof`` binary to watch fo
 
 Finally, you need to tell Solr to create the core "collection1" on startup::
 
-        echo "name=collection1" > /usr/local/solr/solr-7.7.2/server/solr/collection1/core.properties
+        echo "name=collection1" > /usr/local/solr/solr-8.8.1/server/solr/collection1/core.properties
 
 Solr Init Script
 ================

downloads/download.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
 curl -L -O https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2020.6/payara-5.2020.6.zip
-curl -L -O https://archive.apache.org/dist/lucene/solr/7.7.2/solr-7.7.2.tgz
+curl -L -O https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz
 curl -L -O https://search.maven.org/remotecontent?filepath=org/jboss/weld/weld-osgi-bundle/2.2.10.Final/weld-osgi-bundle-2.2.10.Final-glassfish4.jar
 curl -s -L http://sourceforge.net/projects/schemaspy/files/schemaspy/SchemaSpy%205.0.0/schemaSpy_5.0.0.jar/download > schemaSpy_5.0.0.jar

pom.xml

@@ -325,7 +325,7 @@
         <dependency>
             <groupId>org.apache.solr</groupId>
             <artifactId>solr-solrj</artifactId>
-            <version>7.7.2</version>
+            <version>8.8.1</version>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>

scripts/installer/Makefile

@@ -56,13 +56,13 @@ ${JHOVE_SCHEMA}:	../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR}
 	@echo copying jhove schema file
 	/bin/cp ../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR}
 
-${SOLR_SCHEMA}:		../../conf/solr/7.7.2/schema.xml ../../conf/solr/7.7.2/schema_dv_mdb_fields.xml ../../conf/solr/7.7.2/schema_dv_mdb_copies.xml ../../conf/solr/7.7.2/updateSchemaMDB.sh ${INSTALLER_ZIP_DIR}
+${SOLR_SCHEMA}:		../../conf/solr/8.8.1/schema.xml ../../conf/solr/8.8.1/schema_dv_mdb_fields.xml ../../conf/solr/8.8.1/schema_dv_mdb_copies.xml ../../conf/solr/8.8.1/updateSchemaMDB.sh ${INSTALLER_ZIP_DIR}
 	@echo copying Solr schema file
-	/bin/cp ../../conf/solr/7.7.2/schema*.xml ../../conf/solr/7.7.2/updateSchemaMDB.sh ${INSTALLER_ZIP_DIR}
+	/bin/cp ../../conf/solr/8.8.1/schema*.xml ../../conf/solr/8.8.1/updateSchemaMDB.sh ${INSTALLER_ZIP_DIR}
 
-${SOLR_CONFIG}:		../../conf/solr/7.7.2/solrconfig.xml ${INSTALLER_ZIP_DIR}
+${SOLR_CONFIG}:		../../conf/solr/8.8.1/solrconfig.xml ${INSTALLER_ZIP_DIR}
 	@echo copying Solr config file
-	/bin/cp ../../conf/solr/7.7.2/solrconfig.xml ${INSTALLER_ZIP_DIR}
+	/bin/cp ../../conf/solr/8.8.1/solrconfig.xml ${INSTALLER_ZIP_DIR}
 
 ${PYTHON_FILES}: README_python.txt install.py installConfig.py installAppServer.py installUtils.py requirements.txt default.config interactive.config ${INSTALLER_ZIP_DIR}
 	@echo copying Python installer files

scripts/installer/README.txt

@@ -35,7 +35,7 @@ from conf/jhove:
 
 jhove.conf
 
-SOLR schema and config files, from conf/solr/7.7.2: 
+SOLR schema and config files, from conf/solr/8.8.1: 
 
 schema.xml
 schema_dv_cmb_copies.xml

scripts/vagrant/setup-solr.sh

@@ -5,12 +5,12 @@ SOLR_USER=solr
 SOLR_HOME=/usr/local/solr
 mkdir $SOLR_HOME
 chown $SOLR_USER:$SOLR_USER $SOLR_HOME
-su $SOLR_USER -s /bin/sh -c "cp /dataverse/downloads/solr-7.7.2.tgz $SOLR_HOME"
-su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-7.7.2.tgz"
-su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.7.2/server/solr && cp -r configsets/_default . && mv _default collection1"
-su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/7.7.2/schema*.xml $SOLR_HOME/solr-7.7.2/server/solr/collection1/conf/"
-su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/7.7.2/solrconfig.xml $SOLR_HOME/solr-7.7.2/server/solr/collection1/conf/solrconfig.xml"
-su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.7.2 && bin/solr start && bin/solr create_core -c collection1 -d server/solr/collection1/conf/"
+su $SOLR_USER -s /bin/sh -c "cp /dataverse/downloads/solr-8.8.1.tgz $SOLR_HOME"
+su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-8.8.1.tgz"
+su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-8.8.1/server/solr && cp -r configsets/_default . && mv _default collection1"
+su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/8.8.1/schema*.xml $SOLR_HOME/solr-8.8.1/server/solr/collection1/conf/"
+su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/8.8.1/solrconfig.xml $SOLR_HOME/solr-8.8.1/server/solr/collection1/conf/solrconfig.xml"
+su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-8.8.1 && bin/solr start && bin/solr create_core -c collection1 -d server/solr/collection1/conf/"
 cp /dataverse/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr /etc/init.d/solr
 chmod 755 /etc/init.d/solr
 /etc/init.d/solr stop

scripts/vagrant/setup.sh

@@ -54,7 +54,7 @@ echo "Ensuring Unix user '$SOLR_USER' exists"
 useradd $SOLR_USER || :
 DOWNLOAD_DIR='/dataverse/downloads'
 PAYARA_ZIP="$DOWNLOAD_DIR/payara-5.2020.6.zip"
-SOLR_TGZ="$DOWNLOAD_DIR/solr-7.7.2.tgz"
+SOLR_TGZ="$DOWNLOAD_DIR/solr-8.8.1.tgz"
 if [ ! -f $PAYARA_ZIP ] || [ ! -f $SOLR_TGZ ]; then
     echo "Couldn't find $PAYARA_ZIP or $SOLR_TGZ! Running download script...."
     cd $DOWNLOAD_DIR && ./download.sh && cd