Skip to content

4579 solr 7.3.0 #4621

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kcondon merged 4 commits into from
Apr 30, 2018
Merged

4579 solr 7.3.0 #4621

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ce05eb1
Bump solr version #4579
matthew-a-dunlap Apr 25, 2018
db10923
Point to correct docker location in guides #4579
matthew-a-dunlap Apr 25, 2018
e74f8cd
Add docker-aio file missed via .gitignore #4579
Apr 25, 2018
4f0d7e4
Merge branch 'develop' into 4579-solr-7.3.0
matthew-a-dunlap Apr 25, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion PULL_REQUEST_TEMPLATE.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,6 @@ Welcome! New contributors should at least glance at [CONTRIBUTING.md](/CONTRIBUT

[tests]: http://guides.dataverse.org/en/latest/developers/testing.html
[SQL updates]: https://github.com/IQSS/dataverse/tree/develop/scripts/database/upgrades
[Solr updates]: https://github.com/IQSS/dataverse/blob/develop/conf/solr/7.2.1/schema.xml
[Solr updates]: https://github.com/IQSS/dataverse/blob/develop/conf/solr/7.3.0/schema.xml
[docs]: http://guides.dataverse.org/en/latest/developers/documentation.html
[branch]: http://guides.dataverse.org/en/latest/developers/branching-strategy.html
4 changes: 2 additions & 2 deletions conf/docker-aio/0prep_deps.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@ if [ ! -e dv/deps/glassfish4dv.tgz ]; then
# assuming that folks usually have /tmp auto-clean as needed
fi

if [ ! -e dv/deps/solr-7.2.1dv.tgz ]; then
if [ ! -e dv/deps/solr-7.3.0dv.tgz ]; then
echo "solr dependency prep"
# schema changes *should* be the only ones...
cd dv/deps/
wget https://archive.apache.org/dist/lucene/solr/7.2.1/solr-7.2.1.tgz -O solr-7.2.1dv.tgz
wget https://archive.apache.org/dist/lucene/solr/7.3.0/solr-7.3.0.tgz -O solr-7.3.0dv.tgz
cd ../../
fi

4 changes: 2 additions & 2 deletions conf/docker-aio/1prep.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ export LANG="en_US.UTF-8"
# this was based off the phoenix deployment; and is likely uglier and bulkier than necessary in a perfect world

mkdir -p testdata/doc/sphinx-guides/source/_static/util/
cp ../solr/7.2.1/schema.xml testdata/
cp ../solr/7.2.1/solrconfig.xml testdata/
cp ../solr/7.3.0/schema.xml testdata/
cp ../solr/7.3.0/solrconfig.xml testdata/
cp ../jhove/jhove.conf testdata/
cd ../../
cp -r scripts conf/docker-aio/testdata/
Expand Down
8 changes: 4 additions & 4 deletions conf/docker-aio/c7.dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ COPY testdata/solrconfig.xml /tmp/dv
COPY disableipv6.conf /etc/sysctl.d/
RUN rm /etc/httpd/conf/*
COPY testdata/httpd.conf /etc/httpd/conf
RUN cd /opt ; tar zxf /tmp/dv/deps/solr-7.2.1dv.tgz
RUN cd /opt ; tar zxf /tmp/dv/deps/solr-7.3.0dv.tgz
RUN cd /opt ; tar zxf /tmp/dv/deps/glassfish4dv.tgz

# this copy of domain.xml is the result of running `asadmin set server.monitoring-service.module-monitoring-levels.jvm=LOW` on a default glassfish installation (aka - enable the glassfish REST monitir endpoint for the jvm`
Expand All @@ -21,9 +21,9 @@ RUN sudo -u postgres /usr/bin/initdb -D /var/lib/pgsql/data

# copy configuration related files
RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/data/
RUN cp -r /opt/solr-7.2.1/server/solr/configsets/_default /opt/solr-7.2.1/server/solr/collection1
RUN cp /tmp/dv/schema.xml /opt/solr-7.2.1/server/solr/collection1/conf/schema.xml
RUN cp /tmp/dv/solrconfig.xml /opt/solr-7.2.1/server/solr/collection1/conf/solrconfig.xml
RUN cp -r /opt/solr-7.3.0/server/solr/configsets/_default /opt/solr-7.3.0/server/solr/collection1
RUN cp /tmp/dv/schema.xml /opt/solr-7.3.0/server/solr/collection1/conf/schema.xml
RUN cp /tmp/dv/solrconfig.xml /opt/solr-7.3.0/server/solr/collection1/conf/solrconfig.xml

# skipping glassfish user and solr user (run both as root)

Expand Down
2 changes: 1 addition & 1 deletion conf/docker-aio/entrypoint.bash
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/usr/bin/env bash

sudo -u postgres /usr/bin/postgres -D /var/lib/pgsql/data &
cd /opt/solr-7.2.1/
cd /opt/solr-7.3.0/
# TODO: Run Solr as non-root and remove "-force".
bin/solr start -force
bin/solr create_core -c collection1 -d server/solr/collection1/conf -force
Expand Down
30 changes: 30 additions & 0 deletions conf/docker-aio/testdata/httpd.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@

Include conf.d/*.conf
Include conf.modules.d/*.conf
ServerName localhost
Listen 80 443
PidFile run/httpd.pid
DocumentRoot "/var/www/html"
TypesConfig /etc/mime.types
User apache
Group apache

<VirtualHost *:80>
ServerName localhost
LogLevel debug
ErrorLog logs/error_log
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
CustomLog logs/access_log combined

# proxy config (aka - what to send to glassfish or not)
ProxyPassMatch ^/RApacheInfo$ !
ProxyPassMatch ^/custom !
ProxyPassMatch ^/dataexplore !
ProxyPassMatch ^/Shibboleth.sso !
ProxyPassMatch ^/shibboleth-ds !
# pass everything else to Glassfish
ProxyPass / ajp://localhost:8009/
# glassfish can be slow sometimes
ProxyTimeout 300

</VirtualHost>
Copy link
Copy Markdown
Member

@pdurbin pdurbin Apr 25, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is this file for?

Copy link
Copy Markdown
Contributor Author

@matthew-a-dunlap matthew-a-dunlap Apr 25, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Its a file that was missing from the docker-aio folder. .gitignore has blocked it when the configuration was last committed.

Copy link
Copy Markdown
Member

@pdurbin pdurbin Apr 25, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh. Ok. Thanks for explaining this.

8 changes: 4 additions & 4 deletions conf/docker/solr/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@ MAINTAINER Dataverse (support@dataverse.org)

RUN yum install -y wget unzip perl git java-1.8.0-openjdk-devel postgresql.x86_64

# Install Solr 7.2.1
# Install Solr 7.3.0
# The context of the build is the "conf" directory.
COPY solr/7.2.1/schema.xml /tmp
COPY solr/7.2.1/solrconfig.xml /tmp
COPY solr/7.3.0/schema.xml /tmp
COPY solr/7.3.0/solrconfig.xml /tmp

RUN cd /tmp && wget https://archive.apache.org/dist/lucene/solr/7.2.1/solr-7.2.1.tgz
RUN cd /tmp && wget https://archive.apache.org/dist/lucene/solr/7.3.0/solr-7.3.0.tgz

EXPOSE 8983

Expand Down
4 changes: 2 additions & 2 deletions conf/docker/solr/entrypoint.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@

# FIXME: Don't run Solr out of /tmp!
# Solr is /tmp to avoid AccessDeniedException under Minishift/OpenShift.
SOLR_DIR=/tmp/solr-7.2.1
SOLR_DIR=/tmp/solr-7.3.0

if [ "$1" = 'solr' ]; then
cd /tmp
tar xvfz solr-7.2.1.tgz
tar xvfz solr-7.3.0.tgz
cp -r $SOLR_DIR/server/solr/configsets/_default $SOLR_DIR/server/solr/collection1
cp /tmp/schema.xml $SOLR_DIR/server/solr/collection1/conf
cp /tmp/solrconfig.xml $SOLR_DIR/server/solr/collection1/conf
Expand Down
0 conf/solr/7.2.1/readme.me → conf/solr/7.3.0/readme.me
View file
Open in desktop
File renamed without changes.
2 changes: 1 addition & 1 deletion conf/solr/7.2.1/schema.xml → conf/solr/7.3.0/schema.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@

<!-- catchall field, containing all other searchable text fields (implemented
via copyField further on in this schema -->
<!-- Dataverse solr 7.2.1: for some reason the old text wasn't working so switched to _text_ for copyfields -->
<!-- Dataverse solr 7.3.0: for some reason the old text wasn't working so switched to _text_ for copyfields -->
<!--<field name="text" type="text_general" indexed="true" stored="false" multiValued="true"/>-->
<field name="_text_" type="text_general" indexed="true" stored="false" multiValued="true"/>
<!-- catchall text field that indexes tokens both normally and in reverse for efficient
Expand Down
2 changes: 1 addition & 1 deletion conf/solr/7.2.1/solrconfig.xml → conf/solr/7.3.0/solrconfig.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
that you fully re-index after changing this setting as it can
affect both how text is indexed and queried.
-->
<luceneMatchVersion>7.2.1</luceneMatchVersion>
<luceneMatchVersion>7.3.0</luceneMatchVersion>

<!-- <lib/> directives can be used to instruct Solr to load any Jars
identified and use them to resolve any "plugins" specified in
Expand Down
2 changes: 1 addition & 1 deletion doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
# chkconfig: 35 92 08
# description: Starts and stops Apache Solr

SOLR_DIR="/usr/local/solr/solr-7.2.1"
SOLR_DIR="/usr/local/solr/solr-7.3.0"
SOLR_COMMAND="bin/solr"
SOLR_ARGS="-m 1g"
SOLR_USER=solr
Expand Down
4 changes: 2 additions & 2 deletions doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ After = syslog.target network.target remote-fs.target nss-lookup.target
[Service]
User = solr
Type = simple
WorkingDirectory = /usr/local/solr/solr-7.2.1
ExecStart = /usr/local/solr/solr-7.2.1/bin/solr start -m 1g
WorkingDirectory = /usr/local/solr/solr-7.3.0
ExecStart = /usr/local/solr/solr-7.3.0/bin/solr start -m 1g
Restart=on-failure

[Install]
Expand Down
14 changes: 7 additions & 7 deletions doc/sphinx-guides/source/developers/dev-environment.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ On Linux, you should just install PostgreSQL from your package manager without w
Install Solr
~~~~~~~~~~~~

`Solr <http://lucene.apache.org/solr/>`_ 7.2.1 is required.
`Solr <http://lucene.apache.org/solr/>`_ 7.3.0 is required.

To install Solr, execute the following commands:

Expand All @@ -117,23 +117,23 @@ To install Solr, execute the following commands:

``cd /usr/local/solr``

``curl -O http://archive.apache.org/dist/lucene/solr/7.2.1/solr-7.2.1.tgz``
``curl -O http://archive.apache.org/dist/lucene/solr/7.3.0/solr-7.3.0.tgz``

``tar xvfz solr-7.2.1.tgz``
``tar xvfz solr-7.3.0.tgz``

``cd solr-7.2.1/server/solr``
``cd solr-7.3.0/server/solr``

``cp -r configsets/_default collection1``

``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.2.1/schema.xml``
``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.3.0/schema.xml``

``mv schema.xml collection1/conf``

``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.2.1/solrconfig.xml``
``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.3.0/solrconfig.xml``

``mv solrconfig.xml collection1/conf/solrconfig.xml``

``cd /usr/local/solr/solr-7.2.1``
``cd /usr/local/solr/solr-7.3.0``

``bin/solr start``

Expand Down
2 changes: 1 addition & 1 deletion doc/sphinx-guides/source/developers/testing.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ Unit tests are run automatically on every build, but dev environments and server

The :doc:`dev-environment` section currently refers developers here for advice on getting set up to run REST Assured tests, but we'd like to add some sort of "dev" flag to the installer to put Dataverse in "insecure" mode, with lots of scary warnings that this dev mode should not be used in production.

The instructions below assume a relatively static dev environment on a Mac. There is a newer "all in one" Docker-based approach documented in the :doc:`dev-environment` section under "Docker" that you may like to play with as well.
The instructions below assume a relatively static dev environment on a Mac. There is a newer "all in one" Docker-based approach documented in the :doc:`/developers/containers` section under "Docker" that you may like to play with as well.

The Burrito Key
^^^^^^^^^^^^^^^
Expand Down
14 changes: 7 additions & 7 deletions doc/sphinx-guides/source/installation/prerequisites.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,23 +176,23 @@ Become the ``solr`` user and then download and configure Solr::

su - solr
cd /usr/local/solr
wget https://archive.apache.org/dist/lucene/solr/7.2.1/solr-7.2.1.tgz
tar xvzf solr-7.2.1.tgz
cd solr-7.2.1
wget https://archive.apache.org/dist/lucene/solr/7.3.0/solr-7.3.0.tgz
tar xvzf solr-7.3.0.tgz
cd solr-7.3.0
cp -r server/solr/configsets/_default server/solr/collection1

You should already have a "dvinstall.zip" file that you downloaded from https://github.com/IQSS/dataverse/releases . Unzip it into ``/tmp``. Then copy the files into place::

cp /tmp/dvinstall/schema.xml /usr/local/solr/solr-7.2.1/server/solr/collection1/conf
cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-7.2.1/server/solr/collection1/conf
cp /tmp/dvinstall/schema.xml /usr/local/solr/solr-7.3.0/server/solr/collection1/conf
cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-7.3.0/server/solr/collection1/conf

Note: Dataverse has customized Solr to boost results that come from certain indexed elements inside Dataverse, for example results matching on the name of a dataset. If you would like to remove this, edit your ``solrconfig.xml`` and remove the ``<str name="qf">`` element and its contents.

Dataverse requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-7.2.1/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400``
Dataverse requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-7.3.0/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400``

With the Dataverse-specific config in place, you can now start Solr and create the core that will be used to manage search information::

cd /usr/local/solr/solr-7.2.1
cd /usr/local/solr/solr-7.3.0
bin/solr start
bin/solr create_core -c collection1 -d server/solr/collection1/conf/

Expand Down
2 changes: 1 addition & 1 deletion downloads/.gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
glassfish-4.1.zip
solr-7.2.1.tgz
solr-7.3.0.tgz
weld-osgi-bundle-2.2.10.Final-glassfish4.jar
schemaSpy_5.0.0.jar
2 changes: 1 addition & 1 deletion downloads/download.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/bin/sh
curl -L -O http://download.java.net/glassfish/4.1/release/glassfish-4.1.zip
curl -L -O https://archive.apache.org/dist/lucene/solr/7.2.1/solr-7.2.1.tgz
curl -L -O https://archive.apache.org/dist/lucene/solr/7.3.0/solr-7.3.0.tgz
curl -L -O http://search.maven.org/remotecontent?filepath=org/jboss/weld/weld-osgi-bundle/2.2.10.Final/weld-osgi-bundle-2.2.10.Final-glassfish4.jar
curl -s -L http://sourceforge.net/projects/schemaspy/files/schemaspy/SchemaSpy%205.0.0/schemaSpy_5.0.0.jar/download > schemaSpy_5.0.0.jar
2 changes: 1 addition & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
<version>7.2.1</version>
<version>7.3.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
Expand Down
8 changes: 4 additions & 4 deletions scripts/installer/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,12 @@ ${JHOVE_SCHEMA}: ../../conf/jhove/jhoveConfig.xsd
@mkdir -p ${INSTALLER_ZIP_DIR}
/bin/cp ../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR}

${SOLR_SCHEMA}: ../../conf/solr/7.2.1/schema.xml
${SOLR_SCHEMA}: ../../conf/solr/7.3.0/schema.xml
@echo copying Solr schema file
@mkdir -p ${INSTALLER_ZIP_DIR}
/bin/cp ../../conf/solr/7.2.1/schema.xml ${INSTALLER_ZIP_DIR}
/bin/cp ../../conf/solr/7.3.0/schema.xml ${INSTALLER_ZIP_DIR}

${SOLR_CONFIG}: ../../conf/solr/7.2.1/solrconfig.xml
${SOLR_CONFIG}: ../../conf/solr/7.3.0/solrconfig.xml
@echo copying Solr config file
@mkdir -p ${INSTALLER_ZIP_DIR}
/bin/cp ../../conf/solr/7.2.1/solrconfig.xml ${INSTALLER_ZIP_DIR}
/bin/cp ../../conf/solr/7.3.0/solrconfig.xml ${INSTALLER_ZIP_DIR}
2 changes: 1 addition & 1 deletion scripts/installer/README.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ from conf/jhove:

jhove.conf

SOLR schema and config files, from conf/solr/7.2.1:
SOLR schema and config files, from conf/solr/7.3.0:

schema.xml
solrconfig.xml
12 changes: 6 additions & 6 deletions scripts/vagrant/setup-solr.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@ SOLR_USER=solr
SOLR_HOME=/usr/local/solr
mkdir $SOLR_HOME
chown $SOLR_USER:$SOLR_USER $SOLR_HOME
su $SOLR_USER -s /bin/sh -c "cp /downloads/solr-7.2.1.tgz $SOLR_HOME"
su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-7.2.1.tgz"
su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.2.1/server/solr && cp -r configsets/_default . && mv _default collection1"
su $SOLR_USER -s /bin/sh -c "cp /conf/solr/7.2.1/schema.xml $SOLR_HOME/solr-7.2.1/server/solr/collection1/conf/schema.xml"
su $SOLR_USER -s /bin/sh -c "cp /conf/solr/7.2.1/solrconfig.xml $SOLR_HOME/solr-7.2.1/server/solr/collection1/conf/solrconfig.xml"
su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.2.1 && bin/solr start && bin/solr create_core -c collection1 -d server/solr/collection1/conf/"
su $SOLR_USER -s /bin/sh -c "cp /downloads/solr-7.3.0.tgz $SOLR_HOME"
su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-7.3.0.tgz"
su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.3.0/server/solr && cp -r configsets/_default . && mv _default collection1"
su $SOLR_USER -s /bin/sh -c "cp /conf/solr/7.3.0/schema.xml $SOLR_HOME/solr-7.3.0/server/solr/collection1/conf/schema.xml"
su $SOLR_USER -s /bin/sh -c "cp /conf/solr/7.3.0/solrconfig.xml $SOLR_HOME/solr-7.3.0/server/solr/collection1/conf/solrconfig.xml"
su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.3.0 && bin/solr start && bin/solr create_core -c collection1 -d server/solr/collection1/conf/"
cp /dataverse/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr /etc/init.d/solr
chmod 755 /etc/init.d/solr
/etc/init.d/solr stop
Expand Down
2 changes: 1 addition & 1 deletion scripts/vagrant/setup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ echo "Ensuring Unix user '$SOLR_USER' exists"
useradd $SOLR_USER || :
DOWNLOAD_DIR='/dataverse/downloads'
GLASSFISH_ZIP="$DOWNLOAD_DIR/glassfish-4.1.zip"
SOLR_TGZ="$DOWNLOAD_DIR/solr-7.2.1.tgz"
SOLR_TGZ="$DOWNLOAD_DIR/solr-7.3.0.tgz"
WELD_PATCH="$DOWNLOAD_DIR/weld-osgi-bundle-2.2.10.Final-glassfish4.jar"
if [ ! -f $GLASSFISH_ZIP ] || [ ! -f $SOLR_TGZ ]; then
echo "Couldn't find $GLASSFISH_ZIP or $SOLR_TGZ! Running download script...."
Expand Down