IQSS · pdurbin · Oct 9, 2018 · May 31, 2018 · Jun 25, 2018 · Jun 28, 2018
diff --git a/conf/docker-dcm/0prep.sh b/conf/docker-dcm/0prep.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
 
-wget https://github.com/sbgrid/data-capture-module/releases/download/0.2/dcm-0.2-0.noarch.rpm
+wget https://github.com/sbgrid/data-capture-module/releases/download/0.3/dcm-0.3-0.noarch.rpm
diff --git a/conf/docker-dcm/dcmsrv.dockerfile b/conf/docker-dcm/dcmsrv.dockerfile
@@ -1,7 +1,7 @@
 # build from repo root
 FROM centos:6
 RUN yum install -y epel-release
-ARG RPMFILE=dcm-0.2-0.noarch.rpm
+ARG RPMFILE=dcm-0.3-0.noarch.rpm
 COPY ${RPMFILE} /tmp/
 COPY bashrc /root/.bashrc
 COPY test_install.sh /root/

diff --git a/conf/docker/solr/solrconfig_master.xml b/conf/docker/solr/solrconfig_master.xml
@@ -686,13 +686,55 @@
        of SearchComponents (see below) and supports distributed
        queries across multiple shards
     -->
-  <requestHandler name="/select" class="solr.SearchHandler">
+   <requestHandler name="/select" class="solr.SearchHandler">
     <!-- default values for query parameters can be specified, these
          will be overridden by parameters in the request
       -->
     <lst name="defaults">
       <str name="echoParams">explicit</str>
       <int name="rows">10</int>
+      <str name="defType">edismax</str>
+      <float name="tie">0.075</float>
+        <str name="qf">
+            dvName^400
+            authorName^180
+            dvSubject^190
+            dvDescription^180
+            dvAffiliation^170
+            title^130
+            subject^120
+            keyword^110
+            topicClassValue^100
+            dsDescriptionValue^90
+            authorAffiliation^80
+            publicationCitation^60
+            producerName^50
+            fileName^30
+            fileDescription^30
+            variableLabel^20
+            variableName^10
+            text^1.0
+        </str>
+        <str name="pf">
+            dvName^200
+            authorName^100
+            dvSubject^100
+            dvDescription^100
+            dvAffiliation^100
+            title^75
+            subject^75
+            keyword^75
+            topicClassValue^75
+            dsDescriptionValue^75
+            authorAffiliation^75
+            publicationCitation^75
+            producerName^75
+        </str>
+        <!-- Even though this number is huge it only seems to apply a boost of ~1.5x to final result -MAD 4.9.3 -->
+        <str name="bq">
+            isHarvested:false^25000
+        </str>
+
       <!-- Default search field
          <str name="df">text</str> 
         -->
@@ -924,27 +966,6 @@
       <str name="spellcheck.collateExtendedResults">true</str>
       <str name="spellcheck.maxCollationTries">10</str>
       <str name="spellcheck.maxCollations">5</str>
-      <str name="defType">edismax</str>
-<str name="qf">
-dvName^170
-dvSubject^160
-dvDescription^150
-dvAffiliation^140
-title^130
-subject^120
-keyword^110
-topicClassValue^100
-dsDescriptionValue^90
-authorName^80
-authorAffiliation^70
-publicationCitation^60
-producerName^50
-fileName^40
-fileDescription^30
-variableLabel^20
-variableName^10
-text^1.0
-</str>
     </lst>
     <arr name="last-components">
       <str>spellcheck</str>

diff --git a/conf/docker/solr/solrconfig_slave.xml b/conf/docker/solr/solrconfig_slave.xml
@@ -686,13 +686,55 @@
        of SearchComponents (see below) and supports distributed
        queries across multiple shards
     -->
-  <requestHandler name="/select" class="solr.SearchHandler">
+   <requestHandler name="/select" class="solr.SearchHandler">
     <!-- default values for query parameters can be specified, these
          will be overridden by parameters in the request
       -->
     <lst name="defaults">
       <str name="echoParams">explicit</str>
       <int name="rows">10</int>
+      <str name="defType">edismax</str>
+      <float name="tie">0.075</float>
+        <str name="qf">
+            dvName^400
+            authorName^180
+            dvSubject^190
+            dvDescription^180
+            dvAffiliation^170
+            title^130
+            subject^120
+            keyword^110
+            topicClassValue^100
+            dsDescriptionValue^90
+            authorAffiliation^80
+            publicationCitation^60
+            producerName^50
+            fileName^30
+            fileDescription^30
+            variableLabel^20
+            variableName^10
+            text^1.0
+        </str>
+        <str name="pf">
+            dvName^200
+            authorName^100
+            dvSubject^100
+            dvDescription^100
+            dvAffiliation^100
+            title^75
+            subject^75
+            keyword^75
+            topicClassValue^75
+            dsDescriptionValue^75
+            authorAffiliation^75
+            publicationCitation^75
+            producerName^75
+        </str>
+        <!-- Even though this number is huge it only seems to apply a boost of ~1.5x to final result -MAD 4.9.3 -->
+        <str name="bq">
+            isHarvested:false^25000
+        </str>
+
       <!-- Default search field
          <str name="df">text</str> 
         -->
@@ -924,27 +966,6 @@
       <str name="spellcheck.collateExtendedResults">true</str>
       <str name="spellcheck.maxCollationTries">10</str>
       <str name="spellcheck.maxCollations">5</str>
-      <str name="defType">edismax</str>
-<str name="qf">
-dvName^170
-dvSubject^160
-dvDescription^150
-dvAffiliation^140
-title^130
-subject^120
-keyword^110
-topicClassValue^100
-dsDescriptionValue^90
-authorName^80
-authorAffiliation^70
-publicationCitation^60
-producerName^50
-fileName^40
-fileDescription^30
-variableLabel^20
-variableName^10
-text^1.0
-</str>
     </lst>
     <arr name="last-components">
       <str>spellcheck</str>

diff --git a/conf/solr/7.3.0/solrconfig.xml b/conf/solr/7.3.0/solrconfig.xml
@@ -693,6 +693,48 @@
     <lst name="defaults">
       <str name="echoParams">explicit</str>
       <int name="rows">10</int>
+      <str name="defType">edismax</str>
+      <float name="tie">0.075</float>
+        <str name="qf">
+            dvName^400
+            authorName^180
+            dvSubject^190
+            dvDescription^180
+            dvAffiliation^170
+            title^130
+            subject^120
+            keyword^110
+            topicClassValue^100
+            dsDescriptionValue^90
+            authorAffiliation^80
+            publicationCitation^60
+            producerName^50
+            fileName^30
+            fileDescription^30
+            variableLabel^20
+            variableName^10
+            text^1.0
+        </str>
+        <str name="pf">
+            dvName^200
+            authorName^100
+            dvSubject^100
+            dvDescription^100
+            dvAffiliation^100
+            title^75
+            subject^75
+            keyword^75
+            topicClassValue^75
+            dsDescriptionValue^75
+            authorAffiliation^75
+            publicationCitation^75
+            producerName^75
+        </str>
+        <!-- Even though this number is huge it only seems to apply a boost of ~1.5x to final result -MAD 4.9.3 -->
+        <str name="bq">
+            isHarvested:false^25000
+        </str>
+
       <!-- Default search field
          <str name="df">text</str> 
         -->
@@ -924,27 +966,6 @@
       <str name="spellcheck.collateExtendedResults">true</str>
       <str name="spellcheck.maxCollationTries">10</str>
       <str name="spellcheck.maxCollations">5</str>
-      <str name="defType">edismax</str>
-<str name="qf">
-dvName^170
-dvSubject^160
-dvDescription^150
-dvAffiliation^140
-title^130
-subject^120
-keyword^110
-topicClassValue^100
-dsDescriptionValue^90
-authorName^80
-authorAffiliation^70
-publicationCitation^60
-producerName^50
-fileName^40
-fileDescription^30
-variableLabel^20
-variableName^10
-text^1.0
-</str>
     </lst>
     <arr name="last-components">
       <str>spellcheck</str>

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
@@ -513,11 +513,11 @@ Dataset Locks
 
 To check if a dataset is locked:: 
 
-    curl -H "$SERVER_URL/api/datasets/{database_id}/locks
+    curl "$SERVER_URL/api/datasets/{database_id}/locks
 
 Optionally, you can check if there's a lock of a specific type on the dataset:: 
 
-    curl -H "$SERVER_URL/api/datasets/{database_id}/locks?type={lock_type}
+    curl "$SERVER_URL/api/datasets/{database_id}/locks?type={lock_type}
 
 Currently implemented lock types are ``Ingest, Workflow, InReview, DcmUpload and pidRegister``. 
 

diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py
@@ -64,9 +64,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '4.9.3'
+version = '4.9.4'
 # The full version, including alpha/beta/rc tags.
-release = '4.9.3'
+release = '4.9.4'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst
@@ -25,7 +25,7 @@ Please note that S3 support for DCM is highly experimental. Files can be uploade
 Once you have installed a DCM, you will need to configure two database settings on the Dataverse side. These settings are documented in the :doc:`/installation/config` section of the Installation Guide:
 
 - ``:DataCaptureModuleUrl`` should be set to the URL of a DCM you installed.
-- ``:UploadMethods`` should be set to ``dcm/rsync+ssh``.
+- ``:UploadMethods`` should include ``dcm/rsync+ssh``.
 
 This will allow your Dataverse installation to communicate with your DCM, so that Dataverse can download rsync scripts for your users.
 
@@ -155,7 +155,7 @@ In order to see the rsync URLs, you must run this command:
 
 ``curl -X PUT -d 'rsal/rsync' http://localhost:8080/api/admin/settings/:DownloadMethods``
 
-TODO: Document these in the Installation Guide once they're final.
+..  TODO: Document these in the Installation Guide once they're final.
 
 To specify replication sites that appear in rsync URLs:
 

diff --git a/doc/sphinx-guides/source/installation/advanced.rst b/doc/sphinx-guides/source/installation/advanced.rst
@@ -10,7 +10,11 @@ Advanced installations are not officially supported but here we are at least doc
 Multiple Glassfish Servers
 --------------------------
 
-The main thing to know about running multiple Glassfish servers is that only one can be the dedicated timer server, as explained in the :doc:`/admin/timers` section of the Admin Guide.
+You should be conscious of the following when running multiple Glassfish servers.
+
+- Only one Glassfish server can be the dedicated timer server, as explained in the :doc:`/admin/timers` section of the Admin Guide.
+- When users upload a logo for their dataverse using the "theme" feature described in the :doc:`/user/dataverse-management` section of the User Guide, these logos are stored only on the Glassfish server the user happend to be on when uploading the logo. By default these logos are written to the directory ``/usr/local/glassfish4/glassfish/domains/domain1/docroot/logos``.
+- When a sitemp is created by a Glassfish server it is written to the filesystem of just that Glassfish server. By default the sitemap is written to the directory ``/usr/local/glassfish4/glassfish/domains/domain1/docroot/sitemap``.
 
 Detecting Which Glassfish Server a User Is On
 +++++++++++++++++++++++++++++++++++++++++++++

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
@@ -26,6 +26,8 @@ The :doc:`/api/native-api` contains a useful but potentially dangerous API endpo
 
 By default, all APIs can be operated on remotely and a number of endpoints do not require authentication. https://github.com/IQSS/dataverse/issues/1886 was opened to explore changing these defaults, but until then it is very important to block both the "admin" endpoint (and at least consider blocking ``builtin-users``). For details please see also the section on ``:BlockedApiPolicy`` below.
 
+It's also possible to prevent file uploads via API by adjusting the ``:UploadMethods`` database setting.
+
 Forcing HTTPS
 +++++++++++++
 
@@ -423,6 +425,9 @@ Out of the box, Dataverse attempts to block search engines from crawling your in
 Letting Search Engines Crawl Your Installation
 ++++++++++++++++++++++++++++++++++++++++++++++
 
+Ensure robots.txt Is Not Blocking Search Engines
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
 For a public production Dataverse installation, it is probably desired that search agents be able to index published pages (AKA - pages that are visible to an unauthenticated user).
 Polite crawlers usually respect the `Robots Exclusion Standard <https://en.wikipedia.org/wiki/Robots_exclusion_standard>`_; we have provided an example of a production robots.txt :download:`here </_static/util/robots.txt>`).
 
@@ -437,6 +442,25 @@ For more of an explanation of ``ProxyPassMatch`` see the :doc:`shibboleth` secti
 
 If you are not fronting Glassfish with Apache you'll need to prevent Glassfish from serving the robots.txt file embedded in the war file by overwriting robots.txt after the war file has been deployed. The downside of this technique is that you will have to remember to overwrite robots.txt in the "exploded" war file each time you deploy the war file, which probably means each time you upgrade to a new version of Dataverse. Furthermore, since the version of Dataverse is always incrementing and the version can be part of the file path, you will need to be conscious of where on disk you need to replace the file. For example, for Dataverse 4.6.1 the path to robots.txt may be ``/usr/local/glassfish4/glassfish/domains/domain1/applications/dataverse-4.6.1/robots.txt`` with the version number ``4.6.1`` as part of the path.
 
+Creating a Sitemap and Submitting it to Search Engines
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Search engines have an easier time indexing content when you provide them a sitemap. The Dataverse sitemap includes URLs to all published dataverses and all published datasets that are not harvested or deaccessioned.
+
+Create or update your sitemap by adding the following curl command to cron to run nightly or as you see fit:
+
+``curl -X POST http://localhost:8080/api/admin/sitemap``
+
+This will create or update a file in the following location unless you have customized your installation directory for Glassfish:
+
+``/usr/local/glassfish4/glassfish/domains/domain1/docroot/sitemap/sitemap.xml``
+
+On an installation of Dataverse with many datasets, the creation or updating of the sitemap can take a while. You can check Glassfish's server.log file for "BEGIN updateSiteMap" and "END updateSiteMap" lines to know when the process started and stopped and any errors in between.
+
+https://demo.dataverse.org/sitemap.xml is the sitemap URL for the Dataverse Demo site and yours should be similar. Submit your sitemap URL to Google by following `Google's "submit a sitemap" instructions`_ or similar instructions for other search engines.
+
+.. _Google's "submit a sitemap" instructions: https://support.google.com/webmasters/answer/183668
+
 Putting Your Dataverse Installation on the Map at dataverse.org
 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
@@ -525,7 +549,7 @@ Configuration for :doc:`r-rapache-tworavens`.
 dataverse.dropbox.key
 +++++++++++++++++++++
 
-Dropbox provides a Chooser app, which is a Javascript component that allows you to upload files to Dataverse from Dropbox. It is an optional configuration setting, which requires you to pass it an app key. For more information on setting up your Chooser app, visit https://www.dropbox.com/developers/chooser.
+Dropbox provides a Chooser app, which is a Javascript component that allows you to upload files to Dataverse from Dropbox. It is an optional configuration setting, which requires you to pass it an app key and configure the ``:UploadMethods`` database setting. For more information on setting up your Chooser app, visit https://www.dropbox.com/developers/chooser.
 
 ``./asadmin create-jvm-options "-Ddataverse.dropbox.key={{YOUR_APP_KEY}}"``
 
@@ -1289,9 +1313,18 @@ The URL for your Repository Storage Abstraction Layer (RSAL) installation. This
 :UploadMethods
 ++++++++++++++
 
-This setting is experimental and to be used with the Data Capture Module (DCM). For now, if you set the upload methods to ``dcm/rsync+ssh`` it will allow your users to download rsync scripts from the DCM.
+This setting controls which upload methods are available to users of your installation of Dataverse. The following upload methods are available:
+
+- ``native/http``: Corresponds to "Upload with HTTP via your browser" and APIs that use HTTP (SWORD and native).
+- ``dcm/rsync+ssh``: Corresponds to "Upload with rsync+ssh via Data Capture Module (DCM)". A lot of setup is required, as explained in the :doc:`/developers/big-data-support` section of the Dev Guide.
+
+Out of the box only ``native/http`` is enabled and will work without further configuration. To add multiple upload method, separate them using a comma like this:
+
+``curl -X PUT -d 'native/http,dcm/rsync+ssh' http://localhost:8080/api/admin/settings/:UploadMethods``
+
+You'll always want at least one upload method, so the easiest way to remove one of them is to simply ``PUT`` just the one you want, like this:
 
-``curl -X PUT -d 'dcm/rsync+ssh' http://localhost:8080/api/admin/settings/:UploadMethods``
+``curl -X PUT -d 'native/http' http://localhost:8080/api/admin/settings/:UploadMethods``
 
 :DownloadMethods
 ++++++++++++++++