IQSS · kcondon · Sep 25, 2017 · Jul 20, 2017 · Jul 24, 2017 · Jul 24, 2017
diff --git a/doc/sphinx-guides/source/_static/fontcustom.css b/doc/sphinx-guides/source/_static/fontcustom.css
@@ -4,19 +4,20 @@
 
 @font-face {
   font-family: "fontcustom";
-  src: url("./fontcustom/fontcustom_0cdeefae934823416d24b6c2132ac702.eot");
-  src: url("./fontcustom/fontcustom_0cdeefae934823416d24b6c2132ac702.eot?#iefix") format("embedded-opentype"),
-       url("./fontcustom/fontcustom_0cdeefae934823416d24b6c2132ac702.woff") format("woff"),
-       url("./fontcustom/fontcustom_0cdeefae934823416d24b6c2132ac702.ttf") format("truetype"),
-       url("./fontcustom/fontcustom_0cdeefae934823416d24b6c2132ac702.svg#fontcustom") format("svg");
+  src: url("./fontcustom/fontcustom_fb6bd74e6004cae7db6ffae62177daaf.eot");
+  src: url("./fontcustom/fontcustom_fb6bd74e6004cae7db6ffae62177daaf.eot?#iefix") format("embedded-opentype"),
+       url("./fontcustom/fontcustom_fb6bd74e6004cae7db6ffae62177daaf.woff2") format("woff2"),
+       url("./fontcustom/fontcustom_fb6bd74e6004cae7db6ffae62177daaf.woff") format("woff"),
+       url("./fontcustom/fontcustom_fb6bd74e6004cae7db6ffae62177daaf.ttf") format("truetype"),
+       url("./fontcustom/fontcustom_fb6bd74e6004cae7db6ffae62177daaf.svg#fontcustom") format("svg");
   font-weight: normal;
   font-style: normal;
 }
 
 @media screen and (-webkit-min-device-pixel-ratio:0) {
   @font-face {
     font-family: "fontcustom";
-    src: url("./fontcustom/fontcustom_0cdeefae934823416d24b6c2132ac702.svg#fontcustom") format("svg");
+    src: url("./fontcustom/fontcustom_fb6bd74e6004cae7db6ffae62177daaf.svg#fontcustom") format("svg");
   }
 }
 
@@ -34,6 +35,7 @@
 .icon-image:before,
 .icon-network:before,
 .icon-other:before,
+.icon-package:before,
 .icon-tabular:before,
 .icon-unlock:before,
 .icon-video:before {
@@ -62,6 +64,7 @@
 .icon-image:before { content: "\f103"; }
 .icon-network:before { content: "\f10c"; }
 .icon-other:before { content: "\f10d"; }
+.icon-package:before { content: "\f10f"; }
 .icon-tabular:before { content: "\f108"; }
 .icon-unlock:before { content: "\f10e"; }
 .icon-video:before { content: "\f109"; }
diff --git a/doc/sphinx-guides/source/_static/fontcustom/fontcustom_0cdeefae934823416d24b6c2132ac702.woff b/doc/sphinx-guides/source/_static/fontcustom/fontcustom_0cdeefae934823416d24b6c2132ac702.woff
diff --git a/...stom_0cdeefae934823416d24b6c2132ac702.eot → ...stom_fb6bd74e6004cae7db6ffae62177daaf.eot b/...stom_0cdeefae934823416d24b6c2132ac702.eot → ...stom_fb6bd74e6004cae7db6ffae62177daaf.eot
diff --git a/...stom_0cdeefae934823416d24b6c2132ac702.svg → ...stom_fb6bd74e6004cae7db6ffae62177daaf.svg b/...stom_0cdeefae934823416d24b6c2132ac702.svg → ...stom_fb6bd74e6004cae7db6ffae62177daaf.svg
diff --git a/...stom_0cdeefae934823416d24b6c2132ac702.ttf → ...stom_fb6bd74e6004cae7db6ffae62177daaf.ttf b/...stom_0cdeefae934823416d24b6c2132ac702.ttf → ...stom_fb6bd74e6004cae7db6ffae62177daaf.ttf
diff --git a/doc/sphinx-guides/source/_static/fontcustom/fontcustom_fb6bd74e6004cae7db6ffae62177daaf.woff b/doc/sphinx-guides/source/_static/fontcustom/fontcustom_fb6bd74e6004cae7db6ffae62177daaf.woff
diff --git a/...sphinx-guides/source/_static/fontcustom/fontcustom_fb6bd74e6004cae7db6ffae62177daaf.woff2 b/...sphinx-guides/source/_static/fontcustom/fontcustom_fb6bd74e6004cae7db6ffae62177daaf.woff2
diff --git a/doc/sphinx-guides/source/_static/icon-template.svg b/doc/sphinx-guides/source/_static/icon-template.svg
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
@@ -710,8 +710,9 @@ Sample output appears below.
                     "createdTime":"2000-01-01 00:00:00.0",
                     "lastLoginTime":"2017-07-03 12:22:35.926",
                     "lastApiUseTime":"2017-07-03 12:55:57.186"
-                },
-                **... 22 more user documents ...**
+                }
+
+                // ... 22 more user documents ...
             ]
         }
     }

diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst
@@ -51,6 +51,52 @@ Here's the syntax for sending the JSON.
 
 ``curl -H "X-Dataverse-key: $API_TOKEN" -X POST -H 'Content-type: application/json' --upload-file checksumValidationSuccess.json $DV_BASE_URL/api/datasets/:persistentId/dataCaptureModule/checksumValidation?persistentId=$PERSISTENT_ID``
 
+
+Steps to set up a DCM mock for Development
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Install Flask.
+
+
+Download and run the mock. You will be cloning the https://github.com/sbgrid/data-capture-module repo.
+
+- ``git clone git://github.com/sbgrid/data-capture-module.git``
+- ``cd data-capture-module/api``
+- ``./dev_mock.sh``
+
+If you see an error about not having Flask installed, install it as explained below.
+
+On Mac, you can install Flask with:
+
+- ``mkvirtualenv mockdcm``
+- ``pip install -r requirements-mock.txt``
+
+On Ubuntu/Debian, you can install Flask with:
+
+- ``sudo apt install python-pip`` (will install python as well)
+- ``pip install flask``
+
+Once you have Flask installed, try running the dev mock script again:
+
+- ``./dev_mock.sh``
+
+This should spin up the DCM mock on port 5000.
+
+Add Dataverse settings to use mock (same as using DCM, noted above):
+
+- ``curl http://localhost:8080/api/admin/settings/:DataCaptureModuleUrl -X PUT -d "http://localhost:5000"``
+- ``curl http://localhost:8080/api/admin/settings/:UploadMethods -X PUT -d "dcm/rsync+ssh"``
+
+At this point you should be able to download a placeholder rsync script. Dataverse is then waiting for new from the DCM about if checksum validation has succeeded or not. First, you have to put files in place, which is usually the job of the DCM. You should substitute "X1METO" for the "identifier" of the dataset you create. You must also use the proper path for where you store files in your dev environment.
+
+- ``mkdir /usr/local/glassfish4/glassfish/domains/domain1/files/10.5072/FK2/X1METO``
+- ``mkdir /usr/local/glassfish4/glassfish/domains/domain1/files/10.5072/FK2/X1METO/X1METO``
+- ``cd /usr/local/glassfish4/glassfish/domains/domain1/files/10.5072/FK2/X1METO/X1METO``
+- ``echo "hello" > file1.txt``
+- ``shasum file1.txt > files.sha``
+
+Now the files are in place and you need to send JSON to Dataverse with a success or failure message as described above. Make a copy of ``doc/sphinx-guides/source/_static/installation/files/root/big-data-support/checksumValidationSuccess.json`` and put the identifier in place such as "X1METO" under "uploadFolder"). Then use curl as described above to send the JSON.
+
 Troubleshooting
 ~~~~~~~~~~~~~~~
 
@@ -61,6 +107,9 @@ The following low level command should only be used when troubleshooting the "im
 Repository Storage Abstraction Layer (RSAL)
 -------------------------------------------
 
+Configuring the RSAL Mock
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
 Info for configuring the RSAL Mock: https://github.com/sbgrid/rsal/tree/master/mocks
 
 Also, to configure Dataverse to use the new workflow you must do the following:
@@ -94,3 +143,20 @@ Edit internal-httpSR-workflow.json and replace url and rollbackUrl to be the url
 8. When finished testing, unset the workflow:
 
 ``curl -X DELETE http://localhost:8080/api/admin/workflows/default/PrePublishDataset``
+
+Configuring download via rsync
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In order to see the rsync URLs, you must run this command:
+
+``curl -X PUT -d 'rsal/rsync' http://localhost:8080/api/admin/settings/:DownloadMethods``
+
+TODO: Document these in the Installation Guide once they're final.
+
+To specify replication sites that appear in rsync URLs:
+
+``curl http://localhost:8080/api/admin/settings/:ReplicationSites -X PUT -d "dv.sbgrid.org:Harvard Medical School:USA,sbgrid.icm.uu.se:Uppsala University:Sweden,sbgrid.ncpss.org:Institut Pasteur de Montevideo:Uruguay,sbgrid.ncpss.org:Shanghai Institutes for Biological Sciences:China"``
+
+In the GUI, this is called "Local Access". It's where you can compute on files on your cluster.
+
+``curl http://localhost:8080/api/admin/settings/:LocalDataAccessPath -X PUT -d "/programs/datagrid"``
diff --git a/doc/sphinx-guides/source/developers/tools.rst b/doc/sphinx-guides/source/developers/tools.rst
@@ -81,6 +81,13 @@ MSV
     validating ddi.xml
     the document is valid.
 
+FontCustom
+++++++++++
+
+The custom file type icons were created with the help of `FontCustom <https://github.com/FontCustom/fontcustom>`. Their README provides installation instructions as well as directions for producing your own vector-based icon font.
+
+Here is a vector-based SVG file to start with as a template: :download:`icon-template.svg <../_static/icon-template.svg>`
+
 ----
 
 Previous: :doc:`making-releases` | Next: :doc:`unf/index`
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
@@ -31,6 +31,13 @@ Forcing HTTPS
 
 To avoid having your users send credentials in the clear, it's strongly recommended to force all web traffic to go through HTTPS (port 443) rather than HTTP (port 80). The ease with which one can install a valid SSL cert into Apache compared with the same operation in Glassfish might be a compelling enough reason to front Glassfish with Apache. In addition, Apache can be configured to rewrite HTTP to HTTPS with rules such as those found at https://wiki.apache.org/httpd/RewriteHTTPToHTTPS or in the section on :doc:`shibboleth`.
 
+Privacy Considerations
+++++++++++++++++++++++
+
+Out of the box, Dataverse will list email addresses of the "contacts" for datasets when users visit a dataset page and click the "Export Metadata" button. If you prefer to exclude email addresses of dataset contacts from metadata export, set :ref:`:ExcludeEmailFromExport <:ExcludeEmailFromExport>` to true.
+
+Additional Recommendations
+++++++++++++++++++++++++++
 Run Glassfish as a User Other Than Root
 +++++++++++++++++++++++++++++++++++++++
 
@@ -294,7 +301,7 @@ Once you have acquired the keys, they need to be added to``credentials``. The fo
 | ``aws_access_key_id = <insert key, no brackets>``
 | ``aws_secret_access_key = <insert secret key, no brackets>``
 
-Place this file ina a folder named ``.aws`` under the home directory for the user running your dataverse installation.
+Place this file in a folder named ``.aws`` under the home directory for the user running your dataverse installation.
 
 Setup aws via command line tools
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -775,6 +782,9 @@ Specify a URL where users can read your API Terms of Use.
 
 ``curl -X PUT -d http://best-practices.dataverse.org/harvard-policies/harvard-api-tou.html http://localhost:8080/api/admin/settings/:ApiTermsOfUse``
 
+
+.. _:ExcludeEmailFromExport:
+
 :ExcludeEmailFromExport
 +++++++++++++++++++++++
 
@@ -1175,20 +1185,13 @@ This setting is experimental and to be used with the Data Capture Module (DCM).
 :DownloadMethods
 ++++++++++++++++
 
-This setting is experimental and related to Repository Storage Abstraction Layer (RSAL). As of this writing it has no effect.
+This setting is experimental and related to Repository Storage Abstraction Layer (RSAL).
+
+``curl -X PUT -d 'rsal/rsync' http://localhost:8080/api/admin/settings/:DownloadMethods``
 
 :GuestbookResponsesPageDisplayLimit
 +++++++++++++++++++++++++++++++++++
 
 Limit on how many guestbook entries to display on the guestbook-responses page. By default, only the 5000 most recent entries will be shown. Use the standard settings API in order to change the limit. For example, to set it to 10,000, make the following API call: 
 
-``curl -X PUT -d 10000 http://localhost:8080/api/admin/settings/:GuestbookResponsesPageDisplayLimit``
-
-:CustomDatasetSummaryFields
-+++++++++++++++++++++++++
-
-You can replace the default dataset metadata fields that are displayed above files table on the dataset page with a custom list separated by commas using the curl command below.
-
-``curl http://localhost:8080/api/admin/settings/:CustomDatasetSummaryFields -X PUT -d 'producer,subtitle,alternativeTitle'``
-
-You have to put the datasetFieldType name attribute in the :CustomDatasetSummaryFields setting for this to work. 
+``curl -X PUT -d 10000 http://localhost:8080/api/admin/settings/:GuestbookResponsesPageDisplayLimit`` 
diff --git a/doc/sphinx-guides/source/installation/installation-main.rst b/doc/sphinx-guides/source/installation/installation-main.rst
@@ -82,7 +82,7 @@ Logging In
 Out of the box, Glassfish runs on port 8080 and 8181 rather than 80 and 443, respectively, so visiting http://localhost:8080 (substituting your hostname) should bring up a login page. See the :doc:`shibboleth` page for more on ports, but for now, let's confirm we can log in by using port 8080. Poke a temporary hole in your firewall, if needed. 
 
 Superuser Account
-+++++++++++++++++
+^^^^^^^^^^^^^^^^^
 
 We'll use the superuser account created by the installer to make sure you can log into Dataverse. For more on the difference between being a superuser and having the "Admin" role, read about configuring the root dataverse in the :doc:`config` section.
 
@@ -106,12 +106,12 @@ Troubleshooting
 If the following doesn't apply, please get in touch as explained in the :doc:`intro`. You may be asked to provide ``glassfish4/glassfish/domains/domain1/logs/server.log`` for debugging.
 
 Dataset Cannot Be Published
-+++++++++++++++++++++++++++
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Check to make sure you used a fully qualified domain name when installing Dataverse. You can change the ``dataverse.fqdn`` JVM option after the fact per the :doc:`config` section.
 
 Problems Sending Email
-++++++++++++++++++++++
+^^^^^^^^^^^^^^^^^^^^^^
 
 If your Dataverse installation is not sending system emails, you may need to provide authentication for your mail host. First, double check the SMTP server being used with this Glassfish asadmin command:
 
@@ -124,7 +124,7 @@ If the command returns a host you don't want to use, you can modify your notifyM
 If your mail host requires a username/password for access, continue to the next section.
 
 Mail Host Configuration & Authentication
-++++++++++++++++++++++++++++++++++++++++
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 If you need to alter your mail host address, user, or provide a password to connect with, these settings are easily changed in the Glassfish admin console. 
 
@@ -166,7 +166,7 @@ mail.smtp.socketFactory.class			javax.net.ssl.SSLSocketFactory
 Be sure you save the changes made here and then restart your Glassfish server to test it out.
 
 UnknownHostException While Deploying
-++++++++++++++++++++++++++++++++++++
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 If you are seeing "Caused by: java.net.UnknownHostException: myhost: Name or service not known" in server.log and your hostname is "myhost" the problem is likely that "myhost" doesn't appear in ``/etc/hosts``. See also http://stackoverflow.com/questions/21817809/glassfish-exception-during-deployment-project-with-stateful-ejb/21850873#21850873
 
@@ -176,7 +176,7 @@ Fresh Reinstall
 Early on when you're installing Dataverse, you may think, "I just want to blow away what I've installed and start over." That's fine. You don't have to uninstall the various components like Glassfish, PostgreSQL and Solr, but you should be conscious of how to clear out their data.
 
 Drop database
-+++++++++++++
+^^^^^^^^^^^^^
 
 In order to drop the database, you have to stop Glassfish, which will have open connections. Before you stop Glassfish, you may as well undeploy the war file. First, find the name like this:
 
@@ -195,22 +195,22 @@ With Glassfish down, you should now be able to drop your database and recreate i
 ``psql -U dvnapp -c 'DROP DATABASE "dvndb"' template1``
 
 Clear Solr
-++++++++++
+^^^^^^^^^^
 
 The database is fresh and new but Solr has stale data it in. Clear it out with this command:
 
 ``curl http://localhost:8983/solr/update/json?commit=true -H "Content-type: application/json" -X POST -d "{\"delete\": { \"query\":\"*:*\"}}"``
 
 
-Deleting uploaded files
-+++++++++++++++++++++++
+Deleting Uploaded Files
+^^^^^^^^^^^^^^^^^^^^^^^
 
 The path below will depend on the value for ``dataverse.files.directory`` as described in the :doc:`config` section:
 
 ``rm -rf /usr/local/glassfish4/glassfish/domains/domain1/files``
 
 Rerun Installer
-+++++++++++++++
+^^^^^^^^^^^^^^^
 
 With all the data cleared out, you should be ready to rerun the installer per above.