Make Data Count support: backend #4821

conf/docker-aio/run-test-suite.sh

@@ -8,4 +8,4 @@ fi
 
 # Please note the "dataverse.test.baseurl" is set to run for "all-in-one" Docker environment.
 # TODO: Rather than hard-coding the list of "IT" classes here, add a profile to pom.xml.
-mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT -Ddataverse.test.baseurl=$dvurl
+mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT -Ddataverse.test.baseurl=$dvurl

doc/sphinx-guides/source/_static/admin/counter-processor-config.yaml

@@ -0,0 +1,67 @@
+# currently no other option but to have daily logs and have year-month-day format in the name with
+# 4-digit year and 2-digit month and day
+# /usr/local/glassfish4/glassfish/domains/domain1/logs/counter_2019-01-11.log
+#log_name_pattern: sample_logs/counter_(yyyy-mm-dd).log
+log_name_pattern: /usr/local/glassfish4/glassfish/domains/domain1/logs/counter_(yyyy-mm-dd).log
+
+# path_types regular expressions allow matching to classify page urls as either an investigation or request
+# based on specific URL structure for your system.
+# Dataverse Note: the url matches on this does not include the query params, so dataset.xhtml\S+ will not match
+path_types:
+  investigations:
+    - ^.*/dataset.xhtml\S*$
+    - ^.*/file.xhtml\S*$
+    - ^.*/api/datasets\S*$
+    - ^.*/api/v1/datasets\S*$
+    ## Below historic regex for testing
+    #- ^/api/datasets/[^\/]+$
+    #- ^/api/versions/\d+$
+    #- ^/stash/dataset/\S+$
+    #- ^/stash/data_paper/\S+$
+  requests:
+    - ^.*/api/access/datafile\S+$
+    - ^.*/api/v1/access/datafile\S+$
+    ## Below historic regex for testing
+    #- ^/api/datasets/[^\/]+/download$
+    #- ^/api/versions/\d+/download$
+    #- ^/api/downloads/\d+$
+    #- ^/stash/downloads/download_resource/\d+$
+    #- ^/stash/downloads/file_download/\d+$
+    #- ^/stash/downloads/file_stream/\d+$
+    #- ^/stash/downloads/async_request/\d+$
+    #- ^/stash/share/\S+$
+
+# Robots and machines urls are urls where the script can download a list of regular expressions to determine
+# if something is a robot or machine user-agent.  The text file has one regular expression per line
+robots_url: https://raw.githubusercontent.com/CDLUC3/Make-Data-Count/master/user-agents/lists/robot.txt
+machines_url: https://raw.githubusercontent.com/CDLUC3/Make-Data-Count/master/user-agents/lists/machine.txt
+
+# the year and month for the report you are creating.
+year_month: 2019-01
+
+# Don't put the filename extension, the code will tack on the tsv or json extension for you.
+# Output formats are either tsv or json currently.  TSV is currently broken until anyone accepts reports in that format.
+# FIXME: "/tmp" is fine for a quick test but pick another directory that
+# the "counter" user can write to and that the "glassfish" user can read from.
+output_file: /tmp/make-data-count-report
+output_format: json
+
+# the name of the platform that goes into your reports
+# FIXME: Change "platform" to the brand of your Dataverse installation.
+platform: LibreScholar
+
+# Don't put your api token in here if you're going to commit it, but put in separate secrets.yaml in same
+# directory as the config or else set a environment variable when starting up in order to override the key.
+# yaml key/values set in secrets.yaml will override one from the main config.
+hub_api_token: set_me_in_secrets
+# the test metrics is only for testing
+# hub_base_url: https://metrics.test.datacite.org
+# FIXME: change "hub_base_url" to https://api.datacite.org (production) once you have credentials.
+hub_base_url: https://api.test.datacite.org
+# FIXME: Change "upload_to_hub" to True when you're ready.
+upload_to_hub: False
+
+# only use this to simulate running on a date besides today
+# simulate_date: 2019-01-12
+
+maxmind_geoip_country_path: maxmind_geoip/GeoLite2-Country.mmdb

doc/sphinx-guides/source/admin/img/make-data-count.uml

@@ -0,0 +1,66 @@
+@startuml
+
+skinparam BoxPadding 30
+
+title Make Data Count components in Dataverse
+autonumber "<font color=blue>"
+
+actor user
+box "Dataverse" #DDD
+  participant datasetPage as "dataset\npage"
+  participant downloadApi as "download\nAPI"
+  participant metricsApi as "metrics\nAPI"
+  participant datasetMetrics as "datasetmetrics\ntable"
+  participant datasetExternalCitations as "datasetexternalcitations\ntable"
+  participant sushiParser as "SUSHI\nparser"
+  participant citationParser as "citation\nparser"
+end box
+
+box "filesystem or S3" #DDD
+  participant log as "activity-yyyy-mm.log"
+  participant sushi as "sushi-yyyy-mm.json"
+endbox
+
+box "counter-processor" #DDD
+  participant main.py
+  database sqlite as "yyyy-mm.sqlite3"
+end box
+
+box "DataCite hub" #DDD
+  participant reports as "reports\nendpoint"
+  participant events as "events\nendpoint"
+end box
+
+box "Crossref" #DDD
+  participant crossrefApi as "Crossref\nAPI"
+end box
+
+== Activity Logging ==
+user --> datasetPage : GET /dataset.xhtml
+datasetPage --> log : investigation/view
+user --> downloadApi : GET /api/access/datafile
+downloadApi --> log : request/download
+
+== Log Processing ==
+
+main.py --> log : read log nightly
+main.py --> sqlite : write to SQLite
+main.py --> reports : send SUSHI report to DataCite hub "reports" endpoint
+main.py --> sushi : write SUSHI to filesystem or S3
+sushiParser --> sushi : parse SUSHI file
+sushiParser --> datasetMetrics : write metrics to datasetmetrics table
+
+== Citation Retrieval ==
+
+events --> crossrefApi : retrieve citations periodically 
+citationParser --> events : retrieve citations nightly
+citationParser --> datasetExternalCitations : write citations to datasetexternalcitations table
+
+== Retrieving Usage Metrics ==
+
+user --> metricsApi: request number of views, downloads, and citations
+metricsApi --> datasetMetrics: get data from table
+metricsApi --> datasetExternalCitations: get data from table
+metricsApi --> user: return data
+
+@enduml

doc/sphinx-guides/source/admin/index.rst

@@ -18,6 +18,7 @@ This guide documents the functionality only available to superusers (such as "da
    metadatacustomization
    metadataexport
    timers
+   make-data-count
    integrations
    geoconnect-worldmap
    user-administration

doc/sphinx-guides/source/admin/make-data-count.rst

@@ -0,0 +1,169 @@
+Make Data Count
+===============
+
+`Make Data Count`_ is a project to collect and standardize metrics on data use, especially views, downloads, and citations. Dataverse can integrate Make Data Count to collect and display usage metrics including counts of dataset views, file downloads, and dataset citations.
+
+.. contents:: Contents:
+	:local:
+
+Introduction
+------------
+
+`Make Data Count`_ is part of a broader Research Data Alliance (RDA) `Data Usage Metrics Working Group`_ which helped to produce a specification called the `COUNTER Code of Practice for Research Data`_ (`PDF`_, `HTML`_) that Dataverse makes every effort to comply with. The Code of Practice (CoP) is built on top of existing standards such as COUNTER and SUSHI that come out of the article publishing world.  The Make Data Count project has emphasized that they would like feedback on the code of practice. You can keep up to date on the Make Data Count project by subscribing to their `newsletter`_.
+
+Architecture
+------------
+
+Dataverse installations who would like support for Make Data Count must install `Counter Processor`_, a Python project created by California Digital Library (CDL) which is part of the Make Data Count project and which runs the software in production as part of their `DASH`_ data sharing platform.
+
+.. _Counter Processor: https://github.com/CDLUC3/counter-processor
+.. _DASH: https://cdluc3.github.io/dash/
+
+The diagram below shows how Counter Processor interacts with Dataverse and the DataCite hub, once configured. Installations of Dataverse using Handles rather than DOIs should note the limitations in the next section of this page.
+
+|makedatacount_components|
+
+The most important takeaways from the diagram are:
+
+- Once enabled, Dataverse will log activity (views and downloads) to a specialized date-stamped file.
+- You should run Counter Processor once a day to create reports in SUSHI (JSON) format that are saved to disk for Dataverse to process and that are sent to the DataCite hub.
+- You should set up a cron job to have Dataverse process the daily SUSHI reports, updating the Dataverse database with the latest metrics.
+- You should set up a cron job to have Dataverse pull the latest list of citations for each dataset on a periodic basis, perhaps weekly or daily. These citations come from Crossref via the DataCite hub.
+- APIs are available in Dataverse to retrieve Make Data Count metrics: views, downloads, and citations.
+
+
+Limitations for Dataverse Installations Using Handles Rather Than DOIs
+----------------------------------------------------------------------
+
+Data repositories using Handles and other identifiers are not supported by Make Data Count but in the notes_ following a July 2018 webinar, you can see the Make Data Count project's response on this topic. In short, the DataCite hub does not want to receive reports for non-DOI datasets. Additionally, citations are only available from the DataCite hub for datasets that have DOIs. 
+
+This being said, the Dataverse usage logging can still generate logs and process those logs with Counter Processor to create json that details usage on a dataset level. Dataverse can ingest this locally generated json.
+
+When editing the ``counter-processor-config.yaml`` file mentioned below, make sure that the ``upload_to_hub`` boolean is set to ``False``.
+
+Configuring Dataverse for Make Data Count Views and Downloads
+-------------------------------------------------------------
+
+If you haven't already, follow the steps for installing Counter Processor in the :doc:`/installation/prerequisites` section of the Installation Guide.
+
+Enable Logging for Make Data Count
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To make Dataverse log dataset usage (views and downloads) for Make Data Count, you must set the ``:MDCLogPath`` database setting. See :ref:`MDCLogPath` for details.
+
+After you have your first day of logs, you can process them the next day.
+
+Configure Counter Processor
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* First, become the "counter" Unix user.
+
+  * ``sudo su - counter``
+
+* Change to the directory where you installed Counter Processor.
+
+  * ``cd /usr/local/counter-processor-0.0.1``
+
+* Download :download:`counter-processor-config.yaml <../_static/admin/counter-processor-config.yaml>` to ``/usr/local/counter-processor-0.0.1``.
+
+* Edit the config file and pay particular attention to the FIXME lines.
+
+  * ``vim counter-processor-config.yaml``
+
+Populate Views and Downloads for the First Time
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Soon we will be setting up a cron job to run nightly but we start with a single successful configuration and run of Counter Processor and calls to Dataverse APIs.
+
+* Change to the directory where you installed Counter Processor.
+
+  * ``cd /usr/local/counter-processor-0.0.1``
+
+* If you are running Counter Processor for the first time in the middle of a month, you will need create blank log files for the previous days. e.g.:
+
+  * ``cd /usr/local/glassfish4/glassfish/domains/domain1/logs``
+
+  * ``touch counter_2019-02-01.log``
+
+  * ``...``
+
+  * ``touch counter_2019-02-20.log``
+
+* Run Counter Processor.
+
+  * ``CONFIG_FILE=counter-processor-config.yaml python36 main.py``
+
+  * A JSON file in SUSHI format will be created in the directory you specified under "output_file" in the config file.
+
+* Populate views and downloads for your datasets based on the SUSHI JSON file. The "/tmp" directory is used in the example below.
+
+  * ``curl -X POST "http://localhost:8080/api/admin/makeDataCount/addUsageMetricsFromSushiReport?reportOnDisk=/tmp/make-data-count-report.json"``
+
+* Verify that views and downloads are available via API.
+
+  * Now that views and downloads have been recorded in the Dataverse database, you should make sure you can retrieve them from a dataset or two. Use the :ref:`Dataset Metrics <dataset-metrics-api>` endpoints in the :doc:`/api/native-api` section of the API Guide.
+
+Populate Views and Downloads Nightly
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Running ``main.py`` to create the SUSHI JSON file and the subsequent calling of the Dataverse API to process it should be added as a cron job.
+
+Sending Usage Metrics to the DataCite Hub
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Once you are satisfied with your testing, you should contact support@datacite.org for your JSON Web Token and change "upload_to_hub" to "True" in the config file. The next time you run ``main.py`` the following metrics will be sent to the DataCite hub for each published dataset:
+
+- Views ("investigations" in COUNTER)
+- Downloads ("requests" in COUNTER)
+
+Configuring Dataverse for Make Data Count Citations
+---------------------------------------------------
+
+Please note: as explained in the note above about limitations, this feature is not available to installations of Dataverse that use Handles.
+
+Please note that in the curl example, Bash environment variables are used with the idea that you can set a few environment variables and copy and paste the examples as is. For example, "$DOI" could become "doi:10.5072/FK2/BL2IBM" by issuing the following export command from Bash:
+
+``export DOI="doi:10.5072/FK2/BL2IBM"``
+
+To confirm that the environment variable was set properly, you can use echo like this:
+
+``echo $DOI``
+
+On some periodic basis (perhaps weekly) you should call the following curl command for each published dataset to update the list of citations that have been made for that dataset.
+
+``curl -X POST "http://localhost:8080/api/admin/makeDataCount/:persistentId/updateCitationsForDataset?persistentId=$DOI"``
+
+Citations will be retrieved for each published dataset and recorded in the Dataverse database.
+
+For how to get the citations out of Dataverse, see "Retrieving Citations for a Dataset" under :ref:`Dataset Metrics <dataset-metrics-api>` in the :doc:`/api/native-api` section of the API Guide.
+
+Please note that while Dataverse has a metadata field for "Related Dataset" this information is not currently sent as a citation to Crossref.
+
+Retrieving Make Data Count Metrics from the DataCite Hub
+--------------------------------------------------------
+
+The following metrics can be downloaded directly from the DataCite hub (see https://support.datacite.org/docs/eventdata-guide) for datasets hosted by Dataverse installations that have been configured to send these metrics to the hub:
+
+- Total Views for a Dataset
+- Unique Views for a Dataset
+- Total Downloads for a Dataset
+- Downloads for a Dataset
+- Citations for a Dataset (via Crossref)
+
+Retrieving Make Data Count Metrics from Dataverse
+-------------------------------------------------
+
+The Dataverse API endpoints for retrieving Make Data Count metrics are described below under :ref:`Dataset Metrics <dataset-metrics-api>` in the :doc:`/api/native-api` section of the API Guide.
+
+Please note that it is also possible to retrieve metrics from the DataCite hub itself via https://api.datacite.org
+
+.. _notes: https://docs.google.com/document/d/1b1itytDVDsI_Ni2LoxrG887YGt0zDc96tpyJEgBN9Q8/
+.. _newsletter: https://makedatacount.org/contact/
+.. _COUNTER Code of Practice for Research Data: https://makedatacount.org/counter-code-of-practice-for-research-data/
+.. _PDF: https://doi.org/10.7287/peerj.preprints.26505v1
+.. _HTML: https://www.projectcounter.org/code-of-practice-rd-sections/foreword/
+.. _Make Data Count: https://makedatacount.org
+.. _Data Usage Metrics Working Group: https://www.rd-alliance.org/groups/data-usage-metrics-wg
+
+.. |makedatacount_components| image:: ./img/make-data-count.png
+   :class: img-responsive

doc/sphinx-guides/source/admin/metadataexport.rst

@@ -11,8 +11,10 @@ Publishing a dataset automatically starts a metadata export job, that will run i
 
 - Dublin Core
 - Data Documentation Initiative (DDI)
-- Schema.org JSON-LD
+- DataCite 4
 - native JSON (Dataverse-specific)
+- OAI_ORE
+- Schema.org JSON-LD
 
 A scheduled timer job that runs nightly will attempt to export any published datasets that for whatever reason haven't been exported yet. This timer is activated automatically on the deployment, or restart, of the application. So, again, no need to start or configure it manually. (See the "Application Timers" section of this guide for more information)
 

doc/sphinx-guides/source/api/metrics.rst

@@ -1,6 +1,8 @@
 Metrics API
 ===========
 
+The Metrics API provides counts of downloads, datasets created, files uploaded, and more, as described below. Dataverse also supports Make Data Count, which is described in the :doc:`/admin/make-data-count` section of the Admin Guide.
+
 .. contents:: |toctitle|
     :local: