Refactor Developer Documentation (#3621)

Doxyfile

@@ -905,7 +905,9 @@ FILE_PATTERNS          = *.c \
                          *.ice \
                          *.rst \
                          *.json \
-                         *.info.yml
+                         *.info.yml \
+                         *.routing.yml \
+                         *.services.yml
 
 # The RECURSIVE tag can be used to specify whether or not subdirectories should
 # be searched for input files as well.
@@ -920,8 +922,8 @@ RECURSIVE              = YES
 # Note that relative paths are relative to the directory from which doxygen is
 # run.
 
-EXCLUDE                = "modules/datastore/modules/datastore_mysql_import/README.md" \
-                         "modules/dkan_js_frontend/README.md"
+#EXCLUDE                = "modules/datastore/modules/datastore_mysql_import/README.md" \
+#                         "modules/dkan_js_frontend/README.md"
 
 # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
 # directories that are symbolic links (a Unix file system feature) are excluded

Jenkinsfile

@@ -31,10 +31,10 @@ pipeline {
                         docker container stop $i
                         docker container rm $i
                       done
-		      
+
                       docker network disconnect $qa_network_id proxy || true
                       docker network rm $qa_network_id
-		      
+
                     sudo rm -r $WORKSPACE/*
                     fi
                     '''
@@ -70,11 +70,23 @@ pipeline {
                         dktl demo
                         dktl drush user:password admin mayisnice
                         sudo chown -R 1000:docker $WORKSPACE/dkan-tools/vendor
+                        dktl dkan:docs
                         curl -fI `dktl url`
                     '''
                 }
             }
         }
+        stage('RebuildDocs') {
+            steps {
+            sh '''
+            ssh -o StrictHostKeyChecking=no root@45.33.76.242 <<EOF
+            cd /var/www/demo/docroot/modules/contrib/dkan
+            latest_release=`curl -s https://api.github.com/repos/GetDKAN/dkan/releases/latest%7Cgrep tag_name|awk '{ print $2 }'|cut -d '"' -f2`
+            DKAN_VERSION="v$latest_release" doxygen
+            EOF
+            '''.stripIndent()
+            }
+        }
     }
     post {
         always {
@@ -91,7 +103,7 @@ pipeline {
                 def target_url = "http://$DKTL_SLUG.$WEB_DOMAIN"
                 setBuildStatus("QA site ready at ${target_url}", target_url, "success");
             }
-            
+
         }
     }
 }

docs_assets/css/custom.css

@@ -32,17 +32,21 @@ body {
   right: 35px;
 }
 h1,h2,h3,h4 {
-  font-family: 'Varela Round', 'Lato', Roboto, Droid Sans, Helvetica Neue, sans-serif;
-  color: #0071bc;
+  /* font-family: 'Varela Round', 'Lato', Roboto, Droid Sans, Helvetica Neue, sans-serif; */
+  color: #003366;
+}
+h1 {
+  font-size: 21px;
 }
 h2 {
-  margin-top: 2em;
+  font-size: 18px;
 }
 h2:first-of-type {
   margin-top: 1em;
 }
 h3 {
-  margin-top: 1.5em;
+  font-size: 16;
+  margin-top: 1.15em;
 }
 .PageDoc > .contents {
   margin: 20px;
@@ -144,10 +148,10 @@ a {
   border: none;
 }
 a:hover {
-  color: #205493;
+  color: #0097c9;
 }
 .contents a:visited {
-  color: #205493;
+  color: #0071bc;
 }
 #nav-tree .selected {
   background-image: none;

docs_assets/main.md

@@ -4,7 +4,7 @@ __DKAN__ is an open-source open-data platform inspired by [CKAN](https://ckan.or
 
 ## Structure
 
-%Drupal allows _modules_ to extend its functionality. DKAN is a %Drupal module.
+DKAN is a %Drupal module that adds data publishing functionality.
 
 Modules can have submodules; DKAN utilizes this structure to organize its internal subsystems. Information about the subsystems/components in DKAN can be found in the \ref components page.
 
@@ -17,7 +17,7 @@ DKAN's modules and subsystems are organized around four main data functions:
 
 ### Data Management
 
-The main function of any open data platform is to help manage data. Making data _public_ is simple, anyone can place a file in a web-accessible store these days, but making data _open_ takes a bit more work. True open data is accessible, discoverable, machine-readable, linked to other resources that provide context, published in an open format and under an open license. 
+The main function of any open data platform is to help manage data. Making data _public_ is simple, anyone can place a file in a web-accessible server, but making data _open_ takes a bit more work. True open data is accessible, discoverable, machine-readable, linked to other resources that provide context, published in an open format and under an open license.
 
 This is what we mean by data management: providing tools that empower data publishers to make data open, which empowers data consumers to find and use the data they need.
 
@@ -47,4 +47,4 @@ Data in files isn't naturally searchable, but the \ref datastore module parses a
 
 DKAN is actively maintained by [CivicActions](https://civicactions.com/dkan).
 
-To learn more about the DKAN community visit [GetDKAN.org](https://getdkan.org).
+To learn more about the DKAN community visit [DKAN Discussions](https://github.com/GetDKAN/dkan/discussions).

docs_assets/pages/1.0_start.md

@@ -28,7 +28,7 @@ DKAN tools bases new projects off of a [composer project](https://github.com/Get
 
     composer create-project getdkan/recommended-project my-project
 
-Or simply create a project however you prefer and add a requirement for `getdkan/dkan`. **Do note** that a bug in Drupal core cron may cause problems with data imports, and applying [this patch](https://www.drupal.org/project/drupal/issues/3230541#comment-14206814) is highly recommended. The patch will be applied automatically if you use the recommended project.
+Or simply create a project however you prefer and add a requirement for `getdkan/dkan`. **Do note** that a bug in Drupal core cron may cause problems with data imports, and applying [this patch](https://www.drupal.org/project/drupal/issues/3274931) is highly recommended. The patch will be applied automatically if you use the [recommended project](https://github.com/GetDKAN/recommended-project).
 
 ## Installing DKAN
 

docs_assets/pages/3.1_metastore.md

@@ -1,17 +1,19 @@
-@page metastore Metastore
+# Metastore {#metastore}
 
 DKAN's **Metastore** is what you use to create, retrieve, update, and delete records describing your data. These records are what we refer to as "metadata."
 
-As a data catalog, DKAN's main goal is to help you share a collection of @ref Dataset "dataset" records. A dataset's metadata can follow virtually any _schema_, or format you want. What is important is that it points to the data you are trying to share, and that it gives useful contextual information. This usually includes when the data was released, how often it is updated and who published it, but can include details as precise as the geographic boundaries it applies to.
+As a data catalog, DKAN's main goal is to help you share a collection of @ref Dataset "dataset" records. A dataset's metadata can follow virtually any _schema_, or format you want. What is important is that it points to the data you are trying to share, and that it gives useful contextual information. This usually includes when the data was released, how often it is updated and who published it, but can include details as precise as the geographic boundaries or relevant time period the data applies to.
 
 Some more details of DKAN's metastore:
 
-* The data assets themselves (usually in the form of local files or URLs) are referred to internally in DKAN as _resources_.
+* The data assets themselves (usually in the form of local files or URLs to data files) are referred to internally in DKAN as _resources_.
 * The structure and format of dataset metadata in DKAN are determined by a [JSON schema](https://json-schema.org/). By default, DKAN provides and utilizes the [DCAT-US metadata schema](https://resources.data.gov/resources/dcat-us/) to store datasets, but @ref ChangingYourDatasetSchema "custom schemas" can be added to the codebase to override this.
 * In DCAT-US, resources are placed in a sub-schema of the parent dataset called a *distribution*.
 
+![dataset structure](https://project-open-data.cio.gov/v1.1/schema-diagram.svg)
+
 @note
-  Read the documentation on @ref tut_dataset to get started adding information to the metastore.
+  Read the documentation on @ref guide_dataset to get started adding information to the metastore.
 
 ## Configuration
 @anchor ChangingYourDatasetSchema
@@ -20,7 +22,7 @@ Some more details of DKAN's metastore:
 
 Replacing the dataset schema in DKAN allows you to add fields and conform to additional (or completely different) specifications. As long as you provide a valid JSON schema, any information going into the metastore will be validated against it.
 
-To change the schema being used, copy the contents of the `schema` directory from the DKAN repo and place it in the root of your Drupal installation (src/schema). Then make any modifications necessary to the `dataset.json` file inside the `collections` directory.
+To change the schema being used, copy the contents of the `schema` directory from the DKAN repo and place it in the root of your Drupal installation (src/schema). Then make any modifications necessary to the `dataset.json` file inside the `collections` directory. Note that even if you are only changing the dataset.json schema, it is important to copy ALL of the schema files as DKAN will be expecting all of the schema files to be in the same location.
 
 @warning
   Warning: The schema is actively used by the catalog to verify the validity of the data. Making changes to the schema after data is present in the catalog should be done with care as non-backward-compatible changes to the schema could cause issues. Look at Drupal::metastore::SchemaRetriever::findSchemaDirectory() for context.
@@ -41,4 +43,6 @@ _(Note that only option #1 above has been implemented in the current version of
 
 Data dictionaries will affect the behavior of the @ref datastore.
 
-See @ref tut_data_dictionaries for step-by-step instructions for use.
+By default, all data imported into a datastore will be stored as strings. Use the data dictionary to define which columns should be stored as dates, integers, decimals, etc.
+
+See @ref guide_data_dictionaries for step-by-step instructions for use.

docs_assets/pages/3.2_datastore.md

@@ -1,17 +1,20 @@
-@page datastore Datastore
+# Datastore {#datastore}
 
-The DKAN **Datastore** module provides integration with the [datastore](https://github.com/GetDKAN/datastore) library. With it you are able to parse CSV files and save the data in database tables. This allows users to query the data through a public API. Note that the CSV must be in UTF-8 format to parse correctly.
+The DKAN **Datastore** provides a service to parse CSV files and save the data in database tables. This allows users to query the data through a [public API](https://demo.getdkan.org/api).
 
-When a dataset is added to the catalog, the mimetype value of each distribution object is checked to determine if the distribution is a CSV file or not. CSV distributions will be added to the import queue to be parsed and imported into the datastore.
+When a dataset is added to the catalog, the mimetype value of each distribution object is checked to determine if the distribution contains a CSV file. The CSV distributions will be added to the import queue to be parsed and imported into the datastore.
+- When providing a remote URL for the distribution object, be sure to include the mimetype property: `"mediaType": "text/csv"`.
+- Note that the CSV must be in UTF-8 format to parse correctly.
 
-The import will happen in batches during cron runs. The import will iterate over the file, if it does not finish, the job will be returned to the queue. Be sure to have cron running on a regular basis so that large file imports will complete. If there is a change to the metadata it will trigger a new import to the datastore.
+By default, the import will happen in batches during cron runs. The import will iterate over the file, if it does not finish, the job will be returned to the queue. Be sure to have cron running on a regular basis so that large file imports will complete. You can view the status of datastore imports at `/admin/dkan/datastore/status`.
 
-You can manually import file data into the datastore via drush with the identifier of the distribution. Use the [API](https://demo.getdkan.org/api/1/metastore/schemas/dataset/items?show-reference-ids) to get the identifier of the file you want to import. The identifier will be at ``distribution.0.data.%Ref:downloadURL.0.data.identifier``
+If you enable the @subpage mysqlimport module, the file will be imported in a single step using MySQL's native LOAD DATA function.
 
-## Drush Commands
-* @ref dkandatastorelist
-* @ref dkandatastoreimport
-* @ref dkandatastoredrop
+If there is a change to the _distribution: downloadURL_ property, the existing datastore will be dropped and a new import will be triggered.
+
+## Datastore Settings
+
+See @ref guide_datastore_settings for configuration information.
 
 ## Data dictionary integration
 
@@ -21,4 +24,4 @@ If you are using DKAN Data Dictionaries on your site, the datastore will refer t
 2. Create a new job in the queue to alter the new table in the database to match its columns to the field types defined in the dictionary, and change the column description/comment in the database to match the "title" in the dictionary.
 3. Report the status of the alter job once it has run.
 
-See @ref metastore and @ref tut_data_dictionaries for more on how to use this feature.
+See @ref metastore and @ref guide_data_dictionaries for more on how to use this feature.

docs_assets/pages/3.3_harvest.md

@@ -1,19 +1,7 @@
-@page harvest Harvest
+# Harvest {#harvest}
 
-The DKAN **Harvest** module provides integration with the [harvest](https://github.com/GetDKAN/harvest) library. This enables you to use the public feed or API of another data portal and import items from that portal’s catalog into your own.
+The DKAN **Harvest** module enables you to use the public feed or API of another data portal and import items from that portal’s catalog into your own.
 
 For example, [Data.gov](https://data.gov/) harvests all of its datasets from the [data.json](https://project-open-data.cio.gov/v1.1/schema/) files of [hundreds of U.S. federal, state and local data portals](https://catalog.data.gov/harvest).
 
-A [harvest](glossary.html#term_harvest) is the execution of a [Harvest Plan](glossary.html#term_harvestplan).
-
-## Drush Commands
-* @ref dkanharvestregister
-* @ref dkanharvestderegister
-* @ref dkanharvestrun
-* @ref dkanharvestrunall
-* @ref dkanharvestrevert
-* @ref dkanharveststatus
-* @ref dkanharvestinfo
-* @ref dkanharvestlist
-* @ref dkanharvestpublish
-* @ref dkanharvestarchive
+See @ref guide_harvest

docs_assets/pages/7.0_guide.md

@@ -0,0 +1,11 @@
+@page guide User Guide
+
+@subpage guide_dataset
+
+@subpage guide_metastore_properties
+
+@subpage guide_datastore_settings
+
+@subpage guide_harvest
+
+@subpage guide_data_dictionaries

docs_assets/pages/8.0_developer.md

@@ -0,0 +1,5 @@
+@page developer Developer
+
+@subpage dev_metastore
+
+@subpage dev_frontend

docs_assets/pages/community_code-of-conduct.md

@@ -1,4 +1,4 @@
-@page conduct Code of conduct
+# Code of conduct {#conduct}
 
 ## Purpose
 

docs_assets/pages/dkan_dataset_info.md

@@ -1,4 +1,4 @@
-@page dkandatasetinfo dkan:dataset-info
+# dkan:dataset-info {#dkandatasetinfo}
 
 View information about a dataset.
 

docs_assets/pages/dkan_datastore_drop.md

@@ -1,4 +1,4 @@
-@page dkandatastoredrop dkan:datastore:drop
+# dkan:datastore:drop {#dkandatastoredrop}
 
 Drop a datastore.
 

docs_assets/pages/dkan_datastore_dropall.md

@@ -1,3 +1,3 @@
-@page dkandatastoredropall dkan:datastore:drop-all
+# dkan:datastore:drop-all {#dkandatastoredropall}
 
 Drop a ALL datastore tables.

docs_assets/pages/dkan_datastore_import.md

@@ -1,6 +1,8 @@
-@page dkandatastoreimport dkan:datastore:import
+# dkan:datastore:import {#dkandatastoreimport}
 
-Import a datastore.
+You can manually import file data into the datastore with the identifier of the distribution. There are two main ways to get the distribution uuid:
+- Use the [API](https://demo.getdkan.org/api/1/metastore/schemas/dataset/items?show-reference-ids) to get the identifier of the file you want to import. The identifier will be at ``distribution.0.data.%Ref:downloadURL.0.data.identifier``
+- Use @ref dkandatasetinfo
 
 #### Arguments
 

docs_assets/pages/dkan_datastore_list.md

@@ -1,4 +1,4 @@
-@page dkandatastorelist dkan:datastore:list
+# dkan:datastore:list {#dkandatastorelist}
 
 List information about all datastores.
 

docs_assets/pages/dkan_harvest_archive.md

@@ -1,4 +1,4 @@
-@page dkanharvestarchive dkan:harvest:archive
+# dkan:harvest:archive {#dkanharvestarchive}
 
 Archives (unpublish) harvested entities.
 

docs_assets/pages/dkan_harvest_deregister.md

@@ -1,4 +1,4 @@
-@page dkanharvestderegister dkan:harvest:deregister
+# dkan:harvest:deregister {#dkanharvestderegister}
 
 Deregister a harvest.
 

docs_assets/pages/dkan_harvest_info.md

@@ -1,4 +1,4 @@
-@page  dkanharvestinfo dkan:harvest:info
+# dkan:harvest:info {#dkanharvestinfo}
 
 Give information about a previous harvest run.
 

docs_assets/pages/dkan_harvest_list.md

@@ -1,4 +1,4 @@
-@page dkanharvestlist dkan:harvest:list
+# dkan:harvest:list {#dkanharvestlist}
 
 List available harvests.
 

docs_assets/pages/dkan_harvest_publish.md

@@ -1,4 +1,4 @@
-@page dkanharvestpublish dkan:harvest:publish
+# dkan:harvest:publish {#dkanharvestpublish}
 
 Publishes harvested entities.
 

docs_assets/pages/dkan_harvest_register.md

@@ -1,4 +1,4 @@
-@page dkanharvestregister dkan:harvest:register
+# dkan:harvest:register {#dkanharvestregister}
 
 Register a new harvest.
 

docs_assets/pages/dkan_harvest_revert.md

@@ -1,4 +1,4 @@
-@page dkanharvestrevert dkan:harvest:revert
+# dkan:harvest:revert {#dkanharvestrevert}
 
 Revert a harvest, i.e. remove harvested entities and unpublish orhpaned keywords, themes, and distributions.
 

docs_assets/pages/dkan_harvest_run-all.md

@@ -1,4 +1,4 @@
-@page dkanharvestrunall dkan:harvest:run-all
+# dkan:harvest:run-all {#dkanharvestrunall}
 
 Run all pending harvests.
 

docs_assets/pages/dkan_harvest_run.md

@@ -1,4 +1,4 @@
-@page dkanharvestrun dkan:harvest:run
+# dkan:harvest:run {#dkanharvestrun}
 
 Run a harvest.
 

docs_assets/pages/dkan_harvest_status.md

@@ -1,4 +1,4 @@
-@page dkanharveststatus dkan:harvest:status
+# dkan:harvest:status {#dkanharveststatus}
 
 Show status of of a particular harvest run.
 

docs_assets/pages/dkan_metadata-form_sync.md

@@ -1,4 +1,4 @@
-@page dkanmetadataformsync dkan:metadata-form:sync
+# dkan:metadata-form:sync {#dkanmetadataformsync}
 
 Synchronize the module with the React app.
 

docs_assets/pages/dkan_metastore-search_rebuild-tracker.md

@@ -1,3 +1,3 @@
-@page dkanmetastoresearchrebuildtracker dkan:metastore-search:rebuild-tracker
+# dkan:metastore-search:rebuild-tracker {#dkanmetastoresearchrebuildtracker}
 
 Rebuild the search api tracker for the dkan index.

docs_assets/pages/dkan_metastore_publish.md

@@ -1,4 +1,4 @@
-@page dkanmetastorepublish dkan:metastore:publish
+# dkan:metastore:publish {#dkanmetastorepublish}
 
 Publish the latest version of a dataset.
 

docs_assets/pages/dkan_sample-content_create.md

@@ -1,3 +1,3 @@
-@page dkansamplecontentcreate dkan:sample-content:create
+# dkan:sample-content:create {#dkansamplecontentcreate}
 
 Create sample content.

docs_assets/pages/dkan_sample-content_remove.md

@@ -1,3 +1,3 @@
-@page dkansamplecontentremove dkan:sample-content:remove
+# dkan:sample-content:remove {#dkansamplecontentremove}
 
 Remove sample content.