Added formatId lookup table

DataONEorg · Sep 24, 2019 · d2566bb · d2566bb
1 parent cbc6805
commit d2566bb
Show file tree

Hide file tree

Showing 3 changed files with 111 additions and 33 deletions.
diff --git a/idxprocdoc.py b/idxprocdoc.py
@@ -1053,7 +1053,7 @@ def toText(self, dest_folder=None):
             for c in range(0, len(field_format_matrix["colmeta"])):
                 v = " "
                 if field in sysmeta_fields:
-                    v = "X"
+                    v = "S"
                 else:
                     # lookup field for each format Id
                     formatId = field_format_matrix["colmeta"][c]
@@ -1062,7 +1062,7 @@ def toText(self, dest_folder=None):
                 row.append(v)
             field_format_matrix["rc"].append(row)
         # generate the CSV
-        header = ["Field", ]
+        header = ["Field \ FormatId", ]
         c = 1
         for hv in field_format_matrix["colmeta"]:
             header.append(f'"{c}"')
@@ -1079,7 +1079,7 @@ def toText(self, dest_folder=None):
         with codecs.open(dest, mode="wb", encoding="utf-8") as f_dest:
             f_dest.write(
                 templates["fid_field_cross.rst"]["template"].render(
-                    resolver=self, csvheader=csv_header, csvrows=csv_rows
+                    resolver=self, csvheader=csv_header, csvrows=csv_rows, ffmatrix=field_format_matrix
                 )
             )
 

diff --git a/source/generated/field_format.rst b/source/generated/field_format.rst
@@ -3,8 +3,68 @@
 Field x FormatId Cross Reference
 ================================
 
+.. list-table::
+   :header-rows: 1
+   :widths: 1 5
+
+   * - Index
+     - FormatId
+   * - 1
+     - eml://ecoinformatics.org/eml-2.0.0
+   * - 2
+     - eml://ecoinformatics.org/eml-2.0.1
+   * - 3
+     - eml://ecoinformatics.org/eml-2.1.0
+   * - 4
+     - eml://ecoinformatics.org/eml-2.1.1
+   * - 5
+     - https://eml.ecoinformatics.org/eml-2.2.0
+   * - 6
+     - http://www.openarchives.org/ore/terms
+   * - 7
+     - FGDC-STD-001-1998
+   * - 8
+     - FGDC-STD-001.1-1999
+   * - 9
+     - FGDC-STD-001.2-1999
+   * - 10
+     - http://www.esri.com/metadata/esriprof80.dtd
+   * - 11
+     - http://purl.org/ornl/schema/mercury/terms/v1.0
+   * - 12
+     - http://purl.org/dryad/terms/
+   * - 13
+     - http://datadryad.org/profile/v3.1
+   * - 14
+     - http://dublincore.org/schemas/xmls/qdc/2008/02/11/qualifieddc.xsd
+   * - 15
+     - http://ns.dataone.org/metadata/schema/onedcx/v1.0
+   * - 16
+     - http://datacite.org/schema/kernel-3.1
+   * - 17
+     - http://datacite.org/schema/kernel-3.0
+   * - 18
+     - http://www.w3.org/TR/rdf-syntax-grammar
+   * - 19
+     - http://docs.annotatorjs.org/en/v1.2.x/annotation-format.html
+   * - 20
+     - http://www.isotc211.org/2005/gmd
+   * - 21
+     - http://www.isotc211.org/2005/gmd-noaa
+   * - 22
+     - http://www.isotc211.org/2005/gmd-pangaea
+   * - 23
+     - http://www.openarchives.org/OAI/2.0/oai_dc/
+
+
+
+The following table indicates which formatIds have processing rules defined to set index field values when processing
+metadata. In the table, an "S" means the property is set from system metadata, an "X" means there is a rule defined
+to set the value, and blank indicates no rules are setting the field value (though Solr copy fields are not considered
+here).
+
 .. csv-table:: FormatId x Solr Field
-   :header: Field,"1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23"
+   :header: Field \ FormatId,"1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23"
 
    LTERSite, , , , , , , , , , , , , , , , , , , , , , , 
    abstract,X,X,X,X,X, ,X,X,X,X,X,X,X,X,X,X,X, , ,X,X,X,X
@@ -19,32 +79,32 @@ Field x FormatId Cross Reference
    authorLastName,X,X,X,X,X, , , , , , , , , , ,X,X, , , , , , 
    authorSurName,X,X,X,X,X, ,X,X,X,X,X,X,X,X,X,X,X, , ,X,X,X,X
    authorSurNameSort,X,X,X,X,X, ,X,X,X,X,X,X,X,X,X,X,X, , ,X,X,X,X
-   authoritativeMN,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
+   authoritativeMN,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
    awardNumber, , , , ,X, , , , , , , , , , , , , , , , , , 
    awardTitle, , , , ,X, , , , , , , , , , , , , , , , , , 
    beginDate,X,X,X,X,X, ,X,X,X,X,X, , ,X,X,X,X, , ,X,X,X, 
-   blockedReplicationMN,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
-   changePermission,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
-   checksum,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
-   checksumAlgorithm,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
+   blockedReplicationMN,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
+   changePermission,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
+   checksum,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
+   checksumAlgorithm,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
    class,X,X,X,X,X, ,X,X,X,X,X, , , , , , , , , , , , 
    contactOrganization,X,X,X,X,X, ,X,X,X,X,X, , ,X,X,X,X, , ,X,X,X,X
    contactOrganizationText, , , , , , , , , , , , , , , , , , , , , , , 
    dataUrl, , , , , , , , , , , , , , , , , , , , , , , 
-   datasource,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
-   dateModified,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
+   datasource,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
+   dateModified,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
    datePublished, , , , , , , , , , , , , , , , , , , , , , , 
-   dateUploaded,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
+   dateUploaded,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
    decade, , , , , , , , , , , , , , , , , , , , , , , 
    documents, , , , , , , , , , , , , , , , , , , , , , , 
    eastBoundCoord,X,X,X,X,X, ,X,X,X,X,X, , , , , , , , ,X,X,X, 
    edition, , , , , , ,X,X,X,X,X, , , , , , , , , , , , 
    endDate,X,X,X,X,X, ,X,X,X,X,X, , ,X,X,X,X, , ,X,X,X, 
    family,X,X,X,X,X, ,X,X,X,X,X, , , , , , , , , , , , 
-   fileID,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
-   fileName,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
-   formatId,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
-   formatType,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
+   fileID,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
+   fileName,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
+   formatId,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
+   formatType,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
    funderIdentifier, , , , ,X, , , , , , , , , , , , , , , , , , 
    funderName, , , , ,X, , , , , , , , , , , , , , , , , , 
    funding, , , , ,X, , , , , , , , , , , , , , , , , , 
@@ -61,26 +121,26 @@ Field x FormatId Cross Reference
    geohash_7,X,X,X,X,X, ,X,X,X,X,X, , , , , , , , ,X,X,X, 
    geohash_8,X,X,X,X,X, ,X,X,X,X,X, , , , , , , , ,X,X,X, 
    geohash_9,X,X,X,X,X, ,X,X,X,X,X, , , , , , , , ,X,X,X, 
-   id,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
+   id,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
    identifier, , , , , , , , , , , , , , , , , , , , , , , 
    investigator,X,X,X,X,X, ,X,X,X,X,X,X,X,X,X,X,X, , ,X,X,X,X
    investigatorText, , , , , , , , , , , , , , , , , , , , , , , 
    isDocumentedBy, , , , , , , , , , , , , , , , , , , , , , , 
-   isPublic,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
+   isPublic,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
    isService,X,X,X,X,X, , , , , , , , , , , , , , ,X,X,X, 
    isSpatial, , , , , , , , , , , , , , , , , , , , , , , 
    keyConcept, , , , , , , , , , , , , , , , , , , , , , , 
    keywords,X,X,X,X,X, ,X,X,X,X,X,X,X,X,X,X,X, , ,X,X,X,X
    keywordsText, , , , , , , , , , , , , , , , , , , , , , , 
    kingdom,X,X,X,X,X, ,X,X,X,X,X, , , , , , , , , , , , 
-   mediaType,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
-   mediaTypeProperty,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
+   mediaType,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
+   mediaTypeProperty,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
    namedLocation, , , , , , , , , , , , , , , , , , , , , , , 
    noBoundingBox, , , , , , , , , , , , , , , , , , , , , , , 
    northBoundCoord,X,X,X,X,X, ,X,X,X,X,X, , ,X,X, , , , ,X,X,X, 
-   numberReplicas,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
-   obsoletedBy,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
-   obsoletes,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
+   numberReplicas,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
+   obsoletedBy,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
+   obsoletes,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
    ogcUrl, , , , , , , , , , , , , , , , , , , , , , , 
    order,X,X,X,X,X, ,X,X,X,X,X, , , , , , , , , , , , 
    origin,X,X,X,X,X, ,X,X,X,X,X,X,X,X,X,X,X, , ,X,X,X,X
@@ -91,7 +151,7 @@ Field x FormatId Cross Reference
    parameterText, , , , , , , , , , , , , , , , , , , , , , , 
    phylum,X,X,X,X,X, ,X,X,X,X,X, , , , , , , , , , , , 
    placeKey, , , , , , ,X,X,X,X,X, , , , , , , , , , , , 
-   preferredReplicationMN,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
+   preferredReplicationMN,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
    presentationCat, , , , , , ,X,X,X,X,X, , , , , , , , , , , , 
    project,X,X,X,X,X, , , , , , , , , , , , , , , , , , 
    projectText, , , , , , , , , , , , , , , , , , , , , , , 
@@ -112,21 +172,21 @@ Field x FormatId Cross Reference
    prov_wasInformedBy, , , , , ,X, , , , , , , , , , , ,X, , , , , 
    pubDate,X,X,X,X,X, ,X,X,X,X,X,X,X,X,X,X,X, , ,X,X,X,X
    purpose, , , , , , ,X,X,X,X,X, , , , , , , , , , , , 
-   readPermission,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
+   readPermission,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
    relatedOrganizations, , , , , , , , , , , , , , , , , , , , , , , 
-   replicaMN,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
-   replicaVerifiedDate,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
-   replicationAllowed,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
+   replicaMN,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
+   replicaVerifiedDate,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
+   replicationAllowed,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
    resourceMap, , , , , , , , , , , , , , , , , , , , , , , 
-   rightsHolder,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
+   rightsHolder,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
    scientificName,X,X,X,X,X, ,X,X,X,X,X,X,X, , , , , , , , , , 
    sem_annotated_by, , , , , , , , , , , , , , , , , , , , , , , 
    sem_annotates, , , , , , , , , , , , , , , , , , , , , , , 
    sem_annotation, , , , ,X, , , , , , , , , , , , , ,X, , , , 
    sem_comment, , , , , , , , , , , , , , , , , , , , , , , 
    sensor, , , , , , , , , , , , , , , , , , , , , , , 
    sensorText, , , , , , , , , , , , , , , , , , , , , , , 
-   seriesId,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
+   seriesId,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
    serviceCoupling, , , , , , , , , , , , , , , , , , , ,X,X,X, 
    serviceDescription,X,X,X,X,X, , , , , , , , , , , , , , ,X,X,X, 
    serviceEndpoint,X,X,X,X,X, , , , , , , , , , , , , , ,X,X,X,X
@@ -136,12 +196,12 @@ Field x FormatId Cross Reference
    serviceType, , , , , , , , , , , , , , , , , , , ,X,X,X, 
    site,X,X,X,X,X, ,X,X,X,X,X,X,X,X,X,X,X, , , , , , 
    siteText, , , , , , , , , , , , , , , , , , , , , , , 
-   size,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
+   size,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
    source, , , , , , , , , , , , , , , , , , , , , , , 
    sourceText, , , , , , , , , , , , , , , , , , , , , , , 
    southBoundCoord,X,X,X,X,X, ,X,X,X,X,X, , , , , , , , ,X,X,X, 
    species,X,X,X,X,X, ,X,X,X,X,X, , , , , , , , , , , , 
-   submitter,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
+   submitter,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
    term, , , , , , , , , , , , , , , , , , , , , , , 
    termText, , , , , , , , , , , , , , , , , , , , , , , 
    text,X,X,X,X,X, ,X,X,X,X,X,X,X,X,X,X,X, , ,X,X,X,X
@@ -152,5 +212,6 @@ Field x FormatId Cross Reference
    updateDate, , , , , , , , , , , , , , , , , , , , , , , 
    webUrl, , , , , , ,X,X,X,X,X, , , , , , , , , , , , 
    westBoundCoord,X,X,X,X,X, ,X,X,X,X,X, , , , , , , , ,X,X,X, 
-   writePermission,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X,X
+   writePermission,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S,S
 
+
diff --git a/templates/fid_field_cross.rst b/templates/fid_field_cross.rst
@@ -6,9 +6,26 @@ formatId with solr field
 Field x FormatId Cross Reference
 ================================
 
+.. list-table::
+   :header-rows: 1
+   :widths: 1 5
+
+   * - Index
+     - FormatId
+   {%for entry in ffmatrix.colmeta%}* - {{loop.index}}
+     - {{entry}}
+   {% endfor %}
+
+
+The following table indicates which formatIds have processing rules defined to set index field values when processing
+metadata. In the table, an "S" means the property is set from system metadata, an "X" means there is a rule defined
+to set the value, and blank indicates no rules are setting the field value (though Solr copy fields are not considered
+here).
+
 .. csv-table:: FormatId x Solr Field
    :header: {{csvheader}}
 
    {%for row in csvrows%}{{row}}
    {% endfor %}
 
+