skipHidden -> skipFields

src/main/assembly/dist/cfg/config.yml

@@ -27,20 +27,8 @@ server:
       port: 20301
 
 ingestFlow:
-  skipHidden:
-    # see also dd-dtap/provisioning/group_vars/
-    # configure_metadata_blocks.hidden_fields in ds_dataversenl.yml
-    # shared_configure_metadata_blocks_hidden_fields in datastations.yml
-    # do not ignore data vault metadata
-    - topicClassification
-    - publication
-    - dateOfDeposit
-    - relatedMaterial
-    - relatedDatasets
-    - otherReferences
-    - originOfSources
-    - characteristicOfSources
-    - accessToSources
+  skipFields: [] # see hidden-fields in dd-dtap though don't skip data vault metadata
+
   import:
     # apiKey: # overridden key for the import area
     inbox: /var/opt/dans.knaw.nl/tmp/import/inbox

src/main/java/nl/knaw/dans/ingest/core/config/IngestFlowConfig.java

@@ -26,7 +26,7 @@
 
 public class IngestFlowConfig {
 
-    private List<String> skipHidden;
+    private List<String> skipFields;
 
     @NotNull
     @Valid
@@ -197,11 +197,11 @@ public void setDataSuppliers(Map<String, String> dataSuppliers) {
         this.dataSuppliers = dataSuppliers == null ? Map.of() : dataSuppliers;
     }
 
-    public List<String> getSkipHidden() {
-        return skipHidden;
+    public List<String> getSkipFields() {
+        return skipFields;
     }
 
-    public void setSkipHidden(List<String> skipHidden) {
-        this.skipHidden = skipHidden;
+    public void setSkipFields(List<String> skipFields) {
+        this.skipFields = skipFields;
     }
 }

src/main/java/nl/knaw/dans/ingest/core/service/DepositIngestTaskFactoryBuilder.java

@@ -75,7 +75,7 @@ public DepositIngestTaskFactory createTaskFactory(IngestAreaConfig ingestAreaCon
             ingestFlowConfig.getIso2ToDataverseLanguage(),
             ingestFlowConfig.getSpatialCoverageCountryTerms(),
             ingestFlowConfig.getDataSuppliers(),
-            configuration.getIngestFlow().getSkipHidden(), dataverseClient
+            configuration.getIngestFlow().getSkipFields(), dataverseClient
         );
         final var datasetService = new DataverseServiceImpl(
             dataverseClient,

src/main/java/nl/knaw/dans/ingest/core/service/mapper/DepositToDvDatasetMetadataMapper.java

@@ -96,19 +96,19 @@ public class DepositToDvDatasetMetadataMapper {
     private final Map<String, String> iso2ToDataverseLanguage;
     private final List<String> spatialCoverageCountryTerms;
     private final Map<String, String> dataSuppliers;
-    private final List<String> skipHidden;
+    private final List<String> skipFields;
     private final boolean isMigration;
     private final boolean deduplicate;
 
     DepositToDvDatasetMetadataMapper(boolean deduplicate, Set<String> activeMetadataBlocks, Map<String, String> iso1ToDataverseLanguage,
-                                     Map<String, String> iso2ToDataverseLanguage, List<String> spatialCoverageCountryTerms, Map<String, String> dataSuppliers, List<String> skipHidden, boolean isMigration) {
+                                     Map<String, String> iso2ToDataverseLanguage, List<String> spatialCoverageCountryTerms, Map<String, String> dataSuppliers, List<String> skipFields, boolean isMigration) {
         this.deduplicate = deduplicate;
         this.activeMetadataBlocks = activeMetadataBlocks;
         this.iso1ToDataverseLanguage = iso1ToDataverseLanguage;
         this.iso2ToDataverseLanguage = iso2ToDataverseLanguage;
         this.spatialCoverageCountryTerms = spatialCoverageCountryTerms;
         this.dataSuppliers = dataSuppliers;
-        this.skipHidden = skipHidden;
+        this.skipFields = skipFields;
         this.isMigration = isMigration;
     }
 
@@ -249,14 +249,14 @@ public Dataset toDataverseDataset(
             dataVaultFieldBuilder.addDataSupplier(dataSupplier);
         }
 
-        return assembleDataverseDataset(termsOfAccess, skipHidden);
+        return assembleDataverseDataset(termsOfAccess, skipFields);
     }
 
     private Stream<Node> getPersonalData(Document ddm) {
         return XPathEvaluator.nodes(ddm, "/ddm:DDM/ddm:profile/ddm:personalData");
     }
 
-    void processMetadataBlock(boolean deduplicate, Map<String, MetadataBlock> fields, String title, String displayName, FieldBuilder builder, List<String> skipHidden) {
+    void processMetadataBlock(boolean deduplicate, Map<String, MetadataBlock> fields, String title, String displayName, FieldBuilder builder, List<String> skipFields) {
         // TODO figure out how to deduplicate compound fields (just on key, or also on value?)
         var compoundFields = builder.getCompoundFields().values()
                 .stream()
@@ -275,7 +275,7 @@ void processMetadataBlock(boolean deduplicate, Map<String, MetadataBlock> fields
         List<MetadataField> result = Stream.of(compoundFields, primitiveFields)
                 .flatMap(i -> i)
                 .filter(Objects::nonNull)
-                .filter(b -> !skipHidden.contains(b.getTypeName()))
+                .filter(b -> !skipFields.contains(b.getTypeName()))
                 .collect(Collectors.toList());
 
         var block = new MetadataBlock();
@@ -285,15 +285,15 @@ void processMetadataBlock(boolean deduplicate, Map<String, MetadataBlock> fields
         fields.put(title, block);
     }
 
-    private Dataset assembleDataverseDataset(String termsOfAccess, List<String> skipHidden) {
+    private Dataset assembleDataverseDataset(String termsOfAccess, List<String> skipFields) {
         var fields = new HashMap<String, MetadataBlock>();
 
-        processMetadataBlock(deduplicate, fields, "citation", "Citation Metadata", citationFields, skipHidden);
-        processMetadataBlock(deduplicate, fields, "dansRights", "Rights Metadata", rightsFields, skipHidden);
-        processMetadataBlock(deduplicate, fields, "dansRelationMetadata", "Relation Metadata", relationFields, skipHidden);
-        processMetadataBlock(deduplicate, fields, "dansArchaeologyMetadata", "Archaeology-Specific Metadata", archaeologyFields, skipHidden);
-        processMetadataBlock(deduplicate, fields, "dansTemporalSpatial", "Temporal and Spatial Coverage", temporalSpatialFields, skipHidden);
-        processMetadataBlock(deduplicate, fields, "dansDataVaultMetadata", "Dans Vault Metadata", dataVaultFieldBuilder, skipHidden);
+        processMetadataBlock(deduplicate, fields, "citation", "Citation Metadata", citationFields, skipFields);
+        processMetadataBlock(deduplicate, fields, "dansRights", "Rights Metadata", rightsFields, skipFields);
+        processMetadataBlock(deduplicate, fields, "dansRelationMetadata", "Relation Metadata", relationFields, skipFields);
+        processMetadataBlock(deduplicate, fields, "dansArchaeologyMetadata", "Archaeology-Specific Metadata", archaeologyFields, skipFields);
+        processMetadataBlock(deduplicate, fields, "dansTemporalSpatial", "Temporal and Spatial Coverage", temporalSpatialFields, skipFields);
+        processMetadataBlock(deduplicate, fields, "dansDataVaultMetadata", "Dans Vault Metadata", dataVaultFieldBuilder, skipFields);
 
         checkRequiredField(fields, "citation", TITLE);
         checkRequiredField(fields, "citation", SUBJECT);

src/main/java/nl/knaw/dans/ingest/core/service/mapper/DepositToDvDatasetMetadataMapperFactory.java

@@ -34,16 +34,16 @@ public class DepositToDvDatasetMetadataMapperFactory {
     private final Map<String, String> iso2ToDataverseLanguage;
     private final List<String> spatialCoverageCountryTerms;
     private final Map<String, String> dataSuppliers;
-    private final List<String> skipHidden;
+    private final List<String> skipFields;
     private final DataverseClient dataverseClient;
 
     public DepositToDvDatasetMetadataMapperFactory(Map<String, String> iso1ToDataverseLanguage, Map<String, String> iso2ToDataverseLanguage,
-        List<String> spatialCoverageCountryTerms, Map<String, String> dataSuppliers, List<String> skipHidden, DataverseClient dataverseClient) {
+        List<String> spatialCoverageCountryTerms, Map<String, String> dataSuppliers, List<String> skipFields, DataverseClient dataverseClient) {
         this.iso1ToDataverseLanguage = iso1ToDataverseLanguage;
         this.iso2ToDataverseLanguage = iso2ToDataverseLanguage;
         this.spatialCoverageCountryTerms = spatialCoverageCountryTerms;
         this.dataSuppliers = dataSuppliers;
-        this.skipHidden = skipHidden;
+        this.skipFields = skipFields;
         this.dataverseClient = dataverseClient;
     }
 
@@ -55,7 +55,7 @@ public DepositToDvDatasetMetadataMapper createMapper(boolean deduplicate, boolea
             iso2ToDataverseLanguage,
             spatialCoverageCountryTerms,
             dataSuppliers,
-            skipHidden,
+            skipFields,
             isMigration
         );
     }

src/test/java/nl/knaw/dans/ingest/core/DepositStartImportTaskWrapperTest.java

@@ -76,7 +76,7 @@ public class DepositStartImportTaskWrapperTest {
 
     private final Map<String, String> iso1ToDataverseLanguage = new HashMap<>();
     private final Map<String, String> iso2ToDataverseLanguage = new HashMap<>();
-    private final List<String> skipHidden = List.of();
+    private final List<String> skipFields = List.of();
 
     private DepositIngestTask createTaskWrapper(String depositName, String created) {
         var mapper = getMapperFactory();
@@ -130,7 +130,7 @@ DepositToDvDatasetMetadataMapperFactory getMapperFactory() {
             iso1ToDataverseLanguage, iso2ToDataverseLanguage,
             List.of("Netherlands", "United Kingdom", "Belgium", "Germany"),
             Map.of(),
-            skipHidden, Mockito.mock(DataverseClient.class)
+            skipFields, Mockito.mock(DataverseClient.class)
         );
     }
 

src/test/java/nl/knaw/dans/ingest/core/service/mapper/CitationMetadataFromDcmiTest.java

@@ -477,9 +477,9 @@ void mapping_skips_hidden() throws Exception {
             + "        <dc:identifier xsi:type='ISBN'>978-3-16-148410-0</dc:identifier>")
             + "</ddm:DDM>");
 
-        var skipHidden = List.of("dateOfDeposit", "publication");
+        var skipFields = List.of("dateOfDeposit", "publication");
         var activeMetadataBlocks = Set.of("citation", "dansRights", "dansDataVaultMetadata");
-        var result = new DepositToDvDatasetMetadataMapper(true, activeMetadataBlocks, Map.of(), Map.of(), List.of(), config.getDataSuppliers(), skipHidden, true)
+        var result = new DepositToDvDatasetMetadataMapper(true, activeMetadataBlocks, Map.of(), Map.of(), List.of(), config.getDataSuppliers(), skipFields, true)
             .toDataverseDataset(doc, null, "2023-02-27", mockedContact, mockedVaultMetadata,null, false, null, null);
         var field = getCompoundMultiValueField("citation", PUBLICATION, result);
         assertThat(field).isNull();

src/test/java/nl/knaw/dans/ingest/core/service/mapper/DepositToDvDatasetMetadataMapperTest.java

@@ -50,7 +50,7 @@ class DepositToDvDatasetMetadataMapperTest {
     private final Map<String, String> iso2ToDataverseLanguage = new HashMap<>();
     private final List<String> spatialCoverageCountryTerms = List.of("Netherlands", "United Kingdom", "Belgium", "Germany");
     private final Map<String, String> dataSuppliers = new HashMap<>();
-    private final List<String> skipHidden = List.of();
+    private final List<String> skipFields = List.of();
 
     Document readDocument(String name) throws ParserConfigurationException, IOException, SAXException {
         return xmlReader.readXmlFile(Path.of(
@@ -60,12 +60,12 @@ Document readDocument(String name) throws ParserConfigurationException, IOExcept
 
     DepositToDvDatasetMetadataMapper getMigrationMapper() {
         return new DepositToDvDatasetMetadataMapper(
-            true, activeMetadataBlocks, iso1ToDataverseLanguage, iso2ToDataverseLanguage, spatialCoverageCountryTerms, dataSuppliers, skipHidden, true);
+            true, activeMetadataBlocks, iso1ToDataverseLanguage, iso2ToDataverseLanguage, spatialCoverageCountryTerms, dataSuppliers, skipFields, true);
     }
 
     DepositToDvDatasetMetadataMapper getNonMigrationMapper() {
         return new DepositToDvDatasetMetadataMapper(
-            true, activeMetadataBlocks, iso1ToDataverseLanguage, iso2ToDataverseLanguage, spatialCoverageCountryTerms, dataSuppliers, skipHidden, false);
+            true, activeMetadataBlocks, iso1ToDataverseLanguage, iso2ToDataverseLanguage, spatialCoverageCountryTerms, dataSuppliers, skipFields, false);
     }
 
     @BeforeEach
@@ -140,7 +140,7 @@ void test_get_acquisition_methods() throws Exception {
 
     @Test
     void processMetadataBlock_should_deduplicate_items_for_PrimitiveFieldBuilder() {
-        var mapper = new DepositToDvDatasetMetadataMapper(true, Set.of("citation"), Map.of(), Map.of(), spatialCoverageCountryTerms, dataSuppliers, skipHidden, true);
+        var mapper = new DepositToDvDatasetMetadataMapper(true, Set.of("citation"), Map.of(), Map.of(), spatialCoverageCountryTerms, dataSuppliers, skipFields, true);
         var fields = new HashMap<String, MetadataBlock>();
         var builder = new ArchaeologyFieldBuilder();
         builder.addArchisZaakId(Stream.of(
@@ -150,7 +150,7 @@ void processMetadataBlock_should_deduplicate_items_for_PrimitiveFieldBuilder() {
             "TEST"
         ));
 
-        mapper.processMetadataBlock(true, fields, "title", "name", builder, skipHidden);
+        mapper.processMetadataBlock(true, fields, "title", "name", builder, skipFields);
 
         // the fourth item should be removed
         assertThat(fields.get("title").getFields())
@@ -161,7 +161,7 @@ void processMetadataBlock_should_deduplicate_items_for_PrimitiveFieldBuilder() {
     @Test
     void processMetadataBlock_should_deduplicate_items_for_CompoundFieldBuilder() {
         var fields = new HashMap<String, MetadataBlock>();
-        var mapper = new DepositToDvDatasetMetadataMapper(true, Set.of("citation"), Map.of(), Map.of(), spatialCoverageCountryTerms, dataSuppliers, skipHidden, true);
+        var mapper = new DepositToDvDatasetMetadataMapper(true, Set.of("citation"), Map.of(), Map.of(), spatialCoverageCountryTerms, dataSuppliers, skipFields, true);
         var builder = new ArchaeologyFieldBuilder();
         builder.addArchisZaakId(Stream.of(
             "TEST",
@@ -170,7 +170,7 @@ void processMetadataBlock_should_deduplicate_items_for_CompoundFieldBuilder() {
             "TEST"
         ));
 
-        mapper.processMetadataBlock(true, fields, "title", "name", builder, skipHidden);
+        mapper.processMetadataBlock(true, fields, "title", "name", builder, skipFields);
 
         // the fourth item should be removed
         assertThat(fields.get("title").getFields())

src/test/java/nl/knaw/dans/ingest/core/service/mapper/MappingTestHelper.java

@@ -102,7 +102,7 @@ static DepositToDvDatasetMetadataMapper createMapper(boolean isMigration) {
             config.getIso2ToDataverseLanguage(),
             config.getSpatialCoverageCountryTerms(),
             config.getDataSuppliers(),
-            config.getSkipHidden(), isMigration);
+            config.getSkipFields(), isMigration);
     }
 
     public static Document ddmWithCustomProfileContent(String content) throws ParserConfigurationException, IOException, SAXException {

src/test/resources/debug-etc/config.yml

@@ -12,20 +12,8 @@ server:
       port: 20301
 
 ingestFlow:
-  skipHidden:
-    # see also dd-dtap/provisioning/group_vars/
-    # configure_metadata_blocks.hidden_fields in ds_dataversenl.yml
-    # shared_configure_metadata_blocks_hidden_fields in datastations.yml
-    # do not ignore data vault metadata
-    - topicClassification
-    - publication
-    - dateOfDeposit
-    - relatedMaterial
-    - relatedDatasets
-    - otherReferences
-    - originOfSources
-    - characteristicOfSources
-    - accessToSources
+  skipFields: [] # see hidden_fields in dd-dtap though don't skip data vault metadata
+
   import:
     apiKey: 'changeme' # overrides the default
     inbox: data/import/inbox

src/test/resources/test-etc/config.yml

@@ -12,10 +12,10 @@ server:
       port: 20301
 
 ingestFlow:
-  skipHidden:
+  skipFields:
     - topicClassification
-#    - publication
-#    - dateOfDeposit
+#    - publication # would break existing tests
+#    - dateOfDeposit # would break existing tests
     - relatedMaterial
     - relatedDatasets
     - otherReferences