Merge branch 'main' into snyk-fix-e8264e89d591506fede024e9ef787bee

Author: pkiraly

.github/workflows/maven.yml

@@ -15,7 +15,8 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Check out the repo
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
 

README.md

@@ -40,6 +40,8 @@ digital collections.
         * [`unique <boolean>`](#unique-boolean)
         * [`dependencies [id1, id2, ..., idN]`](#dependencies-id1-id2--idn)
         * [`dimension [criteria1, criteria2, ..., criteriaN]`](#dimension-criteria1-criteria2--criterian)
+        * [`hasLanguageTag <anyOf|oneOf|allOf>`](#haslanguagetag-anyofoneofallof)
+        * [`isMultilingual <boolean>`](#ismultilingual-boolean)
       - [General properties](#general-properties)
         * [`id <String>`](#id-string)
         * [`description <String>`](#description-string)
@@ -92,6 +94,9 @@ usage:
       [-f <format>] [-h <arg>] [-o <file>] [-r <path>] [-v <format>] [-w <format>] [-z]
 ```
 * `-i,--input <file>` Input file.
+* `-n,--inputFormat <format>` (optional, String) The format of input file. Right now it supports two JSON variants:
+  * `ndjson`: line delimited JSON in which every line is a new record (the default value)
+  * `json-array`: JSON file that contains an array of objects
 * `-s,--schema <file>` Schema file describing the metadata structure to run assessment against.
 * `-v,--schemaFormat <format>` Format of schema file: json, yaml. Default: based on file extension, else json.
 * `-m,--measurements <file>` Configuration file for measurements.
@@ -505,6 +510,11 @@ a single data elements (a DataELement in the API). Its properties are:
   against
 * `indexField` (String): the name which can be used in a search engine connected
   to the application (at the time of writing Apache Solr is supported)
+* `inactive` (boolean): the data element is inactive, do not run checks on this
+* `identifierField` (boolean): the data element is the identifier of the record
+* `asLanguageTagged` (boolean): treat the data element as language tagged. It works 
+  for JSON where the content of the data element is encoded with an associated 
+  array, where the keys are the language tags.
 
 Optionaly you can set the "canonical list" of categories. It provides
 two additional functionalities 
@@ -534,7 +544,7 @@ One can add constraints to the fields. There are content rules, which
 the tool will check. In this version the tool mimin SHACL constraints.
 
 #### Cardinality
-One can specify with this properties how many occurrences of a data elemens
+One can specify with these constraints how many occurrences of a data element
 a record can have.
 
 ##### `minCount <number>`
@@ -877,6 +887,79 @@ fields:
       minHeight: 200
 ```
 
+##### `hasLanguageTag <anyOf|oneOf|allOf>`
+
+(since v0.9.6)
+
+It checks if the data element value has language tag. In XML the language tag is
+found in `@xml:lang` attribute. In JSON it might be encoded differently. Right now 
+MQAF suppoert the following encoding:
+
+```json
+"description": {
+  "de": ["Porträt"]
+}
+```
+
+Since this kind of structure might be applied not only for the language annotation, at
+the field level we should set that the field is expected to have language annotation:
+
+```yaml
+format: json
+fields:
+  - name: description
+    path: $.['description']
+    asLanguageTagged: true
+```
+
+The parameters defines if any, one or all instances should have language annottation:
+
+* `anyOf`: the test passes if at least one instance has language tag
+* `oneOf`: the test passes if one and only one instance has language tag
+* `allOf`: the test passes if at least all instances have language tag
+
+A full example:
+
+```yaml
+format: json
+fields:
+- name: description
+  path: $.['description']
+  asLanguageTagged: true
+  rules:
+  - hasLanguageTag: allOf
+```
+
+##### `isMultilingual <boolean>`
+
+(since v0.9.6)
+
+It checks if the data element is multilingual, so it has at least two instances with
+different language annotations.
+
+```json
+{
+  "description":{
+    "de":["Portr\u00e4t"],
+    "zh":["\u8096\u50cf"]
+  }
+}
+
+```
+
+an example schema
+
+```yaml
+format: json
+fields:
+  - name: description
+    path: $.['description']
+    asLanguageTagged: true
+    rules:
+      - isMultilingual: true
+```
+
+
 #### General properties
 
 ##### `id <String>`

mqa

@@ -2,5 +2,5 @@
 #-----------------------------------------------------------------------
 # Command line interface for the Metadata Quality Assessment Framework -
 #-----------------------------------------------------------------------
-export VERSION=0.9.4
+export VERSION=0.9.5-SNAPSHOT
 java -jar $(dirname $(readlink -f $0))/metadata-qa-api-${VERSION}-shaded.jar $@

pom.xml

@@ -5,7 +5,7 @@
   <groupId>de.gwdg.metadataqa</groupId>
   <artifactId>metadata-qa-api</artifactId>
   <packaging>jar</packaging>
-  <version>0.9.4</version>
+  <version>0.9.6-SNAPSHOT</version>
   <name>Metadata Quality Assurance Framework API</name>
   <description>
     A metadata quality assurance framework. It checks some metrics of 
@@ -86,12 +86,12 @@
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-lang3</artifactId>
-      <version>3.14.0</version>
+      <version>3.16.0</version>
     </dependency>
     <dependency>
       <groupId>commons-io</groupId>
       <artifactId>commons-io</artifactId>
-      <version>2.16.0</version>
+      <version>2.16.1</version>
     </dependency>
     <dependency>
       <groupId>commons-httpclient</groupId>
@@ -101,12 +101,12 @@
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-simple</artifactId>
-      <version>2.0.12</version>
+      <version>2.0.14</version>
     </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
-      <version>2.0.12</version>
+      <version>2.0.14</version>
     </dependency>
     <!-- CSV reader. Docs: http://opencsv.sourceforge.net -->
     <dependency>
@@ -121,7 +121,7 @@
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-text</artifactId>
-      <version>1.11.0</version>
+      <version>1.12.0</version>
     </dependency>
     <!--
       Hierarchical context runner for jUnit
@@ -150,7 +150,7 @@
     <dependency>
       <groupId>commons-cli</groupId>
       <artifactId>commons-cli</artifactId>
-      <version>1.6.0</version>
+      <version>1.8.0</version>
     </dependency>
     <!-- language detection -->
     <dependency>
@@ -168,7 +168,7 @@
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
-      <version>2.16.2</version>
+      <version>2.17.2</version>
     </dependency>
     <dependency>
       <groupId>com.github.stefanbirkner</groupId>

src/main/java/de/gwdg/metadataqa/api/calculator/FieldExtractor.java

@@ -4,16 +4,21 @@
 import de.gwdg.metadataqa.api.counter.FieldCounter;
 import de.gwdg.metadataqa.api.interfaces.Calculator;
 import de.gwdg.metadataqa.api.interfaces.MetricResult;
+import de.gwdg.metadataqa.api.json.DataElement;
 import de.gwdg.metadataqa.api.model.EdmFieldInstance;
 import de.gwdg.metadataqa.api.model.selector.Selector;
 import de.gwdg.metadataqa.api.model.XmlFieldInstance;
 import de.gwdg.metadataqa.api.problemcatalog.FieldCounterBasedResult;
 import de.gwdg.metadataqa.api.schema.Schema;
 import de.gwdg.metadataqa.api.util.FileUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Set;
+import java.util.logging.Logger;
 
 /**
  * Field extractor
@@ -22,6 +27,8 @@
  */
 public class FieldExtractor implements Calculator, Serializable {
 
+  private static final Logger LOGGER = Logger.getLogger(FieldExtractor.class.getCanonicalName());
+
   public static final String CALCULATOR_NAME = "fieldExtractor";
   public static final String FIELD_NAME = "recordId";
 
@@ -56,30 +63,52 @@ public List<MetricResult> measure(Selector cache)
 
     if (schema != null) {
       String path;
+      DataElement dataELement;
       for (String fieldName : schema.getExtractableFields().keySet()) {
         if (idPath == null || !fieldName.equals(FIELD_NAME)) {
+          dataELement = schema.getPathByLabel(fieldName);
           path = schema.getExtractableFields().get(fieldName);
-          extractSingleField(cache, resultMap, path, fieldName);
+          extractSingleField(cache, resultMap, path, fieldName, dataELement);
         }
       }
     }
     return List.of(new FieldCounterBasedResult<>(getCalculatorName(), resultMap).withNoCompression());
   }
 
   private void extractSingleField(Selector cache, FieldCounter<String> resultMap, String path, String fieldName) {
-    List<XmlFieldInstance> values = cache.get(path);
+    extractSingleField(cache, resultMap, path, fieldName,null);
+  }
+
+  private void extractSingleField(Selector cache,
+                                    FieldCounter<String> resultMap,
+                                    String path,
+                                    String fieldName,
+                                    DataElement dataELement) {
+    List<XmlFieldInstance> fieldInstances;
+    if (dataELement != null) {
+      fieldInstances = cache.get(dataELement);
+    } else {
+      fieldInstances = cache.get(path);
+    }
     String value = null;
-    if (values == null || values.isEmpty() || values.get(0) == null) {
+    if (fieldInstances == null || fieldInstances.isEmpty() || fieldInstances.get(0) == null) {
       value = nullValue;
     } else {
-      XmlFieldInstance instance = values.get(0);
-      boolean isEdm = instance instanceof EdmFieldInstance;
-      if (isEdm && ((EdmFieldInstance)instance).getResource() != null) {
-        value = ((EdmFieldInstance) instance).getResource();
-      } else if (instance.getValue() != null) {
-        value = instance.getValue();
+      Set<String> values = new LinkedHashSet<>();
+      for (XmlFieldInstance instance : fieldInstances) {
+        boolean isEdm = instance instanceof EdmFieldInstance;
+        if (isEdm && ((EdmFieldInstance)instance).getResource() != null) {
+          value = ((EdmFieldInstance) instance).getResource();
+        } else if (instance.getValue() != null) {
+          value = instance.getValue();
+        }
+        // if (!values.contains(values))
+        if (StringUtils.isNotBlank(value))
+          values.add(value);
       }
+      value = StringUtils.join(values, " --- ");
     }
+    // LOGGER.info("value: " + value);
     resultMap.put(fieldName, value);
   }
 

src/main/java/de/gwdg/metadataqa/api/calculator/Indexer.java

@@ -33,7 +33,10 @@ public Indexer(SolrClient solrClient, Schema schema) {
   @Override
   public List<MetricResult> measure(Selector cache) {
     try {
-      String recordId = extractValue(cache, schema.getRecordId().getPath()).get(0);
+      List<String> extractedValues = extractValue(cache, schema.getRecordId().getPath());
+      if (extractedValues.isEmpty())
+        throw new RuntimeException(String.format("Missing record ID (path: %s)", schema.getRecordId().getPath()));
+      String recordId = extractedValues.get(0);
 
       Map<String, List<String>> resultMap = new HashMap<>();
       for (UniquenessField solrField : solrFields) {

src/main/java/de/gwdg/metadataqa/api/cli/App.java

@@ -1,5 +1,6 @@
 package de.gwdg.metadataqa.api.cli;
 
+import com.jayway.jsonpath.InvalidJsonException;
 import com.opencsv.exceptions.CsvValidationException;
 import de.gwdg.metadataqa.api.calculator.CalculatorFacade;
 import de.gwdg.metadataqa.api.io.reader.XMLRecordReader;
@@ -9,6 +10,7 @@
 import de.gwdg.metadataqa.api.interfaces.MetricResult;
 import de.gwdg.metadataqa.api.io.reader.RecordReader;
 import de.gwdg.metadataqa.api.schema.Schema;
+import net.minidev.json.parser.ParseException;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
 import org.apache.commons.cli.DefaultParser;
@@ -42,6 +44,7 @@ public class App {
 
   // Arguments
   private static final String INPUT_FILE = "input";
+  private static final String INPUT_FORMAT = "inputFormat";
   private static final String OUTPUT_FILE = "output";
   private static final String OUTPUT_FORMAT = "outputFormat";
   private static final String SCHEMA_CONFIG = "schema";
@@ -101,7 +104,8 @@ public App(CommandLine cmd) throws IOException, CsvValidationException {
 
     // initialize input
     String inputFile = cmd.getOptionValue(INPUT_FILE);
-    this.inputReader = RecordFactory.getRecordReader(inputFile, calculator, cmd.hasOption(GZIP_FLAG));
+    InputFormat inputFormat = InputFormat.byCode(cmd.getOptionValue(INPUT_FORMAT));
+    this.inputReader = RecordFactory.getRecordReader(inputFile, calculator, cmd.hasOption(GZIP_FLAG), inputFormat);
 
     // initialize output
     String outFormat = cmd.getOptionValue(OUTPUT_FORMAT, NDJSON);
@@ -162,6 +166,14 @@ private static Options buildOptions() {
       .desc("Input file.")
       .build();
 
+    Option inputFormatOption = Option.builder("n")
+      .numberOfArgs(1)
+      .argName("inputFormat")
+      .required(false)
+      .longOpt(INPUT_FORMAT)
+      .desc("Format of the input: json, ndjson (new line delimited JSON), json-array (JSON file that contains an array of objects). Default: ndjson.")
+      .build();
+
     Option outputOption = Option.builder("o")
       .numberOfArgs(1)
       .argName("file")
@@ -233,6 +245,7 @@ private static Options buildOptions() {
       .build();
 
     options.addOption(inputOption);
+    options.addOption(inputFormatOption);
     options.addOption(outputOption);
     options.addOption(outputFormatOption);
     options.addOption(schemaConfigOption);
@@ -253,7 +266,6 @@ private void run() {
       outputWriter.writeHeader(header);
 
       while (inputReader.hasNext()) {
-
         Map<String, List<MetricResult>> measurement = inputReader.next();
         outputWriter.writeResult(measurement);
 
@@ -265,9 +277,10 @@ private void run() {
       }
       logger.info(String.format("Assessment completed successfully with %s records. ", counter));
       outputWriter.close();
-    } catch (IOException e) {
+    } catch (InvalidJsonException | IOException e) {
       logger.severe(String.format("Assessment failed with %s records. ", counter));
       logger.severe(e.getMessage());
+      e.printStackTrace();
     }
   }
 }