Uniqueness of PICA field ranges reported wrongly #247

common-script

@@ -463,6 +463,9 @@ do_all_analyses_for_pica() {
 do_all_solr() {
   do_prepare_solr
   do_index
+  if [[ "${HAS_GROUP_PARAM}" == "1" ]]; then
+    php scripts/sqlite/solr-copy-ids-from-validation.php validation ${NAME}
+  fi
 }
 
 # ---- usage and execution of proccessing steps ----

scripts/sqlite/qa_catalogue.groupping.R

@@ -46,7 +46,7 @@ typesDF <- NULL
 pathsDF <- NULL
 categoriesDF <- NULL
 len <- length(groupIds)
-print(sprintf("get statistics for %d greoups", len))
+print(sprintf("get statistics for %d groups", len))
 for (i in 1:len) {
   currentGroupId = groupIds[i]
   if (i == 1 | i == len | i %% 25 == 0) {

scripts/sqlite/solr-copy-ids-from-validation.php

@@ -0,0 +1,54 @@
+<?php
+
+define('LN', "\n");
+define('UPDATE_URL', 'http://localhost:8983/solr/%s/update');
+define('GET_SOME', 'http://localhost:8983/solr/%s/select?q=*%%3A*&start=%d&rows=%d');
+
+$source_core = $argv[1];
+$target_core = $argv[2];
+
+// $source_core = 'validation';
+// $target_core = 'k10plus_pica_groupped';
+
+$start = 0;
+$rows = 50;
+$count  = sprintf('http://localhost:8983/solr/%s/select?q=*%%3A*&rows=0', $source_core);
+
+$resp = json_decode(file_get_contents($count));
+$numFound = (int) $resp->response->numFound;
+printf("Copy groupId and errorId from %s to %s (in %d records)\n", $source_core, $target_core, $numFound);
+
+while ($start < $numFound) {
+  if ($start == 0 || $start % 1000 == 0)
+    printf("%s> %d\n", date_format(date_create(), "Y-m-d H:i:s"), $start);
+  $records = [];
+  $resp = json_decode(file_get_contents(sprintf(GET_SOME, $source_core, $start, $rows)));
+  foreach ($resp->response->docs as $doc) {
+    $id = $doc->id;
+    $record = (object)[
+      'id' => $doc->id,
+      'groupId_is' => (object)['set' => $doc->groupId_is],
+    ];
+    if (isset($doc->errorId_is))
+      $record->errorId_is = (object)['set' => $doc->errorId_is];
+    $records[] = $record;
+  }
+  update($target_core, $records);
+  $start += $rows;
+}
+update($target_core, (object)["commit" => (object)[]]);
+
+printf("%s> DONE\n", date_format(date_create(), "Y-m-d H:i:s"));
+
+function update($target_core, $data) {
+  $ch = curl_init(sprintf(UPDATE_URL, $target_core)); 
+  curl_setopt($ch, CURLOPT_POST, 1);
+  curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($data));
+  curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); 
+  curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type: application/json'));
+  $result = json_decode(curl_exec($ch));
+  curl_close($ch);
+  if ($result->responseHeader->status != 0)
+    print_r($result);
+}
+

src/main/java/de/gwdg/metadataqa/marc/analysis/AuthorithyAnalyzer.java

@@ -52,7 +52,7 @@ public int process() {
             count += fieldInstanceLevelCount;
             add(category, categoryCounter, fieldInstanceLevelCount);
           } else {
-            logger.log(Level.SEVERE, "Unhandled type: {0}", type);
+            logger.log(Level.SEVERE, "Unhandled type: {0}", new Object[]{type});
           }
         }
       } else if (marcRecord.getSchemaType().equals(SchemaType.PICA)) {

src/main/java/de/gwdg/metadataqa/marc/analysis/ThompsonTraillAnalysis.java

@@ -207,7 +207,7 @@ else if (field.getInd2().equals("2"))
           else if (field.getInd2().equals("7")) {
             List<MarcSubfield> subfield2 = field.getSubfield("2");
             if (subfield2 == null) {
-              logger.log(Level.SEVERE, "Error in {0}: ind2 = 7, but there is no $2", marcRecord.getControl001().getContent());
+              logger.log(Level.SEVERE, "Error in {0}: ind2 = 7, but there is no $2", new Object[]{marcRecord.getControl001().getContent()});
             } else
               switch (field.getSubfield("2").get(0).getValue()) {
                 case "fast": ttScores.count(ThompsonTraillFields.FAST); break;

src/main/java/de/gwdg/metadataqa/marc/cli/ClassificationAnalysis.java

@@ -116,7 +116,7 @@ private void printToFile(File file, String message) {
   private File prepareReportFile(String outputDir, String fileName) {
     File reportFile = new File(outputDir, fileName);
     if (reportFile.exists() && !reportFile.delete())
-      logger.log(Level.SEVERE, "File {0} hasn't been deleted", reportFile.getAbsolutePath());
+      logger.log(Level.SEVERE, "File {0} hasn't been deleted", new Object[]{reportFile.getAbsolutePath()});
     return reportFile;
   }
 

src/main/java/de/gwdg/metadataqa/marc/cli/QACli.java

@@ -144,7 +144,7 @@ private String getFilesWithDate(String[] fileNames) {
   protected File prepareReportFile(String outputDir, String fileName) {
     File reportFile = new File(outputDir, fileName);
     if (reportFile.exists() && !reportFile.delete())
-      logger.log(Level.SEVERE, "File {0} hasn't been deleted", reportFile.getAbsolutePath());
+      logger.log(Level.SEVERE, "File {0} hasn't been deleted", new Object[]{reportFile.getAbsolutePath()});
     return reportFile;
   }
 

src/main/java/de/gwdg/metadataqa/marc/cli/Shacl4bib.java

@@ -95,7 +95,7 @@ public void beforeIteration() {
 
   @Override
   public void fileOpened(Path path) {
-    logger.log(Level.INFO, "file opened: {0}", path);
+    logger.log(Level.INFO, "file opened: {0}", new Object[]{path});
   }
 
   @Override

src/main/java/de/gwdg/metadataqa/marc/cli/utils/MarcJsonToSolr.java

@@ -75,7 +75,7 @@ else if (args[2].equals("doCommit=false"))
     }
     long end = System.currentTimeMillis();
 
-    logger.log(Level.INFO, "Bye! It took: {0} s", String.format("%.1f", (float) (end - start) / 1000));
+    logger.log(Level.INFO, "Bye! It took: {0} s", new Object[]{String.format("%.1f", (float) (end - start) / 1000)});
 
     System.exit(0);
   }

src/main/java/de/gwdg/metadataqa/marc/cli/utils/RecordIterator.java

@@ -112,7 +112,7 @@ private void processFile(String inputFileName) {
       MarcReader reader = getMarcFileReader(processor.getParameters(), path);
       processContent(reader, fileName);
       if (processor.getParameters().doLog())
-        logger.log(Level.INFO, "Finished processing file. Processed {0} records.", decimalFormat.format(i));
+        logger.log(Level.INFO, "Finished processing file. Processed {0} records.", new Object[]{decimalFormat.format(i)});
 
     } catch (SolrServerException ex) {
       if (processor.getParameters().doLog())

src/main/java/de/gwdg/metadataqa/marc/utils/pica/reader/model/PicaLine.java

@@ -115,7 +115,7 @@ private void parse(String raw) {
         parseSubfields();
         valid = true;
       } else {
-        logger.log(Level.WARNING, "Unable to parse line: '{0}'", raw);
+        logger.log(Level.WARNING, "Unable to parse line: '{0}'", new Object[]{raw});
       }
     }
   }