From 0ad742c19c5150c6e0129ac4d3aeca8dc4ff9e20 Mon Sep 17 00:00:00 2001 From: Joana Sousa Date: Tue, 1 Mar 2022 11:28:19 +0100 Subject: [PATCH 01/73] [maven-release-plugin] prepare for next development iteration --- metis-authentication/metis-authentication-common/pom.xml | 2 +- metis-authentication/metis-authentication-rest-client/pom.xml | 2 +- metis-authentication/metis-authentication-rest/pom.xml | 2 +- metis-authentication/metis-authentication-service/pom.xml | 2 +- metis-authentication/pom.xml | 2 +- metis-common/metis-common-mongo/pom.xml | 2 +- metis-common/metis-common-network/pom.xml | 2 +- metis-common/metis-common-solr/pom.xml | 2 +- metis-common/metis-common-utils/pom.xml | 2 +- metis-common/metis-common-zoho/pom.xml | 2 +- metis-common/pom.xml | 2 +- metis-core/metis-core-common/pom.xml | 2 +- metis-core/metis-core-rest/pom.xml | 2 +- metis-core/metis-core-service/pom.xml | 2 +- metis-core/pom.xml | 2 +- metis-dereference/metis-dereference-common/pom.xml | 2 +- metis-dereference/metis-dereference-import/pom.xml | 2 +- metis-dereference/metis-dereference-rest/pom.xml | 2 +- metis-dereference/metis-dereference-service/pom.xml | 2 +- metis-dereference/pom.xml | 2 +- metis-enrichment/metis-enrichment-client/pom.xml | 2 +- metis-enrichment/metis-enrichment-common/pom.xml | 2 +- metis-enrichment/metis-enrichment-rest/pom.xml | 2 +- metis-enrichment/metis-enrichment-service/pom.xml | 2 +- metis-enrichment/pom.xml | 2 +- metis-harvesting/pom.xml | 2 +- metis-indexing/pom.xml | 2 +- metis-media-service/pom.xml | 2 +- metis-normalization/pom.xml | 2 +- metis-repository/pom.xml | 2 +- metis-schema/pom.xml | 2 +- metis-transformation/metis-transformation-service/pom.xml | 2 +- metis-transformation/pom.xml | 2 +- metis-validation/metis-validation-client/pom.xml | 2 +- metis-validation/metis-validation-common/pom.xml | 2 +- metis-validation/metis-validation-rest/pom.xml | 2 +- metis-validation/metis-validation-service/pom.xml | 2 +- metis-validation/pom.xml | 2 +- pom.xml | 4 ++-- 39 files changed, 40 insertions(+), 40 deletions(-) diff --git a/metis-authentication/metis-authentication-common/pom.xml b/metis-authentication/metis-authentication-common/pom.xml index a1ebce213..478002639 100644 --- a/metis-authentication/metis-authentication-common/pom.xml +++ b/metis-authentication/metis-authentication-common/pom.xml @@ -4,7 +4,7 @@ metis-authentication eu.europeana.metis - 6 + 7-SNAPSHOT metis-authentication-common diff --git a/metis-authentication/metis-authentication-rest-client/pom.xml b/metis-authentication/metis-authentication-rest-client/pom.xml index fa5dcbd8f..bc73c59bc 100644 --- a/metis-authentication/metis-authentication-rest-client/pom.xml +++ b/metis-authentication/metis-authentication-rest-client/pom.xml @@ -4,7 +4,7 @@ metis-authentication eu.europeana.metis - 6 + 7-SNAPSHOT metis-authentication-rest-client diff --git a/metis-authentication/metis-authentication-rest/pom.xml b/metis-authentication/metis-authentication-rest/pom.xml index c566c4380..eda05fa57 100644 --- a/metis-authentication/metis-authentication-rest/pom.xml +++ b/metis-authentication/metis-authentication-rest/pom.xml @@ -4,7 +4,7 @@ metis-authentication eu.europeana.metis - 6 + 7-SNAPSHOT metis-authentication-rest war diff --git a/metis-authentication/metis-authentication-service/pom.xml b/metis-authentication/metis-authentication-service/pom.xml index 0dea6f614..59c1fdcba 100644 --- a/metis-authentication/metis-authentication-service/pom.xml +++ b/metis-authentication/metis-authentication-service/pom.xml @@ -4,7 +4,7 @@ metis-authentication eu.europeana.metis - 6 + 7-SNAPSHOT metis-authentication-service diff --git a/metis-authentication/pom.xml b/metis-authentication/pom.xml index a91b9daa4..3c276c648 100644 --- a/metis-authentication/pom.xml +++ b/metis-authentication/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 6 + 7-SNAPSHOT metis-authentication pom diff --git a/metis-common/metis-common-mongo/pom.xml b/metis-common/metis-common-mongo/pom.xml index 9f45d1369..180197dbb 100644 --- a/metis-common/metis-common-mongo/pom.xml +++ b/metis-common/metis-common-mongo/pom.xml @@ -4,7 +4,7 @@ metis-common eu.europeana.metis - 6 + 7-SNAPSHOT metis-common-mongo diff --git a/metis-common/metis-common-network/pom.xml b/metis-common/metis-common-network/pom.xml index 4184a069e..c13c1ed7e 100644 --- a/metis-common/metis-common-network/pom.xml +++ b/metis-common/metis-common-network/pom.xml @@ -4,7 +4,7 @@ metis-common eu.europeana.metis - 6 + 7-SNAPSHOT metis-common-network diff --git a/metis-common/metis-common-solr/pom.xml b/metis-common/metis-common-solr/pom.xml index 041ed3dcc..1ac8f2188 100644 --- a/metis-common/metis-common-solr/pom.xml +++ b/metis-common/metis-common-solr/pom.xml @@ -4,7 +4,7 @@ metis-common eu.europeana.metis - 6 + 7-SNAPSHOT metis-common-solr diff --git a/metis-common/metis-common-utils/pom.xml b/metis-common/metis-common-utils/pom.xml index ae8ccaa39..706b1b911 100644 --- a/metis-common/metis-common-utils/pom.xml +++ b/metis-common/metis-common-utils/pom.xml @@ -4,7 +4,7 @@ metis-common eu.europeana.metis - 6 + 7-SNAPSHOT metis-common-utils diff --git a/metis-common/metis-common-zoho/pom.xml b/metis-common/metis-common-zoho/pom.xml index b3ba39426..728ea62c4 100644 --- a/metis-common/metis-common-zoho/pom.xml +++ b/metis-common/metis-common-zoho/pom.xml @@ -4,7 +4,7 @@ metis-common eu.europeana.metis - 6 + 7-SNAPSHOT metis-common-zoho diff --git a/metis-common/pom.xml b/metis-common/pom.xml index 6e7f111a8..e1059bf25 100644 --- a/metis-common/pom.xml +++ b/metis-common/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 6 + 7-SNAPSHOT metis-common pom diff --git a/metis-core/metis-core-common/pom.xml b/metis-core/metis-core-common/pom.xml index 612171884..50760dd35 100644 --- a/metis-core/metis-core-common/pom.xml +++ b/metis-core/metis-core-common/pom.xml @@ -4,7 +4,7 @@ metis-core eu.europeana.metis - 6 + 7-SNAPSHOT metis-core-common diff --git a/metis-core/metis-core-rest/pom.xml b/metis-core/metis-core-rest/pom.xml index a01e06941..0ae469d7e 100644 --- a/metis-core/metis-core-rest/pom.xml +++ b/metis-core/metis-core-rest/pom.xml @@ -4,7 +4,7 @@ metis-core eu.europeana.metis - 6 + 7-SNAPSHOT metis-core-rest war diff --git a/metis-core/metis-core-service/pom.xml b/metis-core/metis-core-service/pom.xml index 1d8d2a29c..366ae33de 100644 --- a/metis-core/metis-core-service/pom.xml +++ b/metis-core/metis-core-service/pom.xml @@ -4,7 +4,7 @@ metis-core eu.europeana.metis - 6 + 7-SNAPSHOT metis-core-service diff --git a/metis-core/pom.xml b/metis-core/pom.xml index ffba399b3..4cc3c771d 100644 --- a/metis-core/pom.xml +++ b/metis-core/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 6 + 7-SNAPSHOT metis-core pom diff --git a/metis-dereference/metis-dereference-common/pom.xml b/metis-dereference/metis-dereference-common/pom.xml index e485103b5..1b6c4a6d6 100644 --- a/metis-dereference/metis-dereference-common/pom.xml +++ b/metis-dereference/metis-dereference-common/pom.xml @@ -4,7 +4,7 @@ metis-dereference eu.europeana.metis - 6 + 7-SNAPSHOT metis-dereference-common diff --git a/metis-dereference/metis-dereference-import/pom.xml b/metis-dereference/metis-dereference-import/pom.xml index 6201b2a4c..5ef621cc5 100644 --- a/metis-dereference/metis-dereference-import/pom.xml +++ b/metis-dereference/metis-dereference-import/pom.xml @@ -3,7 +3,7 @@ metis-dereference eu.europeana.metis - 6 + 7-SNAPSHOT 4.0.0 metis-dereference-import diff --git a/metis-dereference/metis-dereference-rest/pom.xml b/metis-dereference/metis-dereference-rest/pom.xml index 436e5bed8..87bfb7fe1 100644 --- a/metis-dereference/metis-dereference-rest/pom.xml +++ b/metis-dereference/metis-dereference-rest/pom.xml @@ -4,7 +4,7 @@ metis-dereference eu.europeana.metis - 6 + 7-SNAPSHOT metis-dereference-rest war diff --git a/metis-dereference/metis-dereference-service/pom.xml b/metis-dereference/metis-dereference-service/pom.xml index 5875c1722..62bc2d4f6 100644 --- a/metis-dereference/metis-dereference-service/pom.xml +++ b/metis-dereference/metis-dereference-service/pom.xml @@ -4,7 +4,7 @@ metis-dereference eu.europeana.metis - 6 + 7-SNAPSHOT metis-dereference-service diff --git a/metis-dereference/pom.xml b/metis-dereference/pom.xml index f7a9345af..3e1cc3f38 100644 --- a/metis-dereference/pom.xml +++ b/metis-dereference/pom.xml @@ -3,7 +3,7 @@ 4.0.0 eu.europeana.metis - 6 + 7-SNAPSHOT metis-framework diff --git a/metis-enrichment/metis-enrichment-client/pom.xml b/metis-enrichment/metis-enrichment-client/pom.xml index 55ff01ac5..040678062 100644 --- a/metis-enrichment/metis-enrichment-client/pom.xml +++ b/metis-enrichment/metis-enrichment-client/pom.xml @@ -4,7 +4,7 @@ metis-enrichment eu.europeana.metis - 6 + 7-SNAPSHOT metis-enrichment-client jar diff --git a/metis-enrichment/metis-enrichment-common/pom.xml b/metis-enrichment/metis-enrichment-common/pom.xml index 222ef65b6..2d4736c50 100644 --- a/metis-enrichment/metis-enrichment-common/pom.xml +++ b/metis-enrichment/metis-enrichment-common/pom.xml @@ -4,7 +4,7 @@ metis-enrichment eu.europeana.metis - 6 + 7-SNAPSHOT metis-enrichment-common diff --git a/metis-enrichment/metis-enrichment-rest/pom.xml b/metis-enrichment/metis-enrichment-rest/pom.xml index 06cf038d8..6a8bed946 100644 --- a/metis-enrichment/metis-enrichment-rest/pom.xml +++ b/metis-enrichment/metis-enrichment-rest/pom.xml @@ -4,7 +4,7 @@ metis-enrichment eu.europeana.metis - 6 + 7-SNAPSHOT metis-enrichment-rest war diff --git a/metis-enrichment/metis-enrichment-service/pom.xml b/metis-enrichment/metis-enrichment-service/pom.xml index e7a20c1f0..93508ac5c 100644 --- a/metis-enrichment/metis-enrichment-service/pom.xml +++ b/metis-enrichment/metis-enrichment-service/pom.xml @@ -4,7 +4,7 @@ metis-enrichment eu.europeana.metis - 6 + 7-SNAPSHOT metis-enrichment-service jar diff --git a/metis-enrichment/pom.xml b/metis-enrichment/pom.xml index cc5ebd82f..7dedc1224 100644 --- a/metis-enrichment/pom.xml +++ b/metis-enrichment/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 6 + 7-SNAPSHOT metis-enrichment pom diff --git a/metis-harvesting/pom.xml b/metis-harvesting/pom.xml index 018490942..643242ff5 100644 --- a/metis-harvesting/pom.xml +++ b/metis-harvesting/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 6 + 7-SNAPSHOT metis-harvesting diff --git a/metis-indexing/pom.xml b/metis-indexing/pom.xml index f44956ee8..3c2fd7601 100644 --- a/metis-indexing/pom.xml +++ b/metis-indexing/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 6 + 7-SNAPSHOT metis-indexing diff --git a/metis-media-service/pom.xml b/metis-media-service/pom.xml index eb178dceb..50cb6bc63 100644 --- a/metis-media-service/pom.xml +++ b/metis-media-service/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 6 + 7-SNAPSHOT metis-media-service diff --git a/metis-normalization/pom.xml b/metis-normalization/pom.xml index 8612e2382..d0fe155b5 100644 --- a/metis-normalization/pom.xml +++ b/metis-normalization/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 6 + 7-SNAPSHOT metis-normalization diff --git a/metis-repository/pom.xml b/metis-repository/pom.xml index c3ce88566..d60b3a2a7 100644 --- a/metis-repository/pom.xml +++ b/metis-repository/pom.xml @@ -3,7 +3,7 @@ metis-framework eu.europeana.metis - 6 + 7-SNAPSHOT 4.0.0 metis-repository diff --git a/metis-schema/pom.xml b/metis-schema/pom.xml index 533590b08..f3620553b 100644 --- a/metis-schema/pom.xml +++ b/metis-schema/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 6 + 7-SNAPSHOT metis-schema diff --git a/metis-transformation/metis-transformation-service/pom.xml b/metis-transformation/metis-transformation-service/pom.xml index a425a454e..6435ce338 100644 --- a/metis-transformation/metis-transformation-service/pom.xml +++ b/metis-transformation/metis-transformation-service/pom.xml @@ -4,7 +4,7 @@ metis-transformation eu.europeana.metis - 6 + 7-SNAPSHOT metis-transformation-service diff --git a/metis-transformation/pom.xml b/metis-transformation/pom.xml index 347166941..d7faa76dd 100644 --- a/metis-transformation/pom.xml +++ b/metis-transformation/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 6 + 7-SNAPSHOT metis-transformation pom diff --git a/metis-validation/metis-validation-client/pom.xml b/metis-validation/metis-validation-client/pom.xml index f94e32e56..a08e74249 100644 --- a/metis-validation/metis-validation-client/pom.xml +++ b/metis-validation/metis-validation-client/pom.xml @@ -4,7 +4,7 @@ metis-validation eu.europeana.metis - 6 + 7-SNAPSHOT metis-validation-client diff --git a/metis-validation/metis-validation-common/pom.xml b/metis-validation/metis-validation-common/pom.xml index 11f9cbf88..0e0e7c860 100644 --- a/metis-validation/metis-validation-common/pom.xml +++ b/metis-validation/metis-validation-common/pom.xml @@ -4,7 +4,7 @@ metis-validation eu.europeana.metis - 6 + 7-SNAPSHOT metis-validation-common diff --git a/metis-validation/metis-validation-rest/pom.xml b/metis-validation/metis-validation-rest/pom.xml index 98d21ae0f..f808edc03 100644 --- a/metis-validation/metis-validation-rest/pom.xml +++ b/metis-validation/metis-validation-rest/pom.xml @@ -4,7 +4,7 @@ metis-validation eu.europeana.metis - 6 + 7-SNAPSHOT metis-validation-rest war diff --git a/metis-validation/metis-validation-service/pom.xml b/metis-validation/metis-validation-service/pom.xml index 9d5fe568d..00b46299d 100644 --- a/metis-validation/metis-validation-service/pom.xml +++ b/metis-validation/metis-validation-service/pom.xml @@ -4,7 +4,7 @@ metis-validation eu.europeana.metis - 6 + 7-SNAPSHOT metis-validation-service diff --git a/metis-validation/pom.xml b/metis-validation/pom.xml index ebdd67a72..5c02f40ac 100644 --- a/metis-validation/pom.xml +++ b/metis-validation/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 6 + 7-SNAPSHOT metis-validation pom diff --git a/pom.xml b/pom.xml index 35d42ede1..70d297100 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ eu.europeana.metis metis-framework - 6 + 7-SNAPSHOT pom @@ -27,7 +27,7 @@ scm:git:https://github.com/europeana/metis-framework https://github.com/europeana/metis-framework - v6 + HEAD scm:git:https://github.com/europeana/metis-framework From 656c9338f1b5df680cfe116a26c1f460465ee1d0 Mon Sep 17 00:00:00 2001 From: Joana Sousa Date: Thu, 3 Mar 2022 11:37:51 +0100 Subject: [PATCH 02/73] MET-4159 Created new method that returns HttpIterator with InputStream --- .../metis/harvesting/http/HttpHarvester.java | 2 + .../harvesting/http/HttpHarvesterImpl.java | 62 ++++++++++--------- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvester.java b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvester.java index 0b428f273..4d0658b3f 100644 --- a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvester.java +++ b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvester.java @@ -47,6 +47,8 @@ void harvestRecords(InputStream inputStream, CompressedFileExtension compressedF */ void setMaxNumberOfIterations(int maxOfIterations); + HttpRecordIterator createHttpHarvestIterator(InputStream input, CompressedFileExtension compressedFileType) throws HarvesterException; + /** * An object representing an entry in a file archive. */ diff --git a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java index 7a4078180..f4faf5a23 100644 --- a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java +++ b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java @@ -49,26 +49,10 @@ public class HttpHarvesterImpl implements HttpHarvester { public void harvestRecords(InputStream inputStream, CompressedFileExtension compressedFileType, Consumer action) throws HarvesterException { - // We chose where to store the temporary file. - @SuppressWarnings("findsecbugs:PATH_TRAVERSAL_IN") - Path tempDir = null; - try { - - // Save the zip file in a temporary directory (and close the input stream). - final String prefix = UUID.randomUUID().toString(); - final Path tempFile; - try { - tempDir = Files.createTempDirectory(prefix); - tempFile = Files.createTempFile(tempDir, prefix, compressedFileType.getExtension()); - FileUtils.copyInputStreamToFile(inputStream, tempFile.toFile()); - } catch (IOException e) { - throw new HarvesterException("Problem saving archive.", e); - } - AtomicInteger currentNumberOfIterations = new AtomicInteger(); // Now perform the harvesting - go by each file. - final HttpRecordIterator iterator = harvestRecords(tempFile); + final HttpRecordIterator iterator = createHttpHarvestIterator(inputStream, compressedFileType); List> exception = new ArrayList<>(1); iterator.forEach(path -> { try (InputStream content = Files.newInputStream(path)) { @@ -88,18 +72,6 @@ public void harvestRecords(InputStream inputStream, CompressedFileExtension comp throw new HarvesterException("Could not process path " + exception.get(0).getKey() + ".", exception.get(0).getValue()); } - - } finally { - - // Finally, attempt to delete the files. - if (tempDir != null) { - try { - FileUtils.deleteDirectory(tempDir.toFile()); - } catch (IOException e) { - LOGGER.warn("Could not delete temporary directory.", e); - } - } - } } @Override @@ -126,6 +98,38 @@ public HttpRecordIterator harvestRecords(String archiveUrl, String downloadDirec return harvestRecords(downloadedFile); } + @Override + public HttpRecordIterator createHttpHarvestIterator(InputStream input, CompressedFileExtension compressedFileType) throws HarvesterException { + // We chose where to store the temporary file. + @SuppressWarnings("findsecbugs:PATH_TRAVERSAL_IN") + Path tempDir = null; + final Path tempFile; + try { + // Save the zip file in a temporary directory (and close the input stream). + final String prefix = UUID.randomUUID().toString(); + + tempDir = Files.createTempDirectory(prefix); + tempFile = Files.createTempFile(tempDir, prefix, compressedFileType.getExtension()); + FileUtils.copyInputStreamToFile(input, tempFile.toFile()); + + return harvestRecords(tempFile); + } catch (IOException e) { + throw new HarvesterException("Problem saving archive.", e); + } finally { + + // Finally, attempt to delete the files. + if (tempDir != null) { + try { + FileUtils.deleteDirectory(tempDir.toFile()); + } catch (IOException e) { + LOGGER.warn("Could not delete temporary directory.", e); + } + } + } + + + } + private HttpRecordIterator harvestRecords(Path archiveFile) throws HarvesterException { // Extract the archive. From 597d0f58035202df33a628853487d62990d71051 Mon Sep 17 00:00:00 2001 From: Joana Sousa Date: Thu, 3 Mar 2022 16:33:14 +0100 Subject: [PATCH 03/73] MET-4159 Created a new method for HttpRecordIterator to delete its temporary content --- .../metis/harvesting/http/HttpHarvester.java | 10 +++++- .../harvesting/http/HttpHarvesterImpl.java | 33 ++++++++++--------- .../harvesting/http/HttpRecordIterator.java | 2 ++ 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvester.java b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvester.java index 4d0658b3f..de9fcbd97 100644 --- a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvester.java +++ b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvester.java @@ -47,7 +47,15 @@ void harvestRecords(InputStream inputStream, CompressedFileExtension compressedF */ void setMaxNumberOfIterations(int maxOfIterations); - HttpRecordIterator createHttpHarvestIterator(InputStream input, CompressedFileExtension compressedFileType) throws HarvesterException; + /** + * It creates a {@link HttpRecordIterator} with a InputStream into a temporary file directory. + * It is needed to use the {@link HttpRecordIterator#deleteIteratorContent()} method if this method is used. + * @param input The input stream from which we create the iterator + * @param compressedFileType The type of compressed file type + * @return A HttpRecordIterator based on a temporary file location + * @throws HarvesterException In case there is an issue while using the input stream + */ + HttpRecordIterator createTemporaryHttpHarvestIterator(InputStream input, CompressedFileExtension compressedFileType) throws HarvesterException; /** * An object representing an entry in a file archive. diff --git a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java index f4faf5a23..1b61617e8 100644 --- a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java +++ b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java @@ -52,7 +52,7 @@ public void harvestRecords(InputStream inputStream, CompressedFileExtension comp AtomicInteger currentNumberOfIterations = new AtomicInteger(); // Now perform the harvesting - go by each file. - final HttpRecordIterator iterator = createHttpHarvestIterator(inputStream, compressedFileType); + final HttpRecordIterator iterator = createTemporaryHttpHarvestIterator(inputStream, compressedFileType); List> exception = new ArrayList<>(1); iterator.forEach(path -> { try (InputStream content = Files.newInputStream(path)) { @@ -68,6 +68,9 @@ public void harvestRecords(InputStream inputStream, CompressedFileExtension comp return IterationResult.TERMINATE; } }); + + iterator.deleteIteratorContent(); + if (!exception.isEmpty()) { throw new HarvesterException("Could not process path " + exception.get(0).getKey() + ".", exception.get(0).getValue()); @@ -99,10 +102,10 @@ public HttpRecordIterator harvestRecords(String archiveUrl, String downloadDirec } @Override - public HttpRecordIterator createHttpHarvestIterator(InputStream input, CompressedFileExtension compressedFileType) throws HarvesterException { + public HttpRecordIterator createTemporaryHttpHarvestIterator(InputStream input, CompressedFileExtension compressedFileType) throws HarvesterException { // We chose where to store the temporary file. @SuppressWarnings("findsecbugs:PATH_TRAVERSAL_IN") - Path tempDir = null; + Path tempDir; final Path tempFile; try { // Save the zip file in a temporary directory (and close the input stream). @@ -115,18 +118,7 @@ public HttpRecordIterator createHttpHarvestIterator(InputStream input, Compresse return harvestRecords(tempFile); } catch (IOException e) { throw new HarvesterException("Problem saving archive.", e); - } finally { - - // Finally, attempt to delete the files. - if (tempDir != null) { - try { - FileUtils.deleteDirectory(tempDir.toFile()); - } catch (IOException e) { - LOGGER.warn("Could not delete temporary directory.", e); - } - } - } - + } } @@ -210,6 +202,17 @@ public FileIterator(Path extractedDirectory) { this.extractedDirectory = extractedDirectory; } + @Override + public void deleteIteratorContent() { + if (extractedDirectory != null) { + try { + FileUtils.deleteDirectory(extractedDirectory.toFile()); + } catch (IOException e) { + LOGGER.warn("Could not delete directory.", e); + } + } + } + @Override public void forEach(ReportingIteration action) throws HarvesterException { try { diff --git a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpRecordIterator.java b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpRecordIterator.java index 43be9b0ba..e81fb7b04 100644 --- a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpRecordIterator.java +++ b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpRecordIterator.java @@ -13,6 +13,8 @@ */ public interface HttpRecordIterator { + void deleteIteratorContent(); + /** * Iterate through the decompressed records. * From 68492d10dcdc20cd16c3d884361ed3c9deb36a7c Mon Sep 17 00:00:00 2001 From: Joana Sousa Date: Thu, 3 Mar 2022 16:43:59 +0100 Subject: [PATCH 04/73] MET-4159 Removed maxNumberOfIterations variable. It is not needed --- .../metis/harvesting/http/HttpHarvester.java | 8 -------- .../metis/harvesting/http/HttpHarvesterImpl.java | 14 -------------- 2 files changed, 22 deletions(-) diff --git a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvester.java b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvester.java index de9fcbd97..4a1a64e40 100644 --- a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvester.java +++ b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvester.java @@ -39,14 +39,6 @@ HttpRecordIterator harvestRecords(String archiveUrl, String downloadDirectory) void harvestRecords(InputStream inputStream, CompressedFileExtension compressedFileType, Consumer action) throws HarvesterException; - /** - * Method to set up the maximum number of iterations through records during harvesting. - * If there is none, the harvesting iterate through all records. - * - * @param maxOfIterations The maximum number of iterations - */ - void setMaxNumberOfIterations(int maxOfIterations); - /** * It creates a {@link HttpRecordIterator} with a InputStream into a temporary file directory. * It is needed to use the {@link HttpRecordIterator#deleteIteratorContent()} method if this method is used. diff --git a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java index 1b61617e8..262dc54da 100644 --- a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java +++ b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java @@ -23,7 +23,6 @@ import java.util.List; import java.util.Set; import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.stream.Stream; import org.apache.commons.io.FileUtils; @@ -43,14 +42,10 @@ public class HttpHarvesterImpl implements HttpHarvester { private static final Logger LOGGER = LoggerFactory.getLogger(HttpHarvesterImpl.class); - private int maxNumberOfIterations = 0; - @Override public void harvestRecords(InputStream inputStream, CompressedFileExtension compressedFileType, Consumer action) throws HarvesterException { - AtomicInteger currentNumberOfIterations = new AtomicInteger(); - // Now perform the harvesting - go by each file. final HttpRecordIterator iterator = createTemporaryHttpHarvestIterator(inputStream, compressedFileType); List> exception = new ArrayList<>(1); @@ -58,10 +53,6 @@ public void harvestRecords(InputStream inputStream, CompressedFileExtension comp try (InputStream content = Files.newInputStream(path)) { action.accept(new ArchiveEntryImpl(path.getFileName().toString(), new ByteArrayInputStream(IOUtils.toByteArray(content)))); - currentNumberOfIterations.getAndIncrement(); - if (maxNumberOfIterations > 0 && currentNumberOfIterations.get() > maxNumberOfIterations) { - return IterationResult.TERMINATE; - } return IterationResult.CONTINUE; } catch (IOException | RuntimeException e) { exception.add(new ImmutablePair<>(path, e)); @@ -77,11 +68,6 @@ public void harvestRecords(InputStream inputStream, CompressedFileExtension comp } } - @Override - public void setMaxNumberOfIterations(int maxOfIterations) { - this.maxNumberOfIterations = maxOfIterations; - } - @Override public HttpRecordIterator harvestRecords(String archiveUrl, String downloadDirectory) throws HarvesterException { From af653bc9837e3cd41a205c558cca73129f4275b5 Mon Sep 17 00:00:00 2001 From: Adolfo Peixinho Date: Thu, 3 Mar 2022 17:46:32 +0100 Subject: [PATCH 05/73] MET-4237 Updated pom.xml to point to the correct E-Cloud version. Added implementation of depublication to indexPostProcess method. Refactor of TestObjectFactory to align with E-Cloud SubTaskInfo --- .../core/rest/utils/TestObjectFactory.java | 10 +- .../core/execution/WorkflowExecutor.java | 3 +- .../core/execution/WorkflowPostProcessor.java | 99 +++++++++++++------ .../metis/core/utils/TestObjectFactory.java | 43 ++++---- pom.xml | 2 +- 5 files changed, 96 insertions(+), 61 deletions(-) diff --git a/metis-core/metis-core-rest/src/test/java/eu/europeana/metis/core/rest/utils/TestObjectFactory.java b/metis-core/metis-core-rest/src/test/java/eu/europeana/metis/core/rest/utils/TestObjectFactory.java index fdc1ed04d..4dc8d7598 100644 --- a/metis-core/metis-core-rest/src/test/java/eu/europeana/metis/core/rest/utils/TestObjectFactory.java +++ b/metis-core/metis-core-rest/src/test/java/eu/europeana/metis/core/rest/utils/TestObjectFactory.java @@ -253,11 +253,11 @@ public static MetisUserView createMetisUser(String email) { * @return the created sub task info */ public static List createListOfSubTaskInfo() { - SubTaskInfo subTaskInfo1 = new SubTaskInfo(1, "some_resource_id1", RecordState.SUCCESS, "", - "Sensitive Information"); - final int resourceNum = 2; - SubTaskInfo subTaskInfo2 = new SubTaskInfo(resourceNum, "some_resource_id1", RecordState.SUCCESS, "", - "Sensitive Information"); + + SubTaskInfo subTaskInfo1 = new SubTaskInfo(1, "some_resource_id1", RecordState.SUCCESS, "info", + "additional info", "europeanaId", 0L); + SubTaskInfo subTaskInfo2 = new SubTaskInfo(2, "some_resource_id2", RecordState.SUCCESS, "info", + "additional info", "europeanaId", 0L); ArrayList subTaskInfos = new ArrayList<>(); subTaskInfos.add(subTaskInfo1); subTaskInfos.add(subTaskInfo2); diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowExecutor.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowExecutor.java index 2c9393d0c..6281b87d0 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowExecutor.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowExecutor.java @@ -24,6 +24,7 @@ import eu.europeana.metis.core.workflow.plugins.ExecutablePluginType; import eu.europeana.metis.core.workflow.plugins.PluginStatus; import eu.europeana.metis.core.workflow.plugins.PluginType; +import eu.europeana.metis.exception.BadContentException; import eu.europeana.metis.exception.ExternalTaskException; import eu.europeana.metis.network.ExternalRequestUtil; import java.util.Date; @@ -491,7 +492,7 @@ private boolean applyPostProcessing(MonitorResult monitorResult, AbstractExecuta if (monitorResult.getTaskState() == TaskState.PROCESSED) { try { this.workflowPostProcessor.performPluginPostProcessing(plugin, datasetId); - } catch (DpsException | InvalidIndexPluginException | RuntimeException e) { + } catch (DpsException | InvalidIndexPluginException | BadContentException | RuntimeException e) { processingAppliedOrNotRequired = false; LOGGER.warn("Problem occurred during Metis post-processing.", e); plugin.setFinishedDate(null); diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java index 2cbc2fc9d..ceb8d6046 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java @@ -18,6 +18,8 @@ import eu.europeana.metis.core.dataset.DepublishRecordId.DepublicationStatus; import eu.europeana.metis.core.exceptions.InvalidIndexPluginException; import eu.europeana.metis.core.service.OrchestratorService; +import eu.europeana.metis.core.util.DepublishRecordIdSortField; +import eu.europeana.metis.core.util.SortDirection; import eu.europeana.metis.core.workflow.WorkflowExecution; import eu.europeana.metis.core.workflow.plugins.AbstractExecutablePlugin; import eu.europeana.metis.core.workflow.plugins.AbstractMetisPlugin; @@ -27,6 +29,7 @@ import eu.europeana.metis.core.workflow.plugins.IndexToPublishPlugin; import eu.europeana.metis.core.workflow.plugins.MetisPlugin; import eu.europeana.metis.core.workflow.plugins.PluginType; +import eu.europeana.metis.exception.BadContentException; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -40,7 +43,7 @@ import org.slf4j.LoggerFactory; /** - * This object can perform post processing for workflows. + * This object can perform post-processing for workflows. */ public class WorkflowPostProcessor { @@ -56,10 +59,10 @@ public class WorkflowPostProcessor { /** * Constructor. * - * @param depublishRecordIdDao The DAO for depublished records. - * @param datasetDao The DAO for datasets - * @param workflowExecutionDao The DAO for workflow executions. - * @param dpsClient the dps client + * @param depublishRecordIdDao The DAO for de-published records + * @param datasetDao The DAO for datasets + * @param workflowExecutionDao The DAO for workflow executions + * @param dpsClient the dps client */ public WorkflowPostProcessor(DepublishRecordIdDao depublishRecordIdDao, DatasetDao datasetDao, WorkflowExecutionDao workflowExecutionDao, DpsClient dpsClient) { @@ -70,13 +73,17 @@ public WorkflowPostProcessor(DepublishRecordIdDao depublishRecordIdDao, } /** - * This method performs post processing after an individual workflow step. + * This method performs post-processing after an individual workflow step. * - * @param plugin The plugin that was successfully executed. - * @param datasetId The dataset ID to which the plugin belongs. + * @param plugin The plugin that was successfully executed + * @param datasetId The dataset ID to which the plugin belongs + * @throws DpsException If communication with e-cloud dps failed + * @throws InvalidIndexPluginException If invalid type of plugin + * @throws BadContentException In case the records would violate the maximum number of de-published records that each + * dataset can have. */ void performPluginPostProcessing(AbstractExecutablePlugin plugin, String datasetId) - throws DpsException, InvalidIndexPluginException { + throws DpsException, InvalidIndexPluginException, BadContentException { final PluginType pluginType = plugin.getPluginType(); LOGGER.info("Starting postprocessing of plugin {} in dataset {}.", pluginType, datasetId); @@ -91,12 +98,15 @@ void performPluginPostProcessing(AbstractExecutablePlugin plugin, String data /** * Performs post-processing for indexing plugins * - * @param indexPlugin the index plugin - * @param datasetId the dataset id - * @throws DpsException if communication with ecloud dps failed + * @param indexPlugin The index plugin + * @param datasetId The dataset id + * @throws DpsException If communication with e-cloud dps failed + * @throws InvalidIndexPluginException If invalid type of plugin + * @throws BadContentException In case the records would violate the maximum number of de-published records that each + * dataset can have. */ private void indexPostProcess(AbstractExecutablePlugin indexPlugin, String datasetId) - throws DpsException, InvalidIndexPluginException { + throws DpsException, InvalidIndexPluginException, BadContentException { TargetIndexingDatabase targetIndexingDatabase; TargetIndexingEnvironment targetIndexingEnvironment; if (indexPlugin instanceof IndexToPreviewPlugin) { @@ -105,8 +115,21 @@ private void indexPostProcess(AbstractExecutablePlugin indexPlugin, String da } else if (indexPlugin instanceof IndexToPublishPlugin) { targetIndexingDatabase = ((IndexToPublishPlugin) indexPlugin).getTargetIndexingDatabase(); targetIndexingEnvironment = ((IndexToPublishPlugin) indexPlugin).getTargetIndexingEnvironment(); - //Reset depublish status - depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, null, + + // get all tasks from dataset id and topology name + List taskReport = dpsClient.getDetailedTaskReport(indexPlugin.getTopologyName(), + Long.parseLong(indexPlugin.getExternalTaskId())); + // get all currently de-published records ids + Set depublishedRecordIds = depublishRecordIdDao + .getAllDepublishRecordIdsWithStatus(datasetId, DepublishRecordIdSortField.DEPUBLICATION_STATE, SortDirection.ASCENDING, + DepublicationStatus.DEPUBLISHED); + // filter the record ids that are a part of the given report, to be de-published + Set recordIdsToDepublish = taskReport.stream() + .filter(taskInfo -> depublishedRecordIds.contains(taskInfo.getEuropeanaId())) + .map(SubTaskInfo::getEuropeanaId).collect(Collectors.toSet()); + + // reset de-publish status + depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, recordIdsToDepublish, DepublicationStatus.PENDING_DEPUBLICATION, null); } else { throw new InvalidIndexPluginException("Plugin is not of the types supported"); @@ -118,13 +141,14 @@ private void indexPostProcess(AbstractExecutablePlugin indexPlugin, String da } /** - * Performs post processing for depublish plugins + * Performs post-processing for de-publish plugins * - * @param depublishPlugin the depublish plugin - * @param datasetId the dataset id - * @throws DpsException if communication with ecloud dps failed + * @param depublishPlugin The de-publish plugin + * @param datasetId The dataset id + * @throws DpsException If communication with e-cloud dps failed */ - private void depublishPostProcess(DepublishPlugin depublishPlugin, String datasetId) throws DpsException { + private void depublishPostProcess(DepublishPlugin depublishPlugin, String datasetId) + throws DpsException { if (depublishPlugin.getPluginMetadata().isDatasetDepublish()) { depublishDatasetPostProcess(datasetId); } else { @@ -132,12 +156,14 @@ private void depublishPostProcess(DepublishPlugin depublishPlugin, String datase } } + /** + * @param datasetId The dataset id + */ private void depublishDatasetPostProcess(String datasetId) { // Set all depublished records back to PENDING. depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, null, DepublicationStatus.PENDING_DEPUBLICATION, null); - // Find latest PUBLISH Type Plugin and set dataStatus to DELETED. final PluginWithExecutionId latestSuccessfulPlugin = workflowExecutionDao .getLatestSuccessfulPlugin(datasetId, OrchestratorService.PUBLISH_TYPES); @@ -152,32 +178,45 @@ private void depublishDatasetPostProcess(String datasetId) { workflowExecutionDao.updateWorkflowPlugins(workflowExecutionToUpdate); } } - // Set publication fitness to UNFIT. final Dataset dataset = datasetDao.getDatasetByDatasetId(datasetId); dataset.setPublicationFitness(PublicationFitness.UNFIT); datasetDao.update(dataset); } - private void depublishRecordPostProcess(DepublishPlugin depublishPlugin, String datasetId) throws DpsException { + /** + * @param depublishPlugin The de-publish plugin + * @param datasetId The dataset id + * @throws DpsException If communication with e-cloud dps failed + */ + private void depublishRecordPostProcess(DepublishPlugin depublishPlugin, String datasetId) + throws DpsException { // Retrieve the successfully depublished records. final long externalTaskId = Long.parseLong(depublishPlugin.getExternalTaskId()); final List subTasks = new ArrayList<>(); List subTasksBatch; do { - subTasksBatch = retryableExternalRequestForNetworkExceptionsThrowing(() -> dpsClient.getDetailedTaskReportBetweenChunks( - depublishPlugin.getTopologyName(), externalTaskId, subTasks.size(), - subTasks.size() + ECLOUD_REQUEST_BATCH_SIZE)); + // need to change dpsCline call + subTasksBatch = retryableExternalRequestForNetworkExceptionsThrowing( + () -> dpsClient.getDetailedTaskReportBetweenChunks( + depublishPlugin.getTopologyName(), externalTaskId, subTasks.size(), + subTasks.size() + ECLOUD_REQUEST_BATCH_SIZE)); subTasks.addAll(subTasksBatch); } while (subTasksBatch.size() == ECLOUD_REQUEST_BATCH_SIZE); // Mark the records as DEPUBLISHED. final Map> successfulRecords = subTasks.stream() - .filter(subTask -> subTask.getRecordState() == RecordState.SUCCESS) - .map(SubTaskInfo::getResource).map(DepublishRecordIdUtils::decomposeFullRecordId) - .collect(Collectors.groupingBy(Pair::getLeft, - Collectors.mapping(Pair::getRight, Collectors.toSet()))); + .filter(subTask -> + subTask.getRecordState() + == RecordState.SUCCESS) + .map(SubTaskInfo::getResource).map( + DepublishRecordIdUtils::decomposeFullRecordId) + .collect(Collectors.groupingBy( + Pair::getLeft, + Collectors.mapping( + Pair::getRight, + Collectors.toSet()))); successfulRecords.forEach((dataset, records) -> depublishRecordIdDao.markRecordIdsWithDepublicationStatus(dataset, records, DepublicationStatus.DEPUBLISHED, new Date())); diff --git a/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/utils/TestObjectFactory.java b/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/utils/TestObjectFactory.java index 17b7f9833..38d389e4b 100644 --- a/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/utils/TestObjectFactory.java +++ b/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/utils/TestObjectFactory.java @@ -16,8 +16,8 @@ import eu.europeana.metis.core.common.Language; import eu.europeana.metis.core.dao.WorkflowExecutionDao.ExecutionDatasetPair; import eu.europeana.metis.core.dataset.Dataset; -import eu.europeana.metis.core.dataset.DatasetXslt; import eu.europeana.metis.core.dataset.Dataset.PublicationFitness; +import eu.europeana.metis.core.dataset.DatasetXslt; import eu.europeana.metis.core.rest.Record; import eu.europeana.metis.core.workflow.ScheduleFrequence; import eu.europeana.metis.core.workflow.ScheduledWorkflow; @@ -134,8 +134,7 @@ private static WorkflowExecution createWorkflowExecutionObject(Dataset dataset) } /** - * Create a list of dummy workflow executions. The dataset name will have a suffix number for each - * dataset. + * Create a list of dummy workflow executions. The dataset name will have a suffix number for each dataset. * * @param size the number of dummy workflow executions to create * @return the created list @@ -146,8 +145,7 @@ public static List createListOfWorkflowExecutions(int size) { } /** - * Create a list of dummy execution overviews. The dataset name will have a suffix number for each - * dataset. + * Create a list of dummy execution overviews. The dataset name will have a suffix number for each dataset. * * @param size the number of dummy execution overviews to create * @return the created list @@ -180,8 +178,7 @@ public static ScheduledWorkflow createScheduledWorkflowObject() { } /** - * Create a list of dummy scheduled workflows. The dataset name will have a suffix number for each - * dataset. + * Create a list of dummy scheduled workflows. The dataset name will have a suffix number for each dataset. * * @param size the number of dummy scheduled workflows to create * @return the created list @@ -198,11 +195,11 @@ public static List createListOfScheduledWorkflows(int size) { } /** - * Create a list of dummy scheduled workflows with pointer date and frequency. The dataset name - * will have a suffix number for each dataset. + * Create a list of dummy scheduled workflows with pointer date and frequency. The dataset name will have a suffix number for + * each dataset. * - * @param size the number of dummy scheduled workflows to create - * @param date the pointer date + * @param size the number of dummy scheduled workflows to create + * @param date the pointer date * @param scheduleFrequence the schedule frequence * @return the created list */ @@ -270,17 +267,15 @@ public static MetisUserView createMetisUser(String email) { } /** - * Create a dummy sub task info + * Create a dummy subtask info * - * @return the created sub task info + * @return the created subtask info */ public static List createListOfSubTaskInfo() { - SubTaskInfo subTaskInfo1 = new SubTaskInfo(1, "some_resource_id1", RecordState.SUCCESS, "", - "Sensitive Information"); - final int resourceNum = 2; - SubTaskInfo subTaskInfo2 = new SubTaskInfo(resourceNum, "some_resource_id1", - RecordState.SUCCESS, "", - "Sensitive Information"); + SubTaskInfo subTaskInfo1 = new SubTaskInfo(1, "some_resource_id1", RecordState.SUCCESS, "info", + "additional info", "europeanaId", 0L); + SubTaskInfo subTaskInfo2 = new SubTaskInfo(2, "some_resource_id2", RecordState.SUCCESS, "info", + "additional info", "europeanaId", 0L); ArrayList subTaskInfos = new ArrayList<>(); subTaskInfos.add(subTaskInfo1); subTaskInfos.add(subTaskInfo2); @@ -304,8 +299,8 @@ public static TaskErrorsInfo createTaskErrorsInfoListWithoutIdentifiers(int numb } /** - * Create a task errors info object, which contains a list of {@link TaskErrorInfo} objects. These - * will also contain a list of {@link ErrorDetails} that in turn contain dummy identifiers. + * Create a task errors info object, which contains a list of {@link TaskErrorInfo} objects. These will also contain a list of + * {@link ErrorDetails} that in turn contain dummy identifiers. * * @param numberOfErrorTypes the number of dummy error types * @return the created task errors info @@ -325,11 +320,11 @@ public static TaskErrorsInfo createTaskErrorsInfoListWithIdentifiers(int numberO } /** - * Create a task errors info object, which contains a list of {@link TaskErrorInfo} objects. These - * will also contain a list of {@link ErrorDetails} that in turn contain dummy identifiers. + * Create a task errors info object, which contains a list of {@link TaskErrorInfo} objects. These will also contain a list of + * {@link ErrorDetails} that in turn contain dummy identifiers. * * @param errorType the error type to be used for the internal {@link TaskErrorInfo} - * @param message the message type to be used for the internal {@link TaskErrorInfo} + * @param message the message type to be used for the internal {@link TaskErrorInfo} * @return the created task errors info */ public static TaskErrorsInfo createTaskErrorsInfoWithIdentifiers(String errorType, diff --git a/pom.xml b/pom.xml index 70d297100..15f414c74 100644 --- a/pom.xml +++ b/pom.xml @@ -137,7 +137,7 @@ 1.0 4.2.0 - 6-SNAPSHOT + 7-SNAPSHOT 3.0.0 1.3 From cab7e8acc865cc6b0811c6644d3fae8b5d4e20b1 Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Mon, 7 Mar 2022 14:49:52 +0100 Subject: [PATCH 06/73] MET-4159 Update javadoc --- .../metis/harvesting/http/HttpHarvester.java | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvester.java b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvester.java index 4a1a64e40..f464841e7 100644 --- a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvester.java +++ b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvester.java @@ -6,30 +6,28 @@ import java.util.function.Consumer; /** - * Implementations of this interface provide the functionality to harvest from HTTP (compressed - * archive). + * Implementations of this interface provide the functionality to harvest from HTTP (compressed archive). */ public interface HttpHarvester { /** * Harvest from HTTP (compressed archive). * - * @param archiveUrl The URL location of the compressed archive. The URL can use either the - * http(s) protocol or the file protocol. - * @param downloadDirectory The directory to which we download and extract the archive. Note: the - * class does not clean up the downloaded or decompressed files. The caller is responsible for - * providing a directory that is safe (i.e. on the right file system). + * @param archiveUrl The URL location of the compressed archive. The URL can use either the http(s) protocol or the file + * protocol. + * @param downloadDirectory The directory to which we download and extract the archive. Note: the class does not clean up the + * downloaded or decompressed files. The caller is responsible for providing a directory that is safe (i.e. on the right file + * system). * @return An iterator that provides access to the decompressed records. * @throws HarvesterException In case there was an issue during the harvest. */ HttpRecordIterator harvestRecords(String archiveUrl, String downloadDirectory) - throws HarvesterException; + throws HarvesterException; /** - * Harvest from HTTP (compressed archive). This is a convenience method for {@link - * #harvestRecords(String, String)} that copies the input stream to a temporary file (in the - * system's temporary directory) first. An attempt will be made to remove the temporary file - * before this method returns. + * Harvest from HTTP (compressed archive). This is a convenience method for {@link #harvestRecords(String, String)} that copies + * the input stream to a temporary file (in the system's temporary directory) first. An attempt will be made to remove the + * temporary file before this method returns. * * @param inputStream The input stream containing the compressed archive. * @param compressedFileType The type of the archive. @@ -37,17 +35,19 @@ HttpRecordIterator harvestRecords(String archiveUrl, String downloadDirectory) * @throws HarvesterException In case there was an issue during the harvest. */ void harvestRecords(InputStream inputStream, CompressedFileExtension compressedFileType, - Consumer action) throws HarvesterException; + Consumer action) throws HarvesterException; /** - * It creates a {@link HttpRecordIterator} with a InputStream into a temporary file directory. - * It is needed to use the {@link HttpRecordIterator#deleteIteratorContent()} method if this method is used. + * It creates a {@link HttpRecordIterator} with a InputStream into a temporary file directory. When finished using the created + * iterator, the method {@link HttpRecordIterator#deleteIteratorContent()} should be used to clean up leftover files. + * * @param input The input stream from which we create the iterator * @param compressedFileType The type of compressed file type * @return A HttpRecordIterator based on a temporary file location * @throws HarvesterException In case there is an issue while using the input stream */ - HttpRecordIterator createTemporaryHttpHarvestIterator(InputStream input, CompressedFileExtension compressedFileType) throws HarvesterException; + HttpRecordIterator createTemporaryHttpHarvestIterator(InputStream input, CompressedFileExtension compressedFileType) + throws HarvesterException; /** * An object representing an entry in a file archive. @@ -55,8 +55,7 @@ void harvestRecords(InputStream inputStream, CompressedFileExtension compressedF interface ArchiveEntry { /** - * @return The name of the entry. This is the file name (including extension, excluding the - * path). + * @return The name of the entry. This is the file name (including extension, excluding the path). */ String getEntryName(); From 51eb07d13190ef1cbe76303388d619b3015db054 Mon Sep 17 00:00:00 2001 From: Jorge Ortiz Date: Mon, 7 Mar 2022 17:47:40 +0100 Subject: [PATCH 07/73] MET-4159 increased logging message --- .../eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java index 262dc54da..adccc46fe 100644 --- a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java +++ b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java @@ -196,6 +196,8 @@ public void deleteIteratorContent() { } catch (IOException e) { LOGGER.warn("Could not delete directory.", e); } + } else { + LOGGER.warn("Extracted directory undefined, nothing removed."); } } From 32a19edb1789d54321190ab8d75bbca803a87ce3 Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Tue, 8 Mar 2022 10:56:32 +0100 Subject: [PATCH 08/73] MET-4257 Remove alternative indexing environment fields --- .../plugins/AbstractExecutablePlugin.java | 3 +-- .../plugins/AbstractIndexPluginMetadata.java | 9 --------- .../workflow/plugins/DepublishPlugin.java | 4 ---- .../plugins/DepublishPluginMetadata.java | 9 --------- .../plugins/IndexToPreviewPlugin.java | 12 ----------- .../plugins/IndexToPublishPlugin.java | 12 ----------- .../config/ConfigurationPropertiesHolder.java | 6 ------ .../core/rest/config/OrchestratorConfig.java | 2 -- .../main/resources/metis.properties.example | 3 --- .../core/execution/WorkflowPostProcessor.java | 7 +------ .../service/WorkflowExecutionFactory.java | 20 ------------------- 11 files changed, 2 insertions(+), 85 deletions(-) diff --git a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/AbstractExecutablePlugin.java b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/AbstractExecutablePlugin.java index c6543115f..918ba2cf1 100644 --- a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/AbstractExecutablePlugin.java +++ b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/AbstractExecutablePlugin.java @@ -156,7 +156,7 @@ DpsTask createDpsTaskForProcessPlugin(EcloudBasePluginParameters ecloudBasePlugi } DpsTask createDpsTaskForIndexPlugin(EcloudBasePluginParameters ecloudBasePluginParameters, String datasetId, - boolean incrementalIndexing, Date harvestDate, boolean useAlternativeIndexingEnvironment, boolean preserveTimestamps, + boolean incrementalIndexing, Date harvestDate, boolean preserveTimestamps, List datasetIdsToRedirectFrom, boolean performRedirects, String targetDatabase) { final DateFormat dateFormat = new SimpleDateFormat(CommonStringValues.DATE_FORMAT_Z, Locale.US); dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); @@ -165,7 +165,6 @@ DpsTask createDpsTaskForIndexPlugin(EcloudBasePluginParameters ecloudBasePluginP extraParameters.put(PluginParameterKeys.INCREMENTAL_INDEXING, String.valueOf(incrementalIndexing)); extraParameters.put(PluginParameterKeys.HARVEST_DATE, dateFormat.format(harvestDate)); extraParameters.put(PluginParameterKeys.METIS_TARGET_INDEXING_DATABASE, targetDatabase); - extraParameters.put(PluginParameterKeys.METIS_USE_ALT_INDEXING_ENV, String.valueOf(useAlternativeIndexingEnvironment)); extraParameters.put(PluginParameterKeys.METIS_RECORD_DATE, dateFormat.format(getStartedDate())); extraParameters.put(PluginParameterKeys.METIS_PRESERVE_TIMESTAMPS, String.valueOf(preserveTimestamps)); extraParameters.put(PluginParameterKeys.DATASET_IDS_TO_REDIRECT_FROM, String.join(",", datasetIdsToRedirectFrom)); diff --git a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/AbstractIndexPluginMetadata.java b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/AbstractIndexPluginMetadata.java index 2fd8d4c64..ccf8a7c77 100644 --- a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/AbstractIndexPluginMetadata.java +++ b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/AbstractIndexPluginMetadata.java @@ -10,7 +10,6 @@ */ public abstract class AbstractIndexPluginMetadata extends AbstractExecutablePluginMetadata { - private boolean useAlternativeIndexingEnvironment; private boolean preserveTimestamps; private boolean performRedirects; private List datasetIdsToRedirectFrom = new ArrayList<>(); @@ -21,14 +20,6 @@ public AbstractIndexPluginMetadata() { //Required for json serialization } - public boolean isUseAlternativeIndexingEnvironment() { - return useAlternativeIndexingEnvironment; - } - - public void setUseAlternativeIndexingEnvironment(boolean useAlternativeIndexingEnvironment) { - this.useAlternativeIndexingEnvironment = useAlternativeIndexingEnvironment; - } - public boolean isPreserveTimestamps() { return preserveTimestamps; } diff --git a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/DepublishPlugin.java b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/DepublishPlugin.java index 155e69582..4e4bb705e 100644 --- a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/DepublishPlugin.java +++ b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/DepublishPlugin.java @@ -49,13 +49,9 @@ public String getTopologyName() { @Override public DpsTask prepareDpsTask(String datasetId, EcloudBasePluginParameters ecloudBasePluginParameters) { - boolean useAlternativeIndexingEnvironment = getPluginMetadata() - .isUseAlternativeIndexingEnvironment(); Map extraParameters = new HashMap<>(); extraParameters.put(PluginParameterKeys.METIS_DATASET_ID, datasetId); - extraParameters.put(PluginParameterKeys.METIS_USE_ALT_INDEXING_ENV, - String.valueOf(useAlternativeIndexingEnvironment)); //Do set the records ids parameter only if record ids depublication enabled and there are record ids if (!getPluginMetadata().isDatasetDepublish()) { if (CollectionUtils.isEmpty(getPluginMetadata().getRecordIdsToDepublish())) { diff --git a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/DepublishPluginMetadata.java b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/DepublishPluginMetadata.java index 890a47209..0d615b393 100644 --- a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/DepublishPluginMetadata.java +++ b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/DepublishPluginMetadata.java @@ -15,7 +15,6 @@ public class DepublishPluginMetadata extends AbstractExecutablePluginMetadata { private static final ExecutablePluginType pluginType = ExecutablePluginType.DEPUBLISH; - private boolean useAlternativeIndexingEnvironment; private boolean datasetDepublish; private Set recordIdsToDepublish; @@ -28,14 +27,6 @@ public ExecutablePluginType getExecutablePluginType() { return pluginType; } - public boolean isUseAlternativeIndexingEnvironment() { - return useAlternativeIndexingEnvironment; - } - - public void setUseAlternativeIndexingEnvironment(boolean useAlternativeIndexingEnvironment) { - this.useAlternativeIndexingEnvironment = useAlternativeIndexingEnvironment; - } - public boolean isDatasetDepublish() { return datasetDepublish; } diff --git a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/IndexToPreviewPlugin.java b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/IndexToPreviewPlugin.java index 01bc8aa7f..a828690ae 100644 --- a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/IndexToPreviewPlugin.java +++ b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/IndexToPreviewPlugin.java @@ -2,7 +2,6 @@ import eu.europeana.cloud.service.dps.DpsTask; import eu.europeana.cloud.service.dps.metis.indexing.TargetIndexingDatabase; -import eu.europeana.cloud.service.dps.metis.indexing.TargetIndexingEnvironment; /** * Index to Preview Plugin. @@ -38,7 +37,6 @@ public DpsTask prepareDpsTask(String datasetId, EcloudBasePluginParameters eclou return createDpsTaskForIndexPlugin(ecloudBasePluginParameters, datasetId, getPluginMetadata().isIncrementalIndexing(), getPluginMetadata().getHarvestDate(), - getPluginMetadata().isUseAlternativeIndexingEnvironment(), getPluginMetadata().isPreserveTimestamps(), getPluginMetadata().getDatasetIdsToRedirectFrom(), getPluginMetadata().isPerformRedirects(), getTargetIndexingDatabase().name()); @@ -57,14 +55,4 @@ public String getTopologyName() { public TargetIndexingDatabase getTargetIndexingDatabase() { return TargetIndexingDatabase.PREVIEW; } - - /** - * Get the target indexing environment. - * - * @return the target indexing environment - */ - public TargetIndexingEnvironment getTargetIndexingEnvironment() { - return getPluginMetadata().isUseAlternativeIndexingEnvironment() ? TargetIndexingEnvironment.ALTERNATIVE - : TargetIndexingEnvironment.DEFAULT; - } } diff --git a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/IndexToPublishPlugin.java b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/IndexToPublishPlugin.java index d57c81ba5..368ba7128 100644 --- a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/IndexToPublishPlugin.java +++ b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/IndexToPublishPlugin.java @@ -2,7 +2,6 @@ import eu.europeana.cloud.service.dps.DpsTask; import eu.europeana.cloud.service.dps.metis.indexing.TargetIndexingDatabase; -import eu.europeana.cloud.service.dps.metis.indexing.TargetIndexingEnvironment; /** * Index to Publish Plugin. @@ -37,7 +36,6 @@ public DpsTask prepareDpsTask(String datasetId, EcloudBasePluginParameters eclou return createDpsTaskForIndexPlugin(ecloudBasePluginParameters, datasetId, getPluginMetadata().isIncrementalIndexing(), getPluginMetadata().getHarvestDate(), - getPluginMetadata().isUseAlternativeIndexingEnvironment(), getPluginMetadata().isPreserveTimestamps(), getPluginMetadata().getDatasetIdsToRedirectFrom(), getPluginMetadata().isPerformRedirects(), getTargetIndexingDatabase().name()); @@ -56,14 +54,4 @@ public String getTopologyName() { public TargetIndexingDatabase getTargetIndexingDatabase() { return TargetIndexingDatabase.PUBLISH; } - - /** - * Get the target indexing environment. - * - * @return the target indexing environment - */ - public TargetIndexingEnvironment getTargetIndexingEnvironment() { - return getPluginMetadata().isUseAlternativeIndexingEnvironment() ? TargetIndexingEnvironment.ALTERNATIVE - : TargetIndexingEnvironment.DEFAULT; - } } diff --git a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/ConfigurationPropertiesHolder.java b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/ConfigurationPropertiesHolder.java index dafee7c30..151297a1f 100644 --- a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/ConfigurationPropertiesHolder.java +++ b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/ConfigurationPropertiesHolder.java @@ -95,8 +95,6 @@ public class ConfigurationPropertiesHolder { private int maxDepublishRecordIdsPerDataset; // Ecloud configuration - @Value("${metis.use.alternative.indexing.environment}") - private boolean metisUseAlternativeIndexingEnvironment; @Value("${metis.link.checking.default.sampling.size}") private int metisLinkCheckingDefaultSamplingSize; @Value("${solr.commit.period.in.mins}") @@ -304,10 +302,6 @@ public int getMaxDepublishRecordIdsPerDataset() { return maxDepublishRecordIdsPerDataset; } - public boolean isMetisUseAlternativeIndexingEnvironment() { - return metisUseAlternativeIndexingEnvironment; - } - public int getMetisLinkCheckingDefaultSamplingSize() { return metisLinkCheckingDefaultSamplingSize; } diff --git a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/OrchestratorConfig.java b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/OrchestratorConfig.java index c69fbd5b5..a3069107b 100644 --- a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/OrchestratorConfig.java +++ b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/OrchestratorConfig.java @@ -139,8 +139,6 @@ public WorkflowExecutionFactory getWorkflowExecutionFactory( .setValidationExternalProperties(propertiesHolder.getValidationExternalProperties()); workflowExecutionFactory .setValidationInternalProperties(propertiesHolder.getValidationInternalProperties()); - workflowExecutionFactory.setMetisUseAlternativeIndexingEnvironment( - propertiesHolder.isMetisUseAlternativeIndexingEnvironment()); workflowExecutionFactory.setDefaultSamplingSizeForLinkChecking( propertiesHolder.getMetisLinkCheckingDefaultSamplingSize()); return workflowExecutionFactory; diff --git a/metis-core/metis-core-rest/src/main/resources/metis.properties.example b/metis-core/metis-core-rest/src/main/resources/metis.properties.example index f7f7050da..07bec7656 100644 --- a/metis-core/metis-core-rest/src/main/resources/metis.properties.example +++ b/metis-core/metis-core-rest/src/main/resources/metis.properties.example @@ -88,7 +88,4 @@ metis.core.baseUrl= #Metis Core (regardless on whether the list is paginated). metis.core.max.served.execution.list.length= metis.core.max.depublish.record.ids.per.dataset= -#In the combination of TEST and ACCEPTANCE, TEST=false, ACCEPTANCE=true -#For the production environment it should be false -metis.use.alternative.indexing.environment= metis.link.checking.default.sampling.size= \ No newline at end of file diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java index 2cbc2fc9d..d6b27d2e4 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java @@ -7,7 +7,6 @@ import eu.europeana.cloud.common.model.dps.SubTaskInfo; import eu.europeana.cloud.service.dps.exception.DpsException; import eu.europeana.cloud.service.dps.metis.indexing.TargetIndexingDatabase; -import eu.europeana.cloud.service.dps.metis.indexing.TargetIndexingEnvironment; import eu.europeana.metis.core.common.DepublishRecordIdUtils; import eu.europeana.metis.core.dao.DatasetDao; import eu.europeana.metis.core.dao.DepublishRecordIdDao; @@ -98,13 +97,10 @@ void performPluginPostProcessing(AbstractExecutablePlugin plugin, String data private void indexPostProcess(AbstractExecutablePlugin indexPlugin, String datasetId) throws DpsException, InvalidIndexPluginException { TargetIndexingDatabase targetIndexingDatabase; - TargetIndexingEnvironment targetIndexingEnvironment; if (indexPlugin instanceof IndexToPreviewPlugin) { targetIndexingDatabase = ((IndexToPreviewPlugin) indexPlugin).getTargetIndexingDatabase(); - targetIndexingEnvironment = ((IndexToPreviewPlugin) indexPlugin).getTargetIndexingEnvironment(); } else if (indexPlugin instanceof IndexToPublishPlugin) { targetIndexingDatabase = ((IndexToPublishPlugin) indexPlugin).getTargetIndexingDatabase(); - targetIndexingEnvironment = ((IndexToPublishPlugin) indexPlugin).getTargetIndexingEnvironment(); //Reset depublish status depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, null, DepublicationStatus.PENDING_DEPUBLICATION, null); @@ -112,8 +108,7 @@ private void indexPostProcess(AbstractExecutablePlugin indexPlugin, String da throw new InvalidIndexPluginException("Plugin is not of the types supported"); } final Integer databaseTotalRecords = retryableExternalRequestForNetworkExceptionsThrowing(() -> - (int) dpsClient.getTotalMetisDatabaseRecords(datasetId, targetIndexingDatabase, - targetIndexingEnvironment)); + (int) dpsClient.getTotalMetisDatabaseRecords(datasetId, targetIndexingDatabase)); indexPlugin.getExecutionProgress().setTotalDatabaseRecords(databaseTotalRecords); } diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/WorkflowExecutionFactory.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/WorkflowExecutionFactory.java index bac712f4a..d0f0c644f 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/WorkflowExecutionFactory.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/WorkflowExecutionFactory.java @@ -50,7 +50,6 @@ public class WorkflowExecutionFactory { private ValidationProperties validationExternalProperties; // Use getter and setter! private ValidationProperties validationInternalProperties; // Use getter and setter! - private boolean metisUseAlternativeIndexingEnvironment; // Use getter and setter for this field! private int defaultSamplingSizeForLinkChecking; // Use getter and setter for this field! /** @@ -112,24 +111,18 @@ private AbstractExecutablePlugin createWorkflowExecutionPlugin(Dataset datase this.setupValidationInternalForPluginMetadata( (ValidationInternalPluginMetadata) pluginMetadata, getValidationInternalProperties()); } else if (pluginMetadata instanceof IndexToPreviewPluginMetadata) { - ((IndexToPreviewPluginMetadata) pluginMetadata) - .setUseAlternativeIndexingEnvironment(isMetisUseAlternativeIndexingEnvironment()); ((IndexToPreviewPluginMetadata) pluginMetadata) .setDatasetIdsToRedirectFrom(dataset.getDatasetIdsToRedirectFrom()); boolean performRedirects = shouldRedirectsBePerformed(dataset, workflowPredecessor, ExecutablePluginType.PREVIEW, typesInWorkflowBeforeThisPlugin); ((IndexToPreviewPluginMetadata) pluginMetadata).setPerformRedirects(performRedirects); } else if (pluginMetadata instanceof IndexToPublishPluginMetadata) { - ((IndexToPublishPluginMetadata) pluginMetadata) - .setUseAlternativeIndexingEnvironment(isMetisUseAlternativeIndexingEnvironment()); ((IndexToPublishPluginMetadata) pluginMetadata) .setDatasetIdsToRedirectFrom(dataset.getDatasetIdsToRedirectFrom()); boolean performRedirects = shouldRedirectsBePerformed(dataset, workflowPredecessor, ExecutablePluginType.PUBLISH, typesInWorkflowBeforeThisPlugin); ((IndexToPublishPluginMetadata) pluginMetadata).setPerformRedirects(performRedirects); } else if (pluginMetadata instanceof DepublishPluginMetadata) { - ((DepublishPluginMetadata) pluginMetadata) - .setUseAlternativeIndexingEnvironment(isMetisUseAlternativeIndexingEnvironment()); setupDepublishPluginMetadata(dataset, ((DepublishPluginMetadata) pluginMetadata)); } else if (pluginMetadata instanceof LinkCheckingPluginMetadata) { ((LinkCheckingPluginMetadata) pluginMetadata) @@ -306,19 +299,6 @@ public void setValidationInternalProperties(ValidationProperties validationInter } } - private boolean isMetisUseAlternativeIndexingEnvironment() { - synchronized (this) { - return metisUseAlternativeIndexingEnvironment; - } - } - - public void setMetisUseAlternativeIndexingEnvironment( - boolean metisUseAlternativeIndexingEnvironment) { - synchronized (this) { - this.metisUseAlternativeIndexingEnvironment = metisUseAlternativeIndexingEnvironment; - } - } - private int getDefaultSamplingSizeForLinkChecking() { synchronized (this) { return defaultSamplingSizeForLinkChecking; From 9961a3d807d587feeb345f55e1ef9d386e6e8bad Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Tue, 8 Mar 2022 14:20:55 +0100 Subject: [PATCH 09/73] MET-4310 Reduce connection pool on mongo clients --- .../mongo/connection/MongoClientProvider.java | 92 ++++++++++++------- .../mongo/connection/MongoProperties.java | 22 ++++- .../connection/MongoClientProviderTest.java | 43 +++++---- .../mongo/connection/MongoPropertiesTest.java | 3 +- 4 files changed, 105 insertions(+), 55 deletions(-) diff --git a/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoClientProvider.java b/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoClientProvider.java index 7b6ae4333..fb79ff0ed 100644 --- a/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoClientProvider.java +++ b/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoClientProvider.java @@ -9,6 +9,7 @@ import com.mongodb.ServerAddress; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; +import com.mongodb.connection.ConnectionPoolSettings; import eu.europeana.metis.mongo.connection.MongoProperties.ReadPreferenceValue; import java.util.List; import java.util.Optional; @@ -17,8 +18,7 @@ import java.util.function.Supplier; /** - * This class can set up and provide a Mongo client given the Mongo properties. It applies the - * following default values: + * This class can set up and provide a Mongo client given the Mongo properties. It applies the following default values: *
    *
  • * The read preference for the connection is defaulted to {@link ReadPreference#secondaryPreferred()}. @@ -46,6 +46,7 @@ public class MongoClientProvider { private static final ReadPreference DEFAULT_READ_PREFERENCE = ReadPreference.secondaryPreferred(); private static final int DEFAULT_MAX_CONNECTION_IDLE_MILLIS = 30_000; + private static final int DEFAULT_MAX_CONNECTIONS = 20; private static final boolean DEFAULT_RETRY_WRITES = false; private static final String DEFAULT_APPLICATION_NAME = "Europeana Application Suite"; @@ -53,8 +54,8 @@ public class MongoClientProvider { private final String authenticationDatabase; /** - * Constructor from a connection URI string (see the documentation of {@link MongoClientURI} for - * the details). The connection URL can provide settings that will override the default settings. + * Constructor from a connection URI string (see the documentation of {@link MongoClientURI} for the details). The connection + * URL can provide settings that will override the default settings. * * @param connectionUri The connection URI as a string * @param exceptionCreator How to report exceptions. @@ -79,28 +80,27 @@ public MongoClientProvider(String connectionUri, Function exceptionCr } /** - * Constructor from a {@link MongoProperties} object. The caller needs to provide settings that - * will be used instead of the default settings. + * Constructor from a {@link MongoProperties} object. The caller needs to provide settings that will be used instead of the + * default settings. * - * @param properties The properties of the Mongo connection. Note that if the passed properties - * object is changed after calling this method, those changes will not be reflected when creating - * mongo clients. - * @param clientSettingsBuilder The settings to be applied. The default settings will not be used. - * The caller can however choose to incorporate the default settings as needed by using a client - * settings builder obtained from {@link #getDefaultClientSettingsBuilder()} as input. + * @param properties The properties of the Mongo connection. Note that if the passed properties object is changed after calling + * this method, those changes will not be reflected when creating mongo clients. + * @param clientSettingsBuilder The settings to be applied. The default settings will not be used. The caller can however choose + * to incorporate the default settings as needed by using a client settings builder obtained from {@link + * #getDefaultClientSettingsBuilder()} as input. * @throws E In case the properties are wrong */ public MongoClientProvider(MongoProperties properties, Builder clientSettingsBuilder) throws E { final ReadPreference readPreference = Optional.ofNullable(properties.getReadPreferenceValue()) - .map(ReadPreferenceValue::getReadPreferenceSupplier).map(Supplier::get) - .orElse(DEFAULT_READ_PREFERENCE); + .map(ReadPreferenceValue::getReadPreferenceSupplier).map(Supplier::get) + .orElse(DEFAULT_READ_PREFERENCE); clientSettingsBuilder.readPreference(readPreference); final List mongoHosts = properties.getMongoHosts(); final MongoCredential mongoCredential = properties.getMongoCredentials(); this.authenticationDatabase = Optional.ofNullable(mongoCredential) - .map(MongoCredential::getSource).orElse(null); + .map(MongoCredential::getSource).orElse(null); clientSettingsBuilder .applyToSslSettings(builder -> builder.enabled(properties.mongoEnableSsl())); clientSettingsBuilder.applyToClusterSettings(builder -> builder.hosts(mongoHosts)); @@ -109,6 +109,9 @@ public MongoClientProvider(MongoProperties properties, Builder clientSettings } Optional.ofNullable(properties.getApplicationName()).filter(name -> !name.isBlank()) .ifPresent(clientSettingsBuilder::applicationName); + + clientSettingsBuilder.applyToConnectionPoolSettings( + builder -> builder.applySettings(createConnectionPoolSettings(properties.getMaxConnectionPoolSize()))); final MongoClientSettings mongoClientSettings = clientSettingsBuilder.build(); this.creator = () -> MongoClients.create(mongoClientSettings); @@ -117,9 +120,8 @@ public MongoClientProvider(MongoProperties properties, Builder clientSettings /** * Constructor from a {@link MongoProperties} object, using the default settings. * - * @param properties The properties of the Mongo connection. Note that if the passed properties - * object is changed after calling this method, those changes will not be reflected when calling - * {@link #createMongoClient()}. + * @param properties The properties of the Mongo connection. Note that if the passed properties object is changed after calling + * this method, those changes will not be reflected when calling {@link #createMongoClient()}. * @throws E In case the properties are wrong */ public MongoClientProvider(MongoProperties properties) throws E { @@ -131,19 +133,17 @@ public MongoClientProvider(MongoProperties properties) throws E { * * @return A new instance of {@link Builder} with the default settings. */ - public static Builder getDefaultClientSettingsBuilder() { + public static MongoClientSettings.Builder getDefaultClientSettingsBuilder() { return MongoClientSettings.builder() - // TODO: 7/16/20 Remove default retry writes after upgrade to mongo server version 4.2 - .retryWrites(DEFAULT_RETRY_WRITES) - .applyToConnectionPoolSettings(builder -> builder - .maxConnectionIdleTime(DEFAULT_MAX_CONNECTION_IDLE_MILLIS, TimeUnit.MILLISECONDS)) - .readPreference(DEFAULT_READ_PREFERENCE) - .applicationName(DEFAULT_APPLICATION_NAME); + // TODO: 7/16/20 Remove default retry writes after upgrade to mongo server version 4.2 + .retryWrites(DEFAULT_RETRY_WRITES) + .applyToConnectionPoolSettings(builder -> builder.applySettings(getDefaultConnectionPoolSettings())) + .readPreference(DEFAULT_READ_PREFERENCE) + .applicationName(DEFAULT_APPLICATION_NAME); } /** - * Convenience method for {@link #MongoClientProvider(String, Function)}. See that - * constructor for the details. + * Convenience method for {@link #MongoClientProvider(String, Function)}. See that constructor for the details. * * @param connectionUri The connection URI. * @return An instance. @@ -153,8 +153,7 @@ public static MongoClientProvider create(String connec } /** - * Convenience method for {@link #MongoClientProvider(String, Function)}. See that - * constructor for the details. + * Convenience method for {@link #MongoClientProvider(String, Function)}. See that constructor for the details. * * @param connectionUri The connection URI. * @return A supplier for {@link MongoClient} instances based on this class. @@ -164,8 +163,17 @@ public static Supplier createAsSupplier(String connectionUri) { } /** - * Returns the authentication database for mongo connections that are provided. Can be null - * (signifying that the default is to be used or that no authentication is specified). + * Get the default connection pool settings + * + * @return the default connection pool settings + */ + private static ConnectionPoolSettings getDefaultConnectionPoolSettings() { + return createConnectionPoolSettings(null); + } + + /** + * Returns the authentication database for mongo connections that are provided. Can be null (signifying that the default is to + * be used or that no authentication is specified). * * @return The authentication database. */ @@ -174,9 +182,8 @@ public final String getAuthenticationDatabase() { } /** - * Creates a Mongo client. This method can be called multiple times and will create and return a - * different client each time. The calling code is responsible for properly closing the created - * client. + * Creates a Mongo client. This method can be called multiple times and will create and return a different client each time. The + * calling code is responsible for properly closing the created client. * * @return A mongo client. * @throws E In case there is a problem with creating the client. @@ -185,6 +192,23 @@ public final MongoClient createMongoClient() throws E { return creator.createMongoClient(); } + /** + * Create a connection pool settings object. Settings that are null will be set to default settings. + * + * @param maxPoolSize the maximum connection pool size + * @return the connection pool settings + */ + static ConnectionPoolSettings createConnectionPoolSettings(Integer maxPoolSize) { + final ConnectionPoolSettings.Builder builder = ConnectionPoolSettings.builder(); + builder.maxConnectionIdleTime(DEFAULT_MAX_CONNECTION_IDLE_MILLIS, TimeUnit.MILLISECONDS); + if (maxPoolSize != null && maxPoolSize > 0) { + builder.maxSize(maxPoolSize); + } else { + builder.maxSize(DEFAULT_MAX_CONNECTIONS); + } + return builder.build(); + } + private interface MongoClientCreator { MongoClient createMongoClient() throws E; diff --git a/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoProperties.java b/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoProperties.java index 497215e66..e71ee360d 100644 --- a/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoProperties.java +++ b/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoProperties.java @@ -28,6 +28,7 @@ public class MongoProperties { private MongoCredential mongoCredentials; private boolean mongoEnableSsl; private ReadPreferenceValue readPreferenceValue; + private Integer maxConnectionPoolSize; private String applicationName; /** @@ -150,8 +151,16 @@ public void setReadPreferenceValue(ReadPreferenceValue readPreferenceValue) { } /** - * Set the application name. Can be null, in which case a default generic application name is - * to be used. + * Get the maximum connection pol size + * + * @return the maximum connection pool size + */ + public Integer getMaxConnectionPoolSize() { + return maxConnectionPoolSize; + } + + /** + * Set the application name. Can be null, in which case a default generic application name is to be used. * * @param applicationName The application name, or null for the default. */ @@ -221,6 +230,15 @@ public ReadPreferenceValue getReadPreferenceValue() { return readPreferenceValue; } + /** + * Set the maximum connection poll size. Can be null, in which case the default applies. + * + * @param maxConnectionPoolSize the maximum connection pool size + */ + public void setMaxConnectionPoolSize(Integer maxConnectionPoolSize) { + this.maxConnectionPoolSize = maxConnectionPoolSize; + } + /** * This method returns the value of the application name (or null for the default). * diff --git a/metis-common/metis-common-mongo/src/test/java/eu/europeana/metis/mongo/connection/MongoClientProviderTest.java b/metis-common/metis-common-mongo/src/test/java/eu/europeana/metis/mongo/connection/MongoClientProviderTest.java index 4b68389e5..145bba9fc 100644 --- a/metis-common/metis-common-mongo/src/test/java/eu/europeana/metis/mongo/connection/MongoClientProviderTest.java +++ b/metis-common/metis-common-mongo/src/test/java/eu/europeana/metis/mongo/connection/MongoClientProviderTest.java @@ -8,8 +8,7 @@ import com.mongodb.MongoClientSettings; import com.mongodb.ReadPreference; import com.mongodb.client.MongoClient; -import eu.europeana.metis.mongo.connection.MongoClientProvider; -import eu.europeana.metis.mongo.connection.MongoProperties; +import com.mongodb.connection.ConnectionPoolSettings; import eu.europeana.metis.mongo.embedded.EmbeddedLocalhostMongo; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -40,14 +39,26 @@ static void tearDown() { embeddedLocalhostMongo.stop(); } - @Test - void getDefaultClientSettingsBuilder() { - MongoClientSettings.Builder actual = MongoClientProvider.getDefaultClientSettingsBuilder(); + private static MongoProperties getMongoProperties() { + final String mongoHost = embeddedLocalhostMongo.getMongoHost(); + final int mongoPort = embeddedLocalhostMongo.getMongoPort(); + final MongoProperties mongoProperties = new MongoProperties<>( + IllegalArgumentException::new); + mongoProperties.setMongoHosts(new String[]{mongoHost}, new int[]{mongoPort}); + mongoProperties.setMongoCredentials("user", "wachtwoord", "authenticationDB"); + mongoProperties.setApplicationName(DATABASE_NAME); + mongoProperties.setMaxConnectionPoolSize(10); + return mongoProperties; + } - assertFalse(actual.build().getRetryWrites()); - assertEquals(ReadPreference.secondaryPreferred(), actual.build().getReadPreference()); - assertEquals("Europeana Application Suite", actual.build().getApplicationName()); - assertEquals(30_000, actual.build().getConnectionPoolSettings().getMaxConnectionIdleTime(TimeUnit.MILLISECONDS)); + @Test + void getClientSettingsBuilder() { + final MongoClientSettings mongoClientSettings = MongoClientProvider.getDefaultClientSettingsBuilder().build(); + assertFalse(mongoClientSettings.getRetryWrites()); + assertEquals(ReadPreference.secondaryPreferred(), mongoClientSettings.getReadPreference()); + assertEquals("Europeana Application Suite", mongoClientSettings.getApplicationName()); + assertEquals(30_000, mongoClientSettings.getConnectionPoolSettings().getMaxConnectionIdleTime(TimeUnit.MILLISECONDS)); + assertEquals(20, mongoClientSettings.getConnectionPoolSettings().getMaxSize()); } @Test @@ -87,14 +98,10 @@ void createMongoClient() { assertTrue(mongoClient instanceof MongoClient); } - private static MongoProperties getMongoProperties() { - final String mongoHost = embeddedLocalhostMongo.getMongoHost(); - final int mongoPort = embeddedLocalhostMongo.getMongoPort(); - final MongoProperties mongoProperties = new MongoProperties<>( - IllegalArgumentException::new); - mongoProperties.setMongoHosts(new String[]{mongoHost}, new int[]{mongoPort}); - mongoProperties.setMongoCredentials("user","wachtwoord","authenticationDB"); - mongoProperties.setApplicationName(DATABASE_NAME); - return mongoProperties; + @Test + void createConnectionPoolSettings() { + final ConnectionPoolSettings connectionPoolSettings = MongoClientProvider.createConnectionPoolSettings(10); + assertEquals(30_000, connectionPoolSettings.getMaxConnectionIdleTime(TimeUnit.MILLISECONDS)); + assertEquals(10, connectionPoolSettings.getMaxSize()); } } \ No newline at end of file diff --git a/metis-common/metis-common-mongo/src/test/java/eu/europeana/metis/mongo/connection/MongoPropertiesTest.java b/metis-common/metis-common-mongo/src/test/java/eu/europeana/metis/mongo/connection/MongoPropertiesTest.java index 4df4b749a..321694e9a 100644 --- a/metis-common/metis-common-mongo/src/test/java/eu/europeana/metis/mongo/connection/MongoPropertiesTest.java +++ b/metis-common/metis-common-mongo/src/test/java/eu/europeana/metis/mongo/connection/MongoPropertiesTest.java @@ -5,7 +5,6 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import eu.europeana.metis.mongo.connection.MongoProperties; import eu.europeana.metis.mongo.connection.MongoProperties.ReadPreferenceValue; import java.net.InetSocketAddress; import org.junit.jupiter.api.Test; @@ -33,6 +32,8 @@ void setAllProperties() throws Exception { "testAplication"); assertMongoProperties(mongoProperties); + mongoProperties.setMaxConnectionPoolSize(10); + assertEquals(10, mongoProperties.getMaxConnectionPoolSize()); } @Test From 65086ba80a8ce3d3d5fefca54450b286096d25e2 Mon Sep 17 00:00:00 2001 From: Adolfo Peixinho Date: Thu, 10 Mar 2022 15:44:37 +0100 Subject: [PATCH 10/73] MET-4237 Refactored implementation to get chunked tasks from e-cloud client. --- .../core/execution/WorkflowPostProcessor.java | 64 +++++++++++-------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java index ceb8d6046..26046cd6f 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java @@ -49,7 +49,7 @@ public class WorkflowPostProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(WorkflowPostProcessor.class); - private static final int ECLOUD_REQUEST_BATCH_SIZE = 100; + private static final int ECLOUD_REQUEST_BATCH_SIZE = 1000; private final DepublishRecordIdDao depublishRecordIdDao; private final DatasetDao datasetDao; @@ -116,21 +116,35 @@ private void indexPostProcess(AbstractExecutablePlugin indexPlugin, String da targetIndexingDatabase = ((IndexToPublishPlugin) indexPlugin).getTargetIndexingDatabase(); targetIndexingEnvironment = ((IndexToPublishPlugin) indexPlugin).getTargetIndexingEnvironment(); - // get all tasks from dataset id and topology name - List taskReport = dpsClient.getDetailedTaskReport(indexPlugin.getTopologyName(), - Long.parseLong(indexPlugin.getExternalTaskId())); - // get all currently de-published records ids - Set depublishedRecordIds = depublishRecordIdDao - .getAllDepublishRecordIdsWithStatus(datasetId, DepublishRecordIdSortField.DEPUBLICATION_STATE, SortDirection.ASCENDING, - DepublicationStatus.DEPUBLISHED); - // filter the record ids that are a part of the given report, to be de-published - Set recordIdsToDepublish = taskReport.stream() - .filter(taskInfo -> depublishedRecordIds.contains(taskInfo.getEuropeanaId())) - .map(SubTaskInfo::getEuropeanaId).collect(Collectors.toSet()); - - // reset de-publish status - depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, recordIdsToDepublish, - DepublicationStatus.PENDING_DEPUBLICATION, null); + final long totalRecords = dpsClient.getTotalMetisDatabaseRecords(indexPlugin.getExternalTaskId(), + ((IndexToPublishPlugin) indexPlugin).getTargetIndexingDatabase()); + List subTaskInfoList; + + // get chunked tasks from dataset id and topology name + for (int i = 0; i < totalRecords; i = +ECLOUD_REQUEST_BATCH_SIZE) { + subTaskInfoList = dpsClient.getDetailedTaskReportBetweenChunks(indexPlugin.getTopologyName(), + Long.parseLong(indexPlugin.getExternalTaskId()), i, i + ECLOUD_REQUEST_BATCH_SIZE); + if (i >= totalRecords) { + subTaskInfoList = dpsClient.getDetailedTaskReportBetweenChunks(indexPlugin.getTopologyName(), + Long.parseLong(indexPlugin.getExternalTaskId()), (int) (totalRecords - (totalRecords % ECLOUD_REQUEST_BATCH_SIZE)), + (int) totalRecords); + } + // get all currently de-published records ids + Set depublishedRecordIds = depublishRecordIdDao + .getAllDepublishRecordIdsWithStatus(datasetId, DepublishRecordIdSortField.DEPUBLICATION_STATE, + SortDirection.ASCENDING, + DepublicationStatus.DEPUBLISHED); + + // TODO: what if it's incremental + // filter the record ids that are a part of the given report, to be de-published + Set recordIdsToDepublish = subTaskInfoList.stream() + .filter(taskInfo -> depublishedRecordIds.contains( + taskInfo.getEuropeanaId())) + .map(SubTaskInfo::getEuropeanaId).collect(Collectors.toSet()); + // reset de-publish status + depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, recordIdsToDepublish, + DepublicationStatus.PENDING_DEPUBLICATION, null); + } } else { throw new InvalidIndexPluginException("Plugin is not of the types supported"); } @@ -207,16 +221,16 @@ private void depublishRecordPostProcess(DepublishPlugin depublishPlugin, String // Mark the records as DEPUBLISHED. final Map> successfulRecords = subTasks.stream() - .filter(subTask -> - subTask.getRecordState() - == RecordState.SUCCESS) - .map(SubTaskInfo::getResource).map( + .filter(subTask -> + subTask.getRecordState() + == RecordState.SUCCESS) + .map(SubTaskInfo::getResource).map( DepublishRecordIdUtils::decomposeFullRecordId) - .collect(Collectors.groupingBy( - Pair::getLeft, - Collectors.mapping( - Pair::getRight, - Collectors.toSet()))); + .collect(Collectors.groupingBy( + Pair::getLeft, + Collectors.mapping( + Pair::getRight, + Collectors.toSet()))); successfulRecords.forEach((dataset, records) -> depublishRecordIdDao.markRecordIdsWithDepublicationStatus(dataset, records, DepublicationStatus.DEPUBLISHED, new Date())); From 53812ed805c3aa1ce81718346c6319cf612860b4 Mon Sep 17 00:00:00 2001 From: Adolfo Peixinho Date: Thu, 10 Mar 2022 17:57:00 +0100 Subject: [PATCH 11/73] MET-4237 Wrong logic in increment. Math is not Programming. Commutative Identity does not apply a+b==b+a --- .../europeana/metis/core/execution/WorkflowPostProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java index 26046cd6f..3e022327b 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java @@ -121,7 +121,7 @@ private void indexPostProcess(AbstractExecutablePlugin indexPlugin, String da List subTaskInfoList; // get chunked tasks from dataset id and topology name - for (int i = 0; i < totalRecords; i = +ECLOUD_REQUEST_BATCH_SIZE) { + for (int i = 0; i < totalRecords; i +=ECLOUD_REQUEST_BATCH_SIZE) { subTaskInfoList = dpsClient.getDetailedTaskReportBetweenChunks(indexPlugin.getTopologyName(), Long.parseLong(indexPlugin.getExternalTaskId()), i, i + ECLOUD_REQUEST_BATCH_SIZE); if (i >= totalRecords) { From ae7cd0c914517c7a1bb2b40fd5b73adf552eec0a Mon Sep 17 00:00:00 2001 From: Adolfo Peixinho Date: Thu, 10 Mar 2022 18:01:13 +0100 Subject: [PATCH 12/73] MET-4237 Code cleanup --- .../eu/europeana/metis/core/execution/WorkflowPostProcessor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java index 3e022327b..264efbe81 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java @@ -211,7 +211,6 @@ private void depublishRecordPostProcess(DepublishPlugin depublishPlugin, String final List subTasks = new ArrayList<>(); List subTasksBatch; do { - // need to change dpsCline call subTasksBatch = retryableExternalRequestForNetworkExceptionsThrowing( () -> dpsClient.getDetailedTaskReportBetweenChunks( depublishPlugin.getTopologyName(), externalTaskId, subTasks.size(), From 0cd58faab18093045d1595e76eb88bc410706e1d Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Tue, 15 Mar 2022 14:16:43 +0100 Subject: [PATCH 13/73] Debt/met 4250 refactor code to remove mock maker inline (#508) * MET-4250 Update NetworkUtil * MET-4250 Update RdfConversionUtils * MET-4250 Javadocs and cleanup * MET-4250 Remove mockito inline from root pom --- .../rest/client/TestAuthenticationClient.java | 2 +- .../embedded/EmbeddedLocalhostMongo.java | 2 +- metis-common/metis-common-network/pom.xml | 5 - .../europeana/metis/network/NetworkUtil.java | 25 +-- .../network/ExternalRequestUtilTest.java | 22 +-- .../metis/network/NetworkUtilTest.java | 21 +-- .../metis/network/StringHttpClientTest.java | 14 +- .../europeana/metis/zoho/ZohoUtilsTest.java | 4 +- .../core/service/TestDatasetService.java | 4 +- .../core/service/TestProxiesService.java | 2 +- .../rest/client/EnrichmentWorkerImpl.java | 9 +- .../enrichment/MetisRecordParserTest.java | 7 +- .../utils/EntityMergeEngineTest.java | 6 +- .../oaipmh/CloseableHttpOaiClientTest.java | 2 +- .../oaipmh/OaiHarvesterImplTest.java | 2 +- .../eu/europeana/indexing/IndexerImpl.java | 101 ++++++------ .../RecordTierCalculationViewGenerator.java | 15 +- .../metadata/ContextualClassesClassifier.java | 4 +- .../metadata/EnablingElementsClassifier.java | 6 +- .../tiers/metadata/LanguageClassifier.java | 4 +- .../media/AbstractMediaClassifierTest.java | 4 +- .../extraction/AudioVideoProcessorTest.java | 10 +- metis-schema/pom.xml | 6 - .../schema/convert/RdfConversionUtils.java | 145 +++++++++--------- .../convert/RdfConversionUtilsTest.java | 79 ++++------ .../src/test/java/TestValidationClient.java | 2 +- .../src/test/java/TestApplication.java | 2 +- .../src/test/java/TestApplication.java | 2 +- .../src/test/java/TestSchemaProvider.java | 2 +- .../test/java/TestValidationExecution.java | 1 - pom.xml | 6 - 31 files changed, 250 insertions(+), 266 deletions(-) diff --git a/metis-authentication/metis-authentication-rest-client/src/test/java/eu/europeana/metis/authentication/rest/client/TestAuthenticationClient.java b/metis-authentication/metis-authentication-rest-client/src/test/java/eu/europeana/metis/authentication/rest/client/TestAuthenticationClient.java index 07e4633b5..16fd1c40b 100644 --- a/metis-authentication/metis-authentication-rest-client/src/test/java/eu/europeana/metis/authentication/rest/client/TestAuthenticationClient.java +++ b/metis-authentication/metis-authentication-rest-client/src/test/java/eu/europeana/metis/authentication/rest/client/TestAuthenticationClient.java @@ -28,7 +28,7 @@ class TestAuthenticationClient { static { try { - portForWireMock = NetworkUtil.getAvailableLocalPort(); + portForWireMock = new NetworkUtil().getAvailableLocalPort(); } catch (IOException e) { e.printStackTrace(); } diff --git a/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/embedded/EmbeddedLocalhostMongo.java b/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/embedded/EmbeddedLocalhostMongo.java index 3667fb795..5249b59a3 100644 --- a/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/embedded/EmbeddedLocalhostMongo.java +++ b/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/embedded/EmbeddedLocalhostMongo.java @@ -39,7 +39,7 @@ public EmbeddedLocalhostMongo() { public void start() { if (mongodExecutable == null) { try { - mongoPort = NetworkUtil.getAvailableLocalPort(); + mongoPort = new NetworkUtil().getAvailableLocalPort(); RuntimeConfig runtimeConfig = Defaults.runtimeConfigFor(Command.MongoD, LOGGER) .processOutput(ProcessOutput.getDefaultInstanceSilent()) .build(); diff --git a/metis-common/metis-common-network/pom.xml b/metis-common/metis-common-network/pom.xml index c13c1ed7e..362a47011 100644 --- a/metis-common/metis-common-network/pom.xml +++ b/metis-common/metis-common-network/pom.xml @@ -52,11 +52,6 @@ org.mockito mockito-core - - org.mockito - mockito-inline - test - org.glassfish.jersey.core jersey-common diff --git a/metis-common/metis-common-network/src/main/java/eu/europeana/metis/network/NetworkUtil.java b/metis-common/metis-common-network/src/main/java/eu/europeana/metis/network/NetworkUtil.java index efce93017..ee3ac060d 100644 --- a/metis-common/metis-common-network/src/main/java/eu/europeana/metis/network/NetworkUtil.java +++ b/metis-common/metis-common-network/src/main/java/eu/europeana/metis/network/NetworkUtil.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.net.InetAddress; import java.net.ServerSocket; +import javax.net.ServerSocketFactory; import javax.net.ssl.SSLServerSocketFactory; /** @@ -11,26 +12,30 @@ * @author Simon Tzanakis (Simon.Tzanakis@europeana.eu) * @since 2017-02-24 */ -public final class NetworkUtil { +public class NetworkUtil { private static final int BACKLOG = 100; - private NetworkUtil() { - } - /** - * This method can be used in JUnit tests to get a random available port on localhost to run a - * service. It should not be used for normal operation, otherwise ssl checks should be followed to - * avoid man-in-the-middle attacks. + * This method can be used in JUnit tests to get a random available port on localhost to run a service. It should not be used + * for normal operation, otherwise ssl checks should be followed to avoid man-in-the-middle attacks. * * @return the available port number * @throws IOException if the specified localhost is not available */ - public static int getAvailableLocalPort() throws IOException { - ServerSocket s = SSLServerSocketFactory.getDefault() - .createServerSocket(0, BACKLOG, InetAddress.getByName("localhost")); + public int getAvailableLocalPort() throws IOException { + ServerSocket s = getServerSocketFactory().createServerSocket(0, BACKLOG, InetAddress.getByName("localhost")); int localPort = s.getLocalPort(); s.close(); return localPort; } + + /** + * Get a server socket factory. + * + * @return the server socket factory + */ + ServerSocketFactory getServerSocketFactory() { + return SSLServerSocketFactory.getDefault(); + } } diff --git a/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/ExternalRequestUtilTest.java b/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/ExternalRequestUtilTest.java index 8a06425b5..eb254c8f0 100644 --- a/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/ExternalRequestUtilTest.java +++ b/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/ExternalRequestUtilTest.java @@ -64,25 +64,19 @@ void testRetryableExternalRequestWithMap() { @Test void testRetryableExternalRequestThrowsExceptionOutOfSpecifiedMap() { - assertThrows(RuntimeException.class, () -> { - ExternalRequestUtil.retryableExternalRequest( - () -> { - throw new RuntimeException(new ClassNotFoundException("Class pointer test exception")); - }, - UNMODIFIABLE_MAP_WITH_TEST_EXCEPTIONS); - }); + assertThrows(RuntimeException.class, () -> ExternalRequestUtil.retryableExternalRequest( + () -> { + throw new RuntimeException(new ClassNotFoundException("Class pointer test exception")); + }, UNMODIFIABLE_MAP_WITH_TEST_EXCEPTIONS)); } @Disabled("TODO: MET-4255 Improve execution time") @Test void testRetryableExternalRequestThrowsException() { - assertThrows(RuntimeException.class, () -> { - ExternalRequestUtil.retryableExternalRequest( - () -> { - throw new RuntimeException(new ClassNotFoundException("Class pointer test exception")); - }, - null); - }); + assertThrows(RuntimeException.class, () -> ExternalRequestUtil.retryableExternalRequest( + () -> { + throw new RuntimeException(new ClassNotFoundException("Class pointer test exception")); + }, null)); } @Test diff --git a/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/NetworkUtilTest.java b/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/NetworkUtilTest.java index 3d47f33a7..ae091ef99 100644 --- a/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/NetworkUtilTest.java +++ b/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/NetworkUtilTest.java @@ -3,16 +3,13 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import java.io.IOException; import java.net.InetAddress; import javax.net.ServerSocketFactory; -import javax.net.ssl.SSLServerSocketFactory; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import org.mockito.MockedStatic; /** * Unit test for {@link NetworkUtil} @@ -24,21 +21,17 @@ class NetworkUtilTest { @Test void getAvailableLocalPort() throws IOException { - int availableLocalPort = NetworkUtil.getAvailableLocalPort(); - + int availableLocalPort = new NetworkUtil().getAvailableLocalPort(); assertTrue(availableLocalPort > 0); } - @Disabled("TODO: MET-4250 Handle MockMaker in Jenkins") @Test void getAvailableLocalPortWithException() throws IOException { final int BACKLOG = 100; - try (MockedStatic sslServerSocketFactory = mockStatic(SSLServerSocketFactory.class)) { - ServerSocketFactory serverSocketFactory = mock(ServerSocketFactory.class); - sslServerSocketFactory.when(SSLServerSocketFactory::getDefault).thenReturn(serverSocketFactory); - when(serverSocketFactory.createServerSocket(0, BACKLOG, InetAddress.getByName("localhost"))).thenThrow(IOException.class); - - assertThrows(IOException.class, () -> NetworkUtil.getAvailableLocalPort()); - } + final ServerSocketFactory sslServerSocketFactory = mock(ServerSocketFactory.class); + when(sslServerSocketFactory.createServerSocket(0, BACKLOG, InetAddress.getByName("localhost"))).thenThrow(IOException.class); + final NetworkUtil networkUtil = spy(NetworkUtil.class); + when(networkUtil.getServerSocketFactory()).thenReturn(sslServerSocketFactory); + assertThrows(IOException.class, networkUtil::getAvailableLocalPort); } } \ No newline at end of file diff --git a/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/StringHttpClientTest.java b/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/StringHttpClientTest.java index c58837c58..4181574f7 100644 --- a/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/StringHttpClientTest.java +++ b/metis-common/metis-common-network/src/test/java/eu/europeana/metis/network/StringHttpClientTest.java @@ -10,7 +10,6 @@ import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -19,7 +18,6 @@ import org.apache.hc.core5.http.HttpEntity; import org.apache.hc.core5.http.io.entity.BasicHttpEntity; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; /** @@ -56,8 +54,7 @@ void getResourceUrlWithException() { void createResult() throws URISyntaxException, IOException { List closeables = new ArrayList<>(); HttpEntity responseEntity = new BasicHttpEntity(new ByteArrayInputStream("content".getBytes()), ContentType.TEXT_PLAIN); - final ContentRetriever contentRetriever = ContentRetriever.forNonCloseableContent( - responseEntity == null ? InputStream::nullInputStream : responseEntity::getContent, + final ContentRetriever contentRetriever = ContentRetriever.forNonCloseableContent(responseEntity::getContent, closeables::add); StringContent actualContent = stringHttpClient.createResult(new URI("/resource/provided"), new URI("/resource/actual"), @@ -65,18 +62,17 @@ void createResult() throws URISyntaxException, IOException { assertEquals("content", actualContent.getContent()); assertEquals("text/plain", actualContent.getContentType()); + assertEquals(1, closeables.size()); } - @Disabled("TODO: MET-4250 Handle MockMaker in Jenkins") @Test void createResultWithException() throws IOException { final ContentRetriever contentRetriever = mock(ContentRetriever.class); when(contentRetriever.getContent()).thenThrow(IOException.class); - assertThrows(IOException.class, () -> { - stringHttpClient.createResult(new URI("/resource/provided"), new URI("/resource/actual"), - "text/plain", 7L, contentRetriever); - }); + assertThrows(IOException.class, + () -> stringHttpClient.createResult(new URI("/resource/provided"), new URI("/resource/actual"), + "text/plain", 7L, contentRetriever)); } @Test diff --git a/metis-common/metis-common-zoho/src/test/java/eu/europeana/metis/zoho/ZohoUtilsTest.java b/metis-common/metis-common-zoho/src/test/java/eu/europeana/metis/zoho/ZohoUtilsTest.java index 54ae0b2df..dc638df13 100644 --- a/metis-common/metis-common-zoho/src/test/java/eu/europeana/metis/zoho/ZohoUtilsTest.java +++ b/metis-common/metis-common-zoho/src/test/java/eu/europeana/metis/zoho/ZohoUtilsTest.java @@ -33,9 +33,7 @@ void stringListSupplier() { final Record recordOrganization = new Record(); final List expectedChoiceList = List.of("Organization1Role", "Organization2Role"); recordOrganization.addKeyValue(ZohoConstants.ORGANIZATION_ROLE_FIELD, - expectedChoiceList.stream() - .map(choice -> new Choice<>(choice)) - .collect(Collectors.toList())); + expectedChoiceList.stream().map(Choice::new).collect(Collectors.toList())); final List organizationRoleStringList = ZohoUtils.stringListSupplier( recordOrganization.getKeyValue(ZohoConstants.ORGANIZATION_ROLE_FIELD)); diff --git a/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestDatasetService.java b/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestDatasetService.java index 678e3a3ff..74f6754e4 100644 --- a/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestDatasetService.java +++ b/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestDatasetService.java @@ -22,7 +22,6 @@ import static org.mockito.Mockito.when; import com.github.tomakehurst.wiremock.WireMockServer; -import eu.europeana.metis.utils.RestEndpoints; import eu.europeana.metis.authentication.user.MetisUserView; import eu.europeana.metis.core.dao.DatasetDao; import eu.europeana.metis.core.dao.DatasetXsltDao; @@ -41,6 +40,7 @@ import eu.europeana.metis.exception.GenericMetisException; import eu.europeana.metis.exception.UserUnauthorizedException; import eu.europeana.metis.network.NetworkUtil; +import eu.europeana.metis.utils.RestEndpoints; import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; @@ -66,7 +66,7 @@ class TestDatasetService { static { try { - portForWireMock = NetworkUtil.getAvailableLocalPort(); + portForWireMock = new NetworkUtil().getAvailableLocalPort(); } catch (IOException e) { e.printStackTrace(); } diff --git a/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestProxiesService.java b/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestProxiesService.java index 0a004009b..c42e800a3 100644 --- a/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestProxiesService.java +++ b/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestProxiesService.java @@ -564,7 +564,7 @@ void testGetRecord() throws MCSException, ExternalTaskException { // Create representation final Representation representation = mock(Representation.class); - final String contentUri = "http://example.com"; + final String contentUri = "https://example.com"; final File file = new File(); file.setContentUri(URI.create(contentUri)); when(representation.getFiles()).thenReturn(Collections.singletonList(file)); diff --git a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/EnrichmentWorkerImpl.java b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/EnrichmentWorkerImpl.java index 7f944fd97..316eabc5f 100644 --- a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/EnrichmentWorkerImpl.java +++ b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/EnrichmentWorkerImpl.java @@ -20,6 +20,7 @@ public class EnrichmentWorkerImpl implements EnrichmentWorker { private static final Logger LOGGER = LoggerFactory.getLogger(EnrichmentWorkerImpl.class); + private static final RdfConversionUtils rdfConversionUtils = new RdfConversionUtils(); private final Enricher enricher; private final Dereferencer dereferencer; @@ -169,18 +170,18 @@ private String convertRdfToStringForLogging(final RDF rdf) { String convertRdfToString(RDF rdf) throws SerializationException { - return RdfConversionUtils.convertRdfToString(rdf); + return rdfConversionUtils.convertRdfToString(rdf); } byte[] convertRdfToBytes(RDF rdf) throws SerializationException { - return RdfConversionUtils.convertRdfToBytes(rdf); + return rdfConversionUtils.convertRdfToBytes(rdf); } RDF convertStringToRdf(String xml) throws SerializationException { - return RdfConversionUtils.convertStringToRdf(xml); + return rdfConversionUtils.convertStringToRdf(xml); } RDF convertInputStreamToRdf(InputStream xml) throws SerializationException { - return RdfConversionUtils.convertInputStreamToRdf(xml); + return rdfConversionUtils.convertInputStreamToRdf(xml); } } diff --git a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/MetisRecordParserTest.java b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/MetisRecordParserTest.java index d83e6874c..6cf493987 100644 --- a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/MetisRecordParserTest.java +++ b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/MetisRecordParserTest.java @@ -24,6 +24,7 @@ import eu.europeana.metis.schema.jibx.Subject; import eu.europeana.metis.schema.jibx.Temporal; import eu.europeana.metis.schema.jibx.Type; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Set; import org.apache.commons.io.IOUtils; @@ -31,6 +32,8 @@ public class MetisRecordParserTest { + private static final RdfConversionUtils rdfConversionUtils = new RdfConversionUtils(); + @Test public void testExtractedFieldValuesForEnrichment() { RDF rdf = new RDF(); @@ -177,8 +180,8 @@ public void testExtractedFieldValuesForEnrichment() { @Test public void testSetAdditionalData() throws Exception { String xml = IOUtils - .toString(getClass().getClassLoader().getResourceAsStream("sample_completeness.rdf"), "UTF-8"); - RDF rdf = RdfConversionUtils.convertStringToRdf(xml); + .toString(getClass().getClassLoader().getResourceAsStream("sample_completeness.rdf"), StandardCharsets.UTF_8); + RDF rdf = rdfConversionUtils.convertStringToRdf(xml); EnrichmentUtils.setAdditionalData(rdf); EuropeanaAggregationType europeanaAggregationType = rdf.getEuropeanaAggregationList().stream() .findAny().orElse(null); diff --git a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/utils/EntityMergeEngineTest.java b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/utils/EntityMergeEngineTest.java index 1760a1a6a..3dc87169d 100644 --- a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/utils/EntityMergeEngineTest.java +++ b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/utils/EntityMergeEngineTest.java @@ -44,6 +44,8 @@ public class EntityMergeEngineTest { + private static final RdfConversionUtils rdfConversionUtils = new RdfConversionUtils(); + private static Place createPlace() { Place place = new Place(); @@ -605,7 +607,7 @@ public void testMergePlace() throws SerializationException { verifyPlace((Place) inputList.get(2), rdf.getPlaceList().get(2)); // Convert RDF to string as extra test that everything is OK. - RdfConversionUtils.convertRdfToString(rdf); + rdfConversionUtils.convertRdfToString(rdf); } @Test @@ -645,7 +647,7 @@ public void testMergeOtherTypes() throws SerializationException { verifyOrganization((Organization) inputList.get(5), rdf.getOrganizationList().get(0)); // Convert RDF to string as extra test that everything is OK. - RdfConversionUtils.convertRdfToString(rdf); + rdfConversionUtils.convertRdfToString(rdf); } @Test diff --git a/metis-harvesting/src/test/java/eu/europeana/metis/harvesting/oaipmh/CloseableHttpOaiClientTest.java b/metis-harvesting/src/test/java/eu/europeana/metis/harvesting/oaipmh/CloseableHttpOaiClientTest.java index e6cc24087..458dcc7e6 100644 --- a/metis-harvesting/src/test/java/eu/europeana/metis/harvesting/oaipmh/CloseableHttpOaiClientTest.java +++ b/metis-harvesting/src/test/java/eu/europeana/metis/harvesting/oaipmh/CloseableHttpOaiClientTest.java @@ -29,7 +29,7 @@ public class CloseableHttpOaiClientTest { @BeforeAll static void prepare() throws IOException { - int portForWireMock = NetworkUtil.getAvailableLocalPort(); + int portForWireMock = new NetworkUtil().getAvailableLocalPort(); final String localhostUrl = "http://127.0.0.1:" + portForWireMock; URL = localhostUrl + PATH; CONNECTION_CLIENT_FACTORY = () -> TestHelper.CONNECTION_CLIENT_FACTORY.apply(ENDPOINT); diff --git a/metis-harvesting/src/test/java/eu/europeana/metis/harvesting/oaipmh/OaiHarvesterImplTest.java b/metis-harvesting/src/test/java/eu/europeana/metis/harvesting/oaipmh/OaiHarvesterImplTest.java index bea665715..2fee29bf5 100644 --- a/metis-harvesting/src/test/java/eu/europeana/metis/harvesting/oaipmh/OaiHarvesterImplTest.java +++ b/metis-harvesting/src/test/java/eu/europeana/metis/harvesting/oaipmh/OaiHarvesterImplTest.java @@ -29,7 +29,7 @@ class OaiHarvesterImplTest { @BeforeAll static void prepare() throws IOException { - int portForWireMock = NetworkUtil.getAvailableLocalPort(); + int portForWireMock = new NetworkUtil().getAvailableLocalPort(); final String localhostUrl = "http://127.0.0.1:" + portForWireMock; OAI_PMH_ENDPOINT = localhostUrl + "/oai-phm/"; CONNECTION_CLIENT_FACTORY = TestHelper.CONNECTION_CLIENT_FACTORY::apply; diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/IndexerImpl.java b/metis-indexing/src/main/java/eu/europeana/indexing/IndexerImpl.java index 9e48a6638..4c2d29b11 100644 --- a/metis-indexing/src/main/java/eu/europeana/indexing/IndexerImpl.java +++ b/metis-indexing/src/main/java/eu/europeana/indexing/IndexerImpl.java @@ -5,6 +5,11 @@ import eu.europeana.indexing.exception.SetupRelatedIndexingException; import eu.europeana.indexing.fullbean.StringToFullBeanConverter; import eu.europeana.indexing.tiers.ClassifierFactory; +import eu.europeana.indexing.tiers.model.MediaTier; +import eu.europeana.indexing.tiers.model.MetadataTier; +import eu.europeana.indexing.tiers.model.TierClassifier; +import eu.europeana.indexing.tiers.view.ContentTierBreakdown; +import eu.europeana.indexing.tiers.view.MetadataTierBreakdown; import eu.europeana.indexing.utils.RdfTierUtils; import eu.europeana.indexing.utils.RdfWrapper; import eu.europeana.metis.schema.jibx.RDF; @@ -31,6 +36,8 @@ class IndexerImpl implements Indexer { private final AbstractConnectionProvider connectionProvider; private final IndexingSupplier stringToRdfConverterSupplier; + private final TierClassifier mediaClassifier = ClassifierFactory.getMediaClassifier(); + private final TierClassifier metadataClassifier = ClassifierFactory.getMetadataClassifier(); /** * Constructor. @@ -45,8 +52,8 @@ class IndexerImpl implements Indexer { * Constructor for testing purposes. * * @param connectionProvider The connection provider for this indexer. - * @param stringToRdfConverterSupplier Supplies an instance of {@link StringToFullBeanConverter} - * used to parse strings to instances of {@link RDF}. Will be called once during every index. + * @param stringToRdfConverterSupplier Supplies an instance of {@link StringToFullBeanConverter} used to convert a string to an + * instance of {@link RDF}. Will be called once during every index. */ IndexerImpl(AbstractConnectionProvider connectionProvider, IndexingSupplier stringToRdfConverterSupplier) { @@ -54,8 +61,38 @@ class IndexerImpl implements Indexer { this.stringToRdfConverterSupplier = stringToRdfConverterSupplier; } + @Override + public void indexRdfs(List records, IndexingProperties indexingProperties) + throws IndexingException { + indexRecords(records, indexingProperties); + } + + @Override + public void index(List records, IndexingProperties indexingProperties) + throws IndexingException { + LOGGER.info("Parsing {} records...", records.size()); + final StringToFullBeanConverter stringToRdfConverter = stringToRdfConverterSupplier.get(); + final List wrappedRecords = new ArrayList<>(records.size()); + for (String record : records) { + wrappedRecords.add(stringToRdfConverter.convertStringToRdf(record)); + } + indexRecords(wrappedRecords, indexingProperties); + } + + @Override + public void index(InputStream record, IndexingProperties indexingProperties) + throws IndexingException { + final StringToFullBeanConverter stringToRdfConverter = stringToRdfConverterSupplier.get(); + indexRdf(stringToRdfConverter.convertToRdf(record), indexingProperties); + } + + @Override + public void indexRdf(RDF record, IndexingProperties indexingProperties) throws IndexingException { + indexRdfs(List.of(record), indexingProperties); + } + private void indexRecords(List records, IndexingProperties properties) - throws IndexingException { + throws IndexingException { if (properties.isPerformRedirects() && connectionProvider.getRecordRedirectDao() == null) { throw new SetupRelatedIndexingException( "Record redirect dao has not been initialized and performing redirects is requested"); @@ -68,60 +105,30 @@ private void indexRecords(List records, IndexingProperties properties) preprocessRecord(record, properties.isPerformTierCalculation()); if (properties.isPerformRedirects()) { publisher.publishWithRedirects(new RdfWrapper(record), properties.getRecordDate(), - properties.getDatasetIdsForRedirection()); + properties.getDatasetIdsForRedirection()); } else { publisher.publish(new RdfWrapper(record), properties.getRecordDate(), - properties.getDatasetIdsForRedirection()); + properties.getDatasetIdsForRedirection()); } } LOGGER.info("Successfully processed {} records.", records.size()); } - private static void preprocessRecord(RDF rdf, boolean performTierCalculation) - throws IndexingException { - - // Perform the tier classification - if (performTierCalculation) { - final RdfWrapper rdfWrapper = new RdfWrapper(rdf); - RdfTierUtils.setTier(rdf, ClassifierFactory.getMediaClassifier().classify(rdfWrapper).getTier()); - RdfTierUtils.setTier(rdf, ClassifierFactory.getMetadataClassifier().classify(rdfWrapper).getTier()); - } - } - - @Override - public void indexRdfs(List records, IndexingProperties indexingProperties) - throws IndexingException { - indexRecords(records, indexingProperties); - } - - @Override - public void indexRdf(RDF record, IndexingProperties indexingProperties) throws IndexingException { - indexRdfs(List.of(record), indexingProperties); - } - - @Override - public void index(List records, IndexingProperties indexingProperties) - throws IndexingException { - LOGGER.info("Parsing {} records...", records.size()); - final StringToFullBeanConverter stringToRdfConverter = stringToRdfConverterSupplier.get(); - final List wrappedRecords = new ArrayList<>(records.size()); - for (String record : records) { - wrappedRecords.add(stringToRdfConverter.convertStringToRdf(record)); - } - indexRecords(wrappedRecords, indexingProperties); - } - @Override public void index(String record, IndexingProperties indexingProperties) throws IndexingException { index(List.of(record), indexingProperties); } - @Override - public void index(InputStream record, IndexingProperties indexingProperties) - throws IndexingException { - final StringToFullBeanConverter stringToRdfConverter = stringToRdfConverterSupplier.get(); - indexRdf(stringToRdfConverter.convertToRdf(record), indexingProperties); + private void preprocessRecord(RDF rdf, boolean performTierCalculation) + throws IndexingException { + + // Perform the tier classification + if (performTierCalculation) { + final RdfWrapper rdfWrapper = new RdfWrapper(rdf); + RdfTierUtils.setTier(rdf, mediaClassifier.classify(rdfWrapper).getTier()); + RdfTierUtils.setTier(rdf, metadataClassifier.classify(rdfWrapper).getTier()); + } } @Override @@ -168,8 +175,7 @@ public long countRecords(String datasetId) { } /** - * Similar to the Java interface {@link Supplier}, but one that may throw an {@link - * IndexerRelatedIndexingException}. + * Similar to the Java interface {@link Supplier}, but one that may throw an {@link IndexerRelatedIndexingException}. * * @param The type of the object to be supplied. * @author jochen @@ -181,8 +187,7 @@ interface IndexingSupplier { * Gets a result. * * @return A result. - * @throws IndexerRelatedIndexingException In case something went wrong while getting the - * result. + * @throws IndexerRelatedIndexingException In case something went wrong while getting the result. */ T get() throws IndexerRelatedIndexingException; } diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/tiers/RecordTierCalculationViewGenerator.java b/metis-indexing/src/main/java/eu/europeana/indexing/tiers/RecordTierCalculationViewGenerator.java index 010afc645..c50b3ee9c 100644 --- a/metis-indexing/src/main/java/eu/europeana/indexing/tiers/RecordTierCalculationViewGenerator.java +++ b/metis-indexing/src/main/java/eu/europeana/indexing/tiers/RecordTierCalculationViewGenerator.java @@ -3,6 +3,7 @@ import eu.europeana.indexing.exception.TierCalculationException; import eu.europeana.indexing.tiers.model.MediaTier; import eu.europeana.indexing.tiers.model.MetadataTier; +import eu.europeana.indexing.tiers.model.TierClassifier; import eu.europeana.indexing.tiers.model.TierClassifier.TierClassification; import eu.europeana.indexing.tiers.view.ContentTierBreakdown; import eu.europeana.indexing.tiers.view.MetadataTierBreakdown; @@ -22,6 +23,10 @@ */ public class RecordTierCalculationViewGenerator { + private static final RdfConversionUtils rdfConversionUtils = new RdfConversionUtils(); + private static final TierClassifier mediaClassifier = ClassifierFactory.getMediaClassifier(); + private static final TierClassifier metadataClassifier = ClassifierFactory.getMetadataClassifier(); + private final String europeanaId; private final String providerId; private final String stringRdf; @@ -59,13 +64,11 @@ private RecordTierCalculationView tierClassification(final String xml) { final RDF rdf; try { // Perform the tier classification - rdf = RdfConversionUtils.convertStringToRdf(xml); + rdf = rdfConversionUtils.convertStringToRdf(xml); final RdfWrapper rdfWrapper = new RdfWrapper(rdf); - final TierClassification mediaTierClassification = ClassifierFactory.getMediaClassifier() - .classify(rdfWrapper); - final TierClassification metadataTierClassification = ClassifierFactory.getMetadataClassifier() - .classify( - rdfWrapper); + final TierClassification mediaTierClassification = mediaClassifier.classify(rdfWrapper); + final TierClassification metadataTierClassification = metadataClassifier.classify( + rdfWrapper); RecordTierCalculationSummary recordTierCalculationSummary = new RecordTierCalculationSummary(); recordTierCalculationSummary.setEuropeanaRecordId(europeanaId); recordTierCalculationSummary.setProviderRecordId(providerId); diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/tiers/metadata/ContextualClassesClassifier.java b/metis-indexing/src/main/java/eu/europeana/indexing/tiers/metadata/ContextualClassesClassifier.java index 4363dcc8c..dcb53a3cd 100644 --- a/metis-indexing/src/main/java/eu/europeana/indexing/tiers/metadata/ContextualClassesClassifier.java +++ b/metis-indexing/src/main/java/eu/europeana/indexing/tiers/metadata/ContextualClassesClassifier.java @@ -33,6 +33,8 @@ */ public class ContextualClassesClassifier implements TierClassifierBreakdown { + private static final RdfConversionUtils rdfConversionUtils = new RdfConversionUtils(); + private static Set getResourceLinks(ProxyType proxy) { return Stream.of(ResourceLinkFromProxy.values()) .map(ResourceLinkFromProxy::getLinkAndValueGetter) @@ -64,7 +66,7 @@ public ContextualClassesBreakdown classifyBreakdown(RdfWrapper entity) { final Set uniqueContextualClasses = contextualClassesStatistics.getDistinctClassesSet().stream() .map(ContextualClassGroup::getContextualClass) .map( - RdfConversionUtils::getQualifiedElementNameForClass) + rdfConversionUtils::getQualifiedElementNameForClass) .collect(Collectors.toSet()); return new ContextualClassesBreakdown(contextualClassesStatistics.getCompleteContextualResources(), uniqueContextualClasses, diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/tiers/metadata/EnablingElementsClassifier.java b/metis-indexing/src/main/java/eu/europeana/indexing/tiers/metadata/EnablingElementsClassifier.java index ac218d229..162d6577c 100644 --- a/metis-indexing/src/main/java/eu/europeana/indexing/tiers/metadata/EnablingElementsClassifier.java +++ b/metis-indexing/src/main/java/eu/europeana/indexing/tiers/metadata/EnablingElementsClassifier.java @@ -21,6 +21,8 @@ */ public class EnablingElementsClassifier implements TierClassifierBreakdown { + private static final RdfConversionUtils rdfConversionUtils = new RdfConversionUtils(); + private static final int MIN_ELEMENTS_TIER_A = 1; private static final int MIN_ELEMENTS_TIER_B = 3; private static final int MIN_ELEMENTS_TIER_C = 4; @@ -36,10 +38,10 @@ public EnablingElementsBreakdown classifyBreakdown(RdfWrapper entity) { final MetadataTier metadataTier = calculateMetadataTier(inventory); final Set distinctEnablingElementsList = inventory.getElements().stream().map(EnablingElement::getTypedClass) - .map(RdfConversionUtils::getQualifiedElementNameForClass) + .map(rdfConversionUtils::getQualifiedElementNameForClass) .collect(Collectors.toSet()); final Set metadataGroupsList = inventory.getGroups().stream().map(ContextualClassGroup::getContextualClass) - .map(RdfConversionUtils::getQualifiedElementNameForClass) + .map(rdfConversionUtils::getQualifiedElementNameForClass) .collect(Collectors.toSet()); return new EnablingElementsBreakdown(distinctEnablingElementsList, metadataGroupsList, metadataTier); diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/tiers/metadata/LanguageClassifier.java b/metis-indexing/src/main/java/eu/europeana/indexing/tiers/metadata/LanguageClassifier.java index 18a883bd9..3eb4d821a 100644 --- a/metis-indexing/src/main/java/eu/europeana/indexing/tiers/metadata/LanguageClassifier.java +++ b/metis-indexing/src/main/java/eu/europeana/indexing/tiers/metadata/LanguageClassifier.java @@ -19,6 +19,8 @@ */ public class LanguageClassifier implements TierClassifierBreakdown { + private static final RdfConversionUtils rdfConversionUtils = new RdfConversionUtils(); + private static final float MIN_RATE_FOR_T1 = 0.25F; private static final float MIN_RATE_FOR_T2 = 0.5F; private static final float MIN_RATE_FOR_T3 = 0.75F; @@ -42,7 +44,7 @@ public LanguageBreakdown classifyBreakdown(RdfWrapper entity) { return new LanguageBreakdown(qualifiedProperties.size(), qualifiedPropertiesWithoutLanguage.stream().map(PropertyType::getTypedClass) - .map(RdfConversionUtils::getQualifiedElementNameForClass).collect(Collectors.toSet()), + .map(rdfConversionUtils::getQualifiedElementNameForClass).collect(Collectors.toSet()), metadataTier); } diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/media/AbstractMediaClassifierTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/media/AbstractMediaClassifierTest.java index 49d2e3953..6e9fdfbd4 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/media/AbstractMediaClassifierTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/media/AbstractMediaClassifierTest.java @@ -89,7 +89,7 @@ void testClassify_WithoutWebResources() { doReturn(Collections.emptyList()).when(entity).getWebResourceWrappers( EnumSet.of(WebResourceLinkType.HAS_VIEW, WebResourceLinkType.IS_SHOWN_BY)); //Has embeddable media will be true - doReturn(Set.of("http://soundcloud.com/")).when(entity).getUrlsOfTypes(Set.of(WebResourceLinkType.IS_SHOWN_BY)); + doReturn(Set.of("https://soundcloud.com/")).when(entity).getUrlsOfTypes(Set.of(WebResourceLinkType.IS_SHOWN_BY)); doReturn(MediaTier.T0).when(classifier).classifyEntityWithoutWebResources(entity, hasLandingPage); assertEquals(MediaTier.T0, classifier.classify(entity).getTier()); } @@ -126,7 +126,7 @@ void testClassify_WithWebResources() { doReturn(entityLicense).when(entity).getLicenseType(); doReturn(null).when(classifier).preClassifyEntity(entity); //Has embeddable media will be true - doReturn(Set.of("http://soundcloud.com/")).when(entity).getUrlsOfTypes(Set.of(WebResourceLinkType.IS_SHOWN_BY)); + doReturn(Set.of("https://soundcloud.com/")).when(entity).getUrlsOfTypes(Set.of(WebResourceLinkType.IS_SHOWN_BY)); // Create web resources. doReturn(mediaResourceTechnicalMetadata1).when(classifier) diff --git a/metis-media-service/src/test/java/eu/europeana/metis/mediaprocessing/extraction/AudioVideoProcessorTest.java b/metis-media-service/src/test/java/eu/europeana/metis/mediaprocessing/extraction/AudioVideoProcessorTest.java index 0c79c7e86..567daee30 100644 --- a/metis-media-service/src/test/java/eu/europeana/metis/mediaprocessing/extraction/AudioVideoProcessorTest.java +++ b/metis-media-service/src/test/java/eu/europeana/metis/mediaprocessing/extraction/AudioVideoProcessorTest.java @@ -33,8 +33,8 @@ import eu.europeana.metis.mediaprocessing.model.Resource; import eu.europeana.metis.mediaprocessing.model.ResourceExtractionResultImpl; import eu.europeana.metis.mediaprocessing.model.VideoResourceMetadata; -import eu.europeana.metis.schema.model.MediaType; import eu.europeana.metis.network.NetworkUtil; +import eu.europeana.metis.schema.model.MediaType; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -62,7 +62,7 @@ class AudioVideoProcessorTest { static { try { - portForWireMock = NetworkUtil.getAvailableLocalPort(); + portForWireMock = new NetworkUtil().getAvailableLocalPort(); } catch (IOException e) { e.printStackTrace(); } @@ -130,7 +130,7 @@ void testCreateAudioVideoAnalysisCommand() throws IOException, MediaExtractionEx // Create resource final Resource resource = mock(Resource.class); - doReturn("http://valid.url.nl/test").when(resource).getResourceUrl(); + doReturn("https://valid.url.nl/test").when(resource).getResourceUrl(); doReturn(Paths.get("content path")).when(resource).getContentPath(); // test resource with content @@ -158,10 +158,10 @@ void testCreateAudioVideoAnalysisCommand() throws IOException, MediaExtractionEx doReturn("valid.without.prefix.nl/").when(resource).getResourceUrl(); assertThrows(MediaExtractionException.class, () -> audioVideoProcessor.createAudioVideoAnalysisCommand(resource)); - doReturn("http://invalid.characters.nl/!@#$%^&*()_").when(resource).getResourceUrl(); + doReturn("https://invalid.characters.nl/!@#$%^&*()_").when(resource).getResourceUrl(); assertThrows(MediaExtractionException.class, () -> audioVideoProcessor.createAudioVideoAnalysisCommand(resource)); - doReturn("http://valid.url.nl/test").when(resource).getResourceUrl(); + doReturn("https://valid.url.nl/test").when(resource).getResourceUrl(); // test if hasContent fails. doThrow(new IOException()).when(resource).hasContent(); diff --git a/metis-schema/pom.xml b/metis-schema/pom.xml index f3620553b..66d77bdb4 100644 --- a/metis-schema/pom.xml +++ b/metis-schema/pom.xml @@ -86,10 +86,6 @@ org.junit.jupiter junit-jupiter-engine - - mockito-inline - org.mockito - @@ -169,12 +165,10 @@ src/main/java - true true false - diff --git a/metis-schema/src/main/java/eu/europeana/metis/schema/convert/RdfConversionUtils.java b/metis-schema/src/main/java/eu/europeana/metis/schema/convert/RdfConversionUtils.java index 78826b40c..e15e3788d 100644 --- a/metis-schema/src/main/java/eu/europeana/metis/schema/convert/RdfConversionUtils.java +++ b/metis-schema/src/main/java/eu/europeana/metis/schema/convert/RdfConversionUtils.java @@ -23,20 +23,36 @@ /** * Utility class for converting {@link RDF} to String and vice versa. */ -public final class RdfConversionUtils { +public class RdfConversionUtils { private static final int INDENTATION_SPACE = 2; private static final String UTF8 = StandardCharsets.UTF_8.name(); - private static IBindingFactory rdfBindingFactory; - private static Map rdfXmlElementMetadataMap; @SuppressWarnings("java:S5852") //This regex is safe, and it's only meant for internal use without use input private static final Pattern complexTypePattern = Pattern.compile("^\\{(.*)}:(.*)$"); + private final IBindingFactory rdfBindingFactory; + private final Map rdfXmlElementMetadataMap; - static { - initializeStaticComponents(); + /** + * Default constructor + */ + public RdfConversionUtils() { + this(RDF.class); } - private RdfConversionUtils() { + /** + * Constructor supplying class type for the binding factory. + *

    At the current state this is used for assisting testing

    + * + * @param classType the class object type + * @param the class type + */ + RdfConversionUtils(Class classType) { + try { + rdfBindingFactory = BindingDirectory.getFactory(classType); + rdfXmlElementMetadataMap = initializeRdfXmlElementMetadataMap(); + } catch (JiBXException e) { + throw new IllegalStateException("No binding factory available.", e); + } } /** @@ -46,7 +62,7 @@ private RdfConversionUtils() { * @return An XML string representation of the RDF object * @throws SerializationException if during marshalling there is a failure */ - public static byte[] convertRdfToBytes(RDF rdf) throws SerializationException { + public byte[] convertRdfToBytes(RDF rdf) throws SerializationException { try { IMarshallingContext context = rdfBindingFactory.createMarshallingContext(); context.setIndent(INDENTATION_SPACE); @@ -66,7 +82,7 @@ public static byte[] convertRdfToBytes(RDF rdf) throws SerializationException { * @param objectClass the jibx object class to search for * @return the xml representation */ - public static String getQualifiedElementNameForClass(Class objectClass) { + public String getQualifiedElementNameForClass(Class objectClass) { final RdfXmlElementMetadata rdfXmlElementMetadata = rdfXmlElementMetadataMap.get(objectClass.getCanonicalName()); Objects.requireNonNull(rdfXmlElementMetadata, String.format("Element metadata not found for class: %s", objectClass.getCanonicalName())); @@ -80,7 +96,7 @@ public static String getQualifiedElementNameForClass(Class objectClass) { * @return the RDF object * @throws SerializationException if during unmarshalling there is a failure */ - public static RDF convertInputStreamToRdf(InputStream inputStream) throws SerializationException { + public RDF convertInputStreamToRdf(InputStream inputStream) throws SerializationException { try { final IUnmarshallingContext context = rdfBindingFactory.createUnmarshallingContext(); return (RDF) context.unmarshalDocument(inputStream, UTF8); @@ -91,33 +107,18 @@ public static RDF convertInputStreamToRdf(InputStream inputStream) throws Serial } /** - * Collect all information that we can get for jibx classes from the {@link IBindingFactory}. + * Convert an {@link RDF} to a UTF-8 encoded XML + * + * @param rdf The RDF object to convert + * @return An XML string representation of the RDF object + * @throws SerializationException if during marshalling there is a failure */ - private static Map initializeRdfXmlElementMetadataMap() { - Map rdfXmlElementMetadataMap = new HashMap<>(); - for (int i = 0; i < rdfBindingFactory.getMappedClasses().length; i++) { - final String canonicalName; - final String elementNamespace; - final String elementName; - final Matcher matcher = complexTypePattern.matcher(rdfBindingFactory.getMappedClasses()[i]); - if (matcher.matches()) { - //Complex type search - elementNamespace = matcher.group(1); - elementName = matcher.group(2); - final Pattern canonicalClassNamePattern = Pattern.compile(String.format("^(.*)\\.(%s)$", elementName)); - canonicalName = Arrays.stream(rdfBindingFactory.getAbstractMappings()).flatMap(Arrays::stream) - .filter(Objects::nonNull) - .filter(input -> canonicalClassNamePattern.matcher(input).matches()) - .findFirst().orElse(null); - } else { - //Simple type search - elementNamespace = rdfBindingFactory.getElementNamespaces()[i]; - elementName = rdfBindingFactory.getElementNames()[i]; - canonicalName = rdfBindingFactory.getMappedClasses()[i]; - } - checkAndStoreMetadataInMap(rdfXmlElementMetadataMap, canonicalName, elementNamespace, elementName); + public String convertRdfToString(RDF rdf) throws SerializationException { + try { + return new String(convertRdfToBytes(rdf), UTF8); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("Unexpected exception - should not occur.", e); } - return rdfXmlElementMetadataMap; } static class RdfXmlElementMetadata { @@ -152,21 +153,52 @@ public String getName() { } /** - * Convert an {@link RDF} to a UTF-8 encoded XML + * Convert a UTF-8 encoded XML to {@link RDF} * - * @param rdf The RDF object to convert - * @return An XML string representation of the RDF object - * @throws SerializationException if during marshalling there is a failure + * @param xml the xml string + * @return the RDF object + * @throws SerializationException if during unmarshalling there is a failure */ - public static String convertRdfToString(RDF rdf) throws SerializationException { - try { - return new String(convertRdfToBytes(rdf), UTF8); - } catch (UnsupportedEncodingException e) { - throw new IllegalStateException("Unexpected exception - should not occur.", e); + public RDF convertStringToRdf(String xml) throws SerializationException { + try (final InputStream inputStream = new ByteArrayInputStream( + xml.getBytes(StandardCharsets.UTF_8))) { + return convertInputStreamToRdf(inputStream); + } catch (IOException e) { + throw new SerializationException("Unexpected issue with byte stream.", e); + } + } + + /** + * Collect all information that we can get for jibx classes from the {@link IBindingFactory}. + */ + private Map initializeRdfXmlElementMetadataMap() { + Map elementMetadataMap = new HashMap<>(); + for (int i = 0; i < rdfBindingFactory.getMappedClasses().length; i++) { + final String canonicalName; + final String elementNamespace; + final String elementName; + final Matcher matcher = complexTypePattern.matcher(rdfBindingFactory.getMappedClasses()[i]); + if (matcher.matches()) { + //Complex type search + elementNamespace = matcher.group(1); + elementName = matcher.group(2); + final Pattern canonicalClassNamePattern = Pattern.compile(String.format("^(.*)\\.(%s)$", elementName)); + canonicalName = Arrays.stream(rdfBindingFactory.getAbstractMappings()).flatMap(Arrays::stream) + .filter(Objects::nonNull) + .filter(input -> canonicalClassNamePattern.matcher(input).matches()) + .findFirst().orElse(null); + } else { + //Simple type search + elementNamespace = rdfBindingFactory.getElementNamespaces()[i]; + elementName = rdfBindingFactory.getElementNames()[i]; + canonicalName = rdfBindingFactory.getMappedClasses()[i]; + } + checkAndStoreMetadataInMap(elementMetadataMap, canonicalName, elementNamespace, elementName); } + return elementMetadataMap; } - private static void checkAndStoreMetadataInMap(final Map rdfXmlElementMetadataMap, + private void checkAndStoreMetadataInMap(final Map rdfXmlElementMetadataMap, String canonicalName, String elementNamespace, String elementName) { //Store only if we could find the canonical name properly if (canonicalName != null) { @@ -179,29 +211,4 @@ private static void checkAndStoreMetadataInMap(final Map bindingDirectoryMockedStatic = Mockito.mockStatic(BindingDirectory.class)) { - bindingDirectoryMockedStatic.when(() -> BindingDirectory.getFactory(RDF.class)).thenThrow(JiBXException.class); - try { - RdfConversionUtils.initializeStaticComponents(); - } catch (Throwable throwable) { - //Check for two possibilities because based on the execution order of the tests this can throw a different exception - assertTrue(throwable instanceof ExceptionInInitializerError || throwable instanceof IllegalStateException); - assertTrue(throwable.getCause() instanceof IllegalStateException || throwable.getCause() instanceof JiBXException); - } - } + void failRdfConversionUtilsInitialization() { + //Force failure + assertThrows(IllegalStateException.class, () -> new RdfConversionUtils(RdfConversionUtils.class)); } @Test void getQualifiedElementNameForClass_ContextualClasses() { //Check contextual classes - assertEquals("edm:AgentType", RdfConversionUtils.getQualifiedElementNameForClass(AgentType.class)); - assertEquals("edm:TimeSpanType", RdfConversionUtils.getQualifiedElementNameForClass(TimeSpanType.class)); - assertEquals("edm:PlaceType", RdfConversionUtils.getQualifiedElementNameForClass(PlaceType.class)); - assertEquals("skos:Concept", RdfConversionUtils.getQualifiedElementNameForClass(Concept.class)); + final RdfConversionUtils rdfConversionUtils = new RdfConversionUtils(); + assertEquals("edm:AgentType", rdfConversionUtils.getQualifiedElementNameForClass(AgentType.class)); + assertEquals("edm:TimeSpanType", rdfConversionUtils.getQualifiedElementNameForClass(TimeSpanType.class)); + assertEquals("edm:PlaceType", rdfConversionUtils.getQualifiedElementNameForClass(PlaceType.class)); + assertEquals("skos:Concept", rdfConversionUtils.getQualifiedElementNameForClass(Concept.class)); } @Test void getQualifiedElementNameForClass_Dc() { //Check dc elements - assertEquals("dc:coverage", RdfConversionUtils.getQualifiedElementNameForClass(Coverage.class)); - assertEquals("dc:description", RdfConversionUtils.getQualifiedElementNameForClass(Description.class)); - assertEquals("dc:format", RdfConversionUtils.getQualifiedElementNameForClass(Format.class)); - assertEquals("dc:relation", RdfConversionUtils.getQualifiedElementNameForClass(Relation.class)); - assertEquals("dc:rights", RdfConversionUtils.getQualifiedElementNameForClass(Rights.class)); - assertEquals("dc:source", RdfConversionUtils.getQualifiedElementNameForClass(Source.class)); - assertEquals("dc:subject", RdfConversionUtils.getQualifiedElementNameForClass(Subject.class)); - assertEquals("dc:title", RdfConversionUtils.getQualifiedElementNameForClass(Title.class)); - assertEquals("dc:type", RdfConversionUtils.getQualifiedElementNameForClass(Type.class)); + final RdfConversionUtils rdfConversionUtils = new RdfConversionUtils(); + assertEquals("dc:coverage", rdfConversionUtils.getQualifiedElementNameForClass(Coverage.class)); + assertEquals("dc:description", rdfConversionUtils.getQualifiedElementNameForClass(Description.class)); + assertEquals("dc:format", rdfConversionUtils.getQualifiedElementNameForClass(Format.class)); + assertEquals("dc:relation", rdfConversionUtils.getQualifiedElementNameForClass(Relation.class)); + assertEquals("dc:rights", rdfConversionUtils.getQualifiedElementNameForClass(Rights.class)); + assertEquals("dc:source", rdfConversionUtils.getQualifiedElementNameForClass(Source.class)); + assertEquals("dc:subject", rdfConversionUtils.getQualifiedElementNameForClass(Subject.class)); + assertEquals("dc:title", rdfConversionUtils.getQualifiedElementNameForClass(Title.class)); + assertEquals("dc:type", rdfConversionUtils.getQualifiedElementNameForClass(Type.class)); } @Test void getQualifiedElementNameForClass_Dcterms() { //Check dcterms elements - assertEquals("dcterms:alternative", RdfConversionUtils.getQualifiedElementNameForClass(Alternative.class)); - assertEquals("dcterms:hasPart", RdfConversionUtils.getQualifiedElementNameForClass(HasPart.class)); - assertEquals("dcterms:isPartOf", RdfConversionUtils.getQualifiedElementNameForClass(IsPartOf.class)); - assertEquals("dcterms:isReferencedBy", RdfConversionUtils.getQualifiedElementNameForClass(IsReferencedBy.class)); - assertEquals("dcterms:medium", RdfConversionUtils.getQualifiedElementNameForClass(Medium.class)); - assertEquals("dcterms:provenance", RdfConversionUtils.getQualifiedElementNameForClass(Provenance.class)); - assertEquals("dcterms:references", RdfConversionUtils.getQualifiedElementNameForClass(References.class)); - assertEquals("dcterms:spatial", RdfConversionUtils.getQualifiedElementNameForClass(Spatial.class)); - assertEquals("dcterms:tableOfContents", RdfConversionUtils.getQualifiedElementNameForClass(TableOfContents.class)); - assertEquals("dcterms:temporal", RdfConversionUtils.getQualifiedElementNameForClass(Temporal.class)); + final RdfConversionUtils rdfConversionUtils = new RdfConversionUtils(); + assertEquals("dcterms:alternative", rdfConversionUtils.getQualifiedElementNameForClass(Alternative.class)); + assertEquals("dcterms:hasPart", rdfConversionUtils.getQualifiedElementNameForClass(HasPart.class)); + assertEquals("dcterms:isPartOf", rdfConversionUtils.getQualifiedElementNameForClass(IsPartOf.class)); + assertEquals("dcterms:isReferencedBy", rdfConversionUtils.getQualifiedElementNameForClass(IsReferencedBy.class)); + assertEquals("dcterms:medium", rdfConversionUtils.getQualifiedElementNameForClass(Medium.class)); + assertEquals("dcterms:provenance", rdfConversionUtils.getQualifiedElementNameForClass(Provenance.class)); + assertEquals("dcterms:references", rdfConversionUtils.getQualifiedElementNameForClass(References.class)); + assertEquals("dcterms:spatial", rdfConversionUtils.getQualifiedElementNameForClass(Spatial.class)); + assertEquals("dcterms:tableOfContents", rdfConversionUtils.getQualifiedElementNameForClass(TableOfContents.class)); + assertEquals("dcterms:temporal", rdfConversionUtils.getQualifiedElementNameForClass(Temporal.class)); } @Test void getQualifiedElementNameForClass_Edm() { //Check edm elements - assertEquals("edm:currentLocation", RdfConversionUtils.getQualifiedElementNameForClass(CurrentLocation.class)); - assertEquals("edm:hasType", RdfConversionUtils.getQualifiedElementNameForClass(HasType.class)); - assertEquals("edm:isRelatedTo", RdfConversionUtils.getQualifiedElementNameForClass(IsRelatedTo.class)); + final RdfConversionUtils rdfConversionUtils = new RdfConversionUtils(); + assertEquals("edm:currentLocation", rdfConversionUtils.getQualifiedElementNameForClass(CurrentLocation.class)); + assertEquals("edm:hasType", rdfConversionUtils.getQualifiedElementNameForClass(HasType.class)); + assertEquals("edm:isRelatedTo", rdfConversionUtils.getQualifiedElementNameForClass(IsRelatedTo.class)); } } \ No newline at end of file diff --git a/metis-validation/metis-validation-client/src/test/java/TestValidationClient.java b/metis-validation/metis-validation-client/src/test/java/TestValidationClient.java index 90574be1c..0a9ceb84b 100644 --- a/metis-validation/metis-validation-client/src/test/java/TestValidationClient.java +++ b/metis-validation/metis-validation-client/src/test/java/TestValidationClient.java @@ -28,7 +28,7 @@ class TestValidationClient { static { try { - portForWireMock = NetworkUtil.getAvailableLocalPort(); + portForWireMock = new NetworkUtil().getAvailableLocalPort(); } catch (IOException e) { e.printStackTrace(); } diff --git a/metis-validation/metis-validation-rest/src/test/java/TestApplication.java b/metis-validation/metis-validation-rest/src/test/java/TestApplication.java index ccb2dabe0..eaaf89029 100644 --- a/metis-validation/metis-validation-rest/src/test/java/TestApplication.java +++ b/metis-validation/metis-validation-rest/src/test/java/TestApplication.java @@ -19,7 +19,7 @@ public class TestApplication { static { try { - portForWireMock = NetworkUtil.getAvailableLocalPort(); + portForWireMock = new NetworkUtil().getAvailableLocalPort(); } catch (IOException e) { e.printStackTrace(); } diff --git a/metis-validation/metis-validation-service/src/test/java/TestApplication.java b/metis-validation/metis-validation-service/src/test/java/TestApplication.java index 51888196d..52db600d1 100644 --- a/metis-validation/metis-validation-service/src/test/java/TestApplication.java +++ b/metis-validation/metis-validation-service/src/test/java/TestApplication.java @@ -19,7 +19,7 @@ public class TestApplication { static { try { - portForWireMock = NetworkUtil.getAvailableLocalPort(); + portForWireMock = new NetworkUtil().getAvailableLocalPort(); } catch (IOException e) { e.printStackTrace(); } diff --git a/metis-validation/metis-validation-service/src/test/java/TestSchemaProvider.java b/metis-validation/metis-validation-service/src/test/java/TestSchemaProvider.java index 6e0ec4020..2ffa8c3ad 100644 --- a/metis-validation/metis-validation-service/src/test/java/TestSchemaProvider.java +++ b/metis-validation/metis-validation-service/src/test/java/TestSchemaProvider.java @@ -30,7 +30,7 @@ class TestSchemaProvider { static { try { - portForWireMock = NetworkUtil.getAvailableLocalPort(); + portForWireMock = new NetworkUtil().getAvailableLocalPort(); } catch (IOException e) { e.printStackTrace(); } diff --git a/metis-validation/metis-validation-service/src/test/java/TestValidationExecution.java b/metis-validation/metis-validation-service/src/test/java/TestValidationExecution.java index d0f19151e..85f045c1a 100644 --- a/metis-validation/metis-validation-service/src/test/java/TestValidationExecution.java +++ b/metis-validation/metis-validation-service/src/test/java/TestValidationExecution.java @@ -26,7 +26,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import net.lingala.zip4j.ZipFile; -import net.lingala.zip4j.exception.ZipException; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.AfterAll; diff --git a/pom.xml b/pom.xml index 15f414c74..57995f0eb 100644 --- a/pom.xml +++ b/pom.xml @@ -483,12 +483,6 @@ ${version.mockito.core} test - - mockito-inline - org.mockito - test - ${version.mockito.core} - org.springframework From 3401cd8022a45449ebc6c780b25b505f6f00d7a8 Mon Sep 17 00:00:00 2001 From: JoanaCMS <70145179+JoanaCMS@users.noreply.github.com> Date: Tue, 15 Mar 2022 14:32:35 +0100 Subject: [PATCH 14/73] MET-4233 Secure Dereference service from url request attacks (#507) * MET-4233 Created DereferenceValidationUtils class * MET-4233 Improvement in exception being thrown Refactpring in DereferenceValidationUtils * MET-4233 Refactoring code * MET-4233 Refactoring of code * MET-4233 Created new unit tests * MET-4233 Added new unit tests Code refactoring * MET-4233 Removing code smells * MET-4233 Removing code smell * MET-4233 Code review changes * MET-4233 Changed application configuration and updated the code accordingly * MET-4233 Removed code smells * MET-4233 Code cleanup * MET-4233 Removed DereferenceValidationUtils class * MET-4233 Code review changes * MET-4233 Created new unit tests * MET-4233 Removed code smells --- .../VocabularyCollectionImporterFactory.java | 33 +++++++++++++- .../VocabularyCollectionMavenRule.java | 21 ++++++++- ...cabularyCollectionImporterFactoryTest.java | 45 +++++++++++++++++++ .../dereference/rest/config/Application.java | 12 +++++ .../dereferencing.properties.example | 4 +- .../MongoDereferencingManagementService.java | 11 ++++- ...ngoDereferencingManagementServiceTest.java | 4 +- 7 files changed, 123 insertions(+), 7 deletions(-) create mode 100644 metis-dereference/metis-dereference-import/src/test/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactoryTest.java diff --git a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactory.java b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactory.java index 2f491bb68..c7db33197 100644 --- a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactory.java +++ b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactory.java @@ -1,26 +1,46 @@ package eu.europeana.metis.dereference.vocimport; +import eu.europeana.metis.dereference.vocimport.exception.VocabularyImportException; import eu.europeana.metis.dereference.vocimport.model.Location; +import org.apache.commons.collections.CollectionUtils; + import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; /** * This class is the factory for instances of {@link VocabularyCollectionImporter}. */ public class VocabularyCollectionImporterFactory { + private final List validUrlPrefixes; + + /** + * Constructor for the factory + * + * @param validUrlPrefixes The utils class used to verify input values + */ + public VocabularyCollectionImporterFactory(List validUrlPrefixes) { + this.validUrlPrefixes = new ArrayList<>(validUrlPrefixes); + } + /** * Create a vocabulary importer for remote web addresses, indicated by instances of {@link URI}. * Note that this method can only be used for locations that are also a valid {@link * java.net.URL}. * * @param directoryLocation The location of the directory to import. + * @throws VocabularyImportException if a problem occurs when verifying directory * @return A vocabulary importer. */ - public VocabularyCollectionImporter createImporter(URI directoryLocation) { + public VocabularyCollectionImporter createImporter(URI directoryLocation) throws VocabularyImportException { + if(isUrlPrefixNotValid(directoryLocation.toString())){ + throw new VocabularyImportException("The location of the directory to import is not valid."); + } return new VocabularyCollectionImporterImpl(new UriLocation(directoryLocation)); } @@ -47,6 +67,17 @@ public VocabularyCollectionImporter createImporter(Path baseDirectory, Path dire return new VocabularyCollectionImporterImpl(new PathLocation(baseDirectory, directoryLocation)); } + private boolean isUrlPrefixNotValid(String directoryToEvaluate) { + boolean result; + + if (CollectionUtils.isEmpty(validUrlPrefixes)) { + result = true; + } else { + result = validUrlPrefixes.stream().noneMatch(directoryToEvaluate::startsWith); + } + return result; + } + private static final class UriLocation implements Location { private final URI uri; diff --git a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionMavenRule.java b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionMavenRule.java index de4bbd747..f69238b83 100644 --- a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionMavenRule.java +++ b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionMavenRule.java @@ -1,6 +1,7 @@ package eu.europeana.metis.dereference.vocimport; import eu.europeana.metis.dereference.vocimport.exception.VocabularyImportException; + import java.nio.file.Path; import org.apache.maven.enforcer.rule.api.EnforcerRule; import org.apache.maven.enforcer.rule.api.EnforcerRuleException; @@ -8,6 +9,8 @@ import org.apache.maven.plugin.logging.Log; import org.apache.maven.project.MavenProject; import org.codehaus.plexus.component.repository.exception.ComponentLookupException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; /** * This is a Maven-enabled enforcer rule that can be used in a maven project. For an example of how @@ -44,6 +47,7 @@ *     * } */ +@Component public class VocabularyCollectionMavenRule implements EnforcerRule { /** @@ -69,12 +73,24 @@ public class VocabularyCollectionMavenRule implements EnforcerRule { */ private String vocabularyDirectoryFile = null; + private VocabularyCollectionImporterFactory vocabularyCollectionImporterFactory; + /** * No-arguments constructor, required for maven instantiation. */ public VocabularyCollectionMavenRule() { } + /** + * Constructor. Used to inject the factory + * + * @param vocabularyCollectionImporterFactory The vocabulary collection importer factory + */ + @Autowired + public VocabularyCollectionMavenRule(VocabularyCollectionImporterFactory vocabularyCollectionImporterFactory){ + this.vocabularyCollectionImporterFactory = vocabularyCollectionImporterFactory; + } + /** * Constructor. * @@ -113,8 +129,9 @@ public void execute(EnforcerRuleHelper enforcerRuleHelper) throws EnforcerRuleEx final Path baseDirectory = project.getBasedir().toPath(); final Path vocabularyDirectory = baseDirectory.resolve(vocabularyDirectoryFile); + try { // Prepare validation - final VocabularyCollectionImporter importer = new VocabularyCollectionImporterFactory() + final VocabularyCollectionImporter importer = vocabularyCollectionImporterFactory .createImporter(baseDirectory, vocabularyDirectory); final VocabularyCollectionValidatorImpl validator = new VocabularyCollectionValidatorImpl( importer, lenientOnLackOfExamples, lenientOnMappingTestFailures, @@ -123,7 +140,7 @@ public void execute(EnforcerRuleHelper enforcerRuleHelper) throws EnforcerRuleEx log.info("Validating vocabulary collection: " + importer.getDirectoryLocation().toString()); // Perform validation - try { + validator.validate(vocabulary -> log.info(" Vocabulary found: " + vocabulary.getName()), log::warn); } catch (VocabularyImportException e) { diff --git a/metis-dereference/metis-dereference-import/src/test/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactoryTest.java b/metis-dereference/metis-dereference-import/src/test/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactoryTest.java new file mode 100644 index 000000000..a3b76174c --- /dev/null +++ b/metis-dereference/metis-dereference-import/src/test/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactoryTest.java @@ -0,0 +1,45 @@ +package eu.europeana.metis.dereference.vocimport; + +import eu.europeana.metis.dereference.vocimport.exception.VocabularyImportException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class VocabularyCollectionImporterFactoryTest { + + private VocabularyCollectionImporterFactory factory; + + @BeforeEach + void setUp() { + List validUrlsPrefixes = new ArrayList<>(); + validUrlsPrefixes.add("https://validprefix"); + factory = new VocabularyCollectionImporterFactory(validUrlsPrefixes); + } + + @Test + void createImporterWithUri_expectSuccess() throws URISyntaxException, VocabularyImportException { + VocabularyCollectionImporter result = factory.createImporter(new URI("https://validprefix/test/call")); + assertEquals("https://validprefix/test/call", result.getDirectoryLocation().toString()); + } + + @Test + void createImporterWithUri_expectFail() { + assertThrows(VocabularyImportException.class, + () -> factory.createImporter(new URI("https://anotherprefix/test/call"))); + } + + @Test + void createImporterWithPath_expectSuccess() { + VocabularyCollectionImporter result = factory.createImporter(Paths.get("/path/test/random")); + assertEquals("/path/test/random", result.getDirectoryLocation().toString()); + } + +} diff --git a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java index 819a647bd..922cca604 100644 --- a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java +++ b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java @@ -4,8 +4,11 @@ import eu.europeana.corelib.web.socks.SocksProxy; import eu.europeana.metis.dereference.service.dao.ProcessedEntityDao; import eu.europeana.metis.dereference.service.dao.VocabularyDao; +import eu.europeana.metis.dereference.vocimport.VocabularyCollectionImporterFactory; import eu.europeana.metis.mongo.connection.MongoClientProvider; import eu.europeana.metis.mongo.connection.MongoProperties; + +import java.util.Arrays; import java.util.Collections; import javax.annotation.PreDestroy; import org.springframework.beans.factory.InitializingBean; @@ -66,6 +69,10 @@ public class Application implements WebMvcConfigurer, InitializingBean { @Value("${vocabulary.db}") private String vocabularyDb; + //Valid directories list + @Value("${valid.url.prefixes}") + private String[] validUrlPrefixes; + private MongoClient mongoClientEntity; private MongoClient mongoClientVocabulary; @@ -121,6 +128,11 @@ public static PropertySourcesPlaceholderConfigurer propertySourcesPlaceholderCon return new PropertySourcesPlaceholderConfigurer(); } + @Bean + public VocabularyCollectionImporterFactory getVocabularyCollectionImporterFactory(){ + return new VocabularyCollectionImporterFactory(Arrays.asList(validUrlPrefixes)); + } + /** * Closes any connections previous acquired. */ diff --git a/metis-dereference/metis-dereference-rest/src/main/resources/dereferencing.properties.example b/metis-dereference/metis-dereference-rest/src/main/resources/dereferencing.properties.example index 3ee045635..93a843124 100644 --- a/metis-dereference/metis-dereference-rest/src/main/resources/dereferencing.properties.example +++ b/metis-dereference/metis-dereference-rest/src/main/resources/dereferencing.properties.example @@ -12,4 +12,6 @@ mongo.username= mongo.password= mongo.application.name= entity.db= -vocabulary.db= \ No newline at end of file +vocabulary.db= + +valid.url.prefixes= \ No newline at end of file diff --git a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementService.java b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementService.java index 19fd99634..a0d608d3b 100644 --- a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementService.java +++ b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementService.java @@ -8,6 +8,7 @@ import eu.europeana.metis.dereference.vocimport.VocabularyCollectionValidator; import eu.europeana.metis.dereference.vocimport.VocabularyCollectionValidatorImpl; import eu.europeana.metis.dereference.vocimport.exception.VocabularyImportException; + import java.net.URI; import java.util.ArrayList; import java.util.List; @@ -22,6 +23,7 @@ public class MongoDereferencingManagementService implements DereferencingManagem private final VocabularyDao vocabularyDao; private final ProcessedEntityDao processedEntityDao; + private final VocabularyCollectionImporterFactory vocabularyCollectionImporterFactory; /** * Constructor. @@ -30,9 +32,10 @@ public class MongoDereferencingManagementService implements DereferencingManagem */ @Autowired public MongoDereferencingManagementService(VocabularyDao vocabularyDao, - ProcessedEntityDao processedEntityDao) { + ProcessedEntityDao processedEntityDao, VocabularyCollectionImporterFactory vocabularyCollectionImporterFactory) { this.vocabularyDao = vocabularyDao; this.processedEntityDao = processedEntityDao; + this.vocabularyCollectionImporterFactory = vocabularyCollectionImporterFactory; } @Override @@ -48,9 +51,10 @@ public void emptyCache() { @Override public void loadVocabularies(URI directoryUrl) throws VocabularyImportException { + try { // Import and validate the vocabularies final List vocabularies = new ArrayList<>(); - final VocabularyCollectionImporter importer = new VocabularyCollectionImporterFactory() + final VocabularyCollectionImporter importer = vocabularyCollectionImporterFactory .createImporter(directoryUrl); final VocabularyCollectionValidator validator = new VocabularyCollectionValidatorImpl(importer, true, true, true); @@ -58,6 +62,9 @@ public void loadVocabularies(URI directoryUrl) throws VocabularyImportException // All vocabularies are loaded well. Now we replace the vocabularies. vocabularyDao.replaceAll(vocabularies); + } catch (VocabularyImportException e) { + throw new VocabularyImportException("An error as occurred while loading the vocabularies", e); + } } private static Vocabulary convertVocabulary( diff --git a/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementServiceTest.java b/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementServiceTest.java index f77c0a64a..b6439ee87 100644 --- a/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementServiceTest.java +++ b/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementServiceTest.java @@ -9,6 +9,7 @@ import eu.europeana.metis.dereference.Vocabulary; import eu.europeana.metis.dereference.service.dao.ProcessedEntityDao; import eu.europeana.metis.dereference.service.dao.VocabularyDao; +import eu.europeana.metis.dereference.vocimport.VocabularyCollectionImporterFactory; import eu.europeana.metis.mongo.embedded.EmbeddedLocalhostMongo; import java.util.Collections; import java.util.List; @@ -40,7 +41,8 @@ void prepare() { } }; ProcessedEntityDao processedEntityDao = mock(ProcessedEntityDao.class); - service = new MongoDereferencingManagementService(vocDao, processedEntityDao); + VocabularyCollectionImporterFactory vocabularyCollectionImporterFactory = mock(VocabularyCollectionImporterFactory.class); + service = new MongoDereferencingManagementService(vocDao, processedEntityDao, vocabularyCollectionImporterFactory); } @Test From 3a6af4c1942360735a1d0bd4557d5f79b19ccc1f Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Thu, 17 Mar 2022 09:49:37 +0100 Subject: [PATCH 15/73] MET-4233 Secure Dereference service from url request attacks (#515) --- .../VocabularyCollectionImporterFactory.java | 40 ++------------ .../VocabularyCollectionImporterImpl.java | 6 +- .../VocabularyCollectionMavenRule.java | 12 ---- .../VocabularyCollectionValidatorImpl.java | 8 +-- ...cabularyCollectionImporterFactoryTest.java | 55 +++++++------------ .../DereferencingManagementController.java | 22 +++++++- .../dereference/rest/config/Application.java | 12 ++-- ...DereferencingManagementControllerTest.java | 28 ++++++++-- 8 files changed, 81 insertions(+), 102 deletions(-) diff --git a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactory.java b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactory.java index c7db33197..93f2d46d5 100644 --- a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactory.java +++ b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactory.java @@ -1,46 +1,25 @@ package eu.europeana.metis.dereference.vocimport; -import eu.europeana.metis.dereference.vocimport.exception.VocabularyImportException; import eu.europeana.metis.dereference.vocimport.model.Location; -import org.apache.commons.collections.CollectionUtils; - import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; /** * This class is the factory for instances of {@link VocabularyCollectionImporter}. */ public class VocabularyCollectionImporterFactory { - private final List validUrlPrefixes; - - /** - * Constructor for the factory - * - * @param validUrlPrefixes The utils class used to verify input values - */ - public VocabularyCollectionImporterFactory(List validUrlPrefixes) { - this.validUrlPrefixes = new ArrayList<>(validUrlPrefixes); - } - /** - * Create a vocabulary importer for remote web addresses, indicated by instances of {@link URI}. - * Note that this method can only be used for locations that are also a valid {@link - * java.net.URL}. + * Create a vocabulary importer for remote web addresses, indicated by instances of {@link URI}. Note that this method can only + * be used for locations that are also a valid {@link java.net.URL}. * * @param directoryLocation The location of the directory to import. - * @throws VocabularyImportException if a problem occurs when verifying directory * @return A vocabulary importer. */ - public VocabularyCollectionImporter createImporter(URI directoryLocation) throws VocabularyImportException { - if(isUrlPrefixNotValid(directoryLocation.toString())){ - throw new VocabularyImportException("The location of the directory to import is not valid."); - } + public VocabularyCollectionImporter createImporter(URI directoryLocation) { return new VocabularyCollectionImporterImpl(new UriLocation(directoryLocation)); } @@ -67,28 +46,19 @@ public VocabularyCollectionImporter createImporter(Path baseDirectory, Path dire return new VocabularyCollectionImporterImpl(new PathLocation(baseDirectory, directoryLocation)); } - private boolean isUrlPrefixNotValid(String directoryToEvaluate) { - boolean result; - - if (CollectionUtils.isEmpty(validUrlPrefixes)) { - result = true; - } else { - result = validUrlPrefixes.stream().noneMatch(directoryToEvaluate::startsWith); - } - return result; - } - private static final class UriLocation implements Location { private final URI uri; UriLocation(URI uri) { this.uri = uri; + } @Override public InputStream read() throws IOException { return uri.toURL().openStream(); + } @Override diff --git a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterImpl.java b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterImpl.java index 02bbcb5dc..37ce3a47c 100644 --- a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterImpl.java +++ b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterImpl.java @@ -28,16 +28,18 @@ final class VocabularyCollectionImporterImpl implements VocabularyCollectionImpo } @Override - public Iterable importVocabularies() throws VocabularyImportException { + public Iterable importVocabularies() + throws VocabularyImportException { // Obtain the directory entries. final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); final VocabularyDirectoryEntry[] directoryEntries; + try (final InputStream input = directoryLocation.read()) { directoryEntries = mapper.readValue(input, VocabularyDirectoryEntry[].class); } catch (IOException e) { throw new VocabularyImportException( - "Could not read vocabulary directory at [" + directoryLocation + "].", e); + "Could not read vocabulary directory at [" + directoryLocation + "].", e); } // Compile the vocabulary loaders diff --git a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionMavenRule.java b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionMavenRule.java index f69238b83..9d21ffb86 100644 --- a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionMavenRule.java +++ b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionMavenRule.java @@ -1,7 +1,6 @@ package eu.europeana.metis.dereference.vocimport; import eu.europeana.metis.dereference.vocimport.exception.VocabularyImportException; - import java.nio.file.Path; import org.apache.maven.enforcer.rule.api.EnforcerRule; import org.apache.maven.enforcer.rule.api.EnforcerRuleException; @@ -9,7 +8,6 @@ import org.apache.maven.plugin.logging.Log; import org.apache.maven.project.MavenProject; import org.codehaus.plexus.component.repository.exception.ComponentLookupException; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; /** @@ -81,16 +79,6 @@ public class VocabularyCollectionMavenRule implements EnforcerRule { public VocabularyCollectionMavenRule() { } - /** - * Constructor. Used to inject the factory - * - * @param vocabularyCollectionImporterFactory The vocabulary collection importer factory - */ - @Autowired - public VocabularyCollectionMavenRule(VocabularyCollectionImporterFactory vocabularyCollectionImporterFactory){ - this.vocabularyCollectionImporterFactory = vocabularyCollectionImporterFactory; - } - /** * Constructor. * diff --git a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionValidatorImpl.java b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionValidatorImpl.java index dfa92b86b..62ac84952 100644 --- a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionValidatorImpl.java +++ b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionValidatorImpl.java @@ -51,20 +51,18 @@ public void validate(Consumer vocabularyReceiver, Consumer w } @Override - public void validateVocabularyOnly(Consumer vocabularyReceiver) - throws VocabularyImportException { + public void validateVocabularyOnly(Consumer vocabularyReceiver) throws VocabularyImportException { validateInternal(vocabularyReceiver, null, false); } private void validateInternal(Consumer vocabularyReceiver, - Consumer warningReceiver, boolean validateExamples) - throws VocabularyImportException { + Consumer warningReceiver, boolean validateExamples) throws VocabularyImportException { final DuplicationChecker duplicationChecker = new DuplicationChecker(); final Iterable vocabularyLoaders = importer.importVocabularies(); for (VocabularyLoader loader : vocabularyLoaders) { final Vocabulary vocabulary = loader.load(); final IncomingRecordToEdmConverter converter = validateVocabulary(vocabulary, - duplicationChecker); + duplicationChecker); if (validateExamples) { validateExamples(vocabulary, warningReceiver, converter); } diff --git a/metis-dereference/metis-dereference-import/src/test/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactoryTest.java b/metis-dereference/metis-dereference-import/src/test/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactoryTest.java index a3b76174c..7d725eb46 100644 --- a/metis-dereference/metis-dereference-import/src/test/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactoryTest.java +++ b/metis-dereference/metis-dereference-import/src/test/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactoryTest.java @@ -1,45 +1,32 @@ package eu.europeana.metis.dereference.vocimport; -import eu.europeana.metis.dereference.vocimport.exception.VocabularyImportException; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; class VocabularyCollectionImporterFactoryTest { - private VocabularyCollectionImporterFactory factory; - - @BeforeEach - void setUp() { - List validUrlsPrefixes = new ArrayList<>(); - validUrlsPrefixes.add("https://validprefix"); - factory = new VocabularyCollectionImporterFactory(validUrlsPrefixes); - } - - @Test - void createImporterWithUri_expectSuccess() throws URISyntaxException, VocabularyImportException { - VocabularyCollectionImporter result = factory.createImporter(new URI("https://validprefix/test/call")); - assertEquals("https://validprefix/test/call", result.getDirectoryLocation().toString()); - } - - @Test - void createImporterWithUri_expectFail() { - assertThrows(VocabularyImportException.class, - () -> factory.createImporter(new URI("https://anotherprefix/test/call"))); - } - - @Test - void createImporterWithPath_expectSuccess() { - VocabularyCollectionImporter result = factory.createImporter(Paths.get("/path/test/random")); - assertEquals("/path/test/random", result.getDirectoryLocation().toString()); - } + private VocabularyCollectionImporterFactory factory; + + @BeforeEach + void setUp() { + factory = new VocabularyCollectionImporterFactory(); + } + + @Test + void createImporterWithUri_expectSuccess() throws URISyntaxException { + VocabularyCollectionImporter result = factory.createImporter(new URI("https://validprefix/test/call")); + assertEquals("https://validprefix/test/call", result.getDirectoryLocation().toString()); + } + + @Test + void createImporterWithPath_expectSuccess() { + VocabularyCollectionImporter result = factory.createImporter(Paths.get("/path/test/random")); + assertEquals("/path/test/random", result.getDirectoryLocation().toString()); + } } diff --git a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/DereferencingManagementController.java b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/DereferencingManagementController.java index 6d45846b3..90dd5a664 100644 --- a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/DereferencingManagementController.java +++ b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/DereferencingManagementController.java @@ -1,9 +1,9 @@ package eu.europeana.metis.dereference.rest; -import eu.europeana.metis.utils.RestEndpoints; import eu.europeana.metis.dereference.Vocabulary; import eu.europeana.metis.dereference.service.DereferencingManagementService; import eu.europeana.metis.dereference.vocimport.exception.VocabularyImportException; +import eu.europeana.metis.utils.RestEndpoints; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -11,7 +11,9 @@ import io.swagger.annotations.ApiResponses; import java.net.URI; import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.List; +import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -34,10 +36,12 @@ public class DereferencingManagementController { private static final Logger LOGGER = LoggerFactory.getLogger(DereferencingManagementController.class); private final DereferencingManagementService service; + private final List validUrlPrefixes; @Autowired - public DereferencingManagementController(DereferencingManagementService service) { + public DereferencingManagementController(DereferencingManagementService service, List validUrlPrefixes) { this.service = service; + this.validUrlPrefixes = new ArrayList<>(validUrlPrefixes); } /** @@ -79,6 +83,9 @@ public void emptyCache() { }) public ResponseEntity loadVocabularies( @ApiParam("directory_url") @RequestParam("directory_url") String directoryUrl) { try { + if (isUrlPrefixNotValid(directoryUrl)) { + return ResponseEntity.badRequest().body("The url of the directory to import is not valid."); + } service.loadVocabularies(new URI(directoryUrl)); return ResponseEntity.ok().build(); } catch (URISyntaxException e) { @@ -89,4 +96,15 @@ public void emptyCache() { return ResponseEntity.status(HttpStatus.BAD_GATEWAY).body(e.getMessage()); } } + + private boolean isUrlPrefixNotValid(String directoryToEvaluate) { + boolean result; + + if (CollectionUtils.isEmpty(validUrlPrefixes)) { + result = true; + } else { + result = validUrlPrefixes.stream().noneMatch(directoryToEvaluate::startsWith); + } + return result; + } } diff --git a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java index 922cca604..58cb63fb7 100644 --- a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java +++ b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java @@ -4,12 +4,10 @@ import eu.europeana.corelib.web.socks.SocksProxy; import eu.europeana.metis.dereference.service.dao.ProcessedEntityDao; import eu.europeana.metis.dereference.service.dao.VocabularyDao; -import eu.europeana.metis.dereference.vocimport.VocabularyCollectionImporterFactory; import eu.europeana.metis.mongo.connection.MongoClientProvider; import eu.europeana.metis.mongo.connection.MongoProperties; - -import java.util.Arrays; import java.util.Collections; +import java.util.List; import javax.annotation.PreDestroy; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Value; @@ -124,13 +122,13 @@ VocabularyDao getVocabularyDao() { } @Bean - public static PropertySourcesPlaceholderConfigurer propertySourcesPlaceholderConfigurer() { - return new PropertySourcesPlaceholderConfigurer(); + List getValidUrlPrefixes() { + return List.of(validUrlPrefixes); } @Bean - public VocabularyCollectionImporterFactory getVocabularyCollectionImporterFactory(){ - return new VocabularyCollectionImporterFactory(Arrays.asList(validUrlPrefixes)); + public static PropertySourcesPlaceholderConfigurer propertySourcesPlaceholderConfigurer() { + return new PropertySourcesPlaceholderConfigurer(); } /** diff --git a/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingManagementControllerTest.java b/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingManagementControllerTest.java index 96f2f4589..bd950d707 100644 --- a/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingManagementControllerTest.java +++ b/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingManagementControllerTest.java @@ -2,19 +2,24 @@ import static org.hamcrest.core.Is.is; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import eu.europeana.metis.dereference.Vocabulary; import eu.europeana.metis.dereference.rest.exceptions.RestResponseExceptionHandler; import eu.europeana.metis.dereference.service.DereferencingManagementService; +import java.net.URI; import java.util.ArrayList; import java.util.Collections; +import java.util.List; import org.bson.types.ObjectId; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -33,7 +38,7 @@ void setUp() { dereferencingManagementServiceMock = mock(DereferencingManagementService.class); DereferencingManagementController dereferencingManagementController = new DereferencingManagementController( - dereferencingManagementServiceMock); + dereferencingManagementServiceMock, List.of("http://correctUrl")); dereferencingManagementControllerMock = MockMvcBuilders .standaloneSetup(dereferencingManagementController) @@ -60,9 +65,22 @@ void testGetAllVocabularies() throws Exception { when(dereferencingManagementServiceMock.getAllVocabularies()).thenReturn(dummyVocabList); dereferencingManagementControllerMock.perform(get("/vocabularies")) - .andExpect(jsonPath("$[0].uris[0]", is("http://dummy1.org/path1"))) - .andExpect(jsonPath("$[1].uris[0]", is("http://dummy2.org/path2"))) - .andExpect(status().is(200)); + .andExpect(jsonPath("$[0].uris[0]", is("http://dummy1.org/path1"))) + .andExpect(jsonPath("$[1].uris[0]", is("http://dummy2.org/path2"))) + .andExpect(status().is(200)); + } + + @Test + void testLoadVocabularies_validPrefix_expectSuccess() throws Exception { + doNothing().when(dereferencingManagementServiceMock).loadVocabularies(any(URI.class)); + dereferencingManagementControllerMock.perform(post("/load_vocabularies") + .param("directory_url", "http://correctUrl/test/call")).andExpect(status().is(200)); + } + + @Test + void testLoadVocabularies_invalidPrefix_expectFail() throws Exception { + dereferencingManagementControllerMock.perform(post("/load_vocabularies") + .param("directory_url", "http://wrongUrl")).andExpect(status().is(400)); } @Test @@ -73,7 +91,7 @@ void testEmptyCache() throws Exception { }).when(dereferencingManagementServiceMock).emptyCache(); dereferencingManagementControllerMock.perform(delete("/cache")) - .andExpect(status().is(200)); + .andExpect(status().is(200)); assertEquals("OK", testEmptyCacheResult); } From 1aea2e92bc043b03d6dfe96df049af3516cb73f9 Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Thu, 17 Mar 2022 16:57:16 +0100 Subject: [PATCH 16/73] MET-4233 MET-4233B Reproduce sonar issue on branch (#519) * MET-4233 MET-4233B Reproduce sonar issue on branch * MET-4233 MET-4233B Reproduce sonar issue on branch2 * MET-4233 MET-4233B 1st try to fix issue * MET-4233 MET-4233B 2nd try to fix issue * MET-4233 MET-4233B Re-organize code * MET-4233 MET-4233B Cleanup * MET-4233 MET-4233B Push small change, github didn't pick last change on PR --- .../VocabularyCollectionImporterFactory.java | 36 +++++---- .../VocabularyCollectionImporterImpl.java | 46 ++++++----- .../dereference/vocimport/model/Location.java | 9 ++- ...cabularyCollectionImporterFactoryTest.java | 32 -------- .../DereferencingManagementController.java | 81 +++++++++++++------ .../dereference/rest/config/Application.java | 10 +-- .../rest/config/ServletInitializer.java | 4 +- .../dereferencing.properties.example | 3 +- ...DereferencingManagementControllerTest.java | 24 +++--- .../DereferencingManagementService.java | 7 +- .../MongoDereferencingManagementService.java | 19 +++-- 11 files changed, 142 insertions(+), 129 deletions(-) delete mode 100644 metis-dereference/metis-dereference-import/src/test/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactoryTest.java diff --git a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactory.java b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactory.java index 93f2d46d5..97eeea888 100644 --- a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactory.java +++ b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactory.java @@ -1,9 +1,13 @@ package eu.europeana.metis.dereference.vocimport; import eu.europeana.metis.dereference.vocimport.model.Location; +import eu.europeana.metis.exception.BadContentException; import java.io.IOException; import java.io.InputStream; +import java.net.MalformedURLException; import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; @@ -19,8 +23,8 @@ public class VocabularyCollectionImporterFactory { * @param directoryLocation The location of the directory to import. * @return A vocabulary importer. */ - public VocabularyCollectionImporter createImporter(URI directoryLocation) { - return new VocabularyCollectionImporterImpl(new UriLocation(directoryLocation)); + public VocabularyCollectionImporter createImporter(URL directoryLocation) { + return new VocabularyCollectionImporterImpl(new UrlLocation(directoryLocation)); } /** @@ -34,9 +38,8 @@ public VocabularyCollectionImporter createImporter(Path directoryLocation) { } /** - * Create a vocabulary importer for local files, indicated by instances of {@link Path}. This - * method provides a way to set a base directory that will be assumed known (so that output and - * logs will only include the relative location). + * Create a vocabulary importer for local files, indicated by instances of {@link Path}. This method provides a way to set a + * base directory that will be assumed known (so that output and logs will only include the relative location). * * @param baseDirectory The base directory of the project or collection. Can be null. * @param directoryLocation The full location of the directory file to import. @@ -46,29 +49,32 @@ public VocabularyCollectionImporter createImporter(Path baseDirectory, Path dire return new VocabularyCollectionImporterImpl(new PathLocation(baseDirectory, directoryLocation)); } - private static final class UriLocation implements Location { + private static final class UrlLocation implements Location { - private final URI uri; - - UriLocation(URI uri) { - this.uri = uri; + private final URL url; + UrlLocation(URL url) { + this.url = url; } @Override public InputStream read() throws IOException { - return uri.toURL().openStream(); - + return url.openStream(); } @Override - public Location resolve(String relativeLocation) { - return new UriLocation(uri.resolve(relativeLocation)); + public Location resolve(String relativeLocation) throws BadContentException { + try { + return new UrlLocation(url.toURI().resolve(relativeLocation).toURL()); + } catch (URISyntaxException | MalformedURLException e) { + throw new BadContentException( + String.format("Provided url '%s' and relative location %s, failed to parse.", url, relativeLocation), e); + } } @Override public String toString() { - return uri.toString(); + return url.toString(); } } diff --git a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterImpl.java b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterImpl.java index 37ce3a47c..733305b98 100644 --- a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterImpl.java +++ b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterImpl.java @@ -9,6 +9,7 @@ import eu.europeana.metis.dereference.vocimport.model.VocabularyDirectoryEntry; import eu.europeana.metis.dereference.vocimport.model.VocabularyLoader; import eu.europeana.metis.dereference.vocimport.model.VocabularyMetadata; +import eu.europeana.metis.exception.BadContentException; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -21,15 +22,14 @@ */ final class VocabularyCollectionImporterImpl implements VocabularyCollectionImporter { - private Location directoryLocation; + private final Location directoryLocation; VocabularyCollectionImporterImpl(Location directoryLocation) { this.directoryLocation = directoryLocation; } @Override - public Iterable importVocabularies() - throws VocabularyImportException { + public Iterable importVocabularies() throws VocabularyImportException { // Obtain the directory entries. final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); @@ -45,8 +45,16 @@ public Iterable importVocabularies() // Compile the vocabulary loaders final List result = new ArrayList<>(directoryEntries.length); for (VocabularyDirectoryEntry entry : directoryEntries) { - final Location metadataLocation = directoryLocation.resolve(entry.getMetadata()); - final Location mappingLocation = directoryLocation.resolve(entry.getMapping()); + final Location metadataLocation; + final Location mappingLocation; + try { + metadataLocation = directoryLocation.resolve(entry.getMetadata()); + mappingLocation = directoryLocation.resolve(entry.getMapping()); + } catch (BadContentException e) { + throw new VocabularyImportException( + String.format("Could not read vocabulary directory at [%s] and entry metadata [%s], entry mapping [%s].", + directoryLocation, entry.getMetadata(), entry.getMapping()), e); + } result.add(() -> loadVocabulary(metadataLocation, mappingLocation, mapper)); } @@ -55,7 +63,7 @@ public Iterable importVocabularies() } private Vocabulary loadVocabulary(Location metadataLocation, Location mappingLocation, - ObjectMapper mapper) throws VocabularyImportException { + ObjectMapper mapper) throws VocabularyImportException { // Read the metadata file. final VocabularyMetadata metadata; @@ -63,7 +71,7 @@ private Vocabulary loadVocabulary(Location metadataLocation, Location mappingLoc metadata = mapper.readValue(input, VocabularyMetadata.class); } catch (IOException e) { throw new VocabularyImportException( - "Could not read vocabulary metadata at [" + metadataLocation + "].", e); + "Could not read vocabulary metadata at [" + metadataLocation + "].", e); } // Read the mapping file. @@ -72,22 +80,22 @@ private Vocabulary loadVocabulary(Location metadataLocation, Location mappingLoc mapping = IOUtils.toString(input, StandardCharsets.UTF_8); } catch (IOException e) { throw new VocabularyImportException( - "Could not read vocabulary mapping at [" + mappingLocation + "].", e); + "Could not read vocabulary mapping at [" + mappingLocation + "].", e); } // Compile the vocabulary. return Vocabulary.builder() - .setName(metadata.getName()) - .setTypes(metadata.getTypes()) - .setPaths(metadata.getPaths()) - .setParentIterations(metadata.getParentIterations()) - .setSuffix(metadata.getSuffix()) - .setExamples(metadata.getExamples()) - .setCounterExamples(metadata.getCounterExamples()) - .setTransformation(mapping) - .setReadableMetadataLocation(metadataLocation.toString()) - .setReadableMappingLocation(mappingLocation.toString()) - .build(); + .setName(metadata.getName()) + .setTypes(metadata.getTypes()) + .setPaths(metadata.getPaths()) + .setParentIterations(metadata.getParentIterations()) + .setSuffix(metadata.getSuffix()) + .setExamples(metadata.getExamples()) + .setCounterExamples(metadata.getCounterExamples()) + .setTransformation(mapping) + .setReadableMetadataLocation(metadataLocation.toString()) + .setReadableMappingLocation(mappingLocation.toString()) + .build(); } public Location getDirectoryLocation() { diff --git a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/model/Location.java b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/model/Location.java index f6970584b..f2a1039da 100644 --- a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/model/Location.java +++ b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/model/Location.java @@ -1,5 +1,6 @@ package eu.europeana.metis.dereference.vocimport.model; +import eu.europeana.metis.exception.BadContentException; import java.io.IOException; import java.io.InputStream; @@ -14,14 +15,14 @@ public interface Location { InputStream read() throws IOException; /** - * Resolve a relative location against the given location. The given location can be assumed to be - * a file (as opposed to a path/directory) so that essentially the relative location is resolved - * against the parent of the given location. + * Resolve a relative location against the given location. The given location can be assumed to be a file (as opposed to a + * path/directory) so that essentially the relative location is resolved against the parent of the given location. * * @param relativeLocation The relative location to resolve. * @return The resolved location. + * @throws BadContentException if the resolve did not succeed */ - Location resolve(String relativeLocation); + Location resolve(String relativeLocation) throws BadContentException; /** * @return A human-readable representation of the location. diff --git a/metis-dereference/metis-dereference-import/src/test/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactoryTest.java b/metis-dereference/metis-dereference-import/src/test/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactoryTest.java deleted file mode 100644 index 7d725eb46..000000000 --- a/metis-dereference/metis-dereference-import/src/test/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionImporterFactoryTest.java +++ /dev/null @@ -1,32 +0,0 @@ -package eu.europeana.metis.dereference.vocimport; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Paths; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class VocabularyCollectionImporterFactoryTest { - - private VocabularyCollectionImporterFactory factory; - - @BeforeEach - void setUp() { - factory = new VocabularyCollectionImporterFactory(); - } - - @Test - void createImporterWithUri_expectSuccess() throws URISyntaxException { - VocabularyCollectionImporter result = factory.createImporter(new URI("https://validprefix/test/call")); - assertEquals("https://validprefix/test/call", result.getDirectoryLocation().toString()); - } - - @Test - void createImporterWithPath_expectSuccess() { - VocabularyCollectionImporter result = factory.createImporter(Paths.get("/path/test/random")); - assertEquals("/path/test/random", result.getDirectoryLocation().toString()); - } - -} diff --git a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/DereferencingManagementController.java b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/DereferencingManagementController.java index 90dd5a664..43b046ba3 100644 --- a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/DereferencingManagementController.java +++ b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/DereferencingManagementController.java @@ -3,17 +3,21 @@ import eu.europeana.metis.dereference.Vocabulary; import eu.europeana.metis.dereference.service.DereferencingManagementService; import eu.europeana.metis.dereference.vocimport.exception.VocabularyImportException; +import eu.europeana.metis.exception.BadContentException; import eu.europeana.metis.utils.RestEndpoints; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; +import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; -import java.util.ArrayList; +import java.net.URL; +import java.util.HashSet; import java.util.List; -import org.apache.commons.collections.CollectionUtils; +import java.util.Optional; +import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -36,12 +40,16 @@ public class DereferencingManagementController { private static final Logger LOGGER = LoggerFactory.getLogger(DereferencingManagementController.class); private final DereferencingManagementService service; - private final List validUrlPrefixes; + private final Set allowedUrlDomains; + /** + * @param service the dereferencing management service + * @param allowedUrlDomains the allowed valid url prefixes + */ @Autowired - public DereferencingManagementController(DereferencingManagementService service, List validUrlPrefixes) { + public DereferencingManagementController(DereferencingManagementService service, Set allowedUrlDomains) { this.service = service; - this.validUrlPrefixes = new ArrayList<>(validUrlPrefixes); + this.allowedUrlDomains = new HashSet<>(allowedUrlDomains); } /** @@ -57,9 +65,8 @@ public List getAllVocabularies() { } /** - * Empty Cache. This will remove ALL entries in the cache (Redis). If the same redis - * instance/cluster is used for multiple services then the cache for other services is cleared as - * well. + * Empty Cache. This will remove ALL entries in the cache (Redis). If the same redis instance/cluster is used for multiple + * services then the cache for other services is cleared as well. */ @DeleteMapping(value = RestEndpoints.CACHE_EMPTY) @ResponseBody @@ -71,24 +78,27 @@ public void emptyCache() { /** * Load the vocabularies from an online source. This does NOT purge the cache. * - * @param directoryUrl The online location of the vocabulary directory. + * @param directoryUrl The online location of the vocabulary directory + * @return sting containing an error message otherwise empty */ @PostMapping(value = RestEndpoints.LOAD_VOCABULARIES) @ResponseBody @ApiOperation(value = "Load and replace the vocabularies listed by the given vocabulary directory. Does NOT purge the cache.") @ApiResponses(value = { - @ApiResponse(code = 200, message = "Vocabularies loaded successfully."), - @ApiResponse(code = 400, message = "Bad request parameters."), - @ApiResponse(code = 502, message = "Problem accessing vocabulary repository.") - }) public ResponseEntity loadVocabularies( - @ApiParam("directory_url") @RequestParam("directory_url") String directoryUrl) { + @ApiResponse(code = 200, message = "Vocabularies loaded successfully."), + @ApiResponse(code = 400, message = "Bad request parameters."), + @ApiResponse(code = 502, message = "Problem accessing vocabulary repository.") + }) + public ResponseEntity loadVocabularies( + @ApiParam("directory_url") @RequestParam("directory_url") String directoryUrl) { try { - if (isUrlPrefixNotValid(directoryUrl)) { - return ResponseEntity.badRequest().body("The url of the directory to import is not valid."); + final Optional validatedLocationUrl = getValidatedLocationUrl(directoryUrl); + if (validatedLocationUrl.isPresent()) { + service.loadVocabularies(validatedLocationUrl.get()); + return ResponseEntity.ok().build(); } - service.loadVocabularies(new URI(directoryUrl)); - return ResponseEntity.ok().build(); - } catch (URISyntaxException e) { + return ResponseEntity.badRequest().body("The url of the directory to import is not valid."); + } catch (BadContentException e) { LOGGER.warn("Could not load vocabularies", e); return ResponseEntity.badRequest().body(e.getMessage()); } catch (VocabularyImportException e) { @@ -97,14 +107,33 @@ public void emptyCache() { } } - private boolean isUrlPrefixNotValid(String directoryToEvaluate) { - boolean result; + /** + * Validates a String representation of a URL. + *

    The method will check that the url is: + *

      + *
    • valid according to the protocol
      • + *
    • of https scheme
      • + *
    • part of the allowed domains
      • + *
    + * domain for the application to further access it.

    + * + * @param directoryUrl the url to validate + * @return the validated URL class + * @throws BadContentException if the url failed during parsing + */ + private Optional getValidatedLocationUrl(String directoryUrl) throws BadContentException { + try { + URI uri = new URI(directoryUrl); + String scheme = uri.getScheme(); + String remoteHost = uri.getHost(); - if (CollectionUtils.isEmpty(validUrlPrefixes)) { - result = true; - } else { - result = validUrlPrefixes.stream().noneMatch(directoryToEvaluate::startsWith); + if ("https".equals(scheme) && allowedUrlDomains.contains(remoteHost)) { + return Optional.of(uri.toURL()); + } + } catch (URISyntaxException | MalformedURLException e) { + throw new BadContentException(String.format("Provided directoryUrl '%s', failed to parse.", directoryUrl), e); } - return result; + + return Optional.empty(); } } diff --git a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java index 58cb63fb7..d951ae798 100644 --- a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java +++ b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java @@ -7,7 +7,7 @@ import eu.europeana.metis.mongo.connection.MongoClientProvider; import eu.europeana.metis.mongo.connection.MongoProperties; import java.util.Collections; -import java.util.List; +import java.util.Set; import javax.annotation.PreDestroy; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Value; @@ -68,8 +68,8 @@ public class Application implements WebMvcConfigurer, InitializingBean { private String vocabularyDb; //Valid directories list - @Value("${valid.url.prefixes}") - private String[] validUrlPrefixes; + @Value("${allowed.url.domains}") + private String[] allowedUrlDomains; private MongoClient mongoClientEntity; private MongoClient mongoClientVocabulary; @@ -122,8 +122,8 @@ VocabularyDao getVocabularyDao() { } @Bean - List getValidUrlPrefixes() { - return List.of(validUrlPrefixes); + Set getAllowedUrlDomains() { + return Set.of(allowedUrlDomains); } @Bean diff --git a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/ServletInitializer.java b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/ServletInitializer.java index 20d9dc309..22baa6a3a 100644 --- a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/ServletInitializer.java +++ b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/ServletInitializer.java @@ -1,8 +1,9 @@ package eu.europeana.metis.dereference.rest.config; +import eu.europeana.metis.dereference.RdfRetriever; import eu.europeana.metis.dereference.service.MongoDereferenceService; import eu.europeana.metis.dereference.service.MongoDereferencingManagementService; -import eu.europeana.metis.dereference.RdfRetriever; +import eu.europeana.metis.dereference.vocimport.VocabularyCollectionImporterFactory; import org.springframework.util.ClassUtils; import org.springframework.web.context.WebApplicationContext; import org.springframework.web.context.support.AnnotationConfigWebApplicationContext; @@ -19,6 +20,7 @@ protected WebApplicationContext createServletApplicationContext() { context.scan(ClassUtils.getPackageName(getClass())); context.register(MongoDereferenceService.class); context.register(MongoDereferencingManagementService.class); + context.register(VocabularyCollectionImporterFactory.class); context.register(RdfRetriever.class); return context; diff --git a/metis-dereference/metis-dereference-rest/src/main/resources/dereferencing.properties.example b/metis-dereference/metis-dereference-rest/src/main/resources/dereferencing.properties.example index 93a843124..a7f04f76d 100644 --- a/metis-dereference/metis-dereference-rest/src/main/resources/dereferencing.properties.example +++ b/metis-dereference/metis-dereference-rest/src/main/resources/dereferencing.properties.example @@ -14,4 +14,5 @@ mongo.application.name= entity.db= vocabulary.db= -valid.url.prefixes= \ No newline at end of file +#The allowed domains for vocabularies loading without the scheme(always validated against https). e.g. raw.githubusercontent.com +allowed.url.domains= \ No newline at end of file diff --git a/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingManagementControllerTest.java b/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingManagementControllerTest.java index bd950d707..a7e99e76b 100644 --- a/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingManagementControllerTest.java +++ b/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingManagementControllerTest.java @@ -16,10 +16,10 @@ import eu.europeana.metis.dereference.Vocabulary; import eu.europeana.metis.dereference.rest.exceptions.RestResponseExceptionHandler; import eu.europeana.metis.dereference.service.DereferencingManagementService; -import java.net.URI; +import java.net.URL; import java.util.ArrayList; import java.util.Collections; -import java.util.List; +import java.util.Set; import org.bson.types.ObjectId; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -38,7 +38,7 @@ void setUp() { dereferencingManagementServiceMock = mock(DereferencingManagementService.class); DereferencingManagementController dereferencingManagementController = new DereferencingManagementController( - dereferencingManagementServiceMock, List.of("http://correctUrl")); + dereferencingManagementServiceMock, Set.of("valid.domain.com")); dereferencingManagementControllerMock = MockMvcBuilders .standaloneSetup(dereferencingManagementController) @@ -51,12 +51,12 @@ void testGetAllVocabularies() throws Exception { Vocabulary dummyVocab1 = new Vocabulary(); dummyVocab1.setId(new ObjectId()); dummyVocab1.setName("Dummy1"); - dummyVocab1.setUris(Collections.singleton("http://dummy1.org/path1")); + dummyVocab1.setUris(Collections.singleton("https://dummy1.org/path1")); Vocabulary dummyVocab2 = new Vocabulary(); dummyVocab2.setId(new ObjectId()); dummyVocab2.setName("Dummy2"); - dummyVocab2.setUris(Collections.singleton("http://dummy2.org/path2")); + dummyVocab2.setUris(Collections.singleton("https://dummy2.org/path2")); ArrayList dummyVocabList = new ArrayList<>(); dummyVocabList.add(dummyVocab1); @@ -65,22 +65,22 @@ void testGetAllVocabularies() throws Exception { when(dereferencingManagementServiceMock.getAllVocabularies()).thenReturn(dummyVocabList); dereferencingManagementControllerMock.perform(get("/vocabularies")) - .andExpect(jsonPath("$[0].uris[0]", is("http://dummy1.org/path1"))) - .andExpect(jsonPath("$[1].uris[0]", is("http://dummy2.org/path2"))) + .andExpect(jsonPath("$[0].uris[0]", is("https://dummy1.org/path1"))) + .andExpect(jsonPath("$[1].uris[0]", is("https://dummy2.org/path2"))) .andExpect(status().is(200)); } @Test - void testLoadVocabularies_validPrefix_expectSuccess() throws Exception { - doNothing().when(dereferencingManagementServiceMock).loadVocabularies(any(URI.class)); + void testLoadVocabularies_validDomain_expectSuccess() throws Exception { + doNothing().when(dereferencingManagementServiceMock).loadVocabularies(any(URL.class)); dereferencingManagementControllerMock.perform(post("/load_vocabularies") - .param("directory_url", "http://correctUrl/test/call")).andExpect(status().is(200)); + .param("directory_url", "https://valid.domain.com/test/call")).andExpect(status().is(200)); } @Test - void testLoadVocabularies_invalidPrefix_expectFail() throws Exception { + void testLoadVocabularies_invalidDomain_expectFail() throws Exception { dereferencingManagementControllerMock.perform(post("/load_vocabularies") - .param("directory_url", "http://wrongUrl")).andExpect(status().is(400)); + .param("directory_url", "https://invalid.domain.com")).andExpect(status().is(400)); } @Test diff --git a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/DereferencingManagementService.java b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/DereferencingManagementService.java index 7248f6a63..9a057884e 100644 --- a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/DereferencingManagementService.java +++ b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/DereferencingManagementService.java @@ -2,7 +2,7 @@ import eu.europeana.metis.dereference.Vocabulary; import eu.europeana.metis.dereference.vocimport.exception.VocabularyImportException; -import java.net.URI; +import java.net.URL; import java.util.List; /** @@ -26,8 +26,7 @@ public interface DereferencingManagementService { * Load the vocabularies from an online source. This does NOT purge the cache. * * @param directoryUrl The online location of the vocabulary directory. - * @throws VocabularyImportException In case some issue occurred while importing the - * vocabularies. + * @throws VocabularyImportException In case some issue occurred while importing the vocabularies. */ - void loadVocabularies(URI directoryUrl) throws VocabularyImportException; + void loadVocabularies(URL directoryUrl) throws VocabularyImportException; } diff --git a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementService.java b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementService.java index a0d608d3b..b6d98362f 100644 --- a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementService.java +++ b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementService.java @@ -8,8 +8,7 @@ import eu.europeana.metis.dereference.vocimport.VocabularyCollectionValidator; import eu.europeana.metis.dereference.vocimport.VocabularyCollectionValidatorImpl; import eu.europeana.metis.dereference.vocimport.exception.VocabularyImportException; - -import java.net.URI; +import java.net.URL; import java.util.ArrayList; import java.util.List; import org.springframework.beans.factory.annotation.Autowired; @@ -49,16 +48,16 @@ public void emptyCache() { } @Override - public void loadVocabularies(URI directoryUrl) throws VocabularyImportException { + public void loadVocabularies(URL directoryUrl) throws VocabularyImportException { try { - // Import and validate the vocabularies - final List vocabularies = new ArrayList<>(); - final VocabularyCollectionImporter importer = vocabularyCollectionImporterFactory - .createImporter(directoryUrl); - final VocabularyCollectionValidator validator = new VocabularyCollectionValidatorImpl(importer, - true, true, true); - validator.validateVocabularyOnly(vocabulary -> vocabularies.add(convertVocabulary(vocabulary))); + // Import and validate the vocabularies + final List vocabularies = new ArrayList<>(); + final VocabularyCollectionImporter importer = vocabularyCollectionImporterFactory + .createImporter(directoryUrl); + final VocabularyCollectionValidator validator = new VocabularyCollectionValidatorImpl(importer, + true, true, true); + validator.validateVocabularyOnly(vocabulary -> vocabularies.add(convertVocabulary(vocabulary))); // All vocabularies are loaded well. Now we replace the vocabularies. vocabularyDao.replaceAll(vocabularies); From 641b046d7848519844bcd386556902bb6dfc09db Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Thu, 24 Mar 2022 14:12:50 +0100 Subject: [PATCH 17/73] MET-4237 Align depublication status in metis (#520) * MET-4237 Implementation of indexPostProcessing de-publication status, with incremental processing * MET-4237 final implementation --- .../core/execution/WorkflowPostProcessor.java | 152 ++++++++---------- 1 file changed, 70 insertions(+), 82 deletions(-) diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java index 4c5e7e9d9..3eecebde1 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java @@ -31,6 +31,7 @@ import eu.europeana.metis.exception.BadContentException; import java.util.ArrayList; import java.util.Date; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -71,38 +72,15 @@ public WorkflowPostProcessor(DepublishRecordIdDao depublishRecordIdDao, this.dpsClient = dpsClient; } - /** - * This method performs post-processing after an individual workflow step. - * - * @param plugin The plugin that was successfully executed - * @param datasetId The dataset ID to which the plugin belongs - * @throws DpsException If communication with e-cloud dps failed - * @throws InvalidIndexPluginException If invalid type of plugin - * @throws BadContentException In case the records would violate the maximum number of de-published records that each dataset - * can have. - */ - void performPluginPostProcessing(AbstractExecutablePlugin plugin, String datasetId) - throws DpsException, InvalidIndexPluginException, BadContentException { - - final PluginType pluginType = plugin.getPluginType(); - LOGGER.info("Starting postprocessing of plugin {} in dataset {}.", pluginType, datasetId); - if (pluginType == PluginType.PREVIEW || pluginType == PluginType.PUBLISH) { - indexPostProcess(plugin, datasetId); - } else if (pluginType == PluginType.DEPUBLISH) { - depublishPostProcess((DepublishPlugin) plugin, datasetId); - } - LOGGER.info("Finished postprocessing of plugin {} in dataset {}.", pluginType, datasetId); - } - /** * Performs post-processing for indexing plugins * * @param indexPlugin The index plugin - * @param datasetId The dataset id - * @throws DpsException If communication with e-cloud dps failed + * @param datasetId The dataset id + * @throws DpsException If communication with e-cloud dps failed * @throws InvalidIndexPluginException If invalid type of plugin - * @throws BadContentException In case the records would violate the maximum number of de-published records that each dataset - * can have. + * @throws BadContentException In case the records would violate the maximum number of de-published records that each + * dataset can have. */ private void indexPostProcess(AbstractExecutablePlugin indexPlugin, String datasetId) throws DpsException, InvalidIndexPluginException, BadContentException { @@ -111,37 +89,24 @@ private void indexPostProcess(AbstractExecutablePlugin indexPlugin, String da targetIndexingDatabase = ((IndexToPreviewPlugin) indexPlugin).getTargetIndexingDatabase(); } else if (indexPlugin instanceof IndexToPublishPlugin) { targetIndexingDatabase = ((IndexToPublishPlugin) indexPlugin).getTargetIndexingDatabase(); - //Reset depublish status - depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, null, - DepublicationStatus.PENDING_DEPUBLICATION, null); - - final long totalRecords = dpsClient.getTotalMetisDatabaseRecords(indexPlugin.getExternalTaskId(), - ((IndexToPublishPlugin) indexPlugin).getTargetIndexingDatabase()); - List subTaskInfoList; - - // get chunked tasks from dataset id and topology name - for (int i = 0; i < totalRecords; i += ECLOUD_REQUEST_BATCH_SIZE) { - subTaskInfoList = dpsClient.getDetailedTaskReportBetweenChunks(indexPlugin.getTopologyName(), - Long.parseLong(indexPlugin.getExternalTaskId()), i, i + ECLOUD_REQUEST_BATCH_SIZE); - if (i >= totalRecords) { - subTaskInfoList = dpsClient.getDetailedTaskReportBetweenChunks(indexPlugin.getTopologyName(), - Long.parseLong(indexPlugin.getExternalTaskId()), (int) (totalRecords - (totalRecords % ECLOUD_REQUEST_BATCH_SIZE)), - (int) totalRecords); - } + + final boolean isIncremental = ((IndexToPublishPlugin) indexPlugin).getPluginMetadata().isIncrementalIndexing(); + + if (isIncremental) { // get all currently de-published records ids Set depublishedRecordIds = depublishRecordIdDao .getAllDepublishRecordIdsWithStatus(datasetId, DepublishRecordIdSortField.DEPUBLICATION_STATE, SortDirection.ASCENDING, DepublicationStatus.DEPUBLISHED); - // TODO: what if it's incremental - // filter the record ids that are a part of the given report, to be de-published - Set recordIdsToDepublish = subTaskInfoList.stream() - .filter(taskInfo -> depublishedRecordIds.contains( - taskInfo.getEuropeanaId())) - .map(SubTaskInfo::getEuropeanaId).collect(Collectors.toSet()); - // reset de-publish status - depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, recordIdsToDepublish, + List publishedDatasetRecordIds = dpsClient.searchPublishedDatasetRecords(indexPlugin.getExternalTaskId(), + new ArrayList<>(depublishedRecordIds)); + // reset de-publish status, pass recordIds to be de-published + depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, new HashSet<>(publishedDatasetRecordIds), + DepublicationStatus.PENDING_DEPUBLICATION, null); + } else { + // reset de-publish status, pass null, all records will be de-published + depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, null, DepublicationStatus.PENDING_DEPUBLICATION, null); } } else { @@ -156,7 +121,7 @@ private void indexPostProcess(AbstractExecutablePlugin indexPlugin, String da * Performs post-processing for de-publish plugins * * @param depublishPlugin The de-publish plugin - * @param datasetId The dataset id + * @param datasetId The dataset id * @throws DpsException If communication with e-cloud dps failed */ private void depublishPostProcess(DepublishPlugin depublishPlugin, String datasetId) @@ -168,37 +133,9 @@ private void depublishPostProcess(DepublishPlugin depublishPlugin, String datase } } - /** - * @param datasetId The dataset id - */ - private void depublishDatasetPostProcess(String datasetId) { - - // Set all depublished records back to PENDING. - depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, null, - DepublicationStatus.PENDING_DEPUBLICATION, null); - // Find latest PUBLISH Type Plugin and set dataStatus to DELETED. - final PluginWithExecutionId latestSuccessfulPlugin = workflowExecutionDao - .getLatestSuccessfulPlugin(datasetId, OrchestratorService.PUBLISH_TYPES); - if (Objects.nonNull(latestSuccessfulPlugin) && Objects - .nonNull(latestSuccessfulPlugin.getPlugin())) { - final WorkflowExecution workflowExecutionToUpdate = workflowExecutionDao - .getById(latestSuccessfulPlugin.getExecutionId()); - final Optional metisPluginWithType = workflowExecutionToUpdate - .getMetisPluginWithType(latestSuccessfulPlugin.getPlugin().getPluginType()); - if (metisPluginWithType.isPresent()) { - metisPluginWithType.get().setDataStatus(DataStatus.DELETED); - workflowExecutionDao.updateWorkflowPlugins(workflowExecutionToUpdate); - } - } - // Set publication fitness to UNFIT. - final Dataset dataset = datasetDao.getDatasetByDatasetId(datasetId); - dataset.setPublicationFitness(PublicationFitness.UNFIT); - datasetDao.update(dataset); - } - /** * @param depublishPlugin The de-publish plugin - * @param datasetId The dataset id + * @param datasetId The dataset id * @throws DpsException If communication with e-cloud dps failed */ private void depublishRecordPostProcess(DepublishPlugin depublishPlugin, String datasetId) @@ -239,4 +176,55 @@ private void depublishRecordPostProcess(DepublishPlugin depublishPlugin, String datasetDao.update(dataset); } } + + /** + * @param datasetId The dataset id + */ + private void depublishDatasetPostProcess(String datasetId) { + + // Set all depublished records back to PENDING. + depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, null, + DepublicationStatus.PENDING_DEPUBLICATION, null); + // Find latest PUBLISH Type Plugin and set dataStatus to DELETED. + final PluginWithExecutionId latestSuccessfulPlugin = workflowExecutionDao + .getLatestSuccessfulPlugin(datasetId, OrchestratorService.PUBLISH_TYPES); + if (Objects.nonNull(latestSuccessfulPlugin) && Objects + .nonNull(latestSuccessfulPlugin.getPlugin())) { + final WorkflowExecution workflowExecutionToUpdate = workflowExecutionDao + .getById(latestSuccessfulPlugin.getExecutionId()); + final Optional metisPluginWithType = workflowExecutionToUpdate + .getMetisPluginWithType(latestSuccessfulPlugin.getPlugin().getPluginType()); + if (metisPluginWithType.isPresent()) { + metisPluginWithType.get().setDataStatus(DataStatus.DELETED); + workflowExecutionDao.updateWorkflowPlugins(workflowExecutionToUpdate); + } + } + // Set publication fitness to UNFIT. + final Dataset dataset = datasetDao.getDatasetByDatasetId(datasetId); + dataset.setPublicationFitness(PublicationFitness.UNFIT); + datasetDao.update(dataset); + } + + /** + * This method performs post-processing after an individual workflow step. + * + * @param plugin The plugin that was successfully executed + * @param datasetId The dataset ID to which the plugin belongs + * @throws DpsException If communication with e-cloud dps failed + * @throws InvalidIndexPluginException If invalid type of plugin + * @throws BadContentException In case the records would violate the maximum number of de-published records that each dataset + * can have. + */ + void performPluginPostProcessing(AbstractExecutablePlugin plugin, String datasetId) + throws DpsException, InvalidIndexPluginException, BadContentException { + + final PluginType pluginType = plugin.getPluginType(); + LOGGER.info("Starting postprocessing of plugin {} in dataset {}.", pluginType, datasetId); + if (pluginType == PluginType.PREVIEW || pluginType == PluginType.PUBLISH) { + indexPostProcess(plugin, datasetId); + } else if (pluginType == PluginType.DEPUBLISH) { + depublishPostProcess((DepublishPlugin) plugin, datasetId); + } + LOGGER.info("Finished postprocessing of plugin {} in dataset {}.", pluginType, datasetId); + } } \ No newline at end of file From 2dc505c7db12cadb689f421ad74e17970c8ce72d Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Fri, 25 Mar 2022 10:32:41 +0100 Subject: [PATCH 18/73] Update CONTRIBUTING.MD for squash merges --- CONTRIBUTING.MD | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.MD b/CONTRIBUTING.MD index 1e59b1733..e6d450de1 100644 --- a/CONTRIBUTING.MD +++ b/CONTRIBUTING.MD @@ -128,14 +128,38 @@ If there is any doubt on fixing the merge conflicts while merging, the implement worked on the relevant changes that were introduced on the destination branch. The merge can be performed on the github pull request page or manually(especially for conflicts). -To do this manually, checkout the destination branch(usually `develop`) and execute the merge command with the `--no-ff` -parameter: +To do this manually, checkout the destination branch(usually `develop`). +We prefer squash merging or alternatively non fast forward merging. + +- A squash merge can be performed with the `--squash` parameter: + + `git merge --squash ` + + To complete the squash merge, a commit has to also be performed if done locally. + The commit should be formatted as the following template(replicating github squashed commits): + ```/ (#) + (optionally as description) + * List of all commit messages from the pull requst + ``` + Example message: + ``` + Debt/met 4250 refactor code to remove mock maker inline (#508) + * MET-4250 Update NetworkUtil + + * MET-4250 Update RdfConversionUtils + + * MET-4250 Javadocs and cleanup + + * MET-4250 Remove mockito inline from root pom``` + + +- A non fast forward merge can be performed with the `--no-ff` parameter: `git merge --no-ff ` -The merger should check that the local branch is building before and after merging. If there were merge conflicts that were -resolved during the merge, then a local deployment should be triggered and verified. If the build succeeds the destination branch -can be pushed to the remote repository and the pull request will be resolved. +The merger should check that the local branch is building before and after merging. +If there were merge conflicts that were resolved during the merge, then a local deployment should be triggered and verified. +If the build succeeds the destination branch can be pushed to the remote repository and the pull request will be resolved. The reviewer can now move the ticket ahead in the board and re-assign it to the implementor. From 3dc9004fc2dfc895ed28b34143bc7a39d43cc307 Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Fri, 25 Mar 2022 10:49:39 +0100 Subject: [PATCH 19/73] Update CONTRIBUTING.MD to fix line breaks --- CONTRIBUTING.MD | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.MD b/CONTRIBUTING.MD index e6d450de1..1b650ed48 100644 --- a/CONTRIBUTING.MD +++ b/CONTRIBUTING.MD @@ -137,7 +137,8 @@ We prefer squash merging or alternatively non fast forward merging. To complete the squash merge, a commit has to also be performed if done locally. The commit should be formatted as the following template(replicating github squashed commits): - ```/ (#) + ``` + / (#) (optionally as description) * List of all commit messages from the pull requst ``` @@ -150,7 +151,8 @@ We prefer squash merging or alternatively non fast forward merging. * MET-4250 Javadocs and cleanup - * MET-4250 Remove mockito inline from root pom``` + * MET-4250 Remove mockito inline from root pom + ``` - A non fast forward merge can be performed with the `--no-ff` parameter: From dba23ae5a67b486ef60192719fa3a5de78d3cc89 Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Fri, 8 Apr 2022 11:17:38 +0200 Subject: [PATCH 20/73] MET-4374 Secure regex backtracking in dereference (#521) * MET-4374 Reproduce error * MET-4374 Reproduce error * MET-4374 Solve backtracking * MET-4374 Cleanup to remove futher issues * MET-4374 Fix issues and add some tests * MET-4374 Handle review --- .../IncomingRecordToEdmConverter.java | 94 ----------- .../IncomingRecordToEdmTransformer.java | 148 ++++++++++++++++++ .../IncomingRecordToEdmConverterTest.java | 35 ----- .../IncomingRecordToEdmTransformerTest.java | 81 ++++++++++ .../src/test/resources/copy_xml.xslt | 11 ++ .../src/test/resources/invalid_xml.xml | 3 + .../src/test/resources/produce_empty.xslt | 4 + .../test/resources/produce_invalid_xml.xslt | 7 + .../src/test/resources/yso_p105069.xml | 45 ++++++ .../VocabularyCollectionMavenRule.java | 2 +- .../VocabularyCollectionValidatorImpl.java | 110 ++++++------- .../service/MongoDereferenceService.java | 131 ++++++++-------- 12 files changed, 421 insertions(+), 250 deletions(-) delete mode 100644 metis-dereference/metis-dereference-common/src/main/java/eu/europeana/metis/dereference/IncomingRecordToEdmConverter.java create mode 100644 metis-dereference/metis-dereference-common/src/main/java/eu/europeana/metis/dereference/IncomingRecordToEdmTransformer.java delete mode 100644 metis-dereference/metis-dereference-common/src/test/java/eu/europeana/metis/dereference/IncomingRecordToEdmConverterTest.java create mode 100644 metis-dereference/metis-dereference-common/src/test/java/eu/europeana/metis/dereference/IncomingRecordToEdmTransformerTest.java create mode 100644 metis-dereference/metis-dereference-common/src/test/resources/copy_xml.xslt create mode 100644 metis-dereference/metis-dereference-common/src/test/resources/invalid_xml.xml create mode 100644 metis-dereference/metis-dereference-common/src/test/resources/produce_empty.xslt create mode 100644 metis-dereference/metis-dereference-common/src/test/resources/produce_invalid_xml.xslt create mode 100644 metis-dereference/metis-dereference-common/src/test/resources/yso_p105069.xml diff --git a/metis-dereference/metis-dereference-common/src/main/java/eu/europeana/metis/dereference/IncomingRecordToEdmConverter.java b/metis-dereference/metis-dereference-common/src/main/java/eu/europeana/metis/dereference/IncomingRecordToEdmConverter.java deleted file mode 100644 index ed4170896..000000000 --- a/metis-dereference/metis-dereference-common/src/main/java/eu/europeana/metis/dereference/IncomingRecordToEdmConverter.java +++ /dev/null @@ -1,94 +0,0 @@ -package eu.europeana.metis.dereference; - -import java.io.StringReader; -import java.io.StringWriter; -import java.nio.charset.StandardCharsets; -import java.util.regex.Pattern; -import javax.xml.XMLConstants; -import javax.xml.transform.OutputKeys; -import javax.xml.transform.Source; -import javax.xml.transform.Templates; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.stream.StreamResult; -import javax.xml.transform.stream.StreamSource; -import net.sf.saxon.BasicTransformerFactory; - -/** - * Convert an incoming record to EDM. - */ -public class IncomingRecordToEdmConverter { - - private static final String EMPTY_XML_REGEX = "\\A(<\\?.*?\\?>||\\s)*\\Z"; - private static final Pattern EMPTY_XML_CHECKER = Pattern.compile(EMPTY_XML_REGEX, Pattern.DOTALL); - - /** Vocabulary XSLs require the resource ID as a parameter. This is the parameter name. **/ - private static final String TARGET_ID_PARAMETER_NAME = "targetId"; - - private final Templates template; - - /** - * Create a converter for the given vocabulary. - * - * @param vocabulary The vocabulary for which to perform the conversion. - * @throws TransformerException In case the input could not be parsed or the conversion could not - * be set up. - */ - public IncomingRecordToEdmConverter(Vocabulary vocabulary) throws TransformerException { - this(vocabulary.getXslt()); - } - - /** - * Create a converter for the transformation. - * - * @param xslt The xslt representing the conversion to perform. - * @throws TransformerException In case the input could not be parsed or the conversion could not - * be set up. - */ - public IncomingRecordToEdmConverter(String xslt) throws TransformerException { - final Source xsltSource = new StreamSource(new StringReader(xslt)); - // Ensure that the Saxon library is used by choosing the right transformer factory. - final TransformerFactory factory = new BasicTransformerFactory(); - factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - this.template = factory.newTemplates(xsltSource); - } - - /** - * Convert the given record. - * - * @param record The incoming record (that comes from the vocabulary). - * @param recordId The record ID of the incoming record. - * @return The EDM record, or null if the record couldn't be transformed. - * @throws TransformerException In case there is a problem performing the transformation. - */ - public String convert(String record, String recordId) throws TransformerException { - - // Set up the transformer - final Source source = new StreamSource(new StringReader(record)); - final StringWriter stringWriter = new StringWriter(); - final Transformer transformer = template.newTransformer(); - transformer.setParameter(TARGET_ID_PARAMETER_NAME, recordId); - transformer.setOutputProperty(OutputKeys.INDENT, "yes"); - transformer.setOutputProperty(OutputKeys.ENCODING, StandardCharsets.UTF_8.name()); - - // Perform the transformation. - transformer.transform(source, new StreamResult(stringWriter)); - final String result = stringWriter.toString(); - - // Check whether there is a result (any tag in the file). - return isEmptyXml(result) ? null : result; - } - - /** - * This method analyzes the XML file and decides whether or not it has any content. Excluded are - * space characters, the XML header and XML comments. Note: if this method returns true, the input - * is not technically a valid XML as it doesn't have a root node. - * - * @param file The input XML. - * @return Whether the XML has any content. - */ - static boolean isEmptyXml(String file) { - return EMPTY_XML_CHECKER.matcher(file).matches(); - } -} diff --git a/metis-dereference/metis-dereference-common/src/main/java/eu/europeana/metis/dereference/IncomingRecordToEdmTransformer.java b/metis-dereference/metis-dereference-common/src/main/java/eu/europeana/metis/dereference/IncomingRecordToEdmTransformer.java new file mode 100644 index 000000000..8d2677b9a --- /dev/null +++ b/metis-dereference/metis-dereference-common/src/main/java/eu/europeana/metis/dereference/IncomingRecordToEdmTransformer.java @@ -0,0 +1,148 @@ +package eu.europeana.metis.dereference; + +import static eu.europeana.metis.utils.CommonStringValues.CRLF_PATTERN; + +import eu.europeana.metis.exception.BadContentException; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.regex.Pattern; +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Source; +import javax.xml.transform.Templates; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.stream.StreamResult; +import javax.xml.transform.stream.StreamSource; +import net.sf.saxon.BasicTransformerFactory; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.xml.sax.SAXException; + +/** + * Convert an incoming record to EDM. + */ +public class IncomingRecordToEdmTransformer { + + private static final Logger LOGGER = LoggerFactory.getLogger(IncomingRecordToEdmTransformer.class); + private static final Pattern XML_DECLARATION_CHECKER = Pattern.compile("\\A<\\?[^?]*\\?>\\s*\\z"); + + /** + * Vocabulary XSLs require the resource ID as a parameter. This is the parameter name. + **/ + private static final String TARGET_ID_PARAMETER_NAME = "targetId"; + + private final Templates template; + private final DocumentBuilderFactory documentBuilderFactory; + + /** + * Create a converter for the transformation. + * + * @param xslt The xslt representing the conversion to perform. + * @throws TransformerException if the transformer could not be initialized + * @throws ParserConfigurationException if the xml builder could not be initialized + */ + public IncomingRecordToEdmTransformer(String xslt) throws TransformerException, ParserConfigurationException { + final Source xsltSource = new StreamSource(new StringReader(xslt)); + // Ensure that the Saxon library is used by choosing the right transformer factory. + final TransformerFactory transformerFactory = new BasicTransformerFactory(); + transformerFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + this.template = transformerFactory.newTemplates(xsltSource); + + documentBuilderFactory = DocumentBuilderFactory.newInstance(); + documentBuilderFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + documentBuilderFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + documentBuilderFactory.setNamespaceAware(true); + } + + /** + * Transform the given xmlRecord. + * + * @param xmlRecord The incoming xmlRecord (that comes from the vocabulary). + * @param resourceId The xmlRecord ID of the incoming xmlRecord. + * @return The EDM xmlRecord, or null if the xmlRecord couldn't be transformed. + * @throws BadContentException if there was a problem performing the transformation. + */ + public Optional transform(String xmlRecord, String resourceId) throws BadContentException { + // Set up the transformer + final Source source = new StreamSource(new StringReader(xmlRecord)); + final StringWriter transformedXmlWriter = new StringWriter(); + final Transformer transformer; + try { + transformer = template.newTransformer(); + transformer.setParameter(TARGET_ID_PARAMETER_NAME, resourceId); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + transformer.setOutputProperty(OutputKeys.ENCODING, StandardCharsets.UTF_8.name()); + + // Perform the transformation. + transformer.transform(source, new StreamResult(transformedXmlWriter)); + } catch (TransformerException e) { + throw new BadContentException("Transformation failure", e); + } + return getValidatedXml(resourceId, transformedXmlWriter.toString()); + } + + /** + * Returns an optional which is empty if the provided xml is a validated empty xml or contains the xml itself if it's a valid + * parsable xml. + * + * @param resourceId the resource id + * @param xml the xml + * @return the optional being empty or with the xml contents + * @throws BadContentException if the xml parsing failed + */ + @NotNull + private Optional getValidatedXml(String resourceId, String xml) throws BadContentException { + final Optional xmlResponse; + if (isEmptyXml(xml)) { + xmlResponse = Optional.empty(); + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Transformed entity {} results to an empty XML.", + CRLF_PATTERN.matcher(resourceId).replaceAll("")); + } + } else { + try { + assertXmlValidity(xml); + xmlResponse = Optional.of(xml); + } catch (ParserConfigurationException | IOException | SAXException e) { + throw new BadContentException("Transformed xml is not valid", e); + } + } + + return xmlResponse; + } + + /** + * Asserts if the provided xml is valid and can be parsed. + * + * @param xml the xml string + * @throws ParserConfigurationException if xml parsing failed + * @throws IOException if xml parsing failed + * @throws SAXException if xml parsing failed + */ + private void assertXmlValidity(String xml) throws ParserConfigurationException, IOException, SAXException { + documentBuilderFactory.newDocumentBuilder().parse(new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))); + } + + /** + * Checks if the provided xml is empty. + *

    + * Emptiness is verifying if the only the xml header declaration is present. Note: if this method returns true, the input is not + * technically a valid XML as it doesn't have a root node. + *

    + * + * @param xml the input XML. + * @return true if xml is empty + */ + private boolean isEmptyXml(String xml) { + return XML_DECLARATION_CHECKER.matcher(xml).matches(); + } +} diff --git a/metis-dereference/metis-dereference-common/src/test/java/eu/europeana/metis/dereference/IncomingRecordToEdmConverterTest.java b/metis-dereference/metis-dereference-common/src/test/java/eu/europeana/metis/dereference/IncomingRecordToEdmConverterTest.java deleted file mode 100644 index 5e6d2ce78..000000000 --- a/metis-dereference/metis-dereference-common/src/test/java/eu/europeana/metis/dereference/IncomingRecordToEdmConverterTest.java +++ /dev/null @@ -1,35 +0,0 @@ -package eu.europeana.metis.dereference; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import eu.europeana.metis.dereference.IncomingRecordToEdmConverter; -import org.junit.jupiter.api.Test; - -class IncomingRecordToEdmConverterTest { - - - @Test - void testIsEmptyXml() { - - assertTrue(IncomingRecordToEdmConverter.isEmptyXml("")); - assertTrue(IncomingRecordToEdmConverter.isEmptyXml("")); - assertTrue( - IncomingRecordToEdmConverter.isEmptyXml("")); - assertTrue( - IncomingRecordToEdmConverter.isEmptyXml(" ")); - assertTrue(IncomingRecordToEdmConverter - .isEmptyXml("\n")); - assertTrue(IncomingRecordToEdmConverter - .isEmptyXml("\n")); - assertTrue(IncomingRecordToEdmConverter - .isEmptyXml(" \n ")); - - assertFalse(IncomingRecordToEdmConverter.isEmptyXml("A")); - assertFalse(IncomingRecordToEdmConverter.isEmptyXml( - " \n \n \n ")); - assertFalse(IncomingRecordToEdmConverter - .isEmptyXml("")); - - } -} diff --git a/metis-dereference/metis-dereference-common/src/test/java/eu/europeana/metis/dereference/IncomingRecordToEdmTransformerTest.java b/metis-dereference/metis-dereference-common/src/test/java/eu/europeana/metis/dereference/IncomingRecordToEdmTransformerTest.java new file mode 100644 index 000000000..0438de292 --- /dev/null +++ b/metis-dereference/metis-dereference-common/src/test/java/eu/europeana/metis/dereference/IncomingRecordToEdmTransformerTest.java @@ -0,0 +1,81 @@ +package eu.europeana.metis.dereference; + + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import eu.europeana.metis.exception.BadContentException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Objects; +import java.util.Optional; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +class IncomingRecordToEdmTransformerTest { + + private static final String copyXmlXsltFileName = "copy_xml.xslt"; + private static final String produceEmptyXsltFileName = "produce_empty.xslt"; + private static final String produceInvalidXmlXsltFileName = "produce_invalid_xml.xslt"; + private static final String ysoP105069FileName = "yso_p105069.xml"; + private static final String invalidXmlFileName = "invalid_xml.xml"; + + private static String copyXmlXsltString; + private static String produceEmptyXsltString; + private static String produceInvalidXmlXsltString; + private static String ysoP105069String; + private static String invalidXmlString; + + @BeforeAll + static void setUp() throws Exception { + ClassLoader classLoader = IncomingRecordToEdmTransformerTest.class.getClassLoader(); + Path path = Paths.get(Objects.requireNonNull(classLoader.getResource(copyXmlXsltFileName)).toURI()); + copyXmlXsltString = Files.readString(path, StandardCharsets.UTF_8); + + path = Paths.get(Objects.requireNonNull(classLoader.getResource(produceEmptyXsltFileName)).toURI()); + produceEmptyXsltString = Files.readString(path, StandardCharsets.UTF_8); + + path = Paths.get(Objects.requireNonNull(classLoader.getResource(produceInvalidXmlXsltFileName)).toURI()); + produceInvalidXmlXsltString = Files.readString(path, StandardCharsets.UTF_8); + + path = Paths.get(Objects.requireNonNull(classLoader.getResource(ysoP105069FileName)).toURI()); + ysoP105069String = Files.readString(path, StandardCharsets.UTF_8); + + path = Paths.get(Objects.requireNonNull(classLoader.getResource(invalidXmlFileName)).toURI()); + invalidXmlString = Files.readString(path, StandardCharsets.UTF_8); + } + + @Test + void transform() throws Exception { + IncomingRecordToEdmTransformer incomingRecordToEdmTransformer = new IncomingRecordToEdmTransformer(copyXmlXsltString); + final Optional transformedOptional = incomingRecordToEdmTransformer.transform(ysoP105069String, + "http://www.yso.fi/onto/yso/p105069"); + assertTrue(transformedOptional.isPresent()); + } + + @Test + void transform_EmptyXslt() throws Exception { + IncomingRecordToEdmTransformer incomingRecordToEdmTransformer = new IncomingRecordToEdmTransformer(produceEmptyXsltString); + final Optional transformedOptional = incomingRecordToEdmTransformer.transform(ysoP105069String, + "http://www.yso.fi/onto/yso/p105069"); + assertTrue(transformedOptional.isEmpty()); + } + + @Test + void transform_InvalidSourceXml_BadContentException() throws Exception { + IncomingRecordToEdmTransformer incomingRecordToEdmTransformer = new IncomingRecordToEdmTransformer(copyXmlXsltString); + assertThrows(BadContentException.class, () -> incomingRecordToEdmTransformer.transform(invalidXmlString, + "http://www.yso.fi/onto/yso/p105069")); + } + + @Test + void transform_InvalidXml_BadContentException() throws Exception { + IncomingRecordToEdmTransformer incomingRecordToEdmTransformer = new IncomingRecordToEdmTransformer( + produceInvalidXmlXsltString); + assertThrows(BadContentException.class, () -> incomingRecordToEdmTransformer.transform(ysoP105069String, + "http://www.yso.fi/onto/yso/p105069")); + } +} + diff --git a/metis-dereference/metis-dereference-common/src/test/resources/copy_xml.xslt b/metis-dereference/metis-dereference-common/src/test/resources/copy_xml.xslt new file mode 100644 index 000000000..28082a2d9 --- /dev/null +++ b/metis-dereference/metis-dereference-common/src/test/resources/copy_xml.xslt @@ -0,0 +1,11 @@ + + + + + + + + + + \ No newline at end of file diff --git a/metis-dereference/metis-dereference-common/src/test/resources/invalid_xml.xml b/metis-dereference/metis-dereference-common/src/test/resources/invalid_xml.xml new file mode 100644 index 000000000..7f7e28205 --- /dev/null +++ b/metis-dereference/metis-dereference-common/src/test/resources/invalid_xml.xml @@ -0,0 +1,3 @@ + + + diff --git a/metis-dereference/metis-dereference-common/src/test/resources/produce_empty.xslt b/metis-dereference/metis-dereference-common/src/test/resources/produce_empty.xslt new file mode 100644 index 000000000..777a7600d --- /dev/null +++ b/metis-dereference/metis-dereference-common/src/test/resources/produce_empty.xslt @@ -0,0 +1,4 @@ + + + \ No newline at end of file diff --git a/metis-dereference/metis-dereference-common/src/test/resources/produce_invalid_xml.xslt b/metis-dereference/metis-dereference-common/src/test/resources/produce_invalid_xml.xslt new file mode 100644 index 000000000..e81c022af --- /dev/null +++ b/metis-dereference/metis-dereference-common/src/test/resources/produce_invalid_xml.xslt @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/metis-dereference/metis-dereference-common/src/test/resources/yso_p105069.xml b/metis-dereference/metis-dereference-common/src/test/resources/yso_p105069.xml new file mode 100644 index 000000000..8998f86a4 --- /dev/null +++ b/metis-dereference/metis-dereference-common/src/test/resources/yso_p105069.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + Tjeckien + Tšekki + Czech Republic + + + + Källa för positionsinformation: Wikidata. + Location information source: Wikidata. + Sijaintitietojen lähde: Wikidata. + 1990-06-18 + 2016-05-23T16:13:36+03:00 + + + + + + + + + + Praha + Prague + Prag + 50.08333 + 14.41667 + + + + diff --git a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionMavenRule.java b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionMavenRule.java index 9d21ffb86..68195cf59 100644 --- a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionMavenRule.java +++ b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionMavenRule.java @@ -71,7 +71,7 @@ public class VocabularyCollectionMavenRule implements EnforcerRule { */ private String vocabularyDirectoryFile = null; - private VocabularyCollectionImporterFactory vocabularyCollectionImporterFactory; + private final VocabularyCollectionImporterFactory vocabularyCollectionImporterFactory = new VocabularyCollectionImporterFactory(); /** * No-arguments constructor, required for maven instantiation. diff --git a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionValidatorImpl.java b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionValidatorImpl.java index 62ac84952..36b4ef706 100644 --- a/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionValidatorImpl.java +++ b/metis-dereference/metis-dereference-import/src/main/java/eu/europeana/metis/dereference/vocimport/VocabularyCollectionValidatorImpl.java @@ -1,22 +1,27 @@ package eu.europeana.metis.dereference.vocimport; import eu.europeana.enrichment.utils.EnrichmentBaseConverter; -import eu.europeana.metis.dereference.IncomingRecordToEdmConverter; +import eu.europeana.metis.dereference.IncomingRecordToEdmTransformer; import eu.europeana.metis.dereference.RdfRetriever; import eu.europeana.metis.dereference.vocimport.exception.VocabularyImportException; import eu.europeana.metis.dereference.vocimport.model.Vocabulary; import eu.europeana.metis.dereference.vocimport.model.VocabularyLoader; import eu.europeana.metis.dereference.vocimport.utils.NonCollidingPathVocabularyTrie; +import eu.europeana.metis.exception.BadContentException; import java.io.IOException; import java.net.URISyntaxException; import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.function.Consumer; import javax.xml.bind.JAXBException; +import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; -import org.apache.commons.lang3.StringUtils; +/** + * Class that contains functionality to validate vocabularies using a {@link VocabularyCollectionImporter}. + */ public class VocabularyCollectionValidatorImpl implements VocabularyCollectionValidator { private final VocabularyCollectionImporter importer; @@ -28,16 +33,15 @@ public class VocabularyCollectionValidatorImpl implements VocabularyCollectionVa * Constructor. * * @param importer Vocabulary importer. - * @param lenientOnLackOfExamples Whether the the validator is lenient on vocabulary mappings - * without examples. - * @param lenientOnMappingTestFailures Whether the validator is lenient on errors and unmet - * expectations when applying the mapping to the example and counterexample values. - * @param lenientOnExampleRetrievalFailures Whether the validator is lenient on example or - * counterexample retrieval (download) issues. + * @param lenientOnLackOfExamples Whether the the validator is lenient on vocabulary mappings without examples. + * @param lenientOnMappingTestFailures Whether the validator is lenient on errors and unmet expectations when applying the + * mapping to the example and counterexample values. + * @param lenientOnExampleRetrievalFailures Whether the validator is lenient on example or counterexample retrieval (download) + * issues. */ public VocabularyCollectionValidatorImpl(VocabularyCollectionImporter importer, - boolean lenientOnLackOfExamples, boolean lenientOnMappingTestFailures, - boolean lenientOnExampleRetrievalFailures) { + boolean lenientOnLackOfExamples, boolean lenientOnMappingTestFailures, + boolean lenientOnExampleRetrievalFailures) { this.importer = importer; this.lenientOnLackOfExamples = lenientOnLackOfExamples; this.lenientOnMappingTestFailures = lenientOnMappingTestFailures; @@ -46,7 +50,7 @@ public VocabularyCollectionValidatorImpl(VocabularyCollectionImporter importer, @Override public void validate(Consumer vocabularyReceiver, Consumer warningReceiver) - throws VocabularyImportException { + throws VocabularyImportException { validateInternal(vocabularyReceiver, warningReceiver, true); } @@ -61,7 +65,7 @@ private void validateInternal(Consumer vocabularyReceiver, final Iterable vocabularyLoaders = importer.importVocabularies(); for (VocabularyLoader loader : vocabularyLoaders) { final Vocabulary vocabulary = loader.load(); - final IncomingRecordToEdmConverter converter = validateVocabulary(vocabulary, + final IncomingRecordToEdmTransformer converter = validateVocabulary(vocabulary, duplicationChecker); if (validateExamples) { validateExamples(vocabulary, warningReceiver, converter); @@ -70,29 +74,29 @@ private void validateInternal(Consumer vocabularyReceiver, } } - private IncomingRecordToEdmConverter validateVocabulary(Vocabulary vocabulary, - DuplicationChecker duplicationChecker) throws VocabularyImportException { + private IncomingRecordToEdmTransformer validateVocabulary(Vocabulary vocabulary, + DuplicationChecker duplicationChecker) throws VocabularyImportException { // Check the presence of the required fields. if (vocabulary.getName() == null) { throw new VocabularyImportException( - String.format("No vocabulary name given in metadata at [%s].", - vocabulary.getReadableMetadataLocation())); + String.format("No vocabulary name given in metadata at [%s].", + vocabulary.getReadableMetadataLocation())); } if (vocabulary.getTypes().isEmpty()) { throw new VocabularyImportException( - String.format("No vocabulary type(s) given in metadata at [%s].", - vocabulary.getReadableMetadataLocation())); + String.format("No vocabulary type(s) given in metadata at [%s].", + vocabulary.getReadableMetadataLocation())); } if (vocabulary.getPaths().isEmpty()) { throw new VocabularyImportException( - String.format("No vocabulary path(s) given in metadata at [%s].", - vocabulary.getReadableMetadataLocation())); + String.format("No vocabulary path(s) given in metadata at [%s].", + vocabulary.getReadableMetadataLocation())); } if (vocabulary.getTransformation() == null) { throw new VocabularyImportException( - String.format("No transformation given in mapping at [%s].", - vocabulary.getReadableMappingLocation())); + String.format("No transformation given in mapping at [%s].", + vocabulary.getReadableMappingLocation())); } // Check whether name and links are unique. @@ -100,21 +104,21 @@ private IncomingRecordToEdmConverter validateVocabulary(Vocabulary vocabulary, // Verifying the xslt - compile it. try { - return new IncomingRecordToEdmConverter(vocabulary.getTransformation()); - } catch (TransformerException e) { + return new IncomingRecordToEdmTransformer(vocabulary.getTransformation()); + } catch (TransformerException | ParserConfigurationException e) { throw new VocabularyImportException( - String.format("Error in the transformation given in mapping at [%s].", - vocabulary.getReadableMappingLocation()), e); + String.format("Error in the transformation given in mapping at [%s].", + vocabulary.getReadableMappingLocation()), e); } } private void validateExamples(Vocabulary vocabulary, Consumer warningReceiver, - IncomingRecordToEdmConverter converter) throws VocabularyImportException { + IncomingRecordToEdmTransformer converter) throws VocabularyImportException { // Testing the examples (if there are any - otherwise issue warning). if (vocabulary.getExamples().isEmpty()) { final String message = String.format("No examples specified for metadata at [%s].", - vocabulary.getReadableMetadataLocation()); + vocabulary.getReadableMetadataLocation()); if (lenientOnLackOfExamples) { warningReceiver.accept(message); } else { @@ -123,26 +127,26 @@ private void validateExamples(Vocabulary vocabulary, Consumer warningRec } for (String example : vocabulary.getExamples()) { testExample(converter, example, vocabulary.getSuffix(), false, - vocabulary.getReadableMetadataLocation(), warningReceiver); + vocabulary.getReadableMetadataLocation(), warningReceiver); } // Testing the counter examples (if there are any). for (String example : vocabulary.getCounterExamples()) { testExample(converter, example, vocabulary.getSuffix(), true, - vocabulary.getReadableMetadataLocation(), warningReceiver); + vocabulary.getReadableMetadataLocation(), warningReceiver); } } private String getTestErrorMessage(String example, boolean isCounterExample, - String readableMetadataLocation, String sentenceContinuation, Exception exception) { + String readableMetadataLocation, String sentenceContinuation, Exception exception) { final String sentence = String.format("%s '%s' in metadata at [%s] %s.", - isCounterExample ? "Counterexample" : "Example", example, readableMetadataLocation, - sentenceContinuation); - return sentence + (exception == null ? "" : " Error: " + exception.getMessage()); + isCounterExample ? "Counterexample" : "Example", example, readableMetadataLocation, + sentenceContinuation); + return sentence + (exception == null ? "" : String.format(" Error: %s", exception.getMessage())); } private void processTestError(String message, boolean isWarning, Consumer warningReceiver, - Exception originalException) throws VocabularyImportException { + Exception originalException) throws VocabularyImportException { if (isWarning) { warningReceiver.accept(message); } else { @@ -150,9 +154,9 @@ private void processTestError(String message, boolean isWarning, Consumer warningReceiver) throws VocabularyImportException { + private void testExample(IncomingRecordToEdmTransformer incomingRecordToEdmTransformer, String example, String suffix, + boolean isCounterExample, String readableMetadataLocation, + Consumer warningReceiver) throws VocabularyImportException { // Retrieve the example - is not null. final String exampleContent; @@ -160,40 +164,40 @@ private void testExample(IncomingRecordToEdmConverter converter, String example, exampleContent = new RdfRetriever().retrieve(example, suffix); } catch (IOException | URISyntaxException e) { final String message = getTestErrorMessage(example, isCounterExample, - readableMetadataLocation, "could not be retrieved", e); + readableMetadataLocation, "could not be retrieved", e); processTestError(message, lenientOnExampleRetrievalFailures, warningReceiver, e); return; } // Convert the example - final String result; + final Optional result; try { - result = converter.convert(exampleContent, example); - } catch (TransformerException e) { + result = incomingRecordToEdmTransformer.transform(exampleContent, example); + } catch (BadContentException e) { final String message = getTestErrorMessage(example, isCounterExample, - readableMetadataLocation, "could not be mapped", e); + readableMetadataLocation, "could not be mapped", e); processTestError(message, lenientOnMappingTestFailures, warningReceiver, e); return; } // Check whether the example yielded a mapped entity or not - if (StringUtils.isNotBlank(result) && isCounterExample) { + if (result.isPresent() && isCounterExample) { final String message = getTestErrorMessage(example, isCounterExample, - readableMetadataLocation, "yielded a mapped result, but is expected not to", null); + readableMetadataLocation, "yielded a mapped result, but is expected not to", null); processTestError(message, lenientOnMappingTestFailures, warningReceiver, null); - } else if (StringUtils.isBlank(result) && !isCounterExample) { + } else if (result.isEmpty() && !isCounterExample) { final String message = getTestErrorMessage(example, isCounterExample, - readableMetadataLocation, "did not yield a mapped result, but is expected to", null); + readableMetadataLocation, "did not yield a mapped result, but is expected to", null); processTestError(message, lenientOnMappingTestFailures, warningReceiver, null); } // Check whether the example yielded valid XML - if (StringUtils.isNotBlank(result)) { + if (result.isPresent()) { try { - EnrichmentBaseConverter.convertToEnrichmentBase(result); + EnrichmentBaseConverter.convertToEnrichmentBase(result.get()); } catch (JAXBException e) { final String message = getTestErrorMessage(example, isCounterExample, - readableMetadataLocation, "did not yield a valid XML", e); + readableMetadataLocation, "did not yield a valid XML", e); throw new VocabularyImportException(message, e); } } @@ -211,11 +215,11 @@ void checkAndRegister(Vocabulary vocabulary) throws VocabularyImportException { // Handle the name uniqueness final String nameToCheck = vocabulary.getName().trim().replaceAll("\\s", " ") - .toLowerCase(Locale.ENGLISH); + .toLowerCase(Locale.ENGLISH); if (knownNames.containsKey(nameToCheck)) { final String message = String.format("Duplicate name '%s' detected in metadata at [%s]:" - + " metadata at [%s] contains a name that is similar.", vocabulary.getName(), - vocabulary.getReadableMetadataLocation(), knownNames.get(nameToCheck)); + + " metadata at [%s] contains a name that is similar.", vocabulary.getName(), + vocabulary.getReadableMetadataLocation(), knownNames.get(nameToCheck)); throw new VocabularyImportException(message); } knownNames.put(nameToCheck, vocabulary.getReadableMetadataLocation()); diff --git a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferenceService.java b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferenceService.java index 5d8f88ae0..2d0db505f 100644 --- a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferenceService.java +++ b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferenceService.java @@ -9,7 +9,7 @@ import eu.europeana.enrichment.api.external.model.Resource; import eu.europeana.enrichment.api.external.model.TimeSpan; import eu.europeana.enrichment.utils.EnrichmentBaseConverter; -import eu.europeana.metis.dereference.IncomingRecordToEdmConverter; +import eu.europeana.metis.dereference.IncomingRecordToEdmTransformer; import eu.europeana.metis.dereference.ProcessedEntity; import eu.europeana.metis.dereference.RdfRetriever; import eu.europeana.metis.dereference.Vocabulary; @@ -17,6 +17,7 @@ import eu.europeana.metis.dereference.service.dao.VocabularyDao; import eu.europeana.metis.dereference.service.utils.GraphUtils; import eu.europeana.metis.dereference.service.utils.VocabularyCandidates; +import eu.europeana.metis.exception.BadContentException; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -32,6 +33,7 @@ import java.util.function.Function; import java.util.stream.Stream; import javax.xml.bind.JAXBException; +import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -108,7 +110,7 @@ public List dereference(String resourceId) * @return A collection of dereferenced resources. Is not null, but could be empty. */ private Collection dereferenceResource(String resourceId) - throws JAXBException, TransformerException, URISyntaxException { + throws JAXBException, URISyntaxException { // Get the main object to dereference. If null, we are done. final Pair resource = computeEnrichmentBaseVocabularyPair( @@ -123,7 +125,7 @@ private Collection dereferenceResource(String resourceId) try { result = computeEnrichmentBaseVocabularyPair(key); return result == null ? null : result.getLeft(); - } catch (JAXBException | TransformerException | URISyntaxException e) { + } catch (JAXBException | URISyntaxException e) { LOGGER.warn(String.format("Problem occurred while dereferencing broader resource %s.", key), e); return null; @@ -166,25 +168,6 @@ private static Stream getStream(Collection collection) { return collection == null ? Stream.empty() : collection.stream(); } - Pair computeEnrichmentBaseVocabularyPair(String resourceId) - throws JAXBException, TransformerException, URISyntaxException { - - // Try to get the entity and its vocabulary from the cache. - final ProcessedEntity cachedEntity = processedEntityDao.get(resourceId); - final Pair entityVocabularyPair = computeEntityVocabularyPair(resourceId, - cachedEntity); - - // Parse the entity. - final Pair enrichmentBaseVocabularyPair; - if (entityVocabularyPair.getLeft() == null || entityVocabularyPair.getRight() == null) { - enrichmentBaseVocabularyPair = null; - } else { - enrichmentBaseVocabularyPair = convertToEnrichmentBaseVocabularyPair( - entityVocabularyPair.getLeft(), entityVocabularyPair.getRight()); - } - return enrichmentBaseVocabularyPair; - } - /** * Computes the entity and vocabulary. *

    It will use the cache if it's still valid, otherwise it will retrieve(if applicable) the @@ -208,7 +191,7 @@ Pair computeEnrichmentBaseVocabularyPair(String reso * @throws TransformerException if an exception occurred during transformation of the original entity */ private Pair computeEntityVocabularyPair(String resourceId, - ProcessedEntity cachedEntity) throws URISyntaxException, TransformerException { + ProcessedEntity cachedEntity) throws URISyntaxException { final Pair transformedEntityVocabularyPair; @@ -233,35 +216,7 @@ private Pair computeEntityVocabularyPair(String resourceId, return transformedEntityVocabularyPair; } - private void saveEntity(String resourceId, ProcessedEntity cachedEntity, - Pair transformedEntityAndVocabularyPair) { - - final String entityXml = transformedEntityAndVocabularyPair.getLeft(); - final Vocabulary vocabulary = transformedEntityAndVocabularyPair.getRight(); - final String vocabularyIdString = Optional.ofNullable(vocabulary).map(Vocabulary::getId) - .map(ObjectId::toString).orElse(null); - //Save entity - ProcessedEntity entityToCache = (cachedEntity == null) ? new ProcessedEntity() : cachedEntity; - entityToCache.setResourceId(resourceId); - entityToCache.setXml(entityXml); - entityToCache.setVocabularyId(vocabularyIdString); - processedEntityDao.save(entityToCache); - } - - private Pair convertToEnrichmentBaseVocabularyPair(String entityXml, - Vocabulary entityVocabulary) throws JAXBException { - final Pair result; - if (entityXml == null || entityVocabulary == null) { - result = null; - } else { - result = new ImmutablePair<>(EnrichmentBaseConverter.convertToEnrichmentBase(entityXml), - entityVocabulary); - } - return result; - } - - private Pair retrieveAndTransformEntity(String resourceId) - throws TransformerException, URISyntaxException { + private Pair retrieveAndTransformEntity(String resourceId) throws URISyntaxException { final VocabularyCandidates vocabularyCandidates = VocabularyCandidates .findVocabulariesForUrl(resourceId, vocabularyDao::getByUriSearch); @@ -296,6 +251,47 @@ private Pair retrieveAndTransformEntity(String resourceId) return entityVocabularyPair; } + private void saveEntity(String resourceId, ProcessedEntity cachedEntity, + Pair transformedEntityAndVocabularyPair) { + + final String entityXml = transformedEntityAndVocabularyPair.getLeft(); + final Vocabulary vocabulary = transformedEntityAndVocabularyPair.getRight(); + final String vocabularyIdString = Optional.ofNullable(vocabulary).map(Vocabulary::getId) + .map(ObjectId::toString).orElse(null); + //Save entity + ProcessedEntity entityToCache = (cachedEntity == null) ? new ProcessedEntity() : cachedEntity; + entityToCache.setResourceId(resourceId); + entityToCache.setXml(entityXml); + entityToCache.setVocabularyId(vocabularyIdString); + processedEntityDao.save(entityToCache); + } + + private Pair convertToEnrichmentBaseVocabularyPair(String entityXml, + Vocabulary entityVocabulary) throws JAXBException { + final Pair result; + if (entityXml == null || entityVocabulary == null) { + result = null; + } else { + result = new ImmutablePair<>(EnrichmentBaseConverter.convertToEnrichmentBase(entityXml), + entityVocabulary); + } + return result; + } + + private String transformEntity(Vocabulary vocabulary, String originalEntity, String resourceId) { + Optional result; + try { + final IncomingRecordToEdmTransformer incomingRecordToEdmTransformer = new IncomingRecordToEdmTransformer( + vocabulary.getXslt()); + result = incomingRecordToEdmTransformer.transform(originalEntity, resourceId); + } catch (TransformerException | BadContentException | ParserConfigurationException e) { + LOGGER.warn("Error transforming entity: {} with message: {}", resourceId, e.getMessage()); + LOGGER.debug("Transformation issue: ", e); + result = Optional.empty(); + } + return result.orElse(null); + } + private String retrieveOriginalEntity(String resourceId, VocabularyCandidates candidates) throws URISyntaxException { @@ -323,21 +319,22 @@ private String retrieveOriginalEntity(String resourceId, VocabularyCandidates ca return originalEntity; } - private String transformEntity(Vocabulary vocabulary, String originalEntity, String resourceId) - throws TransformerException { - final IncomingRecordToEdmConverter converter = new IncomingRecordToEdmConverter(vocabulary); - final String result; - try { - result = converter.convert(originalEntity, resourceId); - if (result == null && LOGGER.isInfoEnabled()) { - LOGGER.info("Could not transform entity {} as it results is an empty XML.", - CRLF_PATTERN.matcher(resourceId).replaceAll("")); - } - } catch (TransformerException e) { - LOGGER.warn("Error transforming entity: {} with message: {}", resourceId, e.getMessage()); - LOGGER.debug("Transformation issue: ", e); - return null; + Pair computeEnrichmentBaseVocabularyPair(String resourceId) + throws JAXBException, URISyntaxException { + + // Try to get the entity and its vocabulary from the cache. + final ProcessedEntity cachedEntity = processedEntityDao.get(resourceId); + final Pair entityVocabularyPair = computeEntityVocabularyPair(resourceId, + cachedEntity); + + // Parse the entity. + final Pair enrichmentBaseVocabularyPair; + if (entityVocabularyPair.getLeft() == null || entityVocabularyPair.getRight() == null) { + enrichmentBaseVocabularyPair = null; + } else { + enrichmentBaseVocabularyPair = convertToEnrichmentBaseVocabularyPair( + entityVocabularyPair.getLeft(), entityVocabularyPair.getRight()); } - return result; + return enrichmentBaseVocabularyPair; } } From 6ed88d5fa00e3f973f2a9961cc94b8f74751f2c3 Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Fri, 8 Apr 2022 16:41:37 +0200 Subject: [PATCH 21/73] =?UTF-8?q?MET-4285=20Initial=20implementation=20of?= =?UTF-8?q?=20geo=20parsing=20and=20record=20geo=20locatio=E2=80=A6=20(#52?= =?UTF-8?q?3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * MET-4285 Initial implementation of geo parsing and record geo location extraction for solr * MET-4285 Geo parsing support altitude * MET-4285 Add some relevant tests for Solr * MET-4285 Parse geo uris during solr document creation * MET-4285 Fix xmls so that tests can rdf parse * MET-4285 Fix xmls so that tests can rdf parse * MET-4285 Truncate long decimals geo, re-organize solr properties * MET-4285 Cleanup * MET-4285 Fix analysis * MET-4285 Handle review --- .../metis/utils/GeoUriWGS84Parser.java | 219 ++++++++++++++++++ .../metis/utils/GeoUriWGS84ParserTest.java | 77 ++++++ .../eu/europeana/indexing/solr/EdmLabel.java | 27 ++- .../indexing/solr/SolrDocumentPopulator.java | 57 ++--- .../solr/property/FullBeanSolrProperties.java | 183 +++++++++++++++ .../solr/SolrDocumentPopulatorTest.java | 86 +++++++ .../europeana_record_with_geospatial_data.xml | 127 ++++++++++ ...eana_record_with_geospatial_data_wgs84.xml | 128 ++++++++++ 8 files changed, 858 insertions(+), 46 deletions(-) create mode 100644 metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/GeoUriWGS84Parser.java create mode 100644 metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/GeoUriWGS84ParserTest.java create mode 100644 metis-indexing/src/main/java/eu/europeana/indexing/solr/property/FullBeanSolrProperties.java create mode 100644 metis-indexing/src/test/java/eu/europeana/indexing/solr/SolrDocumentPopulatorTest.java create mode 100644 metis-indexing/src/test/resources/europeana_record_with_geospatial_data.xml create mode 100644 metis-indexing/src/test/resources/europeana_record_with_geospatial_data_wgs84.xml diff --git a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/GeoUriWGS84Parser.java b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/GeoUriWGS84Parser.java new file mode 100644 index 000000000..a2981ae50 --- /dev/null +++ b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/GeoUriWGS84Parser.java @@ -0,0 +1,219 @@ +package eu.europeana.metis.utils; + +import eu.europeana.metis.exception.BadContentException; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * Contains functionality to parse and validate geo uri + */ +public final class GeoUriWGS84Parser { + + private static final String DECIMAL_POINT_REGEX = "(?:\\.\\d+)?"; + private static final String ZEROES_DECIMAL_POINT_REGEX = "(?:\\.0+)?"; + private static final String LATITUDE_REGEX = + "^[+-]?(?:90" + ZEROES_DECIMAL_POINT_REGEX + "|(?:[0-9]|[1-8][0-9])" + DECIMAL_POINT_REGEX + ")$"; + private static final Pattern LATITUDE_PATTERN = Pattern.compile(LATITUDE_REGEX); + private static final String LONGITUDE_REGEX = + "^[+-]?(?:180" + ZEROES_DECIMAL_POINT_REGEX + "|(?:[0-9]|[1-9][0-9]|1[0-7][0-9])" + DECIMAL_POINT_REGEX + ")$"; + private static final Pattern LONGITUDE_PATTERN = Pattern.compile(LONGITUDE_REGEX); + private static final String ALTITUDE_REGEX = "^[+-]?\\d+" + DECIMAL_POINT_REGEX + "$"; + private static final Pattern ALTITUDE_PATTERN = Pattern.compile(ALTITUDE_REGEX); + private static final String CRS_WGS_84 = "wgs84"; + private static final int MAX_NUMBER_COORDINATES = 3; + private static final int MAX_DECIMAL_POINTS_TO_KEEP = 7; + + private GeoUriWGS84Parser() { + } + + /** + * Parse a provided geo uri in wgs84 coordinate reference system (CRS) and validate its contents. + *

    The parsing of the string follows closely but not exhaustively the specification located at + * https://datatracker.ietf.org/doc/html/rfc5870

    + *

    The checks that are performed to the provided string are as follows: + *

      + *
    • There should not be any spaces
      • + *
    • It should start with "geo:"
      • + *
    • There should be at least one part after the scheme and that should be the coordinates
      • + *
    • If crs parameter is present it should be "wgs84"
      • + *
    • The "u" parameter should be just after crs if crs is present or just after the coordinates
      • + *
    • The coordinates should have 2 or 3 dimensions
      • + *
    • The coordinates should be of valid structure and valid range
      • + *
    • The coordinates if they have decimal points they will be truncated after 7th point
      • + *
    + *

    + * + * @param geoUriString the geo uri string + * @return the geo coordinates, null will never be returned + * @throws BadContentException if the geo uri parsing encountered an error + */ + public static GeoCoordinates parse(String geoUriString) throws BadContentException { + final String[] geoUriParts = validateGeoUriAndGetParts(geoUriString); + + //Finally, check the coordinates part and validate + return validateGeoCoordinatesAndGet(geoUriParts[0]); + } + + private static String[] validateGeoUriAndGetParts(String geoUriString) throws BadContentException { + //Validate that there aren't any space characters in the URI + if (!geoUriString.matches("^\\S+$")) { + throw new BadContentException("URI cannot have spaces"); + } + //Validate geo URI + if (!geoUriString.matches("^geo:.*$")) { + throw new BadContentException("Invalid scheme value"); + } + + final String[] schemeAndParts = geoUriString.split(":"); + if (schemeAndParts.length <= 1) { + throw new BadContentException("There are no parts in the geo URI"); + } + + //Find all parts + final String[] geoUriParts = schemeAndParts[1].split(";"); + //Must be at least one part available + if (geoUriParts.length < 1) { + throw new BadContentException("Invalid geo uri parts length"); + } + + //Find all other parameters + final LinkedList geoUriParameters = Arrays.stream(geoUriParts, 1, geoUriParts.length).map(s -> { + final String[] split = s.split("="); + return new GeoUriParameter(split[0], split[1]); + }).collect(Collectors.toCollection(LinkedList::new)); + + //If crs present, it must be the exact first after the dimensions. If not present then there is a default + String crs = CRS_WGS_84; + for (int i = 0; i < geoUriParameters.size(); i++) { + if ("crs".equalsIgnoreCase(geoUriParameters.get(i).getName())) { + crs = geoUriParameters.get(i).getValue(); + if (i != 0) { + throw new BadContentException("Invalid geo uri 'crs' parameter position"); + } + } + if ("u".equalsIgnoreCase(geoUriParameters.get(i).getName()) && i > 1) { + throw new BadContentException("Invalid geo uri 'u' parameter position"); + } + } + //Validate value of crs + if (!CRS_WGS_84.equalsIgnoreCase(crs)) { + throw new BadContentException(String.format("Crs parameter value is not %s", CRS_WGS_84)); + } + return geoUriParts; + } + + /** + * Generate a geo coordinates from a geoUriPart string. + *

    The provided string is validated against: + *

      + *
    • the total coordinates available
      • + *
    • the validity of each number and its range
      • + *
    • the convertibility to a {@link Double}
      • + *
    + * The decimal points are also truncated up to a maximum allowed. + *

    + * + * @param geoUriPart the string that should contain the coordinates + * @return the geo coordinates + * @throws BadContentException if the geo coordinates were not valid + */ + private static GeoCoordinates validateGeoCoordinatesAndGet(String geoUriPart) throws BadContentException { + final String[] coordinates = geoUriPart.split(","); + if (coordinates.length < 2 || coordinates.length > MAX_NUMBER_COORDINATES) { + throw new BadContentException("Coordinates are not of valid length"); + } + final Matcher latitudeMatcher = LATITUDE_PATTERN.matcher(coordinates[0]); + final Matcher longitudeMatcher = LONGITUDE_PATTERN.matcher(coordinates[1]); + final GeoCoordinates geoCoordinates; + if (latitudeMatcher.matches() && longitudeMatcher.matches()) { + Double altitude = null; + if (coordinates.length == MAX_NUMBER_COORDINATES) { + final Matcher altitudeMatcher = ALTITUDE_PATTERN.matcher(coordinates[2]); + if (altitudeMatcher.matches()) { + altitude = Double.parseDouble(truncateDecimalPoints(altitudeMatcher.group(0))); + } + } + geoCoordinates = new GeoCoordinates( + Double.parseDouble(truncateDecimalPoints(latitudeMatcher.group(0))), + Double.parseDouble(truncateDecimalPoints(longitudeMatcher.group(0))), altitude); + } else { + throw new BadContentException("Coordinates are invalid"); + } + return geoCoordinates; + } + + private static String truncateDecimalPoints(String decimalNumber) { + final String[] decimalNumberParts = decimalNumber.split("\\."); + final StringBuilder decimalNumberTruncated = new StringBuilder(); + if (decimalNumberParts.length >= 1) { + decimalNumberTruncated.append(decimalNumberParts[0]); + } + if (decimalNumberParts.length > 1) { + decimalNumberTruncated.append("."); + decimalNumberTruncated.append(decimalNumberParts[1], 0, + Math.min(decimalNumberParts[1].length(), MAX_DECIMAL_POINTS_TO_KEEP)); + } + return decimalNumberTruncated.toString(); + } + + /** + * Class containing geo coordinates (latitude, longitude) + */ + public static class GeoCoordinates { + + private final Double latitude; + private final Double longitude; + private final Double altitude; + + /** + * Constructor with required parameters + * + * @param latitude the latitude + * @param longitude the longitude + * @param altitude the altitude + */ + public GeoCoordinates(Double latitude, Double longitude, Double altitude) { + this.latitude = latitude; + this.longitude = longitude; + this.altitude = altitude; + } + + public Double getLatitude() { + return latitude; + } + + public Double getLongitude() { + return longitude; + } + + public Double getAltitude() { + return altitude; + } + } + + /** + * Class wrapping the name and value of geo uri parameters. + */ + private static class GeoUriParameter { + + private final String name; + private final String value; + + public GeoUriParameter(String name, String value) { + this.name = name; + this.value = value; + } + + public String getName() { + return name; + } + + public String getValue() { + return value; + } + } + +} diff --git a/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/GeoUriWGS84ParserTest.java b/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/GeoUriWGS84ParserTest.java new file mode 100644 index 000000000..c060c26bc --- /dev/null +++ b/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/GeoUriWGS84ParserTest.java @@ -0,0 +1,77 @@ +package eu.europeana.metis.utils; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import eu.europeana.metis.exception.BadContentException; +import eu.europeana.metis.utils.GeoUriWGS84Parser.GeoCoordinates; +import org.junit.jupiter.api.Test; + +class GeoUriWGS84ParserTest { + + @Test + void parse_invalid() { + + //URI cannot have spaces + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo: 37.786971,-122.399677")); + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677; u=35")); + + //Non geo + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("test:")); + + //URI cannot be without dimensions + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:")); + //URI must have at least one part + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:;")); + + //Validate order of crs and u parameters + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677;u=35;crs=wgs84")); + assertThrows(BadContentException.class, + () -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677;crs=wgs84;parameter1=value1;u=35")); + assertThrows(BadContentException.class, + () -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677;parameter1=value1;crs=wgs84;u=35")); + + //Validate crs value + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677;crs=Moon-2011;u=35")); + + //Coordinates must be present and of correct length + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:;crs=wgs84")); + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:37.786971,;crs=wgs84")); + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:37.786971;crs=wgs84")); + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:37.786971,100,100,10;crs=wgs84")); + //Invalid coordinate + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:test,-122.399677;crs=wgs84")); + //Invalid range coordinates + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:-100,200;crs=wgs84")); + } + + @Test + void parse_valid() throws Exception { + assertDoesNotThrow(() -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677;crs=wgs84;u=35")); + assertDoesNotThrow(() -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677;u=35")); + assertDoesNotThrow(() -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677;crs=wgs84;u=35;parameter1=value1")); + assertDoesNotThrow(() -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677;u=35;parameter1=value1")); + + assertDoesNotThrow(() -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677")); + assertDoesNotThrow(() -> GeoUriWGS84Parser.parse("geo:37.786971,-122.399677,10")); + assertDoesNotThrow(() -> GeoUriWGS84Parser.parse("geo:37.1234567,-122.1234567,10")); + assertDoesNotThrow(() -> GeoUriWGS84Parser.parse("geo:37,-122")); + + final GeoCoordinates geoCoordinates = GeoUriWGS84Parser.parse("geo:37.786971,-122.399677"); + assertEquals(Double.parseDouble("37.786971"), geoCoordinates.getLatitude()); + assertEquals(Double.parseDouble("-122.399677"), geoCoordinates.getLongitude()); + + final GeoCoordinates geoCoordinatesWithAltitude = GeoUriWGS84Parser.parse("geo:37.786971,-122.399677,1000.500600"); + assertEquals(Double.parseDouble("37.786971"), geoCoordinatesWithAltitude.getLatitude()); + assertEquals(Double.parseDouble("-122.399677"), geoCoordinatesWithAltitude.getLongitude()); + assertEquals(Double.parseDouble("1000.500600"), geoCoordinatesWithAltitude.getAltitude()); + + //Should truncate the extra decimal points + final GeoCoordinates geoCoordinatesWithLongDecimalPoints = GeoUriWGS84Parser.parse( + "geo:40.123456789,45.123456789,1000.123456789"); + assertEquals(Double.parseDouble("40.1234567"), geoCoordinatesWithLongDecimalPoints.getLatitude()); + assertEquals(Double.parseDouble("45.1234567"), geoCoordinatesWithLongDecimalPoints.getLongitude()); + assertEquals(Double.parseDouble("1000.1234567"), geoCoordinatesWithLongDecimalPoints.getAltitude()); + } +} \ No newline at end of file diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/solr/EdmLabel.java b/metis-indexing/src/main/java/eu/europeana/indexing/solr/EdmLabel.java index e94eea556..ef0131b64 100644 --- a/metis-indexing/src/main/java/eu/europeana/indexing/solr/EdmLabel.java +++ b/metis-indexing/src/main/java/eu/europeana/indexing/solr/EdmLabel.java @@ -85,25 +85,30 @@ public enum EdmLabel { PROXY_DCTERMS_HAS_PART("proxy_dcterms_hasPart"), PROXY_DCTERMS_IS_PART_OF("proxy_dcterms_isPartOf"), PROXY_DCTERMS_ISSUED("proxy_dcterms_issued"), - PROXY_DCTERMS_MEDIUM("proxy_dcterms_medium"), - PROXY_DCTERMS_PROVENANCE("proxy_dcterms_provenance"), - PROXY_DCTERMS_SPATIAL("proxy_dcterms_spatial"), - PROXY_DCTERMS_TEMPORAL("proxy_dcterms_temporal"), + PROXY_DCTERMS_MEDIUM("proxy_dcterms_medium"), + PROXY_DCTERMS_PROVENANCE("proxy_dcterms_provenance"), + PROXY_DCTERMS_SPATIAL("proxy_dcterms_spatial"), + PROXY_DCTERMS_TEMPORAL("proxy_dcterms_temporal"), EDM_UGC("edm_UGC"), PROXY_EDM_CURRENT_LOCATION("proxy_edm_currentLocation"), PROXY_EDM_HAS_MET("proxy_edm_hasMet"), - PROXY_EDM_ISRELATEDTO("proxy_edm_isRelatedTo"), + PROXY_EDM_ISRELATEDTO("proxy_edm_isRelatedTo"), PROXY_EDM_YEAR("proxy_edm_year"), PROVIDER_EDM_TYPE("proxy_edm_type"), + //GEO LOCATION FIELDS + CURRENT_LOCATION_WGS("currentLocation_wgs"), + COVERAGE_LOCATION_WGS("coverageLocation_wgs"), + LOCATION_WGS("location_wgs"), + //SKOS_CONCEPT - SKOS_CONCEPT("skos_concept"), - CC_SKOS_PREF_LABEL("cc_skos_prefLabel"), - CC_SKOS_ALT_LABEL("cc_skos_altLabel"), - + SKOS_CONCEPT("skos_concept"), + CC_SKOS_PREF_LABEL("cc_skos_prefLabel"), + CC_SKOS_ALT_LABEL("cc_skos_altLabel"), + //PLACE - EDM_PLACE("edm_place"), - PL_SKOS_PREF_LABEL("pl_skos_prefLabel"), + EDM_PLACE("edm_place"), + PL_SKOS_PREF_LABEL("pl_skos_prefLabel"), PL_SKOS_ALT_LABEL("pl_skos_altLabel"), PL_WGS84_POS_LAT("pl_wgs84_pos_lat"), PL_WGS84_POS_LONG("pl_wgs84_pos_long"), diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/solr/SolrDocumentPopulator.java b/metis-indexing/src/main/java/eu/europeana/indexing/solr/SolrDocumentPopulator.java index d17d8ad4c..a0ca28ad2 100644 --- a/metis-indexing/src/main/java/eu/europeana/indexing/solr/SolrDocumentPopulator.java +++ b/metis-indexing/src/main/java/eu/europeana/indexing/solr/SolrDocumentPopulator.java @@ -12,6 +12,7 @@ import eu.europeana.indexing.solr.property.AggregationSolrCreator; import eu.europeana.indexing.solr.property.ConceptSolrCreator; import eu.europeana.indexing.solr.property.EuropeanaAggregationSolrCreator; +import eu.europeana.indexing.solr.property.FullBeanSolrProperties; import eu.europeana.indexing.solr.property.LicenseSolrCreator; import eu.europeana.indexing.solr.property.PlaceSolrCreator; import eu.europeana.indexing.solr.property.ProvidedChoSolrCreator; @@ -41,39 +42,35 @@ import org.apache.solr.common.SolrInputDocument; /** - * This class provides functionality to populate Solr documents. Both methods in this class should - * be called to fill the Solr document. The method {@link #populateWithProperties(SolrInputDocument, - * FullBeanImpl)} copies properties from the source to the Solr document. The method {@link - * #populateWithFacets(SolrInputDocument, RdfWrapper)} on the other hand performs some analysis and - * sets technical metadata. + * This class provides functionality to populate Solr documents. Both methods in this class should be called to fill the Solr + * document. The method {@link #populateWithProperties(SolrInputDocument, FullBeanImpl)} copies properties from the source to the + * Solr document. The method {@link #populateWithFacets(SolrInputDocument, RdfWrapper)} on the other hand performs some analysis + * and sets technical metadata. * * @author jochen */ public class SolrDocumentPopulator { /** - * Populates a Solr document with the properties of the full bean. Please note: this method should - * only be called once on a given document, otherwise the behavior is not defined. + * Populates a Solr document with the properties of the full bean. Please note: this method should only be called once on a + * given document, otherwise the behavior is not defined. * * @param document The Solr document to populate. * @param fullBean The FullBean to populate from. */ public void populateWithProperties(SolrInputDocument document, FullBeanImpl fullBean) { - // Get the type: filter duplicates - final String[] types = Optional.ofNullable(fullBean.getProxies()).stream().flatMap(List::stream) - .filter(Objects::nonNull).map(ProxyImpl::getEdmType).filter(Objects::nonNull).distinct() - .toArray(String[]::new); - SolrPropertyUtils.addValues(document, EdmLabel.PROVIDER_EDM_TYPE, types); + new FullBeanSolrProperties().setProperties(document, fullBean); // Gather the licenses. final List licenses = Optional.ofNullable(fullBean.getLicenses()).stream() - .flatMap(List::stream).filter(Objects::nonNull).collect(Collectors.toList()); + .flatMap(List::stream).filter(Objects::nonNull).collect(Collectors.toList()); // Gather the quality annotations. final Set acceptableTargets = Optional.ofNullable(fullBean.getAggregations()).stream() - .flatMap(Collection::stream).filter(Objects::nonNull).map(AggregationImpl::getAbout) - .filter(Objects::nonNull).collect(Collectors.toSet()); + .flatMap(Collection::stream).filter(Objects::nonNull) + .map(AggregationImpl::getAbout) + .filter(Objects::nonNull).collect(Collectors.toSet()); final Predicate hasAcceptableTarget = annotation -> Optional .ofNullable(annotation.getTarget()).stream().flatMap(Arrays::stream) .anyMatch(acceptableTargets::contains); @@ -99,23 +96,15 @@ public void populateWithProperties(SolrInputDocument document, FullBeanImpl full // Add the licenses. final Set defRights = fullBean.getAggregations().stream() - .map(AggregationImpl::getEdmRights).filter(Objects::nonNull) - .flatMap(SolrPropertyUtils::getRightsFromMap).collect(Collectors.toSet()); + .map(AggregationImpl::getEdmRights).filter(Objects::nonNull) + .flatMap(SolrPropertyUtils::getRightsFromMap).collect(Collectors.toSet()); new LicenseSolrCreator(license -> defRights.contains(license.getAbout())) .addAllToDocument(document, fullBean.getLicenses()); - - // Add the top-level properties. - document - .addField(EdmLabel.EUROPEANA_COMPLETENESS.toString(), fullBean.getEuropeanaCompleteness()); - document.addField(EdmLabel.EUROPEANA_COLLECTIONNAME.toString(), - fullBean.getEuropeanaCollectionName()[0]); - document.addField(EdmLabel.TIMESTAMP_CREATED.toString(), fullBean.getTimestampCreated()); - document.addField(EdmLabel.TIMESTAMP_UPDATED.toString(), fullBean.getTimestampUpdated()); } /** - * Populates a Solr document with the CRF fields of the RDF. Please note: this method should only - * be called once on a given document, otherwise the behavior is not defined. + * Populates a Solr document with the CRF fields of the RDF. Please note: this method should only be called once on a given + * document, otherwise the behavior is not defined. * * @param document The document to populate. * @param rdf The RDF to populate from. @@ -131,7 +120,7 @@ public void populateWithFacets(SolrInputDocument document, RdfWrapper rdf) { final List webResourcesWithMedia = rdf.getWebResourceWrappers( EnumSet.of(WebResourceLinkType.IS_SHOWN_BY, WebResourceLinkType.HAS_VIEW)); final boolean hasMedia = webResourcesWithMedia.stream().map(WebResourceWrapper::getMediaType) - .anyMatch(type -> type != MediaType.OTHER); + .anyMatch(type -> type != MediaType.OTHER); document.addField(EdmLabel.FACET_HAS_MEDIA.toString(), hasMedia); // has_landingPage is true if and only if there is at least one web resource of type @@ -141,7 +130,7 @@ public void populateWithFacets(SolrInputDocument document, RdfWrapper rdf) { // is_fulltext is true if and only if there is at least one web resource of type 'isShownBy' // or 'hasView' with 'rdf:type' equal to 'edm:FullTextResource'. final boolean isFullText = webResourcesWithMedia.stream().map(WebResourceWrapper::getType) - .anyMatch("http://www.europeana.eu/schemas/edm/FullTextResource"::equals); + .anyMatch("http://www.europeana.eu/schemas/edm/FullTextResource"::equals); document.addField(EdmLabel.FACET_IS_FULL_TEXT.toString(), isFullText); // Compose the filter and facet tags. Only use the web resources of type 'isShownBy' or 'hasView'. @@ -163,14 +152,12 @@ public void populateWithFacets(SolrInputDocument document, RdfWrapper rdf) { } private List getDataProviderAggregations(FullBeanImpl fullBean) { - List proxyInResult = fullBean.getProxies().stream() - .filter(not(ProxyImpl::isEuropeanaProxy)) - .filter(proxy -> ArrayUtils.isEmpty(proxy.getLineage())).map(ProxyImpl::getProxyIn) - .map(Arrays::asList).flatMap(List::stream).collect(Collectors.toList()); + .filter(not(ProxyImpl::isEuropeanaProxy)) + .filter(proxy -> ArrayUtils.isEmpty(proxy.getLineage())).map(ProxyImpl::getProxyIn) + .map(Arrays::asList).flatMap(List::stream).collect(Collectors.toList()); return fullBean.getAggregations().stream().filter(x -> proxyInResult.contains(x.getAbout())) - .collect(Collectors.toList()); - + .collect(Collectors.toList()); } } diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/solr/property/FullBeanSolrProperties.java b/metis-indexing/src/main/java/eu/europeana/indexing/solr/property/FullBeanSolrProperties.java new file mode 100644 index 000000000..8653e5177 --- /dev/null +++ b/metis-indexing/src/main/java/eu/europeana/indexing/solr/property/FullBeanSolrProperties.java @@ -0,0 +1,183 @@ +package eu.europeana.indexing.solr.property; + +import static eu.europeana.indexing.solr.EdmLabel.COVERAGE_LOCATION_WGS; +import static eu.europeana.indexing.solr.EdmLabel.CURRENT_LOCATION_WGS; +import static eu.europeana.indexing.solr.EdmLabel.LOCATION_WGS; +import static java.lang.String.format; + +import eu.europeana.corelib.solr.bean.impl.FullBeanImpl; +import eu.europeana.corelib.solr.entity.PlaceImpl; +import eu.europeana.corelib.solr.entity.ProxyImpl; +import eu.europeana.indexing.solr.EdmLabel; +import eu.europeana.metis.exception.BadContentException; +import eu.europeana.metis.utils.GeoUriWGS84Parser; +import eu.europeana.metis.utils.GeoUriWGS84Parser.GeoCoordinates; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.apache.solr.common.SolrInputDocument; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Class that creates Solr properties related to the FullBean and properties that need to be retrieved and computed from multiple + * sub-elements. + */ +public class FullBeanSolrProperties { + + private static final Logger LOGGER = LoggerFactory.getLogger(FullBeanSolrProperties.class); + + /** + * Computes and creates all properties relevant to fullbean as a whole. + * + * @param document the solr document + * @param fullBean the fullbean to analyze + */ + public void setProperties(SolrInputDocument document, FullBeanImpl fullBean) { + // Get the type: filter duplicates + final String[] types = Optional.ofNullable(fullBean.getProxies()).stream().flatMap(List::stream) + .filter(Objects::nonNull).map(ProxyImpl::getEdmType).filter(Objects::nonNull).distinct() + .toArray(String[]::new); + SolrPropertyUtils.addValues(document, EdmLabel.PROVIDER_EDM_TYPE, types); + + setGeospatialFields(document, fullBean); + + document.addField(EdmLabel.EUROPEANA_COMPLETENESS.toString(), fullBean.getEuropeanaCompleteness()); + document.addField(EdmLabel.EUROPEANA_COLLECTIONNAME.toString(), fullBean.getEuropeanaCollectionName()[0]); + document.addField(EdmLabel.TIMESTAMP_CREATED.toString(), fullBean.getTimestampCreated()); + document.addField(EdmLabel.TIMESTAMP_UPDATED.toString(), fullBean.getTimestampUpdated()); + } + + private void setGeospatialFields(SolrInputDocument document, FullBeanImpl fullBean) { + final List proxies = fullBean.getProxies(); + final Map placesAboutMap = fullBean.getPlaces().stream() + .collect(Collectors.toMap(PlaceImpl::getAbout, Function.identity(), + (place1, place2) -> place1)); + final Set currentLocationStrings = new HashSet<>(); + final Set coverageLocationStrings = new HashSet<>(); + for (ProxyImpl proxy : proxies) { + currentLocationStrings.addAll(getCurrentLocationStrings(proxy)); + coverageLocationStrings.addAll(getCoverageLocationStrings(proxy)); + } + final Set currentLocationPoints = new HashSet<>( + getReferencedPlacesLocationPoints(placesAboutMap, currentLocationStrings)); + currentLocationPoints.addAll(getWGS84LocationPoints(currentLocationStrings)); + + final Set coverageLocationPoints = new HashSet<>( + getReferencedPlacesLocationPoints(placesAboutMap, coverageLocationStrings)); + coverageLocationPoints.addAll(getWGS84LocationPoints(coverageLocationStrings)); + + SolrPropertyUtils.addValues(document, CURRENT_LOCATION_WGS, + currentLocationPoints.stream().map(Object::toString).toArray(String[]::new)); + + SolrPropertyUtils.addValues(document, COVERAGE_LOCATION_WGS, + coverageLocationPoints.stream().map(Object::toString).toArray(String[]::new)); + + Set locationPointsCombined = new HashSet<>(); + locationPointsCombined.addAll(currentLocationPoints); + locationPointsCombined.addAll(coverageLocationPoints); + SolrPropertyUtils.addValues(document, LOCATION_WGS, + locationPointsCombined.stream().map(Object::toString).toArray(String[]::new)); + } + + + private Set getReferencedPlacesLocationPoints(Map placesAboutMap, + Set locationStrings) { + return locationStrings.stream().map(placesAboutMap::get).filter(Objects::nonNull) + .map(this::getPlaceLocationPoint).filter(Objects::nonNull).collect(Collectors.toSet()); + } + + private Set getWGS84LocationPoints(Set locationStrings) { + return locationStrings.stream().map(this::getValidGeoCoordinates).filter(Objects::nonNull) + .map(LocationPoint::new).collect(Collectors.toSet()); + } + + private Set getCurrentLocationStrings(ProxyImpl proxy) { + final Set currentLocations = new HashSet<>(); + Optional.ofNullable(proxy.getEdmCurrentLocation()).map(Map::values).stream().flatMap(Collection::stream) + .flatMap(Collection::stream) + .filter(StringUtils::isNotBlank) + .forEach(currentLocations::add); + return currentLocations; + } + + private Set getCoverageLocationStrings(ProxyImpl proxy) { + final Set coverageLocations = new HashSet<>(); + Optional.ofNullable(proxy.getDctermsSpatial()).map(Map::values).stream().flatMap(Collection::stream) + .flatMap(Collection::stream) + .filter(StringUtils::isNotBlank) + .forEach(coverageLocations::add); + Optional.ofNullable(proxy.getDcCoverage()).map(Map::values).stream().flatMap(Collection::stream) + .flatMap(Collection::stream) + .filter(StringUtils::isNotBlank) + .forEach(coverageLocations::add); + return coverageLocations; + } + + private LocationPoint getPlaceLocationPoint(PlaceImpl place) { + if (place.getLatitude() != null && place.getLongitude() != null) { + return new LocationPoint(place.getLatitude().doubleValue(), place.getLongitude().doubleValue()); + } + return null; + } + + private GeoCoordinates getValidGeoCoordinates(String s) { + try { + return GeoUriWGS84Parser.parse(s); + } catch (BadContentException e) { + LOGGER.debug(format("Geo parsing failed %s", s), e); + } + return null; + } + + + private static class LocationPoint { + + //We allow 7 decimal points + private static final DecimalFormat decimalFormat = new DecimalFormat("#.#######", new DecimalFormatSymbols(Locale.US)); + private final Double latitude; + private final Double longitude; + + public LocationPoint(Double latitude, Double longitude) { + this.latitude = latitude; + this.longitude = longitude; + } + + public LocationPoint(GeoCoordinates geoCoordinates) { + this.latitude = geoCoordinates.getLatitude(); + this.longitude = geoCoordinates.getLongitude(); + } + + @Override + public String toString() { + return format(Locale.US, "%s,%s", decimalFormat.format(latitude), decimalFormat.format(longitude)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LocationPoint that = (LocationPoint) o; + return this.toString().equals(that.toString()); + } + + @Override + public int hashCode() { + return Objects.hash(latitude, longitude); + } + } +} diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/SolrDocumentPopulatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/SolrDocumentPopulatorTest.java new file mode 100644 index 000000000..a0e0fcab3 --- /dev/null +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/SolrDocumentPopulatorTest.java @@ -0,0 +1,86 @@ +package eu.europeana.indexing.solr; + +import static eu.europeana.indexing.solr.EdmLabel.COVERAGE_LOCATION_WGS; +import static eu.europeana.indexing.solr.EdmLabel.CURRENT_LOCATION_WGS; +import static eu.europeana.indexing.solr.EdmLabel.EUROPEANA_ID; +import static eu.europeana.indexing.solr.EdmLabel.LOCATION_WGS; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import eu.europeana.corelib.solr.bean.impl.FullBeanImpl; +import eu.europeana.indexing.fullbean.RdfToFullBeanConverter; +import eu.europeana.indexing.tiers.ClassifierFactory; +import eu.europeana.indexing.utils.RdfTierUtils; +import eu.europeana.indexing.utils.RdfWrapper; +import eu.europeana.metis.schema.convert.RdfConversionUtils; +import eu.europeana.metis.schema.jibx.RDF; +import java.io.File; +import java.nio.file.Files; +import java.util.List; +import java.util.Objects; +import org.apache.commons.collections.CollectionUtils; +import org.apache.solr.common.SolrInputDocument; +import org.junit.jupiter.api.Test; + +class SolrDocumentPopulatorTest { + + @Test + void populateWithProperties_PlaceCoordinates() throws Exception { + ClassLoader classLoader = SolrDocumentPopulatorTest.class.getClassLoader(); + File file = new File(Objects.requireNonNull(classLoader.getResource("europeana_record_with_geospatial_data.xml")).getFile()); + String xml = new String(Files.readAllBytes(file.toPath())); + final RDF rdf = new RdfConversionUtils().convertStringToRdf(xml); + + // Perform the tier classification + final RdfWrapper rdfWrapper = new RdfWrapper(rdf); + RdfTierUtils.setTier(rdf, ClassifierFactory.getMediaClassifier().classify(rdfWrapper).getTier()); + RdfTierUtils.setTier(rdf, ClassifierFactory.getMetadataClassifier().classify(rdfWrapper).getTier()); + + final RdfToFullBeanConverter fullBeanConverter = new RdfToFullBeanConverter(); + final FullBeanImpl fullBean = fullBeanConverter.convertRdfToFullBean(rdfWrapper); + + // Create Solr document. + final SolrDocumentPopulator documentPopulator = new SolrDocumentPopulator(); + final SolrInputDocument document = new SolrInputDocument(); + documentPopulator.populateWithProperties(document, fullBean); + documentPopulator.populateWithFacets(document, rdfWrapper); + + assertTrue(document.get(EUROPEANA_ID.toString()).getValues().contains(fullBean.getAbout())); + assertTrue(CollectionUtils.isEqualCollection(document.get(CURRENT_LOCATION_WGS.toString()).getValues(), + List.of("50.75,4.5"))); + assertTrue(CollectionUtils.isEqualCollection(document.get(COVERAGE_LOCATION_WGS.toString()).getValues(), + List.of("50,50", "40,40"))); + assertTrue(CollectionUtils.isEqualCollection(document.get(LOCATION_WGS.toString()).getValues(), + List.of("50,50", "40,40", "50.75,4.5"))); + } + + @Test + void populateWithProperties_WGS84Coordinates() throws Exception { + ClassLoader classLoader = SolrDocumentPopulatorTest.class.getClassLoader(); + File file = new File( + Objects.requireNonNull(classLoader.getResource("europeana_record_with_geospatial_data_wgs84.xml")).getFile()); + String xml = new String(Files.readAllBytes(file.toPath())); + final RDF rdf = new RdfConversionUtils().convertStringToRdf(xml); + + // Perform the tier classification + final RdfWrapper rdfWrapper = new RdfWrapper(rdf); + RdfTierUtils.setTier(rdf, ClassifierFactory.getMediaClassifier().classify(rdfWrapper).getTier()); + RdfTierUtils.setTier(rdf, ClassifierFactory.getMetadataClassifier().classify(rdfWrapper).getTier()); + + final RdfToFullBeanConverter fullBeanConverter = new RdfToFullBeanConverter(); + final FullBeanImpl fullBean = fullBeanConverter.convertRdfToFullBean(rdfWrapper); + + // Create Solr document. + final SolrDocumentPopulator documentPopulator = new SolrDocumentPopulator(); + final SolrInputDocument document = new SolrInputDocument(); + documentPopulator.populateWithProperties(document, fullBean); + documentPopulator.populateWithFacets(document, rdfWrapper); + + assertTrue(document.get(EUROPEANA_ID.toString()).getValues().contains(fullBean.getAbout())); + assertTrue(CollectionUtils.isEqualCollection(document.get(CURRENT_LOCATION_WGS.toString()).getValues(), + List.of("50.75,4.5"))); + assertTrue(CollectionUtils.isEqualCollection(document.get(COVERAGE_LOCATION_WGS.toString()).getValues(), + List.of("50,50", "40,40", "40.123456,40.1234567"))); + assertTrue(CollectionUtils.isEqualCollection(document.get(LOCATION_WGS.toString()).getValues(), + List.of("50,50", "40,40", "40.123456,40.1234567", "50.75,4.5"))); + } +} \ No newline at end of file diff --git a/metis-indexing/src/test/resources/europeana_record_with_geospatial_data.xml b/metis-indexing/src/test/resources/europeana_record_with_geospatial_data.xml new file mode 100644 index 000000000..456c99b21 --- /dev/null +++ b/metis-indexing/src/test/resources/europeana_record_with_geospatial_data.xml @@ -0,0 +1,127 @@ + + + + + + + text/html + 197506 + + + image/jpeg + 6643 + 640 + 480 + sRGB + #F0E68C + #B22222 + #FF4500 + #2F4F4F + #ADFF2F + #87CEEB + landscape + + + 50.75 + 4.5 + + + 40 + 40 + + + 50 + 50 + + + 40 + 40 + + + + 1914-1919 के मध्य मुख्यतः यूरोप में व्याप्त इस महायुद्ध को प्रथम विश्व युद्ध कहते हैं । यह महायुद्ध + यूरोप, एशिया व अफ्रीका तीन महाद्वीपों और जल, थल तथा आकाश में लड़ा गया। इसमें भाग लेने वाले देशों की संख्या, इसका क्षेत्र + (जिसमें यह लड़ा गया) तथा इससे हुई क्षति के अभूतपूर्व आंकड़ों के कारण ही इसे विश्वयुद्ध कहते हैं । + + Første verdenskrig + + + + + + + + EFG - The European Film Gateway + J.M.P.- Trends + + + + 12944 + nld + + + + + + + true + 1984 + + + + + + PYLYSER, JEAN-MARIE + Newsitems West Flanders. Day of the Navy in Ostend, with a wreath-laying memorial service at the + monument of the sailors on the dike. Thereafter, a military parade takes place at the Wapenplein in the presence of Prince + Albert, Princess Paola, Governor Vanneste, mayor Goekindt and certain naval officers. The navy admiral gives a speech, + followed by a short parade. The princess salutes the flag bearers of the veterans' associations. + + Westvlaams filmjournaal - WAF (Westvlaamse Actualiteiten Films). Dag van de zeemacht te + Oostende, met een kranslegging aan het monument van de zeelieden op de dijk. Naast enkele zeemachtofficieren legt ook + burgemeester Goekindt een krans. Nadien vindt een wapenschouwing plaats op het Wapenplein in aanwezigheid van prins Albert, + prinses Paola, gouverneur Vanneste en de eerder genoemde personaliteiten. De zeemachtadmiraal houdt een toespraak. Er vindt + een korte parade plaats en de prinsen groeten de vaandeldragers van de oud-strijdersverenigingen. + + Newsitems West Flanders. Day of the Navy in Ostend, with a wreath-laying memorial service at the + monument of the sailors on the dike. Thereafter, a military parade takes place at the Wapenplein in the presence of Prince + Albert, Princess Paola, Governor Vanneste, mayor Goekindt and certain naval officers. The navy admiral gives a speech, + followed by a short parade. The princess salutes the flag bearers of the veterans' associations. + + 12944 + nl + EFG1914 + World War I + WESTVLAAMS FILMJOURNAAL - 164 + Newsreel + + + 1984 + Belgium + + false + + + VIDEO + + + + Europeana Foundation + Europeana Foundation + 1_test + Netherlands + + nl + 10 + + + The Royal Belgian Film Archives + Cinémathèque royale de Belgique + Koninklijk Belgisch Filmarchief + + \ No newline at end of file diff --git a/metis-indexing/src/test/resources/europeana_record_with_geospatial_data_wgs84.xml b/metis-indexing/src/test/resources/europeana_record_with_geospatial_data_wgs84.xml new file mode 100644 index 000000000..5ce8b91ab --- /dev/null +++ b/metis-indexing/src/test/resources/europeana_record_with_geospatial_data_wgs84.xml @@ -0,0 +1,128 @@ + + + + + + + text/html + 197506 + + + image/jpeg + 6643 + 640 + 480 + sRGB + #F0E68C + #B22222 + #FF4500 + #2F4F4F + #ADFF2F + #87CEEB + landscape + + + 50.75 + 4.5 + + + 40 + 40 + + + 50 + 50 + + + 40 + 40 + + + + 1914-1919 के मध्य मुख्यतः यूरोप में व्याप्त इस महायुद्ध को प्रथम विश्व युद्ध कहते हैं । यह महायुद्ध + यूरोप, एशिया व अफ्रीका तीन महाद्वीपों और जल, थल तथा आकाश में लड़ा गया। इसमें भाग लेने वाले देशों की संख्या, इसका क्षेत्र + (जिसमें यह लड़ा गया) तथा इससे हुई क्षति के अभूतपूर्व आंकड़ों के कारण ही इसे विश्वयुद्ध कहते हैं । + + Første verdenskrig + + + + + + + + EFG - The European Film Gateway + J.M.P.- Trends + + + + + 12944 + nld + + + + + + + true + 1984 + + + + + + PYLYSER, JEAN-MARIE + Newsitems West Flanders. Day of the Navy in Ostend, with a wreath-laying memorial service at the + monument of the sailors on the dike. Thereafter, a military parade takes place at the Wapenplein in the presence of Prince + Albert, Princess Paola, Governor Vanneste, mayor Goekindt and certain naval officers. The navy admiral gives a speech, + followed by a short parade. The princess salutes the flag bearers of the veterans' associations. + + Westvlaams filmjournaal - WAF (Westvlaamse Actualiteiten Films). Dag van de zeemacht te + Oostende, met een kranslegging aan het monument van de zeelieden op de dijk. Naast enkele zeemachtofficieren legt ook + burgemeester Goekindt een krans. Nadien vindt een wapenschouwing plaats op het Wapenplein in aanwezigheid van prins Albert, + prinses Paola, gouverneur Vanneste en de eerder genoemde personaliteiten. De zeemachtadmiraal houdt een toespraak. Er vindt + een korte parade plaats en de prinsen groeten de vaandeldragers van de oud-strijdersverenigingen. + + Newsitems West Flanders. Day of the Navy in Ostend, with a wreath-laying memorial service at the + monument of the sailors on the dike. Thereafter, a military parade takes place at the Wapenplein in the presence of Prince + Albert, Princess Paola, Governor Vanneste, mayor Goekindt and certain naval officers. The navy admiral gives a speech, + followed by a short parade. The princess salutes the flag bearers of the veterans' associations. + + 12944 + nl + EFG1914 + World War I + WESTVLAAMS FILMJOURNAAL - 164 + Newsreel + + + 1984 + Belgium + + false + + + VIDEO + + + + Europeana Foundation + Europeana Foundation + 1_test + Netherlands + + nl + 10 + + + The Royal Belgian Film Archives + Cinémathèque royale de Belgique + Koninklijk Belgisch Filmarchief + + \ No newline at end of file From 5617f3d73a498386af55016caad92fe74ed03aa4 Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Mon, 11 Apr 2022 11:45:05 +0200 Subject: [PATCH 22/73] MET-4285 Fix path resolution for tests in Jenkins --- .../solr/SolrDocumentPopulatorTest.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/SolrDocumentPopulatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/SolrDocumentPopulatorTest.java index a0e0fcab3..541d5a549 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/solr/SolrDocumentPopulatorTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/SolrDocumentPopulatorTest.java @@ -13,11 +13,11 @@ import eu.europeana.indexing.utils.RdfWrapper; import eu.europeana.metis.schema.convert.RdfConversionUtils; import eu.europeana.metis.schema.jibx.RDF; -import java.io.File; -import java.nio.file.Files; +import java.io.FileInputStream; +import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.Objects; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.IOUtils; import org.apache.solr.common.SolrInputDocument; import org.junit.jupiter.api.Test; @@ -25,9 +25,8 @@ class SolrDocumentPopulatorTest { @Test void populateWithProperties_PlaceCoordinates() throws Exception { - ClassLoader classLoader = SolrDocumentPopulatorTest.class.getClassLoader(); - File file = new File(Objects.requireNonNull(classLoader.getResource("europeana_record_with_geospatial_data.xml")).getFile()); - String xml = new String(Files.readAllBytes(file.toPath())); + String xml = IOUtils.toString(new FileInputStream("src/test/resources/europeana_record_with_geospatial_data.xml"), + StandardCharsets.UTF_8); final RDF rdf = new RdfConversionUtils().convertStringToRdf(xml); // Perform the tier classification @@ -55,10 +54,8 @@ void populateWithProperties_PlaceCoordinates() throws Exception { @Test void populateWithProperties_WGS84Coordinates() throws Exception { - ClassLoader classLoader = SolrDocumentPopulatorTest.class.getClassLoader(); - File file = new File( - Objects.requireNonNull(classLoader.getResource("europeana_record_with_geospatial_data_wgs84.xml")).getFile()); - String xml = new String(Files.readAllBytes(file.toPath())); + String xml = IOUtils.toString(new FileInputStream("src/test/resources/europeana_record_with_geospatial_data_wgs84.xml"), + StandardCharsets.UTF_8); final RDF rdf = new RdfConversionUtils().convertStringToRdf(xml); // Perform the tier classification From 098f611382443081964549a6e55070269e585a27 Mon Sep 17 00:00:00 2001 From: jochen_vermeulen Date: Tue, 12 Apr 2022 10:42:23 +0200 Subject: [PATCH 23/73] MET-4237: Fix bugs in the post-processing code. --- .../core/execution/WorkflowPostProcessor.java | 35 ++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java index 3eecebde1..5864a613c 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java @@ -31,16 +31,17 @@ import eu.europeana.metis.exception.BadContentException; import java.util.ArrayList; import java.util.Date; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.util.CollectionUtils; /** * This object can perform post-processing for workflows. @@ -93,17 +94,27 @@ private void indexPostProcess(AbstractExecutablePlugin indexPlugin, String da final boolean isIncremental = ((IndexToPublishPlugin) indexPlugin).getPluginMetadata().isIncrementalIndexing(); if (isIncremental) { - // get all currently de-published records ids - Set depublishedRecordIds = depublishRecordIdDao - .getAllDepublishRecordIdsWithStatus(datasetId, DepublishRecordIdSortField.DEPUBLICATION_STATE, - SortDirection.ASCENDING, - DepublicationStatus.DEPUBLISHED); - - List publishedDatasetRecordIds = dpsClient.searchPublishedDatasetRecords(indexPlugin.getExternalTaskId(), - new ArrayList<>(depublishedRecordIds)); - // reset de-publish status, pass recordIds to be de-published - depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, new HashSet<>(publishedDatasetRecordIds), - DepublicationStatus.PENDING_DEPUBLICATION, null); + // get all currently de-published records IDs from the database and create their full versions + final Set depublishedRecordIds = depublishRecordIdDao.getAllDepublishRecordIdsWithStatus( + datasetId, DepublishRecordIdSortField.DEPUBLICATION_STATE, SortDirection.ASCENDING, + DepublicationStatus.DEPUBLISHED); + final Map depublishedRecordIdsByFullId = depublishedRecordIds.stream() + .collect(Collectors.toMap(id -> DepublishRecordIdUtils.composeFullRecordId(datasetId, id), + Function.identity())); + + // Check which have been published by the index action - use full record IDs for eCloud. + if (!CollectionUtils.isEmpty(depublishedRecordIdsByFullId)) { + final List publishedRecordIds = dpsClient.searchPublishedDatasetRecords(datasetId, + new ArrayList<>(depublishedRecordIdsByFullId.keySet())); + + // Remove the 'depublished' status. Note: we need to check for an empty result (otherwise + // the DAO call will update all records). Use the simple record IDs again. + if (!CollectionUtils.isEmpty(publishedRecordIds)) { + depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, + publishedRecordIds.stream().map(depublishedRecordIdsByFullId::get) + .collect(Collectors.toSet()), DepublicationStatus.PENDING_DEPUBLICATION, null); + } + } } else { // reset de-publish status, pass null, all records will be de-published depublishRecordIdDao.markRecordIdsWithDepublicationStatus(datasetId, null, From 054fde753354e1a8ee1e8d30ad30cb207a53252a Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Fri, 15 Apr 2022 16:21:18 +0200 Subject: [PATCH 24/73] MET-4449 Initial implementation of record patterns analysis (#524) * MET-4449 Initial implementation of record patterns analysis * MET-4449 Updates on problem patterns * MET-4449 Make ExecutionStep generic * MET-4449 Make ExecutionStep generic and refactor * MET-4449 Fix spelling mistake --- metis-pattern-analysis/pom.xml | 51 ++++++++ .../PatternAnalysisService.java | 92 ++++++++++++++ .../ProblemPatternAnalyzer.java | 111 ++++++++++++++++ .../exception/PatternAnalysisException.java | 19 +++ .../view/DatasetProblemPatternAnalysis.java | 52 ++++++++ .../view/ProblemOccurrence.java | 47 +++++++ .../patternanalysis/view/ProblemPattern.java | 40 ++++++ .../view/ProblemPatternDescription.java | 77 ++++++++++++ .../patternanalysis/view/RecordAnalysis.java | 32 +++++ .../ProblemPatternAnalyzerTest.java | 49 ++++++++ .../DatasetProblemPatternAnalysisTest.java | 42 +++++++ .../view/ProblemOccurrenceTest.java | 24 ++++ .../view/ProblemPatternDescriptionTest.java | 44 +++++++ .../view/ProblemPatternTest.java | 33 +++++ .../view/RecordAnalysisTest.java | 27 ++++ .../resources/europeana_record_with_P2.xml | 119 ++++++++++++++++++ .../resources/europeana_record_with_P6.xml | 102 +++++++++++++++ pom.xml | 2 + 18 files changed, 963 insertions(+) create mode 100644 metis-pattern-analysis/pom.xml create mode 100644 metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/PatternAnalysisService.java create mode 100644 metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/ProblemPatternAnalyzer.java create mode 100644 metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/exception/PatternAnalysisException.java create mode 100644 metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/DatasetProblemPatternAnalysis.java create mode 100644 metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemOccurrence.java create mode 100644 metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemPattern.java create mode 100644 metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemPatternDescription.java create mode 100644 metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/RecordAnalysis.java create mode 100644 metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/ProblemPatternAnalyzerTest.java create mode 100644 metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/DatasetProblemPatternAnalysisTest.java create mode 100644 metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemOccurrenceTest.java create mode 100644 metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemPatternDescriptionTest.java create mode 100644 metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemPatternTest.java create mode 100644 metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/RecordAnalysisTest.java create mode 100644 metis-pattern-analysis/src/test/resources/europeana_record_with_P2.xml create mode 100644 metis-pattern-analysis/src/test/resources/europeana_record_with_P6.xml diff --git a/metis-pattern-analysis/pom.xml b/metis-pattern-analysis/pom.xml new file mode 100644 index 000000000..6dfeaf2e2 --- /dev/null +++ b/metis-pattern-analysis/pom.xml @@ -0,0 +1,51 @@ + + + + metis-framework + eu.europeana.metis + 7-SNAPSHOT + + 4.0.0 + metis-pattern-analysis + + + + eu.europeana.metis + metis-schema + + + com.fasterxml.jackson.core + jackson-annotations + ${version.jackson} + + + commons-collections + commons-collections + ${version.commons.collections} + + + commons-io + commons-io + + + org.apache.commons + commons-lang3 + + + + org.junit.jupiter + junit-jupiter-api + + + org.junit.jupiter + junit-jupiter-engine + + + com.fasterxml.jackson.core + jackson-databind + ${version.jackson} + test + + + + \ No newline at end of file diff --git a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/PatternAnalysisService.java b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/PatternAnalysisService.java new file mode 100644 index 000000000..1847e52d7 --- /dev/null +++ b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/PatternAnalysisService.java @@ -0,0 +1,92 @@ +package eu.europeana.patternanalysis; + +import eu.europeana.metis.schema.jibx.RDF; +import eu.europeana.patternanalysis.exception.PatternAnalysisException; +import eu.europeana.patternanalysis.view.DatasetProblemPatternAnalysis; +import eu.europeana.patternanalysis.view.ProblemPattern; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Optional; + +/** + * Interface with all methods required for a pattern analysis service + * + * @param the type of the execution step + */ +public interface PatternAnalysisService { + + /** + * Generates the analysis of the record in RDF format. + *

    + * It will compute patterns and store all relevant information in the database + *

    + * + * @param datasetId the datasetId + * @param executionStep the constant value of the step (Similar to eu.europeana.metis.core.workflow.plugins.PluginType from + * metis-core and eu.europeana.metis.sandbox.common.Step from metis-sandbox + * @param executionTimestamp the execution timestamp for the execution of the dataset(this should be the same for all records). + * @param rdfRecord the rdf record + * @throws PatternAnalysisException if an error occurred during the analysis + */ + void generateRecordPatternAnalysis(String datasetId, T executionStep, LocalDateTime executionTimestamp, RDF rdfRecord) + throws PatternAnalysisException; + + /** + * Generates the analysis of the record in String format. + *

    + * It will compute patterns and store all relevant information in the database + *

    + * + * @param datasetId the datasetId + * @param executionStep the constant value of the step (Similar to eu.europeana.metis.core.workflow.plugins.PluginType from + * metis-core and eu.europeana.metis.sandbox.common.Step from metis-sandbox + * @param executionTimestamp the execution timestamp for the execution of the dataset(this should be the same for all records). + * @param rdfRecord the rdf record + * @throws PatternAnalysisException if an error occurred during the analysis + */ + void generateRecordPatternAnalysis(String datasetId, T executionStep, LocalDateTime executionTimestamp, String rdfRecord) + throws PatternAnalysisException; + + /** + * Finalizes the computation of the analysis for the dataset. + *

    This method should be called at the end(post-processing) of the dataset execution, to perform the final calculations

    + * + * @param datasetId the datasetId + * @param executionStep the constant value of the step (Similar to eu.europeana.metis.core.workflow.plugins.PluginType from + * metis-core and eu.europeana.metis.sandbox.common.Step from metis-sandbox). + * @param executionTimestamp the execution timestamp for the execution of the dataset(this should be the same for all records). + * @throws PatternAnalysisException if an error occurred during the analysis + */ + void finalizeDatasetPatternAnalysis(String datasetId, T executionStep, LocalDateTime executionTimestamp) + throws PatternAnalysisException; + + /** + * Get the Dataset pattern analysis for a specific execution. + *

    + * This method will generate the dataset pattern analysis for a dataset and a specific execution from the data in the database. + * An in memory cache could be implemented internally. + *

    + * + * @param datasetId the dataset identifier + * @param executionStep the execution step + * @param executionTimestamp the execution timestamp + * @return the dataset pattern analysis + */ + Optional> getDatasetPatternAnalysis(String datasetId, T executionStep, + LocalDateTime executionTimestamp); + + /** + * Get a list of problem patterns for a particular record without storing them in the database. + *

    Internally this method could check first if the analysis is present in the database and retrieve that. + * If not, it should generate it on the fly. An in memory cache could be implemented internally. + *

    + * + * @param datasetId the dataset identifier + * @param executionStep the execution step + * @param executionTimestamp the execution timestamp + * @param rdfRecord the RDF record + * @return the list of problem patterns + */ + List getRecordPatternAnalysis(String datasetId, T executionStep, LocalDateTime executionTimestamp, + RDF rdfRecord); +} diff --git a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/ProblemPatternAnalyzer.java b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/ProblemPatternAnalyzer.java new file mode 100644 index 000000000..665b89823 --- /dev/null +++ b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/ProblemPatternAnalyzer.java @@ -0,0 +1,111 @@ +package eu.europeana.patternanalysis; + +import static java.lang.String.format; +import static java.util.function.Predicate.not; + +import eu.europeana.metis.schema.convert.RdfConversionUtils; +import eu.europeana.metis.schema.convert.SerializationException; +import eu.europeana.metis.schema.jibx.Description; +import eu.europeana.metis.schema.jibx.EuropeanaType; +import eu.europeana.metis.schema.jibx.EuropeanaType.Choice; +import eu.europeana.metis.schema.jibx.ProvidedCHOType; +import eu.europeana.metis.schema.jibx.ProxyType; +import eu.europeana.metis.schema.jibx.RDF; +import eu.europeana.metis.schema.jibx.Title; +import eu.europeana.patternanalysis.view.ProblemOccurrence; +import eu.europeana.patternanalysis.view.ProblemPattern; +import eu.europeana.patternanalysis.view.ProblemPatternDescription; +import eu.europeana.patternanalysis.view.RecordAnalysis; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +/** + * Class that contains functionality to analyze a record and retrieve all problem patterns. + */ +public class ProblemPatternAnalyzer { + + public static final int MIN_TITLE_LENGTH = 2; + + /** + * Analyzes a record for problem patterns. + * + * @param rdfString the rdf record as a string + * @return a list of problem patterns + * @throws SerializationException if the record could not be converted to {@link RDF} + */ + public List analyzeRecord(String rdfString) throws SerializationException { + return analyzeRecord(new RdfConversionUtils().convertStringToRdf(rdfString)); + } + + /** + * Analyzes a record for problem patterns. + * + * @param rdf the rdf record + * @return a list of problem patterns + */ + public List analyzeRecord(RDF rdf) { + final List providerProxies = getProviderProxies(rdf); + final List titles = providerProxies.stream().map(EuropeanaType::getChoiceList).flatMap(Collection::stream) + .filter(Choice::ifTitle).map(Choice::getTitle) + .map(Title::getString) + .filter(StringUtils::isNotBlank) + .map(String::trim) + .collect(Collectors.toList()); + final List descriptions = providerProxies.stream().map(EuropeanaType::getChoiceList).flatMap(Collection::stream) + .filter(Choice::ifDescription).map(Choice::getDescription) + .map(Description::getString) + .filter(StringUtils::isNotBlank) + .map(String::trim) + .collect(Collectors.toList()); + final String rdfAbout = rdf.getProvidedCHOList().stream().filter(Objects::nonNull).findFirst() + .map(ProvidedCHOType::getAbout).orElse(null); + return computeProblemPatterns(rdfAbout, titles, descriptions); + } + + private ArrayList computeProblemPatterns(String rdfAbout, List titles, List descriptions) { + final ArrayList problemPatterns = new ArrayList<>(); + + constructProblemPattern(rdfAbout, ProblemPatternDescription.P2, checkP2(titles, descriptions)).ifPresent( + problemPatterns::add); + constructProblemPattern(rdfAbout, ProblemPatternDescription.P6, checkP6(titles)).ifPresent(problemPatterns::add); + return problemPatterns; + } + + private List getProviderProxies(RDF rdf) { + return rdf.getProxyList().stream().filter(proxyType -> proxyType.getEuropeanaProxy() != null) + .filter(not(proxyType -> proxyType.getEuropeanaProxy().isEuropeanaProxy())).collect(Collectors.toList()); + } + + private List checkP2(List titles, List descriptions) { + final Set uniqueTitles = titles.stream().map(String::toLowerCase).collect(Collectors.toSet()); + final Set uniqueDescriptions = descriptions.stream().map(String::toLowerCase).collect(Collectors.toSet()); + final HashSet equalTitlesAndDescriptions = new HashSet<>(uniqueTitles); + equalTitlesAndDescriptions.retainAll(uniqueDescriptions); + + return equalTitlesAndDescriptions.stream().map( + value -> new ProblemOccurrence(format("Equal(lower cased) title and description: %s", value)) + ).collect(Collectors.toList()); + } + + private List checkP6(List titles) { + return titles.stream().filter(title -> title.length() <= MIN_TITLE_LENGTH) + .map(title -> new ProblemOccurrence(format("Non meaningful title: %s", title))).collect(Collectors.toList()); + } + + private Optional constructProblemPattern(String recordId, ProblemPatternDescription problemPatternDescription, + List problemOccurrences) { + if (CollectionUtils.isNotEmpty(problemOccurrences)) { + return Optional.of(new ProblemPattern( + problemPatternDescription, 1, List.of(new RecordAnalysis(recordId, problemOccurrences)))); + } + return Optional.empty(); + } +} diff --git a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/exception/PatternAnalysisException.java b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/exception/PatternAnalysisException.java new file mode 100644 index 000000000..e2e54bfa4 --- /dev/null +++ b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/exception/PatternAnalysisException.java @@ -0,0 +1,19 @@ +package eu.europeana.patternanalysis.exception; + +/** + * Exception used for a pattern analysis error. + */ +public class PatternAnalysisException extends Exception { + + /** + * Constructs a new exception with the specified detail message and cause. + * + * @param message the detail message (which is saved for later retrieval by the {@link #getMessage()} method). + * @param cause the cause (which is saved for later retrieval by the {@link #getCause()} method). (A null value is + * permitted, and indicates that the cause is nonexistent or unknown.) + */ + public PatternAnalysisException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/DatasetProblemPatternAnalysis.java b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/DatasetProblemPatternAnalysis.java new file mode 100644 index 000000000..286444e60 --- /dev/null +++ b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/DatasetProblemPatternAnalysis.java @@ -0,0 +1,52 @@ +package eu.europeana.patternanalysis.view; + +import com.fasterxml.jackson.annotation.JsonFormat; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; + +/** + * Class containing the dataset analysis for problem patterns. + * + * @param the type of the execution step + */ +public class DatasetProblemPatternAnalysis { + + private final String datasetId; + private final T executionStep; + @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX") + private final LocalDateTime executionTimestamp; + private final List problemPatternList; + + /** + * Constructor with required parameters. + * + * @param datasetId the dataset id + * @param executionTimestamp the execution timestamp + * @param executionStep the execution step + * @param problemPatternList the problem pattern list + */ + public DatasetProblemPatternAnalysis(String datasetId, T executionStep, LocalDateTime executionTimestamp, + List problemPatternList) { + this.datasetId = datasetId; + this.executionStep = executionStep; + this.executionTimestamp = executionTimestamp; + this.problemPatternList = problemPatternList == null ? new ArrayList<>() : new ArrayList<>(problemPatternList); + } + + public String getDatasetId() { + return datasetId; + } + + public T getExecutionStep() { + return executionStep; + } + + public LocalDateTime getExecutionTimestamp() { + return executionTimestamp; + } + + public List getProblemPatternList() { + return new ArrayList<>(problemPatternList); + } +} diff --git a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemOccurrence.java b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemOccurrence.java new file mode 100644 index 000000000..b09aca9a4 --- /dev/null +++ b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemOccurrence.java @@ -0,0 +1,47 @@ +package eu.europeana.patternanalysis.view; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import java.util.ArrayList; +import java.util.List; + +/** + * Class containing the problem occurrence report. + *

    It also contains {@link #affectedRecordIds} which indicate other records are part of this problem with this record and + * problem. It can be null if the problem is only related to the current record.

    + */ +public class ProblemOccurrence { + + private final String messageReport; + @JsonInclude(Include.NON_NULL) + private final List affectedRecordIds; + + /** + * Constructor with required parameters. + * + * @param message the problem message + * @param affectedRecordIds the affected record ids. Can be null if the problem spans only to the current record. + */ + public ProblemOccurrence(String message, List affectedRecordIds) { + this.messageReport = message; + this.affectedRecordIds = affectedRecordIds == null ? new ArrayList<>() : new ArrayList<>(affectedRecordIds); + } + + /** + * Constructor with required parameters. + * + * @param messageReport the message report + */ + public ProblemOccurrence(String messageReport) { + this.messageReport = messageReport; + this.affectedRecordIds = new ArrayList<>(); + } + + public String getMessageReport() { + return messageReport; + } + + public List getAffectedRecordIds() { + return new ArrayList<>(affectedRecordIds); + } +} diff --git a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemPattern.java b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemPattern.java new file mode 100644 index 000000000..4c73a75e1 --- /dev/null +++ b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemPattern.java @@ -0,0 +1,40 @@ +package eu.europeana.patternanalysis.view; + +import java.util.ArrayList; +import java.util.List; + +/** + * Class containing the problem pattern including its {@link RecordAnalysis}. + */ +public class ProblemPattern { + + private final ProblemPatternDescription problemPatternDescription; + private final int recordOccurrences; + private final List recordAnalysisList; + + /** + * Constructor with required parameters. + * + * @param problemPatternId the problem pattern id + * @param recordOccurrences the record occurrences + * @param recordAnalysisList the record analysis list + */ + public ProblemPattern(ProblemPatternDescription problemPatternId, int recordOccurrences, + List recordAnalysisList) { + this.problemPatternDescription = problemPatternId; + this.recordOccurrences = recordOccurrences; + this.recordAnalysisList = recordAnalysisList == null ? new ArrayList<>() : new ArrayList<>(recordAnalysisList); + } + + public ProblemPatternDescription getProblemPatternDescription() { + return problemPatternDescription; + } + + public int getRecordOccurrences() { + return recordOccurrences; + } + + public List getRecordAnalysisList() { + return new ArrayList<>(recordAnalysisList); + } +} diff --git a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemPatternDescription.java b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemPatternDescription.java new file mode 100644 index 000000000..58dc8c886 --- /dev/null +++ b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemPatternDescription.java @@ -0,0 +1,77 @@ +package eu.europeana.patternanalysis.view; + +import com.fasterxml.jackson.annotation.JsonFormat; +import java.util.Arrays; + +/** + * Enum containing all available problem patterns. + */ +@JsonFormat(shape = JsonFormat.Shape.OBJECT) +public enum ProblemPatternDescription { + + P1(ProblemPatternId.P1, ProblemPatternSeverity.WARNING, ProblemPatternQualityDimension.CONCISENESS), + P2(ProblemPatternId.P2, ProblemPatternSeverity.WARNING, ProblemPatternQualityDimension.CONCISENESS), + P3(ProblemPatternId.P3, ProblemPatternSeverity.NOTICE, ProblemPatternQualityDimension.CONCISENESS), + P5(ProblemPatternId.P5, ProblemPatternSeverity.NOTICE, ProblemPatternQualityDimension.ACCURACY), + P6(ProblemPatternId.P6, ProblemPatternSeverity.NOTICE, ProblemPatternQualityDimension.ACCURACY), + P7(ProblemPatternId.P7, ProblemPatternSeverity.NOTICE, ProblemPatternQualityDimension.COMPLETENESS), + P9(ProblemPatternId.P9, ProblemPatternSeverity.WARNING, ProblemPatternQualityDimension.ACCURACY), + P12(ProblemPatternId.P12, ProblemPatternSeverity.NOTICE, ProblemPatternQualityDimension.ACCURACY); + + private final ProblemPatternId problemPatternId; + private final ProblemPatternSeverity problemPatternSeverity; + private final ProblemPatternQualityDimension problemPatternQualityDimension; + + + ProblemPatternDescription(ProblemPatternId problemPatternId, + ProblemPatternSeverity problemPatternSeverity, + ProblemPatternQualityDimension problemPatternQualityDimension) { + this.problemPatternId = problemPatternId; + this.problemPatternSeverity = problemPatternSeverity; + this.problemPatternQualityDimension = problemPatternQualityDimension; + } + + public ProblemPatternId getProblemPatternId() { + return problemPatternId; + } + + public ProblemPatternSeverity getProblemPatternSeverity() { + return problemPatternSeverity; + } + + public ProblemPatternQualityDimension getProblemPatternQualityDimension() { + return problemPatternQualityDimension; + } + + /** + * Retrieves an instance of the enum based on the provided enum name(ignore case) or else throws a runtime exception + * + * @param name the enum name + * @return the enum object + */ + public static ProblemPatternDescription fromName(String name) { + return Arrays.stream(ProblemPatternDescription.values()).filter(value -> value.name().equalsIgnoreCase(name)).findFirst() + .orElseThrow(); + } + + /** + * The problem pattern ids + */ + public enum ProblemPatternId { + P1, P2, P3, P5, P6, P7, P9, P12; + } + + /** + * The problem pattern severities + */ + public enum ProblemPatternSeverity { + NOTICE, WARNING, ERROR, FATAL + } + + /** + * The problem pattern quality dimensions + */ + public enum ProblemPatternQualityDimension { + ACCURACY, AVAILABILITY, COMPLETENESS, CONCISENESS, COMPLIANCE, CONSISTENCY, TIMELINESS, LICENSING, INTERLINKING, UNDERSTANDABILITY, REPRESENTATIONAL + } +} diff --git a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/RecordAnalysis.java b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/RecordAnalysis.java new file mode 100644 index 000000000..7c47ab4ba --- /dev/null +++ b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/RecordAnalysis.java @@ -0,0 +1,32 @@ +package eu.europeana.patternanalysis.view; + +import java.util.ArrayList; +import java.util.List; + +/** + * Class containing the record analysis. + */ +public class RecordAnalysis { + + private final String recordId; + private final List problemOccurrenceList; + + /** + * Constructor with required parameters. + * + * @param recordId the record id + * @param problemOccurrenceList the problem occurrences list + */ + public RecordAnalysis(String recordId, List problemOccurrenceList) { + this.recordId = recordId; + this.problemOccurrenceList = problemOccurrenceList == null ? new ArrayList<>() : new ArrayList<>(problemOccurrenceList); + } + + public String getRecordId() { + return recordId; + } + + public List getProblemOccurrenceList() { + return new ArrayList<>(problemOccurrenceList); + } +} diff --git a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/ProblemPatternAnalyzerTest.java b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/ProblemPatternAnalyzerTest.java new file mode 100644 index 000000000..60d801f47 --- /dev/null +++ b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/ProblemPatternAnalyzerTest.java @@ -0,0 +1,49 @@ +package eu.europeana.patternanalysis; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import eu.europeana.metis.schema.convert.RdfConversionUtils; +import eu.europeana.metis.schema.jibx.RDF; +import eu.europeana.patternanalysis.view.ProblemPattern; +import eu.europeana.patternanalysis.view.ProblemPatternDescription; +import java.io.FileInputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Test; + +class ProblemPatternAnalyzerTest { + + @Test + void analyzeRecord_P2() throws Exception { + //Should contain two provider proxies that each contain a pair of identical title and description. All four values are identical on the two proxies. + String xml = IOUtils.toString(new FileInputStream("src/test/resources/europeana_record_with_P2.xml"), + StandardCharsets.UTF_8); + final RDF rdf = new RdfConversionUtils().convertStringToRdf(xml); + + final ProblemPatternAnalyzer problemPatternAnalyzer = new ProblemPatternAnalyzer(); + final List problemPatterns = problemPatternAnalyzer.analyzeRecord(rdf); + + assertNotNull(problemPatterns); + assertEquals(1, problemPatterns.size()); + assertEquals(ProblemPatternDescription.P2, problemPatterns.get(0).getProblemPatternDescription()); + assertEquals(1, problemPatterns.get(0).getRecordAnalysisList().get(0).getProblemOccurrenceList().size()); + } + + @Test + void analyzeRecord_P6() throws Exception { + //Should contain one title that is not meaningful(too short) + String xml = IOUtils.toString(new FileInputStream("src/test/resources/europeana_record_with_P6.xml"), + StandardCharsets.UTF_8); + final RDF rdf = new RdfConversionUtils().convertStringToRdf(xml); + + final ProblemPatternAnalyzer problemPatternAnalyzer = new ProblemPatternAnalyzer(); + final List problemPatterns = problemPatternAnalyzer.analyzeRecord(rdf); + + assertNotNull(problemPatterns); + assertEquals(1, problemPatterns.size()); + assertEquals(ProblemPatternDescription.P6, problemPatterns.get(0).getProblemPatternDescription()); + assertEquals(1, problemPatterns.get(0).getRecordAnalysisList().get(0).getProblemOccurrenceList().size()); + } +} \ No newline at end of file diff --git a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/DatasetProblemPatternAnalysisTest.java b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/DatasetProblemPatternAnalysisTest.java new file mode 100644 index 000000000..d287b30c6 --- /dev/null +++ b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/DatasetProblemPatternAnalysisTest.java @@ -0,0 +1,42 @@ +package eu.europeana.patternanalysis.view; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.LocalDateTime; +import java.util.List; +import org.apache.commons.collections.CollectionUtils; +import org.junit.jupiter.api.Test; + +class DatasetProblemPatternAnalysisTest { + + @Test + void objectCreationTest() { + final ProblemOccurrence problemOccurrence1 = new ProblemOccurrence("Duplicate titleA", List.of("recordId1", "recordId2")); + final ProblemOccurrence problemOccurrence2 = new ProblemOccurrence("Duplicate titleB"); + final RecordAnalysis recordAnalysis1 = new RecordAnalysis("recordId1", List.of(problemOccurrence1, problemOccurrence2)); + + final RecordAnalysis recordAnalysis2 = new RecordAnalysis("recordId1", null); + + final ProblemPattern problemPattern1 = new ProblemPattern( + ProblemPatternDescription.P2, 2, List.of(recordAnalysis1, recordAnalysis2)); + final ProblemPattern problemPattern2 = new ProblemPattern(ProblemPatternDescription.P2, 2, null); + + final LocalDateTime currentDate = LocalDateTime.now(); + final DatasetProblemPatternAnalysis datasetProblemPatternAnalysis1 = new DatasetProblemPatternAnalysis<>("datasetId1", + "VALIDATION_EXTERNAL", currentDate + , List.of(problemPattern1, problemPattern2)); + + assertEquals("datasetId1", datasetProblemPatternAnalysis1.getDatasetId()); + assertEquals(0, currentDate.compareTo(datasetProblemPatternAnalysis1.getExecutionTimestamp())); + assertEquals("VALIDATION_EXTERNAL", datasetProblemPatternAnalysis1.getExecutionStep()); + assertTrue(CollectionUtils.isEqualCollection(List.of(problemPattern1, problemPattern2), + datasetProblemPatternAnalysis1.getProblemPatternList())); + + final DatasetProblemPatternAnalysis datasetProblemPatternAnalysis2 = new DatasetProblemPatternAnalysis<>("datasetId1", + "VALIDATION_EXTERNAL", currentDate, null); + assertNotNull(datasetProblemPatternAnalysis2.getProblemPatternList()); + } + +} \ No newline at end of file diff --git a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemOccurrenceTest.java b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemOccurrenceTest.java new file mode 100644 index 000000000..a8a73e73f --- /dev/null +++ b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemOccurrenceTest.java @@ -0,0 +1,24 @@ +package eu.europeana.patternanalysis.view; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.apache.commons.collections.CollectionUtils; +import org.junit.jupiter.api.Test; + +class ProblemOccurrenceTest { + + @Test + void objectCreationTest() { + final ProblemOccurrence problemOccurrence1 = new ProblemOccurrence("Duplicate titleA", List.of("recordId1", "recordId2")); + assertEquals("Duplicate titleA", problemOccurrence1.getMessageReport()); + assertTrue(CollectionUtils.isEqualCollection(List.of("recordId2", "recordId1"), problemOccurrence1.getAffectedRecordIds())); + + final ProblemOccurrence problemOccurrence2 = new ProblemOccurrence("Duplicate titleB"); + assertNotNull(problemOccurrence2.getAffectedRecordIds()); + assertEquals(0, problemOccurrence2.getAffectedRecordIds().size()); + } + +} \ No newline at end of file diff --git a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemPatternDescriptionTest.java b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemPatternDescriptionTest.java new file mode 100644 index 000000000..793c7935d --- /dev/null +++ b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemPatternDescriptionTest.java @@ -0,0 +1,44 @@ +package eu.europeana.patternanalysis.view; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class ProblemPatternDescriptionTest { + + @Test + void checkValues() { + assertEquals("P1", ProblemPatternDescription.P1.getProblemPatternId().toString()); + assertEquals("WARNING", ProblemPatternDescription.P1.getProblemPatternSeverity().toString()); + assertEquals("CONCISENESS", ProblemPatternDescription.P1.getProblemPatternQualityDimension().toString()); + + assertEquals("P2", ProblemPatternDescription.P2.getProblemPatternId().toString()); + assertEquals("WARNING", ProblemPatternDescription.P2.getProblemPatternSeverity().toString()); + assertEquals("CONCISENESS", ProblemPatternDescription.P2.getProblemPatternQualityDimension().toString()); + + assertEquals("P3", ProblemPatternDescription.P3.getProblemPatternId().toString()); + assertEquals("NOTICE", ProblemPatternDescription.P3.getProblemPatternSeverity().toString()); + assertEquals("CONCISENESS", ProblemPatternDescription.P3.getProblemPatternQualityDimension().toString()); + + assertEquals("P5", ProblemPatternDescription.P5.getProblemPatternId().toString()); + assertEquals("NOTICE", ProblemPatternDescription.P5.getProblemPatternSeverity().toString()); + assertEquals("ACCURACY", ProblemPatternDescription.P5.getProblemPatternQualityDimension().toString()); + + assertEquals("P6", ProblemPatternDescription.P6.getProblemPatternId().toString()); + assertEquals("NOTICE", ProblemPatternDescription.P6.getProblemPatternSeverity().toString()); + assertEquals("ACCURACY", ProblemPatternDescription.P6.getProblemPatternQualityDimension().toString()); + + assertEquals("P7", ProblemPatternDescription.P7.getProblemPatternId().toString()); + assertEquals("NOTICE", ProblemPatternDescription.P7.getProblemPatternSeverity().toString()); + assertEquals("COMPLETENESS", ProblemPatternDescription.P7.getProblemPatternQualityDimension().toString()); + + assertEquals("P9", ProblemPatternDescription.P9.getProblemPatternId().toString()); + assertEquals("WARNING", ProblemPatternDescription.P9.getProblemPatternSeverity().toString()); + assertEquals("ACCURACY", ProblemPatternDescription.P9.getProblemPatternQualityDimension().toString()); + + assertEquals("P12", ProblemPatternDescription.P12.getProblemPatternId().toString()); + assertEquals("NOTICE", ProblemPatternDescription.P12.getProblemPatternSeverity().toString()); + assertEquals("ACCURACY", ProblemPatternDescription.P12.getProblemPatternQualityDimension().toString()); + } + +} \ No newline at end of file diff --git a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemPatternTest.java b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemPatternTest.java new file mode 100644 index 000000000..d4112cbff --- /dev/null +++ b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemPatternTest.java @@ -0,0 +1,33 @@ +package eu.europeana.patternanalysis.view; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.apache.commons.collections.CollectionUtils; +import org.junit.jupiter.api.Test; + +class ProblemPatternTest { + + @Test + void objectCreationTest() { + final ProblemOccurrence problemOccurrence1 = new ProblemOccurrence("Duplicate titleA", List.of("recordId1", "recordId2")); + final ProblemOccurrence problemOccurrence2 = new ProblemOccurrence("Duplicate titleB"); + final RecordAnalysis recordAnalysis1 = new RecordAnalysis("recordId1", List.of(problemOccurrence1, problemOccurrence2)); + + final RecordAnalysis recordAnalysis2 = new RecordAnalysis("recordId1", null); + + final ProblemPattern problemPattern1 = new ProblemPattern( + ProblemPatternDescription.P2, 2, List.of(recordAnalysis1, recordAnalysis2)); + + assertEquals(ProblemPatternDescription.P2, problemPattern1.getProblemPatternDescription()); + assertEquals(2, problemPattern1.getRecordOccurrences()); + assertTrue( + CollectionUtils.isEqualCollection(List.of(recordAnalysis1, recordAnalysis2), problemPattern1.getRecordAnalysisList())); + + final ProblemPattern problemPattern2 = new ProblemPattern(ProblemPatternDescription.P2, 2, null); + assertNotNull(problemPattern2.getRecordAnalysisList()); + } + +} \ No newline at end of file diff --git a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/RecordAnalysisTest.java b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/RecordAnalysisTest.java new file mode 100644 index 000000000..3bcb60b29 --- /dev/null +++ b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/RecordAnalysisTest.java @@ -0,0 +1,27 @@ +package eu.europeana.patternanalysis.view; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.apache.commons.collections.CollectionUtils; +import org.junit.jupiter.api.Test; + +class RecordAnalysisTest { + + @Test + void objectCreationTest() { + final ProblemOccurrence problemOccurrence1 = new ProblemOccurrence("Duplicate titleA", List.of("recordId1", "recordId2")); + final ProblemOccurrence problemOccurrence2 = new ProblemOccurrence("Duplicate titleB"); + final RecordAnalysis recordAnalysis1 = new RecordAnalysis("recordId1", List.of(problemOccurrence1, problemOccurrence2)); + + assertEquals("recordId1", recordAnalysis1.getRecordId()); + assertTrue(CollectionUtils.isEqualCollection(List.of(problemOccurrence1, problemOccurrence2), + recordAnalysis1.getProblemOccurrenceList())); + + final RecordAnalysis recordAnalysis2 = new RecordAnalysis("recordId1", null); + assertNotNull(recordAnalysis2.getProblemOccurrenceList()); + } + +} \ No newline at end of file diff --git a/metis-pattern-analysis/src/test/resources/europeana_record_with_P2.xml b/metis-pattern-analysis/src/test/resources/europeana_record_with_P2.xml new file mode 100644 index 000000000..129932bc3 --- /dev/null +++ b/metis-pattern-analysis/src/test/resources/europeana_record_with_P2.xml @@ -0,0 +1,119 @@ + + + + + + + text/html + 197506 + + + image/jpeg + 6643 + 640 + 480 + sRGB + #F0E68C + #B22222 + #FF4500 + #2F4F4F + #ADFF2F + #87CEEB + landscape + + + 50.75 + 4.5 + + + + 1914-1919 के मध्य मुख्यतः यूरोप में व्याप्त इस महायुद्ध को प्रथम विश्व युद्ध कहते हैं । यह महायुद्ध + यूरोप, एशिया व अफ्रीका तीन महाद्वीपों और जल, थल तथा आकाश में लड़ा गया। इसमें भाग लेने वाले देशों की संख्या, इसका क्षेत्र + (जिसमें यह लड़ा गया) तथा इससे हुई क्षति के अभूतपूर्व आंकड़ों के कारण ही इसे विश्वयुद्ध कहते हैं । + + Første verdenskrig + + + + + + + + EFG - The European Film Gateway + J.M.P.- Trends + + + + 12944 + nld + + + + true + 1984 + + + + + + PYLYSER, JEAN-MARIE + same title and Description + 12944 + nl + EFG1914 + World War I + WESTVLAAMS FILMJOURNAAL - 164 + Same title and Description + Newsreel + + + 1984 + Belgium + + false + + + VIDEO + + + PYLYSER, JEAN-MARIE + same title and Description + 12944 + nl + EFG1914 + World War I + WESTVLAAMS FILMJOURNAAL - 164 + Same title and Description + Newsreel + + + 1984 + Belgium + + false + + + + VIDEO + + + + Europeana Foundation + Europeana Foundation + 1_test + Netherlands + + nl + 10 + + + The Royal Belgian Film Archives + Cinémathèque royale de Belgique + Koninklijk Belgisch Filmarchief + + \ No newline at end of file diff --git a/metis-pattern-analysis/src/test/resources/europeana_record_with_P6.xml b/metis-pattern-analysis/src/test/resources/europeana_record_with_P6.xml new file mode 100644 index 000000000..381c8b6f9 --- /dev/null +++ b/metis-pattern-analysis/src/test/resources/europeana_record_with_P6.xml @@ -0,0 +1,102 @@ + + + + + + + text/html + 197506 + + + image/jpeg + 6643 + 640 + 480 + sRGB + #F0E68C + #B22222 + #FF4500 + #2F4F4F + #ADFF2F + #87CEEB + landscape + + + 50.75 + 4.5 + + + + 1914-1919 के मध्य मुख्यतः यूरोप में व्याप्त इस महायुद्ध को प्रथम विश्व युद्ध कहते हैं । यह महायुद्ध + यूरोप, एशिया व अफ्रीका तीन महाद्वीपों और जल, थल तथा आकाश में लड़ा गया। इसमें भाग लेने वाले देशों की संख्या, इसका क्षेत्र + (जिसमें यह लड़ा गया) तथा इससे हुई क्षति के अभूतपूर्व आंकड़ों के कारण ही इसे विश्वयुद्ध कहते हैं । + + Første verdenskrig + + + + + + + + EFG - The European Film Gateway + J.M.P.- Trends + + + + 12944 + nld + + + + true + 1984 + + + + + + PYLYSER, JEAN-MARIE + Newsitems West Flanders. Day of the Navy in Ostend, with a wreath-laying memorial service at the + monument of the sailors on the dike. Thereafter, a military parade takes place at the Wapenplein in the presence of Prince + Albert, Princess Paola, Governor Vanneste, mayor Goekindt and certain naval officers. The navy admiral gives a speech, + followed by a short parade. The princess salutes the flag bearers of the veterans' associations. + + 12944 + nl + EFG1914 + World War I + WESTVLAAMS FILMJOURNAAL - 164 + AB + Newsreel + + + 1984 + Belgium + + false + + + VIDEO + + + + Europeana Foundation + Europeana Foundation + 1_test + Netherlands + + nl + 10 + + + The Royal Belgian Film Archives + Cinémathèque royale de Belgique + Koninklijk Belgisch Filmarchief + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 57995f0eb..1e0c90eb6 100644 --- a/pom.xml +++ b/pom.xml @@ -21,6 +21,7 @@ metis-transformation metis-harvesting metis-repository + metis-pattern-analysis @@ -131,6 +132,7 @@ 1.4 2.9.0 3.12.0 + 3.2.2 1.11 2.15.3 From 603923f1fd95733bf37401ce01cf44292ce8ef4a Mon Sep 17 00:00:00 2001 From: Joana Sousa Date: Fri, 22 Apr 2022 12:17:11 +0200 Subject: [PATCH 25/73] MET-4240 Removed input variables from PatternAnalysisService interface --- .../europeana/patternanalysis/PatternAnalysisService.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/PatternAnalysisService.java b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/PatternAnalysisService.java index 1847e52d7..e08b08158 100644 --- a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/PatternAnalysisService.java +++ b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/PatternAnalysisService.java @@ -81,12 +81,8 @@ Optional> getDatasetPatternAnalysis(String data * If not, it should generate it on the fly. An in memory cache could be implemented internally. *

    * - * @param datasetId the dataset identifier - * @param executionStep the execution step - * @param executionTimestamp the execution timestamp * @param rdfRecord the RDF record * @return the list of problem patterns */ - List getRecordPatternAnalysis(String datasetId, T executionStep, LocalDateTime executionTimestamp, - RDF rdfRecord); + List getRecordPatternAnalysis(RDF rdfRecord); } From f75047347b4dbb0fd424232b9a230305cc782e55 Mon Sep 17 00:00:00 2001 From: JoanaCMS <70145179+JoanaCMS@users.noreply.github.com> Date: Tue, 3 May 2022 13:30:58 +0200 Subject: [PATCH 26/73] Debt/MET-4418 Change Macedonia to North Macedonia (#526) * MET-4418 Upate Macedonia to North Macedonia * MET-4418 Fixed Language value --- .../src/main/java/eu/europeana/metis/core/common/Country.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/common/Country.java b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/common/Country.java index 3301b436d..2c2022cdd 100644 --- a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/common/Country.java +++ b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/common/Country.java @@ -44,7 +44,7 @@ public enum Country { LIECHTENSTEIN("Liechtenstein", "LI"), LITHUANIA("Lithuania", "LT"), LUXEMBOURG("Luxembourg", "LU"), - MACEDONIA("Macedonia", "MK"), + NORTH_MACEDONIA("North Macedonia", "MK"), MALTA("Malta", "MT"), MOLDOVA("Moldova", "MD"), MONACO("Monaco", "MC"), From b6f29755383df53b099366684341b6c3dcfe83e6 Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Tue, 3 May 2022 15:54:31 +0200 Subject: [PATCH 27/73] MET-4470 Generify execution point for pattern service (#525) --- .../PatternAnalysisService.java | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/PatternAnalysisService.java b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/PatternAnalysisService.java index e08b08158..b70ac6165 100644 --- a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/PatternAnalysisService.java +++ b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/PatternAnalysisService.java @@ -12,23 +12,33 @@ * Interface with all methods required for a pattern analysis service * * @param the type of the execution step + * @param the type of the execution point */ -public interface PatternAnalysisService { +public interface PatternAnalysisService { /** - * Generates the analysis of the record in RDF format. - *

    - * It will compute patterns and store all relevant information in the database - *

    + * Initializes the pattern analysis execution to create a unique execution point. + *

    This method should be called at the beginning(pre-processing) of the dataset execution once

    * * @param datasetId the datasetId * @param executionStep the constant value of the step (Similar to eu.europeana.metis.core.workflow.plugins.PluginType from * metis-core and eu.europeana.metis.sandbox.common.Step from metis-sandbox * @param executionTimestamp the execution timestamp for the execution of the dataset(this should be the same for all records). + * @return the execution point that can be used on other calls + */ + K initializePatternAnalysisExecution(String datasetId, T executionStep, LocalDateTime executionTimestamp); + + /** + * Generates the analysis of the record in RDF format. + *

    + * It will compute patterns and store all relevant information in the database + *

    + * + * @param executionPoint the execution point * @param rdfRecord the rdf record * @throws PatternAnalysisException if an error occurred during the analysis */ - void generateRecordPatternAnalysis(String datasetId, T executionStep, LocalDateTime executionTimestamp, RDF rdfRecord) + void generateRecordPatternAnalysis(K executionPoint, RDF rdfRecord) throws PatternAnalysisException; /** @@ -37,27 +47,21 @@ void generateRecordPatternAnalysis(String datasetId, T executionStep, LocalDateT * It will compute patterns and store all relevant information in the database *

    * - * @param datasetId the datasetId - * @param executionStep the constant value of the step (Similar to eu.europeana.metis.core.workflow.plugins.PluginType from - * metis-core and eu.europeana.metis.sandbox.common.Step from metis-sandbox - * @param executionTimestamp the execution timestamp for the execution of the dataset(this should be the same for all records). + * @param executionPoint the execution point * @param rdfRecord the rdf record * @throws PatternAnalysisException if an error occurred during the analysis */ - void generateRecordPatternAnalysis(String datasetId, T executionStep, LocalDateTime executionTimestamp, String rdfRecord) + void generateRecordPatternAnalysis(K executionPoint, String rdfRecord) throws PatternAnalysisException; /** * Finalizes the computation of the analysis for the dataset. *

    This method should be called at the end(post-processing) of the dataset execution, to perform the final calculations

    * - * @param datasetId the datasetId - * @param executionStep the constant value of the step (Similar to eu.europeana.metis.core.workflow.plugins.PluginType from - * metis-core and eu.europeana.metis.sandbox.common.Step from metis-sandbox). - * @param executionTimestamp the execution timestamp for the execution of the dataset(this should be the same for all records). + * @param executionPoint the execution point * @throws PatternAnalysisException if an error occurred during the analysis */ - void finalizeDatasetPatternAnalysis(String datasetId, T executionStep, LocalDateTime executionTimestamp) + void finalizeDatasetPatternAnalysis(K executionPoint) throws PatternAnalysisException; /** From d00c518ebb0ad593cd207bb03513feeab3ad2a5e Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Tue, 10 May 2022 09:13:24 +0200 Subject: [PATCH 28/73] Feat/met 4457 implement problem patterns p3 p5 p7 p9 p12 (#527) * MET-4457 Implement P5 * MET-4457 Combine choices code * MET-4457 Implement P7 * MET-4457 Implement P9 * MET-4457 Fix P5 * MET-4457 Refactor tests * MET-4457 Implement P12 * MET-4457 Implement P3 * MET-4457 Cover case where europeanaProxy null * MET-4457 Do not capture identical pairs(Like P2) on P3 * MET-4457 Unit test unicode codes case unicode codes are counted as one character * MET-4457 Update comment * MET-4457 Process review --- metis-pattern-analysis/pom.xml | 10 + .../ProblemPatternAnalyzer.java | 199 ++++++++++++++++-- .../view/ProblemPatternDescription.java | 24 +++ .../ProblemPatternAnalyzerTest.java | 92 +++++--- .../europeana_record_empty_proxy_choices.xml | 100 +++++++++ .../europeana_record_no_problem_patterns.xml | 101 +++++++++ .../resources/europeana_record_with_P12.xml | 100 +++++++++ .../resources/europeana_record_with_P2.xml | 9 +- .../resources/europeana_record_with_P3.xml | 126 +++++++++++ .../resources/europeana_record_with_P5.xml | 114 ++++++++++ .../resources/europeana_record_with_P6.xml | 6 +- .../resources/europeana_record_with_P7.xml | 95 +++++++++ ...eana_record_with_P7_descriptions_empty.xml | 99 +++++++++ .../resources/europeana_record_with_P9.xml | 98 +++++++++ pom.xml | 1 + 15 files changed, 1116 insertions(+), 58 deletions(-) create mode 100644 metis-pattern-analysis/src/test/resources/europeana_record_empty_proxy_choices.xml create mode 100644 metis-pattern-analysis/src/test/resources/europeana_record_no_problem_patterns.xml create mode 100644 metis-pattern-analysis/src/test/resources/europeana_record_with_P12.xml create mode 100644 metis-pattern-analysis/src/test/resources/europeana_record_with_P3.xml create mode 100644 metis-pattern-analysis/src/test/resources/europeana_record_with_P5.xml create mode 100644 metis-pattern-analysis/src/test/resources/europeana_record_with_P7.xml create mode 100644 metis-pattern-analysis/src/test/resources/europeana_record_with_P7_descriptions_empty.xml create mode 100644 metis-pattern-analysis/src/test/resources/europeana_record_with_P9.xml diff --git a/metis-pattern-analysis/pom.xml b/metis-pattern-analysis/pom.xml index 6dfeaf2e2..00ffc9309 100644 --- a/metis-pattern-analysis/pom.xml +++ b/metis-pattern-analysis/pom.xml @@ -31,6 +31,12 @@ org.apache.commons commons-lang3 + + org.apache.commons + commons-text + ${version.apache.commons.text} + + org.junit.jupiter @@ -40,6 +46,10 @@ org.junit.jupiter junit-jupiter-engine + + org.junit.jupiter + junit-jupiter-params + com.fasterxml.jackson.core jackson-databind diff --git a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/ProblemPatternAnalyzer.java b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/ProblemPatternAnalyzer.java index 665b89823..6960374de 100644 --- a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/ProblemPatternAnalyzer.java +++ b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/ProblemPatternAnalyzer.java @@ -2,37 +2,55 @@ import static java.lang.String.format; import static java.util.function.Predicate.not; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; +import static java.util.stream.Collectors.toSet; +import static org.apache.commons.lang3.BooleanUtils.isFalse; import eu.europeana.metis.schema.convert.RdfConversionUtils; import eu.europeana.metis.schema.convert.SerializationException; -import eu.europeana.metis.schema.jibx.Description; import eu.europeana.metis.schema.jibx.EuropeanaType; import eu.europeana.metis.schema.jibx.EuropeanaType.Choice; +import eu.europeana.metis.schema.jibx.LiteralType; import eu.europeana.metis.schema.jibx.ProvidedCHOType; import eu.europeana.metis.schema.jibx.ProxyType; import eu.europeana.metis.schema.jibx.RDF; -import eu.europeana.metis.schema.jibx.Title; +import eu.europeana.metis.schema.jibx.ResourceOrLiteralType; import eu.europeana.patternanalysis.view.ProblemOccurrence; import eu.europeana.patternanalysis.view.ProblemPattern; import eu.europeana.patternanalysis.view.ProblemPatternDescription; import eu.europeana.patternanalysis.view.RecordAnalysis; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.text.similarity.LongestCommonSubsequence; /** * Class that contains functionality to analyze a record and retrieve all problem patterns. */ public class ProblemPatternAnalyzer { - public static final int MIN_TITLE_LENGTH = 2; + private static final int MIN_TITLE_LENGTH = 2; + private static final int MAX_TITLE_LENGTH = 70; + private static final int MIN_DESCRIPTION_LENGTH = 50; + private static final int UNRECOGNIZABLE_CHARACTERS_THRESHOLD = 5; + private static final double LCS_CALCULATION_THRESHOLD = 0.9; + private static final int TITLE_DESCRIPTION_LENGTH_DISTANCE = 20; + // Match anything that is not alphanumeric in all languages or literal spaces. We cannot just use \\w + private static final String UNRECOGNIZABLE_CHARACTERS_REGEX = "[^\\p{IsAlphabetic}\\p{IsDigit} ]"; + private static final Pattern UNRECOGNIZABLE_CHARACTERS_PATTERN = Pattern.compile(UNRECOGNIZABLE_CHARACTERS_REGEX); /** * Analyzes a record for problem patterns. @@ -53,51 +71,186 @@ public List analyzeRecord(String rdfString) throws Serialization */ public List analyzeRecord(RDF rdf) { final List providerProxies = getProviderProxies(rdf); - final List titles = providerProxies.stream().map(EuropeanaType::getChoiceList).flatMap(Collection::stream) - .filter(Choice::ifTitle).map(Choice::getTitle) - .map(Title::getString) - .filter(StringUtils::isNotBlank) - .map(String::trim) - .collect(Collectors.toList()); - final List descriptions = providerProxies.stream().map(EuropeanaType::getChoiceList).flatMap(Collection::stream) - .filter(Choice::ifDescription).map(Choice::getDescription) - .map(Description::getString) - .filter(StringUtils::isNotBlank) - .map(String::trim) - .collect(Collectors.toList()); + final List choices = providerProxies.stream().map(EuropeanaType::getChoiceList) + .filter(Objects::nonNull) + .flatMap(Collection::stream) + .collect(toList()); + + final List titles = getChoicesInStringList(choices, Choice::ifTitle, Choice::getTitle, LiteralType::getString); + final List descriptions = getChoicesInStringList(choices, Choice::ifDescription, Choice::getDescription, + ResourceOrLiteralType::getString); + final List identifiers = getChoicesInStringList(choices, Choice::ifIdentifier, Choice::getIdentifier, + LiteralType::getString); final String rdfAbout = rdf.getProvidedCHOList().stream().filter(Objects::nonNull).findFirst() .map(ProvidedCHOType::getAbout).orElse(null); - return computeProblemPatterns(rdfAbout, titles, descriptions); + return computeProblemPatterns(rdfAbout, titles, descriptions, identifiers); + } + + private List getChoicesInStringList(List choices, Predicate choicePredicate, + Function choiceGetter, Function getString) { + return choices.stream().filter(Objects::nonNull).filter(choicePredicate).map(choiceGetter).map(getString).collect(toList()); } - private ArrayList computeProblemPatterns(String rdfAbout, List titles, List descriptions) { + private ArrayList computeProblemPatterns(String rdfAbout, List titles, List descriptions, + List identifiers) { final ArrayList problemPatterns = new ArrayList<>(); constructProblemPattern(rdfAbout, ProblemPatternDescription.P2, checkP2(titles, descriptions)).ifPresent( problemPatterns::add); + constructProblemPattern(rdfAbout, ProblemPatternDescription.P3, checkP3(titles, descriptions)).ifPresent( + problemPatterns::add); + constructProblemPattern(rdfAbout, ProblemPatternDescription.P5, checkP5(titles, identifiers)).ifPresent(problemPatterns::add); constructProblemPattern(rdfAbout, ProblemPatternDescription.P6, checkP6(titles)).ifPresent(problemPatterns::add); + constructProblemPattern(rdfAbout, ProblemPatternDescription.P7, checkP7(descriptions)).ifPresent(problemPatterns::add); + constructProblemPattern(rdfAbout, ProblemPatternDescription.P9, checkP9(descriptions)).ifPresent(problemPatterns::add); + constructProblemPattern(rdfAbout, ProblemPatternDescription.P12, checkP12(titles)).ifPresent(problemPatterns::add); return problemPatterns; } + private static boolean isProviderProxy(ProxyType proxy) { + return proxy.getEuropeanaProxy() == null || isFalse(proxy.getEuropeanaProxy().isEuropeanaProxy()); + } + private List getProviderProxies(RDF rdf) { - return rdf.getProxyList().stream().filter(proxyType -> proxyType.getEuropeanaProxy() != null) - .filter(not(proxyType -> proxyType.getEuropeanaProxy().isEuropeanaProxy())).collect(Collectors.toList()); + return Optional.ofNullable(rdf.getProxyList()).stream().flatMap(Collection::stream) + .filter(Objects::nonNull).filter(ProblemPatternAnalyzer::isProviderProxy) + .collect(Collectors.toList()); } + /** + * Check whether there is a title - description pair for which the values are equal, ignoring letter (upper or lower) case. + *

    It will report a single occurrence for multiple same fields

    + * + * @param titles the list of titles + * @param descriptions the list of descriptions + * @return the list of problem occurrences encountered + */ private List checkP2(List titles, List descriptions) { - final Set uniqueTitles = titles.stream().map(String::toLowerCase).collect(Collectors.toSet()); - final Set uniqueDescriptions = descriptions.stream().map(String::toLowerCase).collect(Collectors.toSet()); + final Set uniqueTitles = titles.stream().map(String::toLowerCase).collect(toSet()); + final Set uniqueDescriptions = descriptions.stream().map(String::toLowerCase).collect(toSet()); final HashSet equalTitlesAndDescriptions = new HashSet<>(uniqueTitles); equalTitlesAndDescriptions.retainAll(uniqueDescriptions); return equalTitlesAndDescriptions.stream().map( value -> new ProblemOccurrence(format("Equal(lower cased) title and description: %s", value)) - ).collect(Collectors.toList()); + ).collect(toList()); } + /** + * Check whether there is a title - description pair for which the values are too similar. + *

    + * The solution is based on the LCS algorithm(Longest + * Common Subsequence). + *

    + * The formula chosen is: + *

    + * LCS (title, description) / minimum(length(title), length(desc)) >= 0.9 && |length(title)-length(desc)| <= 20 + *

    + * Blank values are filtered out. Titles and descriptions that are equal, ignoring letter (upper or lower) case are filtered + * out. Same titles will be reported once and will not have a duplicate of it self with same near identical descriptions. + * + * @param titles the list of titles + * @param descriptions the list of descriptions + * @return the list of problem occurrences encountered + */ + private List checkP3(List titles, List descriptions) { + final Map> nearIdenticalTitleDescriptionsMap = + titles.stream().filter(StringUtils::isNotBlank) + .collect(toMap(title -> title, title -> nearIdenticalDescriptions(title, descriptions), (t1, t2) -> t1)); + + return nearIdenticalTitleDescriptionsMap.entrySet().stream().flatMap( + entry -> entry.getValue().stream().map( + value -> new ProblemOccurrence(format("Near-Identical title and description fields: %s | %s", entry.getKey(), value)) + ) + ).collect(toList()); + } + + private List nearIdenticalDescriptions(String title, List descriptions) { + final LongestCommonSubsequence longestCommonSubsequence = new LongestCommonSubsequence(); + final Predicate lcsPredicate = description -> + ((double) longestCommonSubsequence.apply(title, description) / Math.min(title.length(), description.length())) + >= LCS_CALCULATION_THRESHOLD; + final Predicate distancePredicate = description -> Math.abs(title.length() - description.length()) + <= TITLE_DESCRIPTION_LENGTH_DISTANCE; + return descriptions.stream().filter(StringUtils::isNotBlank).filter(not(title::equalsIgnoreCase)) + .filter(lcsPredicate.and(distancePredicate)).collect(toList()); + } + + /** + * Check whether a title is not human-readable. + *

    + * We check this by: + *

      + *
    • Whether there are more than 5 characters that are not valid. + * Non valid characters are considered characters that are not alphanumeric and are not simple "literal" spaces(tabs, + * new lines etc. are considered invalid characters). + * This is performed with regex unicode matching {@link #UNRECOGNIZABLE_CHARACTERS_REGEX} and should support all languages. + * For more information check unicode regex
      • + *
    • The title does not fully contain an identifier
      • + *
    + *

    + * + * @param titles the list of titles + * @param identifiers the list of identifiers + * @return the list of problem occurrences encountered + */ + private List checkP5(List titles, List identifiers) { + final Predicate moreThanThresholdUnrecognizableCharacters = s -> + UNRECOGNIZABLE_CHARACTERS_PATTERN.matcher(s).results().count() > UNRECOGNIZABLE_CHARACTERS_THRESHOLD; + final Predicate containsIdentifier = s -> identifiers.stream().anyMatch(s::contains); + return titles.stream().filter(moreThanThresholdUnrecognizableCharacters.or(containsIdentifier)) + .map(title -> new ProblemOccurrence(format("Unrecognized title: %s", title)) + ).collect(toList()); + } + + /** + * Check whether the record has titles of {@link #MIN_TITLE_LENGTH} characters or fewer. + * + * @param titles the list of titles + * @return the list of problem occurrences encountered + */ private List checkP6(List titles) { return titles.stream().filter(title -> title.length() <= MIN_TITLE_LENGTH) - .map(title -> new ProblemOccurrence(format("Non meaningful title: %s", title))).collect(Collectors.toList()); + .map(title -> new ProblemOccurrence(format("Non meaningful title: %s", title))).collect(toList()); + } + + /** + * Check whether the record is lacking a description (or only has empty descriptions). + * + * @param descriptions the list of descriptions + * @return the list of problem occurrences encountered + */ + private List checkP7(List descriptions) { + if (CollectionUtils.isEmpty(descriptions) || descriptions.stream().allMatch(StringUtils::isBlank)) { + return List.of(new ProblemOccurrence("Missing description fields")); + } + return Collections.emptyList(); + } + + /** + * Check whether the record has descriptions of {@link #MIN_DESCRIPTION_LENGTH} characters or fewer. + *

    Blank values are filtered out

    + * + * @param descriptions the list of descriptions + * @return the list of problem occurrences encountered + */ + private List checkP9(List descriptions) { + return descriptions.stream().filter(StringUtils::isNotBlank) + .filter(description -> description.length() <= MIN_DESCRIPTION_LENGTH) + .map(description -> new ProblemOccurrence(format("Very short description: %s", description))) + .collect(toList()); + } + + /** + * Check whether the record has titles of more than {@link #MAX_TITLE_LENGTH} characters. + *

    Unicode codes are converted to relevant characters(counted as one character) and the length of that is checked.

    + * + * @param titles the list of titles + * @return the list of problem occurrences encountered + */ + private List checkP12(List titles) { + return titles.stream().filter(title -> title.length() > MAX_TITLE_LENGTH) + .map(title -> new ProblemOccurrence(format("Extremely long title: %s", title))).collect(toList()); } private Optional constructProblemPattern(String recordId, ProblemPatternDescription problemPatternDescription, diff --git a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemPatternDescription.java b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemPatternDescription.java index 58dc8c886..3983d9d64 100644 --- a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemPatternDescription.java +++ b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemPatternDescription.java @@ -9,13 +9,37 @@ @JsonFormat(shape = JsonFormat.Shape.OBJECT) public enum ProblemPatternDescription { + /** + * Systematic use of the same title + */ P1(ProblemPatternId.P1, ProblemPatternSeverity.WARNING, ProblemPatternQualityDimension.CONCISENESS), + /** + * Equal title and description fields + */ P2(ProblemPatternId.P2, ProblemPatternSeverity.WARNING, ProblemPatternQualityDimension.CONCISENESS), + /** + * Near-Identical title and description fields + */ P3(ProblemPatternId.P3, ProblemPatternSeverity.NOTICE, ProblemPatternQualityDimension.CONCISENESS), + /** + * Unrecognizable title + */ P5(ProblemPatternId.P5, ProblemPatternSeverity.NOTICE, ProblemPatternQualityDimension.ACCURACY), + /** + * Non-meaningful title + */ P6(ProblemPatternId.P6, ProblemPatternSeverity.NOTICE, ProblemPatternQualityDimension.ACCURACY), + /** + * Missing description fields + */ P7(ProblemPatternId.P7, ProblemPatternSeverity.NOTICE, ProblemPatternQualityDimension.COMPLETENESS), + /** + * Very short description + */ P9(ProblemPatternId.P9, ProblemPatternSeverity.WARNING, ProblemPatternQualityDimension.ACCURACY), + /** + * Extremely long values + */ P12(ProblemPatternId.P12, ProblemPatternSeverity.NOTICE, ProblemPatternQualityDimension.ACCURACY); private final ProblemPatternId problemPatternId; diff --git a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/ProblemPatternAnalyzerTest.java b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/ProblemPatternAnalyzerTest.java index 60d801f47..b23f8c499 100644 --- a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/ProblemPatternAnalyzerTest.java +++ b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/ProblemPatternAnalyzerTest.java @@ -3,47 +3,89 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import eu.europeana.metis.schema.convert.RdfConversionUtils; -import eu.europeana.metis.schema.jibx.RDF; +import eu.europeana.metis.schema.convert.SerializationException; import eu.europeana.patternanalysis.view.ProblemPattern; import eu.europeana.patternanalysis.view.ProblemPatternDescription; import java.io.FileInputStream; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.stream.Stream; import org.apache.commons.io.IOUtils; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; class ProblemPatternAnalyzerTest { - @Test - void analyzeRecord_P2() throws Exception { - //Should contain two provider proxies that each contain a pair of identical title and description. All four values are identical on the two proxies. - String xml = IOUtils.toString(new FileInputStream("src/test/resources/europeana_record_with_P2.xml"), - StandardCharsets.UTF_8); - final RDF rdf = new RdfConversionUtils().convertStringToRdf(xml); + public static final String FILE_XML_NO_PROBLEM_PATTERNS_LOCATION = "src/test/resources/europeana_record_no_problem_patterns.xml"; + public static final String FILE_XML_EMPTY_CHOICES_LOCATION = "src/test/resources/europeana_record_empty_proxy_choices.xml"; + public static final String FILE_XML_P2_LOCATION = "src/test/resources/europeana_record_with_P2.xml"; + public static final String FILE_XML_P3_LOCATION = "src/test/resources/europeana_record_with_P3.xml"; + public static final String FILE_XML_P5_LOCATION = "src/test/resources/europeana_record_with_P5.xml"; + public static final String FILE_XML_P6_LOCATION = "src/test/resources/europeana_record_with_P6.xml"; + public static final String FILE_XML_P7_LOCATION = "src/test/resources/europeana_record_with_P7.xml"; + public static final String FILE_XML_P7_DESCRIPTIONS_EMPTY_LOCATION = "src/test/resources/europeana_record_with_P7_descriptions_empty.xml"; + public static final String FILE_XML_P9_LOCATION = "src/test/resources/europeana_record_with_P9.xml"; + public static final String FILE_XML_P12_LOCATION = "src/test/resources/europeana_record_with_P12.xml"; - final ProblemPatternAnalyzer problemPatternAnalyzer = new ProblemPatternAnalyzer(); - final List problemPatterns = problemPatternAnalyzer.analyzeRecord(rdf); + private static Stream analyzeRecord() { + return Stream.of( + //Should not have any problem patterns generated + Arguments.of(FILE_XML_NO_PROBLEM_PATTERNS_LOCATION, 0, null, 0), + //Should not have any choices on the provider proxy(to check for null list), therefore reporting only P7 + Arguments.of(FILE_XML_EMPTY_CHOICES_LOCATION, 1, ProblemPatternDescription.P7, 1), + //Should contain two provider proxies that each contain a pair of identical title and description. All four values are identical on the two proxies. + Arguments.of(FILE_XML_P2_LOCATION, 1, ProblemPatternDescription.P2, 1), + //Should contain identical titles, very similar ones and also completely different ones + Arguments.of(FILE_XML_P3_LOCATION, 2, ProblemPatternDescription.P3, 1), + //Should contain valid titles in different languages and unrecognizable titles + Arguments.of(FILE_XML_P5_LOCATION, 1, ProblemPatternDescription.P5, 3), + //Should contain one title that is not meaningful(too short) + Arguments.of(FILE_XML_P6_LOCATION, 1, ProblemPatternDescription.P6, 1), + //Should not contain any descriptions + Arguments.of(FILE_XML_P7_LOCATION, 1, ProblemPatternDescription.P7, 1), + //Should contain multiple descriptions that are "empty" + Arguments.of(FILE_XML_P7_DESCRIPTIONS_EMPTY_LOCATION, 1, ProblemPatternDescription.P7, 1), + //Should contain a description with length less than threshold + Arguments.of(FILE_XML_P9_LOCATION, 1, ProblemPatternDescription.P9, 2), + //Should contain a title with length more than threshold + Arguments.of(FILE_XML_P12_LOCATION, 1, ProblemPatternDescription.P12, 1) + ); + } - assertNotNull(problemPatterns); - assertEquals(1, problemPatterns.size()); - assertEquals(ProblemPatternDescription.P2, problemPatterns.get(0).getProblemPatternDescription()); - assertEquals(1, problemPatterns.get(0).getRecordAnalysisList().get(0).getProblemOccurrenceList().size()); + private ProblemPatternDescription getRequestedProblemPattern(ProblemPatternDescription problemPatternDescription, + List problemPatterns) { + return problemPatterns.stream() + .map(ProblemPattern::getProblemPatternDescription) + .filter(patternDescription -> patternDescription == problemPatternDescription).findFirst().orElse(null); } - @Test - void analyzeRecord_P6() throws Exception { - //Should contain one title that is not meaningful(too short) - String xml = IOUtils.toString(new FileInputStream("src/test/resources/europeana_record_with_P6.xml"), - StandardCharsets.UTF_8); - final RDF rdf = new RdfConversionUtils().convertStringToRdf(xml); + private List analyzeProblemPatternsForFile(String fileLocation) throws IOException, SerializationException { + String xml = IOUtils.toString(new FileInputStream(fileLocation), StandardCharsets.UTF_8); final ProblemPatternAnalyzer problemPatternAnalyzer = new ProblemPatternAnalyzer(); - final List problemPatterns = problemPatternAnalyzer.analyzeRecord(rdf); + return problemPatternAnalyzer.analyzeRecord(xml); + } + + private int getRequestedProblemOccurrencesSize(ProblemPatternDescription problemPatternDescription, + List problemPatterns) { + return problemPatterns.stream() + .filter(problemPattern -> problemPattern.getProblemPatternDescription() + == problemPatternDescription) + .map(problemPattern -> problemPattern.getRecordAnalysisList().get(0).getProblemOccurrenceList().size()) + .findFirst().orElse(0); + } + + @ParameterizedTest(name = "[{index}] - For file:{0}, totalPatterns:{1}, patternId:{2}, totalOccurrences:{3}") + @MethodSource + void analyzeRecord(String fileLocation, int totalPatterns, ProblemPatternDescription problemPatternDescription, + int totalOccurrences) throws Exception { + final List problemPatterns = analyzeProblemPatternsForFile(fileLocation); assertNotNull(problemPatterns); - assertEquals(1, problemPatterns.size()); - assertEquals(ProblemPatternDescription.P6, problemPatterns.get(0).getProblemPatternDescription()); - assertEquals(1, problemPatterns.get(0).getRecordAnalysisList().get(0).getProblemOccurrenceList().size()); + assertEquals(totalPatterns, problemPatterns.size()); + assertEquals(problemPatternDescription, getRequestedProblemPattern(problemPatternDescription, problemPatterns)); + assertEquals(totalOccurrences, getRequestedProblemOccurrencesSize(problemPatternDescription, problemPatterns)); } } \ No newline at end of file diff --git a/metis-pattern-analysis/src/test/resources/europeana_record_empty_proxy_choices.xml b/metis-pattern-analysis/src/test/resources/europeana_record_empty_proxy_choices.xml new file mode 100644 index 000000000..de7343397 --- /dev/null +++ b/metis-pattern-analysis/src/test/resources/europeana_record_empty_proxy_choices.xml @@ -0,0 +1,100 @@ + + + + + + + text/html + 197506 + + + image/jpeg + 6643 + 640 + 480 + sRGB + #F0E68C + #B22222 + #FF4500 + #2F4F4F + #ADFF2F + #87CEEB + landscape + + + 50.75 + 4.5 + + + + 1914-1919 के मध्य मुख्यतः यूरोप में व्याप्त इस महायुद्ध को प्रथम विश्व युद्ध कहते हैं । यह महायुद्ध + यूरोप, एशिया व अफ्रीका तीन महाद्वीपों और जल, थल तथा आकाश में लड़ा गया। इसमें भाग लेने वाले देशों की संख्या, इसका क्षेत्र + (जिसमें यह लड़ा गया) तथा इससे हुई क्षति के अभूतपूर्व आंकड़ों के कारण ही इसे विश्वयुद्ध कहते हैं । + + Første verdenskrig + + + + + + + + EFG - The European Film Gateway + J.M.P.- Trends + + + + 12944 + nld + + + + true + 1984 + + + + + + + + + + PYLYSER, JEAN-MARIE + 12944 + nl + EFG1914 + World War I + WESTVLAAMS FILMJOURNAAL - 164 + Newsreel + + + 1984 + Belgium + + + + + VIDEO + + + + Europeana Foundation + Europeana Foundation + 1_test + Netherlands + + nl + 10 + + + The Royal Belgian Film Archives + Cinémathèque royale de Belgique + Koninklijk Belgisch Filmarchief + + \ No newline at end of file diff --git a/metis-pattern-analysis/src/test/resources/europeana_record_no_problem_patterns.xml b/metis-pattern-analysis/src/test/resources/europeana_record_no_problem_patterns.xml new file mode 100644 index 000000000..de7da14ec --- /dev/null +++ b/metis-pattern-analysis/src/test/resources/europeana_record_no_problem_patterns.xml @@ -0,0 +1,101 @@ + + + + + + + text/html + 197506 + + + image/jpeg + 6643 + 640 + 480 + sRGB + #F0E68C + #B22222 + #FF4500 + #2F4F4F + #ADFF2F + #87CEEB + landscape + + + 50.75 + 4.5 + + + + 1914-1919 के मध्य मुख्यतः यूरोप में व्याप्त इस महायुद्ध को प्रथम विश्व युद्ध कहते हैं । यह महायुद्ध + यूरोप, एशिया व अफ्रीका तीन महाद्वीपों और जल, थल तथा आकाश में लड़ा गया। इसमें भाग लेने वाले देशों की संख्या, इसका क्षेत्र + (जिसमें यह लड़ा गया) तथा इससे हुई क्षति के अभूतपूर्व आंकड़ों के कारण ही इसे विश्वयुद्ध कहते हैं । + + Første verdenskrig + + + + + + + + EFG - The European Film Gateway + J.M.P.- Trends + + + + 12944 + nld + + + + true + 1984 + + + + + + This description is more 50 charactersAAAAAAAAAAAAA + + + + + PYLYSER, JEAN-MARIE + 12944 + nl + EFG1914 + World War I + WESTVLAAMS FILMJOURNAAL - 164 + Newsreel + + + 1984 + Belgium + + + + + VIDEO + + + + Europeana Foundation + Europeana Foundation + 1_test + Netherlands + + nl + 10 + + + The Royal Belgian Film Archives + Cinémathèque royale de Belgique + Koninklijk Belgisch Filmarchief + + \ No newline at end of file diff --git a/metis-pattern-analysis/src/test/resources/europeana_record_with_P12.xml b/metis-pattern-analysis/src/test/resources/europeana_record_with_P12.xml new file mode 100644 index 000000000..aa9be5b82 --- /dev/null +++ b/metis-pattern-analysis/src/test/resources/europeana_record_with_P12.xml @@ -0,0 +1,100 @@ + + + + + + + text/html + 197506 + + + image/jpeg + 6643 + 640 + 480 + sRGB + #F0E68C + #B22222 + #FF4500 + #2F4F4F + #ADFF2F + #87CEEB + landscape + + + 50.75 + 4.5 + + + + 1914-1919 के मध्य मुख्यतः यूरोप में व्याप्त इस महायुद्ध को प्रथम विश्व युद्ध कहते हैं । यह महायुद्ध + यूरोप, एशिया व अफ्रीका तीन महाद्वीपों और जल, थल तथा आकाश में लड़ा गया। इसमें भाग लेने वाले देशों की संख्या, इसका क्षेत्र + (जिसमें यह लड़ा गया) तथा इससे हुई क्षति के अभूतपूर्व आंकड़ों के कारण ही इसे विश्वयुद्ध कहते हैं । + + Første verdenskrig + + + + + + + + EFG - The European Film Gateway + J.M.P.- Trends + + + + 12944 + nld + + + + true + 1984 + + + + + + PYLYSER, JEAN-MARIE + This description is more than 50 charactersAAAAAAAA + + 12944 + nl + EFG1914 + World War I + WESTVLAAMS FILMJOURNAAL - 164 + This title is more than 70 characters long AAAAAAAAAAAAAAAAAAAAAAAAAAAA + + This title is more than 70 characters long + Newsreel + + + 1984 + Belgium + + + + VIDEO + + + + Europeana Foundation + Europeana Foundation + 1_test + Netherlands + + nl + 10 + + + The Royal Belgian Film Archives + Cinémathèque royale de Belgique + Koninklijk Belgisch Filmarchief + + \ No newline at end of file diff --git a/metis-pattern-analysis/src/test/resources/europeana_record_with_P2.xml b/metis-pattern-analysis/src/test/resources/europeana_record_with_P2.xml index 129932bc3..0fbd36718 100644 --- a/metis-pattern-analysis/src/test/resources/europeana_record_with_P2.xml +++ b/metis-pattern-analysis/src/test/resources/europeana_record_with_P2.xml @@ -62,13 +62,13 @@ PYLYSER, JEAN-MARIE - same title and Description + same title and Element more than 50 charactersAAAAA 12944 nl EFG1914 World War I WESTVLAAMS FILMJOURNAAL - 164 - Same title and Description + Same title and Element more than 50 charactersAAAAA Newsreel @@ -82,20 +82,19 @@ PYLYSER, JEAN-MARIE - same title and Description + 12944 nl EFG1914 World War I WESTVLAAMS FILMJOURNAAL - 164 - Same title and Description + Newsreel 1984 Belgium - false diff --git a/metis-pattern-analysis/src/test/resources/europeana_record_with_P3.xml b/metis-pattern-analysis/src/test/resources/europeana_record_with_P3.xml new file mode 100644 index 000000000..be1f7e116 --- /dev/null +++ b/metis-pattern-analysis/src/test/resources/europeana_record_with_P3.xml @@ -0,0 +1,126 @@ + + + + + + + text/html + 197506 + + + image/jpeg + 6643 + 640 + 480 + sRGB + #F0E68C + #B22222 + #FF4500 + #2F4F4F + #ADFF2F + #87CEEB + landscape + + + 50.75 + 4.5 + + + + 1914-1919 के मध्य मुख्यतः यूरोप में व्याप्त इस महायुद्ध को प्रथम विश्व युद्ध कहते हैं । यह महायुद्ध + यूरोप, एशिया व अफ्रीका तीन महाद्वीपों और जल, थल तथा आकाश में लड़ा गया। इसमें भाग लेने वाले देशों की संख्या, इसका क्षेत्र + (जिसमें यह लड़ा गया) तथा इससे हुई क्षति के अभूतपूर्व आंकड़ों के कारण ही इसे विश्वयुद्ध कहते हैं । + + Første verdenskrig + + + + + + + + EFG - The European Film Gateway + J.M.P.- Trends + + + + 12944 + nld + + + + true + 1984 + + + + + + PYLYSER, JEAN-MARIE + same title and Element more than 50 charactersAAAAA + 12944 + nl + EFG1914 + World War I + WESTVLAAMS FILMJOURNAAL - 164 + Same title and Element more than 50 charactersAAAAA + Newsreel + + + 1984 + Belgium + + false + + + VIDEO + + + PYLYSER, JEAN-MARIE + same title and Element more than 50 charactersAAAAA + + AThis is an element to try B a near identical occurrence C + + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + 12944 + nl + EFG1914 + World War I + WESTVLAAMS FILMJOURNAAL - 164 + Same title and Element more than 50 charactersAAAAA + This is an element to try a near identical occurrence + This is title not achieving near identicality + + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + Newsreel + + + 1984 + Belgium + + + + + VIDEO + + + + Europeana Foundation + Europeana Foundation + 1_test + Netherlands + + nl + 10 + + + The Royal Belgian Film Archives + Cinémathèque royale de Belgique + Koninklijk Belgisch Filmarchief + + \ No newline at end of file diff --git a/metis-pattern-analysis/src/test/resources/europeana_record_with_P5.xml b/metis-pattern-analysis/src/test/resources/europeana_record_with_P5.xml new file mode 100644 index 000000000..89b90d661 --- /dev/null +++ b/metis-pattern-analysis/src/test/resources/europeana_record_with_P5.xml @@ -0,0 +1,114 @@ + + + + + + + text/html + 197506 + + + image/jpeg + 6643 + 640 + 480 + sRGB + #F0E68C + #B22222 + #FF4500 + #2F4F4F + #ADFF2F + #87CEEB + landscape + + + 50.75 + 4.5 + + + + 1914-1919 के मध्य मुख्यतः यूरोप में व्याप्त इस महायुद्ध को प्रथम विश्व युद्ध कहते हैं । यह महायुद्ध + यूरोप, एशिया व अफ्रीका तीन महाद्वीपों और जल, थल तथा आकाश में लड़ा गया। इसमें भाग लेने वाले देशों की संख्या, इसका क्षेत्र + (जिसमें यह लड़ा गया) तथा इससे हुई क्षति के अभूतपूर्व आंकड़ों के कारण ही इसे विश्वयुद्ध कहते हैं । + + Første verdenskrig + + + + + + + + EFG - The European Film Gateway + J.M.P.- Trends + + + + 12944 + nld + + + + true + 1984 + + + + + + PYLYSER, JEAN-MARIE + This description is more than 50 charactersAAAAAAAA + + 12944 + nl + EFG1914 + World War I + WESTVLAAMS FILMJOURNAAL - 164 + के मध्य मुख्यतः यूरोप में व्याप्त इस + Καλημέρα Ελλάδα + Καλημέρα - Ελλάδα + + Καλημέρα + Ελλάδα + + Contains identifier 12944 + Contains more than 5 non-alphanumeric ?!.?!. + Contains more than 5 non-alphanumeric(new lines) + + + + + + + မြန်မာဘာသာ + Newsreel + + + 1984 + Belgium + + + + VIDEO + + + + Europeana Foundation + Europeana Foundation + 1_test + Netherlands + + nl + 10 + + + The Royal Belgian Film Archives + Cinémathèque royale de Belgique + Koninklijk Belgisch Filmarchief + + \ No newline at end of file diff --git a/metis-pattern-analysis/src/test/resources/europeana_record_with_P6.xml b/metis-pattern-analysis/src/test/resources/europeana_record_with_P6.xml index 381c8b6f9..e5b598e5a 100644 --- a/metis-pattern-analysis/src/test/resources/europeana_record_with_P6.xml +++ b/metis-pattern-analysis/src/test/resources/europeana_record_with_P6.xml @@ -62,10 +62,7 @@ PYLYSER, JEAN-MARIE - Newsitems West Flanders. Day of the Navy in Ostend, with a wreath-laying memorial service at the - monument of the sailors on the dike. Thereafter, a military parade takes place at the Wapenplein in the presence of Prince - Albert, Princess Paola, Governor Vanneste, mayor Goekindt and certain naval officers. The navy admiral gives a speech, - followed by a short parade. The princess salutes the flag bearers of the veterans' associations. + This description is more than 50 charactersAAAAAAAA 12944 nl @@ -79,7 +76,6 @@ 1984 Belgium - false VIDEO diff --git a/metis-pattern-analysis/src/test/resources/europeana_record_with_P7.xml b/metis-pattern-analysis/src/test/resources/europeana_record_with_P7.xml new file mode 100644 index 000000000..527bdf088 --- /dev/null +++ b/metis-pattern-analysis/src/test/resources/europeana_record_with_P7.xml @@ -0,0 +1,95 @@ + + + + + + + text/html + 197506 + + + image/jpeg + 6643 + 640 + 480 + sRGB + #F0E68C + #B22222 + #FF4500 + #2F4F4F + #ADFF2F + #87CEEB + landscape + + + 50.75 + 4.5 + + + + 1914-1919 के मध्य मुख्यतः यूरोप में व्याप्त इस महायुद्ध को प्रथम विश्व युद्ध कहते हैं । यह महायुद्ध + यूरोप, एशिया व अफ्रीका तीन महाद्वीपों और जल, थल तथा आकाश में लड़ा गया। इसमें भाग लेने वाले देशों की संख्या, इसका क्षेत्र + (जिसमें यह लड़ा गया) तथा इससे हुई क्षति के अभूतपूर्व आंकड़ों के कारण ही इसे विश्वयुद्ध कहते हैं । + + Første verdenskrig + + + + + + + + EFG - The European Film Gateway + J.M.P.- Trends + + + + 12944 + nld + + + + true + 1984 + + + + + + PYLYSER, JEAN-MARIE + 12944 + nl + EFG1914 + World War I + WESTVLAAMS FILMJOURNAAL - 164 + Newsreel + + + 1984 + Belgium + + + + VIDEO + + + + Europeana Foundation + Europeana Foundation + 1_test + Netherlands + + nl + 10 + + + The Royal Belgian Film Archives + Cinémathèque royale de Belgique + Koninklijk Belgisch Filmarchief + + \ No newline at end of file diff --git a/metis-pattern-analysis/src/test/resources/europeana_record_with_P7_descriptions_empty.xml b/metis-pattern-analysis/src/test/resources/europeana_record_with_P7_descriptions_empty.xml new file mode 100644 index 000000000..4b5913567 --- /dev/null +++ b/metis-pattern-analysis/src/test/resources/europeana_record_with_P7_descriptions_empty.xml @@ -0,0 +1,99 @@ + + + + + + + text/html + 197506 + + + image/jpeg + 6643 + 640 + 480 + sRGB + #F0E68C + #B22222 + #FF4500 + #2F4F4F + #ADFF2F + #87CEEB + landscape + + + 50.75 + 4.5 + + + + 1914-1919 के मध्य मुख्यतः यूरोप में व्याप्त इस महायुद्ध को प्रथम विश्व युद्ध कहते हैं । यह महायुद्ध + यूरोप, एशिया व अफ्रीका तीन महाद्वीपों और जल, थल तथा आकाश में लड़ा गया। इसमें भाग लेने वाले देशों की संख्या, इसका क्षेत्र + (जिसमें यह लड़ा गया) तथा इससे हुई क्षति के अभूतपूर्व आंकड़ों के कारण ही इसे विश्वयुद्ध कहते हैं । + + Første verdenskrig + + + + + + + + EFG - The European Film Gateway + J.M.P.- Trends + + + + 12944 + nld + + + + true + 1984 + + + + + + PYLYSER, JEAN-MARIE + + + + + 12944 + nl + EFG1914 + World War I + WESTVLAAMS FILMJOURNAAL - 164 + Newsreel + + + 1984 + Belgium + + + + VIDEO + + + + Europeana Foundation + Europeana Foundation + 1_test + Netherlands + + nl + 10 + + + The Royal Belgian Film Archives + Cinémathèque royale de Belgique + Koninklijk Belgisch Filmarchief + + \ No newline at end of file diff --git a/metis-pattern-analysis/src/test/resources/europeana_record_with_P9.xml b/metis-pattern-analysis/src/test/resources/europeana_record_with_P9.xml new file mode 100644 index 000000000..ca2e4dd83 --- /dev/null +++ b/metis-pattern-analysis/src/test/resources/europeana_record_with_P9.xml @@ -0,0 +1,98 @@ + + + + + + + text/html + 197506 + + + image/jpeg + 6643 + 640 + 480 + sRGB + #F0E68C + #B22222 + #FF4500 + #2F4F4F + #ADFF2F + #87CEEB + landscape + + + 50.75 + 4.5 + + + + 1914-1919 के मध्य मुख्यतः यूरोप में व्याप्त इस महायुद्ध को प्रथम विश्व युद्ध कहते हैं । यह महायुद्ध + यूरोप, एशिया व अफ्रीका तीन महाद्वीपों और जल, थल तथा आकाश में लड़ा गया। इसमें भाग लेने वाले देशों की संख्या, इसका क्षेत्र + (जिसमें यह लड़ा गया) तथा इससे हुई क्षति के अभूतपूर्व आंकड़ों के कारण ही इसे विश्वयुद्ध कहते हैं । + + Første verdenskrig + + + + + + + + EFG - The European Film Gateway + J.M.P.- Trends + + + + 12944 + nld + + + + true + 1984 + + + + + + PYLYSER, JEAN-MARIE + This description is less than 50 characters + This description is exactly 50 charactersAAAAAAAAA + This description is more 50 charactersAAAAAAAAAAAAA + 12944 + nl + EFG1914 + World War I + WESTVLAAMS FILMJOURNAAL - 164 + Newsreel + + + 1984 + Belgium + + + + VIDEO + + + + Europeana Foundation + Europeana Foundation + 1_test + Netherlands + + nl + 10 + + + The Royal Belgian Film Archives + Cinémathèque royale de Belgique + Koninklijk Belgisch Filmarchief + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 1e0c90eb6..d4aedc5ed 100644 --- a/pom.xml +++ b/pom.xml @@ -133,6 +133,7 @@ 2.9.0 3.12.0 3.2.2 + 1.9 1.11 2.15.3 From 7a6244ca8779950fd08c465f03534ab303304a3c Mon Sep 17 00:00:00 2001 From: JoanaCMS <70145179+JoanaCMS@users.noreply.github.com> Date: Wed, 11 May 2022 14:57:51 +0200 Subject: [PATCH 29/73] MET-4418 Fix sorting of Countries list (#529) --- .../eu/europeana/metis/core/common/Country.java | 15 +++++++++++++++ .../metis/core/rest/DatasetController.java | 3 ++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/common/Country.java b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/common/Country.java index 2c2022cdd..ac842da5a 100644 --- a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/common/Country.java +++ b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/common/Country.java @@ -1,5 +1,9 @@ package eu.europeana.metis.core.common; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + /** * Countries supported by METIS */ @@ -115,4 +119,15 @@ public static Country getCountryFromIsoCode(String isoCode) { } return null; } + + /** + * Provides the countries sorted by the {@link #getName()} field + * + * @return the list of countries sorted + */ + public static List getCountryListSortedByName() { + List countries = Arrays.asList(Country.values()); + countries.sort(Comparator.comparing(Country::getName)); + return countries; + } } diff --git a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/DatasetController.java b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/DatasetController.java index 28c1d5074..9f7b82e0d 100644 --- a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/DatasetController.java +++ b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/DatasetController.java @@ -613,7 +613,8 @@ public ResponseListWrapper getAllDatasetsByOrganizationName( public List getDatasetsCountries( @RequestHeader("Authorization") String authorization) throws GenericMetisException { authenticationClient.getUserByAccessTokenInHeader(authorization); - return Arrays.stream(Country.values()).map(CountryView::new).collect(Collectors.toList()); + return Country.getCountryListSortedByName().stream().map(CountryView::new) + .collect(Collectors.toList()); } /** From 7e724784088de6774c77b348118c760a012ac779 Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Fri, 13 May 2022 10:08:22 +0200 Subject: [PATCH 30/73] MET-4457 Set default truncate on elements for report (#532) --- .../ProblemPatternAnalyzer.java | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/ProblemPatternAnalyzer.java b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/ProblemPatternAnalyzer.java index 6960374de..7df496029 100644 --- a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/ProblemPatternAnalyzer.java +++ b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/ProblemPatternAnalyzer.java @@ -6,6 +6,7 @@ import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toSet; import static org.apache.commons.lang3.BooleanUtils.isFalse; +import static org.apache.commons.lang3.StringUtils.truncate; import eu.europeana.metis.schema.convert.RdfConversionUtils; import eu.europeana.metis.schema.convert.SerializationException; @@ -48,6 +49,7 @@ public class ProblemPatternAnalyzer { private static final int UNRECOGNIZABLE_CHARACTERS_THRESHOLD = 5; private static final double LCS_CALCULATION_THRESHOLD = 0.9; private static final int TITLE_DESCRIPTION_LENGTH_DISTANCE = 20; + private static final int DEFAULT_MAX_CHARACTERS_ELEMENT_LENGTH_FOR_REPORT = 50; // Match anything that is not alphanumeric in all languages or literal spaces. We cannot just use \\w private static final String UNRECOGNIZABLE_CHARACTERS_REGEX = "[^\\p{IsAlphabetic}\\p{IsDigit} ]"; private static final Pattern UNRECOGNIZABLE_CHARACTERS_PATTERN = Pattern.compile(UNRECOGNIZABLE_CHARACTERS_REGEX); @@ -132,7 +134,8 @@ private List checkP2(List titles, List descri equalTitlesAndDescriptions.retainAll(uniqueDescriptions); return equalTitlesAndDescriptions.stream().map( - value -> new ProblemOccurrence(format("Equal(lower cased) title and description: %s", value)) + value -> new ProblemOccurrence(format("Equal(lower cased) title and description: %s(...)", + truncate(value, DEFAULT_MAX_CHARACTERS_ELEMENT_LENGTH_FOR_REPORT))) ).collect(toList()); } @@ -160,7 +163,9 @@ private List checkP3(List titles, List descri return nearIdenticalTitleDescriptionsMap.entrySet().stream().flatMap( entry -> entry.getValue().stream().map( - value -> new ProblemOccurrence(format("Near-Identical title and description fields: %s | %s", entry.getKey(), value)) + value -> new ProblemOccurrence(format("Near-Identical title and description fields: %s(...) | %s(...)", + truncate(entry.getKey(), DEFAULT_MAX_CHARACTERS_ELEMENT_LENGTH_FOR_REPORT), + truncate(value, DEFAULT_MAX_CHARACTERS_ELEMENT_LENGTH_FOR_REPORT))) ) ).collect(toList()); } @@ -199,7 +204,8 @@ private List checkP5(List titles, List identi UNRECOGNIZABLE_CHARACTERS_PATTERN.matcher(s).results().count() > UNRECOGNIZABLE_CHARACTERS_THRESHOLD; final Predicate containsIdentifier = s -> identifiers.stream().anyMatch(s::contains); return titles.stream().filter(moreThanThresholdUnrecognizableCharacters.or(containsIdentifier)) - .map(title -> new ProblemOccurrence(format("Unrecognized title: %s", title)) + .map(title -> new ProblemOccurrence( + format("Unrecognized title: %s(...)", truncate(title, DEFAULT_MAX_CHARACTERS_ELEMENT_LENGTH_FOR_REPORT))) ).collect(toList()); } @@ -211,7 +217,9 @@ private List checkP5(List titles, List identi */ private List checkP6(List titles) { return titles.stream().filter(title -> title.length() <= MIN_TITLE_LENGTH) - .map(title -> new ProblemOccurrence(format("Non meaningful title: %s", title))).collect(toList()); + .map(title -> new ProblemOccurrence( + format("Non meaningful title: %s(...)", truncate(title, DEFAULT_MAX_CHARACTERS_ELEMENT_LENGTH_FOR_REPORT)))) + .collect(toList()); } /** @@ -237,7 +245,8 @@ private List checkP7(List descriptions) { private List checkP9(List descriptions) { return descriptions.stream().filter(StringUtils::isNotBlank) .filter(description -> description.length() <= MIN_DESCRIPTION_LENGTH) - .map(description -> new ProblemOccurrence(format("Very short description: %s", description))) + .map(description -> new ProblemOccurrence(format("Very short description: %s(...)", + truncate(description, DEFAULT_MAX_CHARACTERS_ELEMENT_LENGTH_FOR_REPORT)))) .collect(toList()); } @@ -250,7 +259,9 @@ private List checkP9(List descriptions) { */ private List checkP12(List titles) { return titles.stream().filter(title -> title.length() > MAX_TITLE_LENGTH) - .map(title -> new ProblemOccurrence(format("Extremely long title: %s", title))).collect(toList()); + .map(title -> new ProblemOccurrence( + format("Extremely long title: %s(...)", truncate(title, DEFAULT_MAX_CHARACTERS_ELEMENT_LENGTH_FOR_REPORT)))) + .collect(toList()); } private Optional constructProblemPattern(String recordId, ProblemPatternDescription problemPatternDescription, From e2c3aeab0f3f43d01b97d3285dedaf2f6e162829 Mon Sep 17 00:00:00 2001 From: Joana Sousa Date: Fri, 13 May 2022 12:23:39 +0200 Subject: [PATCH 31/73] MET-4418 Fixing the schema for North Macedonia --- metis-schema/src/main/resources/schema_xsds/EDM-COMMON-MAIN.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metis-schema/src/main/resources/schema_xsds/EDM-COMMON-MAIN.xsd b/metis-schema/src/main/resources/schema_xsds/EDM-COMMON-MAIN.xsd index 039e9b802..31c9cf21f 100644 --- a/metis-schema/src/main/resources/schema_xsds/EDM-COMMON-MAIN.xsd +++ b/metis-schema/src/main/resources/schema_xsds/EDM-COMMON-MAIN.xsd @@ -168,7 +168,7 @@ - + From cfbefed31e723b25df538cf54137544365056e93 Mon Sep 17 00:00:00 2001 From: JoanaCMS <70145179+JoanaCMS@users.noreply.github.com> Date: Wed, 18 May 2022 08:33:18 +0200 Subject: [PATCH 32/73] MET-4472 New Tier Calculation for 3D (#530) * MET-4472 Started implementing tier calculation for 3D * MET-4472 Fixed unit tests * MET-4472 Changes for tier 1 calculation * MET-4472 Changed code to remove bug from sonarqube * MET-4472 Trying to fix bug from sonarqube * MET-4472 Fixing unit tests and code * MET-4472 Added new unit test for null values * MET-4472 Code review changes Changed classification of webresource for 3D. Updated unit tests * MET-4472 Removed code smell --- .../tiers/media/ThreeDClassifier.java | 37 ++++++++++++-- .../tiers/media/ThreeDClassifierTest.java | 48 +++++++++++++------ 2 files changed, 67 insertions(+), 18 deletions(-) diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/tiers/media/ThreeDClassifier.java b/metis-indexing/src/main/java/eu/europeana/indexing/tiers/media/ThreeDClassifier.java index 0d2deb16c..ac827ad73 100644 --- a/metis-indexing/src/main/java/eu/europeana/indexing/tiers/media/ThreeDClassifier.java +++ b/metis-indexing/src/main/java/eu/europeana/indexing/tiers/media/ThreeDClassifier.java @@ -4,9 +4,11 @@ import eu.europeana.indexing.tiers.view.ResolutionTierMetadata; import eu.europeana.indexing.tiers.view.ResolutionTierMetadata.ResolutionTierMetadataBuilder; import eu.europeana.indexing.utils.RdfWrapper; +import eu.europeana.indexing.utils.WebResourceLinkType; import eu.europeana.indexing.utils.WebResourceWrapper; import eu.europeana.metis.schema.model.MediaType; -import org.apache.commons.lang3.StringUtils; + +import java.util.Set; /** * Classifier for 3D objects. @@ -30,8 +32,19 @@ MediaTier classifyEntityWithoutWebResources(RdfWrapper entity, boolean hasLandin @Override MediaTier classifyWebResource(WebResourceWrapper webResource, boolean hasLandingPage, boolean hasEmbeddableMedia) { - // T2-T4 if there is a mime type (any whatsoever), T0 otherwise. - return StringUtils.isNotBlank(webResource.getMimeType()) ? MediaTier.T4 : MediaTier.T0; + final MediaTier result; + + if(webResource == null){ + result = MediaTier.T0; + } else if(mimeTypeIsNotImageOrApplicationPdf(webResource) && containsIsShownByOrHasViewWebResource(webResource)){ + result = MediaTier.T4; + } else if(hasLandingPage && onlyContainsShownAtWebResource(webResource)){ + result = MediaTier.T1; + } else { + result = MediaTier.T0; + } + + return result; } @Override @@ -43,4 +56,22 @@ ResolutionTierMetadata extractResolutionTierMetadata(WebResourceWrapper webResou MediaType getMediaType() { return MediaType.THREE_D; } + + private boolean mimeTypeIsNotImageOrApplicationPdf(WebResourceWrapper webResource){ + String mimeType = webResource.getMimeType(); + return mimeType != null && webResource.getMediaType() != MediaType.IMAGE && !mimeType.startsWith("application/pdf"); + } + + private boolean containsIsShownByOrHasViewWebResource(WebResourceWrapper webResource){ + Set extractedLinkTypes = webResource.getLinkTypes(); + return extractedLinkTypes != null && (extractedLinkTypes.contains(WebResourceLinkType.IS_SHOWN_BY) || + extractedLinkTypes.contains(WebResourceLinkType.HAS_VIEW)); + } + + private boolean onlyContainsShownAtWebResource(WebResourceWrapper webResource){ + Set linkTypes = webResource.getLinkTypes(); + return linkTypes.contains(WebResourceLinkType.IS_SHOWN_AT) && + !linkTypes.contains(WebResourceLinkType.IS_SHOWN_BY) && + !linkTypes.contains(WebResourceLinkType.HAS_VIEW); + } } diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/media/ThreeDClassifierTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/media/ThreeDClassifierTest.java index 53efbc152..6493c6fbe 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/media/ThreeDClassifierTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/media/ThreeDClassifierTest.java @@ -3,22 +3,20 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.*; import eu.europeana.indexing.tiers.model.MediaTier; import eu.europeana.indexing.utils.RdfWrapper; +import eu.europeana.indexing.utils.WebResourceLinkType; import eu.europeana.indexing.utils.WebResourceWrapper; import eu.europeana.metis.schema.model.MediaType; + +import java.util.Set; import java.util.stream.Stream; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; class ThreeDClassifierTest { @@ -65,16 +63,36 @@ private static Stream testClassifyWebResource() { ); } - @ParameterizedTest(name = "[{index}] - expectedTier:{0} for mimeType:{1}") - @MethodSource("testClassifyWebResource") - void testClassifyWebResource(MediaTier expectedTier, String mimeType) { + @Test + void testClassifyWebResource_tier4Result() { + final WebResourceWrapper webResource = mock(WebResourceWrapper.class); + Set mockSetResponse = Set.of(WebResourceLinkType.HAS_VIEW, WebResourceLinkType.IS_SHOWN_BY); + when(webResource.getLinkTypes()).thenReturn(mockSetResponse); + when(webResource.getMimeType()).thenReturn("video"); + assertEquals(MediaTier.T4, classifier.classifyWebResource(webResource, true, false)); + } + + @Test + void testClassifyWebResource_tier1Result() { + final WebResourceWrapper webResource = mock(WebResourceWrapper.class); + when(webResource.getLinkTypes()).thenReturn(Set.of(WebResourceLinkType.IS_SHOWN_AT)); + when(webResource.getMimeType()).thenReturn("video"); + assertEquals(MediaTier.T1, classifier.classifyWebResource(webResource, true, false)); + } + + @Test + void testClassifyWebResource_tier0Result() { + final WebResourceWrapper webResource = mock(WebResourceWrapper.class); + when(webResource.getLinkTypes()).thenReturn(Set.of()); + when(webResource.getMimeType()).thenReturn("video"); + assertEquals(MediaTier.T0, classifier.classifyWebResource(webResource, false, false)); + } + + @Test + void testClassifyWebResource_tier0NullValuesResult() { final WebResourceWrapper webResource = mock(WebResourceWrapper.class); - doReturn(mimeType).when(webResource).getMimeType(); - //Any combination of hasLandingPage and hasEmbeddableMedia should not change the result - assertEquals(expectedTier, classifier.classifyWebResource(webResource, false, false)); - assertEquals(expectedTier, classifier.classifyWebResource(webResource, true, false)); - assertEquals(expectedTier, classifier.classifyWebResource(webResource, false, true)); - assertEquals(expectedTier, classifier.classifyWebResource(webResource, true, true)); + when(webResource.getLinkTypes()).thenReturn(null); + assertEquals(MediaTier.T0, classifier.classifyWebResource(webResource, false, false)); } @Test From 98bc8b3f0ee217270363abe1f3fcb948b21367d6 Mon Sep 17 00:00:00 2001 From: Jorge Ortiz Date: Thu, 19 May 2022 11:08:53 +0200 Subject: [PATCH 33/73] MET-4452 depublication counter after incremental processing (#533) --- .../eu/europeana/metis/core/service/OrchestratorService.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/OrchestratorService.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/OrchestratorService.java index 8a237a8a3..37447732f 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/OrchestratorService.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/OrchestratorService.java @@ -774,11 +774,9 @@ private void setPublishInformation(DatasetExecutionInformation executionInfo, final int depublishedRecordCount; if (datasetCurrentlyDepublished) { depublishedRecordCount = executionInfo.getLastPublishedRecords(); - } else if (depublishHappenedAfterLatestExecutablePublish) { + } else { depublishedRecordCount = (int) depublishRecordIdDao .countSuccessfullyDepublishedRecordIdsForDataset(datasetId); - } else { - depublishedRecordCount = 0; } //Compute more general information of the plugin From 7d5dfc97930ffea20aeec91fc383b71e8dd5c686 Mon Sep 17 00:00:00 2001 From: JoanaCMS <70145179+JoanaCMS@users.noreply.github.com> Date: Fri, 20 May 2022 09:59:55 +0200 Subject: [PATCH 34/73] MET-4531 Started writing unit tests (#535) * MET-4531 Started writing unit tests * MET-4531 Added new unit tests * MET-4531 Removed code smell --- metis-repository/pom.xml | 10 +- .../metis/repository/dao/RecordTest.java | 92 +++++++++++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 metis-repository/src/test/java/eu/europeana/metis/repository/dao/RecordTest.java diff --git a/metis-repository/pom.xml b/metis-repository/pom.xml index d60b3a2a7..49f2783a0 100644 --- a/metis-repository/pom.xml +++ b/metis-repository/pom.xml @@ -77,6 +77,14 @@ metis-harvesting ${project.version}     + + org.junit.jupiter + junit-jupiter-api + + + org.junit.jupiter + junit-jupiter-engine + @@ -90,4 +98,4 @@ - \ No newline at end of file + diff --git a/metis-repository/src/test/java/eu/europeana/metis/repository/dao/RecordTest.java b/metis-repository/src/test/java/eu/europeana/metis/repository/dao/RecordTest.java new file mode 100644 index 000000000..13c4b657e --- /dev/null +++ b/metis-repository/src/test/java/eu/europeana/metis/repository/dao/RecordTest.java @@ -0,0 +1,92 @@ +package eu.europeana.metis.repository.dao; + +import org.bson.types.ObjectId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Instant; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNull; + +class RecordTest { + + private Record recordToTest; + private final Instant instantUsedForTest = Instant.now(); + + @BeforeEach + void setUp(){ + recordToTest = new Record("recordId", "datasetId", instantUsedForTest, false, "edmRecord"); + } + + @Test + void testGetRecordId(){ + assertEquals("recordId",recordToTest.getRecordId()); + } + + @Test + void testGetDatasetId(){ + assertEquals("datasetId",recordToTest.getDatasetId()); + } + + @Test + void testGetDateStamp(){ + assertEquals(instantUsedForTest,recordToTest.getDateStamp()); + } + + @Test + void testIsDeleted(){ + assertFalse(recordToTest.isDeleted()); + } + + @Test + void testEdmRecord(){ + assertEquals("edmRecord",recordToTest.getEdmRecord()); + } + + @Test + void testSetAndGetId(){ + assertNull(recordToTest.getId()); + ObjectId objectId = new ObjectId(); + recordToTest.setId(objectId); + assertEquals(objectId, recordToTest.getId()); + } + + @Test + void testSetRecordId(){ + assertEquals("recordId",recordToTest.getRecordId()); + recordToTest.setRecordId("newRecordId"); + assertEquals("newRecordId", recordToTest.getRecordId()); + } + + @Test + void testSetDatasetId(){ + assertEquals("datasetId",recordToTest.getDatasetId()); + recordToTest.setDatasetId("newDatasetId"); + assertEquals("newDatasetId", recordToTest.getDatasetId()); + } + + @Test + void testSetDateStamp(){ + assertEquals(instantUsedForTest,recordToTest.getDateStamp()); + Instant instantNew = Instant.now(); + recordToTest.setDateStamp(instantNew); + assertEquals(instantNew, recordToTest.getDateStamp()); + } + + @Test + void testSetDeleted(){ + assertFalse(recordToTest.isDeleted()); + recordToTest.setDeleted(true); + assertTrue( recordToTest.isDeleted()); + } + + @Test + void testSetEdmRecord(){ + assertEquals("edmRecord",recordToTest.getEdmRecord()); + recordToTest.setEdmRecord("newEdmRecord"); + assertEquals("newEdmRecord", recordToTest.getEdmRecord()); + } +} From f0234bf997fedbdc571596253a87954b280ba25b Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Fri, 20 May 2022 12:14:40 +0200 Subject: [PATCH 35/73] Feat/met 4458 implement p1 problem pattern (#537) * MET-4458 Introduce wrapper ProblemPatternAnalysis class * MET-4458 Centralize patterns titles * MET-4458 Compute static sets of global and non-global patterns * MET-4458 Add more unit tests * MET-4458 Fix sonar issues --- .../ProblemPatternAnalyzer.java | 74 +++++++++++-------- .../view/ProblemPatternAnalysis.java | 44 +++++++++++ .../view/ProblemPatternDescription.java | 52 +++++-------- .../ProblemPatternAnalyzerTest.java | 15 ++-- .../view/ProblemOccurrenceTest.java | 9 ++- .../view/ProblemPatternAnalysisTest.java | 27 +++++++ .../view/ProblemPatternDescriptionTest.java | 69 +++++++++-------- 7 files changed, 187 insertions(+), 103 deletions(-) create mode 100644 metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemPatternAnalysis.java create mode 100644 metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemPatternAnalysisTest.java diff --git a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/ProblemPatternAnalyzer.java b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/ProblemPatternAnalyzer.java index 7df496029..047b69a65 100644 --- a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/ProblemPatternAnalyzer.java +++ b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/ProblemPatternAnalyzer.java @@ -6,7 +6,7 @@ import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toSet; import static org.apache.commons.lang3.BooleanUtils.isFalse; -import static org.apache.commons.lang3.StringUtils.truncate; +import static org.apache.commons.lang3.StringUtils.abbreviate; import eu.europeana.metis.schema.convert.RdfConversionUtils; import eu.europeana.metis.schema.convert.SerializationException; @@ -19,11 +19,14 @@ import eu.europeana.metis.schema.jibx.ResourceOrLiteralType; import eu.europeana.patternanalysis.view.ProblemOccurrence; import eu.europeana.patternanalysis.view.ProblemPattern; +import eu.europeana.patternanalysis.view.ProblemPatternAnalysis; import eu.europeana.patternanalysis.view.ProblemPatternDescription; +import eu.europeana.patternanalysis.view.ProblemPatternDescription.ProblemPatternId; import eu.europeana.patternanalysis.view.RecordAnalysis; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -54,6 +57,11 @@ public class ProblemPatternAnalyzer { private static final String UNRECOGNIZABLE_CHARACTERS_REGEX = "[^\\p{IsAlphabetic}\\p{IsDigit} ]"; private static final Pattern UNRECOGNIZABLE_CHARACTERS_PATTERN = Pattern.compile(UNRECOGNIZABLE_CHARACTERS_REGEX); + public static final Set globalProblemPatterns = Collections.unmodifiableSet(EnumSet.of(ProblemPatternId.P1)); + public static final Set nonGlobalProblemPatterns = Collections.unmodifiableSet( + EnumSet.complementOf(EnumSet.of(ProblemPatternId.P1))); + + /** * Analyzes a record for problem patterns. * @@ -61,7 +69,7 @@ public class ProblemPatternAnalyzer { * @return a list of problem patterns * @throws SerializationException if the record could not be converted to {@link RDF} */ - public List analyzeRecord(String rdfString) throws SerializationException { + public ProblemPatternAnalysis analyzeRecord(String rdfString) throws SerializationException { return analyzeRecord(new RdfConversionUtils().convertStringToRdf(rdfString)); } @@ -71,7 +79,7 @@ public List analyzeRecord(String rdfString) throws Serialization * @param rdf the rdf record * @return a list of problem patterns */ - public List analyzeRecord(RDF rdf) { + public ProblemPatternAnalysis analyzeRecord(RDF rdf) { final List providerProxies = getProviderProxies(rdf); final List choices = providerProxies.stream().map(EuropeanaType::getChoiceList) .filter(Objects::nonNull) @@ -85,7 +93,8 @@ public List analyzeRecord(RDF rdf) { LiteralType::getString); final String rdfAbout = rdf.getProvidedCHOList().stream().filter(Objects::nonNull).findFirst() .map(ProvidedCHOType::getAbout).orElse(null); - return computeProblemPatterns(rdfAbout, titles, descriptions, identifiers); + final ArrayList problemPatterns = computeProblemPatterns(rdfAbout, titles, descriptions, identifiers); + return new ProblemPatternAnalysis(rdfAbout, problemPatterns, Set.copyOf(titles)); } private List getChoicesInStringList(List choices, Predicate choicePredicate, @@ -97,6 +106,7 @@ private ArrayList computeProblemPatterns(String rdfAbout, List identifiers) { final ArrayList problemPatterns = new ArrayList<>(); + //We can only compute non-global patterns here constructProblemPattern(rdfAbout, ProblemPatternDescription.P2, checkP2(titles, descriptions)).ifPresent( problemPatterns::add); constructProblemPattern(rdfAbout, ProblemPatternDescription.P3, checkP3(titles, descriptions)).ifPresent( @@ -119,6 +129,18 @@ private List getProviderProxies(RDF rdf) { .collect(Collectors.toList()); } + /** + * Abbreviate(based on {@link StringUtils#abbreviate(String, int)}) an element up to a default max length {@link + * #DEFAULT_MAX_CHARACTERS_ELEMENT_LENGTH_FOR_REPORT}. + *

    Is used locally and can be used publicly for global problem patterns like P1.

    + * + * @param element the string element + * @return the truncated string + */ + public String abbreviateElement(String element) { + return abbreviate(element, DEFAULT_MAX_CHARACTERS_ELEMENT_LENGTH_FOR_REPORT); + } + /** * Check whether there is a title - description pair for which the values are equal, ignoring letter (upper or lower) case. *

    It will report a single occurrence for multiple same fields

    @@ -134,11 +156,21 @@ private List checkP2(List titles, List descri equalTitlesAndDescriptions.retainAll(uniqueDescriptions); return equalTitlesAndDescriptions.stream().map( - value -> new ProblemOccurrence(format("Equal(lower cased) title and description: %s(...)", - truncate(value, DEFAULT_MAX_CHARACTERS_ELEMENT_LENGTH_FOR_REPORT))) + value -> new ProblemOccurrence(abbreviateElement(value)) ).collect(toList()); } + private List nearIdenticalDescriptions(String title, List descriptions) { + final LongestCommonSubsequence longestCommonSubsequence = new LongestCommonSubsequence(); + final Predicate lcsPredicate = description -> + ((double) longestCommonSubsequence.apply(title, description) / Math.min(title.length(), description.length())) + >= LCS_CALCULATION_THRESHOLD; + final Predicate distancePredicate = description -> Math.abs(title.length() - description.length()) + <= TITLE_DESCRIPTION_LENGTH_DISTANCE; + return descriptions.stream().filter(StringUtils::isNotBlank).filter(not(title::equalsIgnoreCase)) + .filter(lcsPredicate.and(distancePredicate)).collect(toList()); + } + /** * Check whether there is a title - description pair for which the values are too similar. *

    @@ -163,24 +195,11 @@ private List checkP3(List titles, List descri return nearIdenticalTitleDescriptionsMap.entrySet().stream().flatMap( entry -> entry.getValue().stream().map( - value -> new ProblemOccurrence(format("Near-Identical title and description fields: %s(...) | %s(...)", - truncate(entry.getKey(), DEFAULT_MAX_CHARACTERS_ELEMENT_LENGTH_FOR_REPORT), - truncate(value, DEFAULT_MAX_CHARACTERS_ELEMENT_LENGTH_FOR_REPORT))) + value -> new ProblemOccurrence(format("%s <--> %s", abbreviateElement(entry.getKey()), abbreviateElement(value))) ) ).collect(toList()); } - private List nearIdenticalDescriptions(String title, List descriptions) { - final LongestCommonSubsequence longestCommonSubsequence = new LongestCommonSubsequence(); - final Predicate lcsPredicate = description -> - ((double) longestCommonSubsequence.apply(title, description) / Math.min(title.length(), description.length())) - >= LCS_CALCULATION_THRESHOLD; - final Predicate distancePredicate = description -> Math.abs(title.length() - description.length()) - <= TITLE_DESCRIPTION_LENGTH_DISTANCE; - return descriptions.stream().filter(StringUtils::isNotBlank).filter(not(title::equalsIgnoreCase)) - .filter(lcsPredicate.and(distancePredicate)).collect(toList()); - } - /** * Check whether a title is not human-readable. *

    @@ -204,8 +223,7 @@ private List checkP5(List titles, List identi UNRECOGNIZABLE_CHARACTERS_PATTERN.matcher(s).results().count() > UNRECOGNIZABLE_CHARACTERS_THRESHOLD; final Predicate containsIdentifier = s -> identifiers.stream().anyMatch(s::contains); return titles.stream().filter(moreThanThresholdUnrecognizableCharacters.or(containsIdentifier)) - .map(title -> new ProblemOccurrence( - format("Unrecognized title: %s(...)", truncate(title, DEFAULT_MAX_CHARACTERS_ELEMENT_LENGTH_FOR_REPORT))) + .map(title -> new ProblemOccurrence(abbreviateElement(title)) ).collect(toList()); } @@ -217,8 +235,7 @@ private List checkP5(List titles, List identi */ private List checkP6(List titles) { return titles.stream().filter(title -> title.length() <= MIN_TITLE_LENGTH) - .map(title -> new ProblemOccurrence( - format("Non meaningful title: %s(...)", truncate(title, DEFAULT_MAX_CHARACTERS_ELEMENT_LENGTH_FOR_REPORT)))) + .map(title -> new ProblemOccurrence(abbreviateElement(title))) .collect(toList()); } @@ -230,7 +247,7 @@ private List checkP6(List titles) { */ private List checkP7(List descriptions) { if (CollectionUtils.isEmpty(descriptions) || descriptions.stream().allMatch(StringUtils::isBlank)) { - return List.of(new ProblemOccurrence("Missing description fields")); + return List.of(new ProblemOccurrence(abbreviateElement("Missing description fields"))); } return Collections.emptyList(); } @@ -245,8 +262,7 @@ private List checkP7(List descriptions) { private List checkP9(List descriptions) { return descriptions.stream().filter(StringUtils::isNotBlank) .filter(description -> description.length() <= MIN_DESCRIPTION_LENGTH) - .map(description -> new ProblemOccurrence(format("Very short description: %s(...)", - truncate(description, DEFAULT_MAX_CHARACTERS_ELEMENT_LENGTH_FOR_REPORT)))) + .map(description -> new ProblemOccurrence(abbreviateElement(description))) .collect(toList()); } @@ -259,8 +275,7 @@ private List checkP9(List descriptions) { */ private List checkP12(List titles) { return titles.stream().filter(title -> title.length() > MAX_TITLE_LENGTH) - .map(title -> new ProblemOccurrence( - format("Extremely long title: %s(...)", truncate(title, DEFAULT_MAX_CHARACTERS_ELEMENT_LENGTH_FOR_REPORT)))) + .map(title -> new ProblemOccurrence(abbreviateElement(title))) .collect(toList()); } @@ -272,4 +287,5 @@ private Optional constructProblemPattern(String recordId, Proble } return Optional.empty(); } + } diff --git a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemPatternAnalysis.java b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemPatternAnalysis.java new file mode 100644 index 000000000..338ed8107 --- /dev/null +++ b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemPatternAnalysis.java @@ -0,0 +1,44 @@ +package eu.europeana.patternanalysis.view; + +import static java.util.Objects.requireNonNull; +import static java.util.Objects.requireNonNullElseGet; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Class containing the problem pattern analysis for a record. + */ +public class ProblemPatternAnalysis { + + private final String rdfAbout; + private final List problemPatterns; + private final Set titles; + + /** + * Constructor with required parameters. + * + * @param rdfAbout the rdf about + * @param problemPatterns the problem patterns + * @param titles the record titles + */ + public ProblemPatternAnalysis(String rdfAbout, List problemPatterns, Set titles) { + this.rdfAbout = requireNonNull(rdfAbout); + this.problemPatterns = requireNonNullElseGet(problemPatterns, ArrayList::new); + this.titles = requireNonNullElseGet(titles, HashSet::new); + } + + public String getRdfAbout() { + return rdfAbout; + } + + public List getProblemPatterns() { + return new ArrayList<>(problemPatterns); + } + + public Set getTitles() { + return new HashSet<>(titles); + } +} diff --git a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemPatternDescription.java b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemPatternDescription.java index 3983d9d64..d94d7b6ec 100644 --- a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemPatternDescription.java +++ b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/view/ProblemPatternDescription.java @@ -9,48 +9,30 @@ @JsonFormat(shape = JsonFormat.Shape.OBJECT) public enum ProblemPatternDescription { - /** - * Systematic use of the same title - */ - P1(ProblemPatternId.P1, ProblemPatternSeverity.WARNING, ProblemPatternQualityDimension.CONCISENESS), - /** - * Equal title and description fields - */ - P2(ProblemPatternId.P2, ProblemPatternSeverity.WARNING, ProblemPatternQualityDimension.CONCISENESS), - /** - * Near-Identical title and description fields - */ - P3(ProblemPatternId.P3, ProblemPatternSeverity.NOTICE, ProblemPatternQualityDimension.CONCISENESS), - /** - * Unrecognizable title - */ - P5(ProblemPatternId.P5, ProblemPatternSeverity.NOTICE, ProblemPatternQualityDimension.ACCURACY), - /** - * Non-meaningful title - */ - P6(ProblemPatternId.P6, ProblemPatternSeverity.NOTICE, ProblemPatternQualityDimension.ACCURACY), - /** - * Missing description fields - */ - P7(ProblemPatternId.P7, ProblemPatternSeverity.NOTICE, ProblemPatternQualityDimension.COMPLETENESS), - /** - * Very short description - */ - P9(ProblemPatternId.P9, ProblemPatternSeverity.WARNING, ProblemPatternQualityDimension.ACCURACY), - /** - * Extremely long values - */ - P12(ProblemPatternId.P12, ProblemPatternSeverity.NOTICE, ProblemPatternQualityDimension.ACCURACY); + P1(ProblemPatternId.P1, "Systematic use of the same title", ProblemPatternSeverity.WARNING, + ProblemPatternQualityDimension.CONCISENESS), + P2(ProblemPatternId.P2, "Equal title and description fields", ProblemPatternSeverity.WARNING, + ProblemPatternQualityDimension.CONCISENESS), + P3(ProblemPatternId.P3, "Near-Identical title and description fields", ProblemPatternSeverity.NOTICE, + ProblemPatternQualityDimension.CONCISENESS), + P5(ProblemPatternId.P5, "Unrecognizable title", ProblemPatternSeverity.NOTICE, ProblemPatternQualityDimension.ACCURACY), + P6(ProblemPatternId.P6, "Non-meaningful title", ProblemPatternSeverity.NOTICE, ProblemPatternQualityDimension.ACCURACY), + P7(ProblemPatternId.P7, "Missing description fields", ProblemPatternSeverity.NOTICE, + ProblemPatternQualityDimension.COMPLETENESS), + P9(ProblemPatternId.P9, "Very short description", ProblemPatternSeverity.WARNING, ProblemPatternQualityDimension.ACCURACY), + P12(ProblemPatternId.P12, "Extremely long values", ProblemPatternSeverity.NOTICE, ProblemPatternQualityDimension.ACCURACY); private final ProblemPatternId problemPatternId; + private final String problemPatternTitle; private final ProblemPatternSeverity problemPatternSeverity; private final ProblemPatternQualityDimension problemPatternQualityDimension; ProblemPatternDescription(ProblemPatternId problemPatternId, - ProblemPatternSeverity problemPatternSeverity, + String problemPatternTitle, ProblemPatternSeverity problemPatternSeverity, ProblemPatternQualityDimension problemPatternQualityDimension) { this.problemPatternId = problemPatternId; + this.problemPatternTitle = problemPatternTitle; this.problemPatternSeverity = problemPatternSeverity; this.problemPatternQualityDimension = problemPatternQualityDimension; } @@ -59,6 +41,10 @@ public ProblemPatternId getProblemPatternId() { return problemPatternId; } + public String getProblemPatternTitle() { + return problemPatternTitle; + } + public ProblemPatternSeverity getProblemPatternSeverity() { return problemPatternSeverity; } diff --git a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/ProblemPatternAnalyzerTest.java b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/ProblemPatternAnalyzerTest.java index b23f8c499..1d84a4189 100644 --- a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/ProblemPatternAnalyzerTest.java +++ b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/ProblemPatternAnalyzerTest.java @@ -5,6 +5,7 @@ import eu.europeana.metis.schema.convert.SerializationException; import eu.europeana.patternanalysis.view.ProblemPattern; +import eu.europeana.patternanalysis.view.ProblemPatternAnalysis; import eu.europeana.patternanalysis.view.ProblemPatternDescription; import java.io.FileInputStream; import java.io.IOException; @@ -61,7 +62,7 @@ private ProblemPatternDescription getRequestedProblemPattern(ProblemPatternDescr .filter(patternDescription -> patternDescription == problemPatternDescription).findFirst().orElse(null); } - private List analyzeProblemPatternsForFile(String fileLocation) throws IOException, SerializationException { + private ProblemPatternAnalysis analyzeProblemPatternsForFile(String fileLocation) throws IOException, SerializationException { String xml = IOUtils.toString(new FileInputStream(fileLocation), StandardCharsets.UTF_8); final ProblemPatternAnalyzer problemPatternAnalyzer = new ProblemPatternAnalyzer(); @@ -81,11 +82,13 @@ private int getRequestedProblemOccurrencesSize(ProblemPatternDescription problem @MethodSource void analyzeRecord(String fileLocation, int totalPatterns, ProblemPatternDescription problemPatternDescription, int totalOccurrences) throws Exception { - final List problemPatterns = analyzeProblemPatternsForFile(fileLocation); + final ProblemPatternAnalysis problemPatternAnalysis = analyzeProblemPatternsForFile(fileLocation); - assertNotNull(problemPatterns); - assertEquals(totalPatterns, problemPatterns.size()); - assertEquals(problemPatternDescription, getRequestedProblemPattern(problemPatternDescription, problemPatterns)); - assertEquals(totalOccurrences, getRequestedProblemOccurrencesSize(problemPatternDescription, problemPatterns)); + assertNotNull(problemPatternAnalysis); + assertEquals(totalPatterns, problemPatternAnalysis.getProblemPatterns().size()); + assertEquals(problemPatternDescription, + getRequestedProblemPattern(problemPatternDescription, problemPatternAnalysis.getProblemPatterns())); + assertEquals(totalOccurrences, + getRequestedProblemOccurrencesSize(problemPatternDescription, problemPatternAnalysis.getProblemPatterns())); } } \ No newline at end of file diff --git a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemOccurrenceTest.java b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemOccurrenceTest.java index a8a73e73f..f92d04a3e 100644 --- a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemOccurrenceTest.java +++ b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemOccurrenceTest.java @@ -16,9 +16,12 @@ void objectCreationTest() { assertEquals("Duplicate titleA", problemOccurrence1.getMessageReport()); assertTrue(CollectionUtils.isEqualCollection(List.of("recordId2", "recordId1"), problemOccurrence1.getAffectedRecordIds())); - final ProblemOccurrence problemOccurrence2 = new ProblemOccurrence("Duplicate titleB"); - assertNotNull(problemOccurrence2.getAffectedRecordIds()); + final ProblemOccurrence problemOccurrence2 = new ProblemOccurrence("Duplicate titleA", null); + assertEquals("Duplicate titleA", problemOccurrence2.getMessageReport()); assertEquals(0, problemOccurrence2.getAffectedRecordIds().size()); - } + final ProblemOccurrence problemOccurrence3 = new ProblemOccurrence("Duplicate titleB"); + assertNotNull(problemOccurrence3.getAffectedRecordIds()); + assertEquals(0, problemOccurrence3.getAffectedRecordIds().size()); + } } \ No newline at end of file diff --git a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemPatternAnalysisTest.java b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemPatternAnalysisTest.java new file mode 100644 index 000000000..3ce6ae647 --- /dev/null +++ b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemPatternAnalysisTest.java @@ -0,0 +1,27 @@ +package eu.europeana.patternanalysis.view; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.List; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class ProblemPatternAnalysisTest { + + @Test + void objectCreationTest() { + final List problemPatterns = List.of(new ProblemPattern(ProblemPatternDescription.P2, 1, + List.of(new RecordAnalysis("recordId", List.of(new ProblemOccurrence("message")))))); + final Set titles = Set.of("titleA"); + final ProblemPatternAnalysis problemPatternAnalysis = new ProblemPatternAnalysis("rdfAbout", problemPatterns, titles); + + assertEquals("rdfAbout", problemPatternAnalysis.getRdfAbout()); + assertEquals(problemPatternAnalysis.getProblemPatterns().size(), problemPatterns.size()); + assertEquals(problemPatternAnalysis.getTitles().size(), titles.size()); + + assertThrows(NullPointerException.class, () -> new ProblemPatternAnalysis(null, problemPatterns, titles)); + assertEquals(0, new ProblemPatternAnalysis("rdfAbout", null, titles).getProblemPatterns().size()); + assertEquals(0, new ProblemPatternAnalysis("rdfAbout", problemPatterns, null).getTitles().size()); + } +} \ No newline at end of file diff --git a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemPatternDescriptionTest.java b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemPatternDescriptionTest.java index 793c7935d..fe1b82e70 100644 --- a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemPatternDescriptionTest.java +++ b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemPatternDescriptionTest.java @@ -1,44 +1,49 @@ package eu.europeana.patternanalysis.view; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import java.util.NoSuchElementException; import org.junit.jupiter.api.Test; class ProblemPatternDescriptionTest { + private void assertProblemPatternDescription(ProblemPatternDescription problemPatternDescription, String patternId, + String problemTitle, + String problemSeverity, String problemAccuracy) { + assertEquals(patternId, problemPatternDescription.getProblemPatternId().toString()); + assertEquals(problemTitle, problemPatternDescription.getProblemPatternTitle()); + assertEquals(problemSeverity, problemPatternDescription.getProblemPatternSeverity().toString()); + assertEquals(problemAccuracy, problemPatternDescription.getProblemPatternQualityDimension().toString()); + } + + @Test + void checkValuesTest() { + assertProblemPatternDescription(ProblemPatternDescription.P1, "P1", "Systematic use of the same title", "WARNING", + "CONCISENESS"); + assertProblemPatternDescription(ProblemPatternDescription.P2, "P2", "Equal title and description fields", "WARNING", + "CONCISENESS"); + assertProblemPatternDescription(ProblemPatternDescription.P3, "P3", "Near-Identical title and description fields", "NOTICE", + "CONCISENESS"); + assertProblemPatternDescription(ProblemPatternDescription.P5, "P5", "Unrecognizable title", "NOTICE", "ACCURACY"); + assertProblemPatternDescription(ProblemPatternDescription.P6, "P6", "Non-meaningful title", "NOTICE", "ACCURACY"); + assertProblemPatternDescription(ProblemPatternDescription.P7, "P7", "Missing description fields", "NOTICE", "COMPLETENESS"); + assertProblemPatternDescription(ProblemPatternDescription.P9, "P9", "Very short description", "WARNING", "ACCURACY"); + assertProblemPatternDescription(ProblemPatternDescription.P12, "P12", "Extremely long values", "NOTICE", "ACCURACY"); + } + @Test - void checkValues() { - assertEquals("P1", ProblemPatternDescription.P1.getProblemPatternId().toString()); - assertEquals("WARNING", ProblemPatternDescription.P1.getProblemPatternSeverity().toString()); - assertEquals("CONCISENESS", ProblemPatternDescription.P1.getProblemPatternQualityDimension().toString()); - - assertEquals("P2", ProblemPatternDescription.P2.getProblemPatternId().toString()); - assertEquals("WARNING", ProblemPatternDescription.P2.getProblemPatternSeverity().toString()); - assertEquals("CONCISENESS", ProblemPatternDescription.P2.getProblemPatternQualityDimension().toString()); - - assertEquals("P3", ProblemPatternDescription.P3.getProblemPatternId().toString()); - assertEquals("NOTICE", ProblemPatternDescription.P3.getProblemPatternSeverity().toString()); - assertEquals("CONCISENESS", ProblemPatternDescription.P3.getProblemPatternQualityDimension().toString()); - - assertEquals("P5", ProblemPatternDescription.P5.getProblemPatternId().toString()); - assertEquals("NOTICE", ProblemPatternDescription.P5.getProblemPatternSeverity().toString()); - assertEquals("ACCURACY", ProblemPatternDescription.P5.getProblemPatternQualityDimension().toString()); - - assertEquals("P6", ProblemPatternDescription.P6.getProblemPatternId().toString()); - assertEquals("NOTICE", ProblemPatternDescription.P6.getProblemPatternSeverity().toString()); - assertEquals("ACCURACY", ProblemPatternDescription.P6.getProblemPatternQualityDimension().toString()); - - assertEquals("P7", ProblemPatternDescription.P7.getProblemPatternId().toString()); - assertEquals("NOTICE", ProblemPatternDescription.P7.getProblemPatternSeverity().toString()); - assertEquals("COMPLETENESS", ProblemPatternDescription.P7.getProblemPatternQualityDimension().toString()); - - assertEquals("P9", ProblemPatternDescription.P9.getProblemPatternId().toString()); - assertEquals("WARNING", ProblemPatternDescription.P9.getProblemPatternSeverity().toString()); - assertEquals("ACCURACY", ProblemPatternDescription.P9.getProblemPatternQualityDimension().toString()); - - assertEquals("P12", ProblemPatternDescription.P12.getProblemPatternId().toString()); - assertEquals("NOTICE", ProblemPatternDescription.P12.getProblemPatternSeverity().toString()); - assertEquals("ACCURACY", ProblemPatternDescription.P12.getProblemPatternQualityDimension().toString()); + void fromNameTest() { + assertEquals(ProblemPatternDescription.P1, ProblemPatternDescription.fromName("P1")); + assertEquals(ProblemPatternDescription.P2, ProblemPatternDescription.fromName("P2")); + assertEquals(ProblemPatternDescription.P3, ProblemPatternDescription.fromName("P3")); + assertEquals(ProblemPatternDescription.P5, ProblemPatternDescription.fromName("P5")); + assertEquals(ProblemPatternDescription.P6, ProblemPatternDescription.fromName("P6")); + assertEquals(ProblemPatternDescription.P7, ProblemPatternDescription.fromName("P7")); + assertEquals(ProblemPatternDescription.P9, ProblemPatternDescription.fromName("P9")); + assertEquals(ProblemPatternDescription.P12, ProblemPatternDescription.fromName("P12")); + + assertThrows(NoSuchElementException.class, () -> ProblemPatternDescription.fromName("invalid")); } } \ No newline at end of file From 7530667d811826cbdc34b0fdc39e7589aa2e896e Mon Sep 17 00:00:00 2001 From: Jorge Ortiz Date: Fri, 20 May 2022 12:34:51 +0200 Subject: [PATCH 36/73] MET-4375_MET-4541 RecordView unit test (#534) --- metis-repository/pom.xml | 199 +++++++++--------- .../metis/repository/rest/RecordViewTest.java | 47 +++++ 2 files changed, 146 insertions(+), 100 deletions(-) create mode 100644 metis-repository/src/test/java/eu/europeana/metis/repository/rest/RecordViewTest.java diff --git a/metis-repository/pom.xml b/metis-repository/pom.xml index 49f2783a0..418f2a7b6 100644 --- a/metis-repository/pom.xml +++ b/metis-repository/pom.xml @@ -1,101 +1,100 @@ - - - metis-framework - eu.europeana.metis - 7-SNAPSHOT - - 4.0.0 - metis-repository - war - - - javax.xml.bind - jaxb-api - - - org.glassfish.jaxb - jaxb-runtime - - - org.apache.logging.log4j - log4j-slf4j-impl - - - org.springframework - spring-core - ${version.spring} - - - org.springframework - spring-webmvc - ${version.spring} - - - javax.servlet - javax.servlet-api - ${version.servlet.api} - provided - - - io.springfox - springfox-swagger2 - ${version.swagger} - - - io.springfox - springfox-swagger-ui - ${version.swagger} - - - com.fasterxml.jackson.core - jackson-annotations - ${version.jackson} - - - com.fasterxml.jackson.dataformat - jackson-dataformat-xml - ${version.jackson} - - - com.fasterxml.jackson.datatype - jackson-datatype-jsr310 - ${version.jackson} - - - eu.europeana.metis - metis-common-utils - ${project.version} - - - eu.europeana.metis - metis-common-mongo - ${project.version} - - - eu.europeana.metis - metis-harvesting - ${project.version} - - - org.junit.jupiter - junit-jupiter-api - - - org.junit.jupiter - junit-jupiter-engine - - - - - - org.apache.maven.plugins - maven-war-plugin - ${version.maven.war.plugin} - - false - - - - - + + + metis-framework + eu.europeana.metis + 7-SNAPSHOT + + 4.0.0 + metis-repository + war + + + javax.xml.bind + jaxb-api + + + org.glassfish.jaxb + jaxb-runtime + + + org.apache.logging.log4j + log4j-slf4j-impl + + + org.springframework + spring-core + ${version.spring} + + + org.springframework + spring-webmvc + ${version.spring} + + + javax.servlet + javax.servlet-api + ${version.servlet.api} + provided + + + io.springfox + springfox-swagger2 + ${version.swagger} + + + io.springfox + springfox-swagger-ui + ${version.swagger} + + + com.fasterxml.jackson.core + jackson-annotations + ${version.jackson} + + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + ${version.jackson} + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + ${version.jackson} + + + eu.europeana.metis + metis-common-utils + ${project.version} + + + eu.europeana.metis + metis-common-mongo + ${project.version} + + + eu.europeana.metis + metis-harvesting + ${project.version} + + + org.junit.jupiter + junit-jupiter + ${version.junit} + test + + + + + + org.apache.maven.plugins + maven-war-plugin + ${version.maven.war.plugin} + + false + + + + + \ No newline at end of file diff --git a/metis-repository/src/test/java/eu/europeana/metis/repository/rest/RecordViewTest.java b/metis-repository/src/test/java/eu/europeana/metis/repository/rest/RecordViewTest.java new file mode 100644 index 000000000..ead7b8c8a --- /dev/null +++ b/metis-repository/src/test/java/eu/europeana/metis/repository/rest/RecordViewTest.java @@ -0,0 +1,47 @@ +package eu.europeana.metis.repository.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.time.Instant; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit test for {@link RecordView} class + */ +class RecordViewTest { + + private RecordView recordView; + + @BeforeEach + void setup() { + final Instant instant = Instant.parse("2022-05-19T08:28:00.823118Z"); + recordView = new RecordView("recordId", "datasetId", instant, false, "edmRecord"); + } + + @Test + void getRecordId() { + assertEquals("recordId", recordView.getRecordId()); + } + + @Test + void getDatasetId() { + assertEquals("datasetId", recordView.getDatasetId()); + } + + @Test + void getDateStamp() { + assertEquals("2022-05-19T08:28:00.823118Z", recordView.getDateStamp().toString()); + } + + @Test + void isMarkedAsDeleted() { + assertFalse(recordView.isMarkedAsDeleted()); + } + + @Test + void getEdmRecord() { + assertEquals("edmRecord", recordView.getEdmRecord()); + } +} From a563905fe9b0af34f284e6ffc7c032fe23722dc4 Mon Sep 17 00:00:00 2001 From: Jorge Ortiz Date: Fri, 20 May 2022 12:49:57 +0200 Subject: [PATCH 37/73] MET-4375_MET-4539 RecordController unit tests (#536) * MET-4375_MET-4539 RecordController unit tests * MET-4375_MET-4539 resource files for unit tests * MET-4375_MET-4539 remove mockito code smells --- metis-repository/pom.xml | 18 ++ .../repository/rest/RecordControllerTest.java | 281 ++++++++++++++++++ .../test/resources/repository-test-error.zip | Bin 0 -> 3274 bytes .../src/test/resources/repository-test.zip | Bin 0 -> 3282 bytes 4 files changed, 299 insertions(+) create mode 100644 metis-repository/src/test/java/eu/europeana/metis/repository/rest/RecordControllerTest.java create mode 100644 metis-repository/src/test/resources/repository-test-error.zip create mode 100644 metis-repository/src/test/resources/repository-test.zip diff --git a/metis-repository/pom.xml b/metis-repository/pom.xml index 418f2a7b6..13f332383 100644 --- a/metis-repository/pom.xml +++ b/metis-repository/pom.xml @@ -84,6 +84,24 @@ ${version.junit} test + + org.mockito + mockito-core + + + com.jayway.jsonpath + json-path + test + + + org.springframework + spring-webmvc + ${version.spring} + + + org.springframework + spring-test + diff --git a/metis-repository/src/test/java/eu/europeana/metis/repository/rest/RecordControllerTest.java b/metis-repository/src/test/java/eu/europeana/metis/repository/rest/RecordControllerTest.java new file mode 100644 index 000000000..681293507 --- /dev/null +++ b/metis-repository/src/test/java/eu/europeana/metis/repository/rest/RecordControllerTest.java @@ -0,0 +1,281 @@ +package eu.europeana.metis.repository.rest; + +import static org.hamcrest.core.Is.is; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.put; +import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import eu.europeana.metis.repository.dao.Record; +import eu.europeana.metis.repository.dao.RecordDao; +import eu.europeana.metis.utils.RestEndpoints; +import java.io.InputStream; +import java.time.Instant; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.springframework.http.MediaType; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.server.ResponseStatusException; + +/** + * Unit test for {@link RecordController} class + */ +class RecordControllerTest { + + private static RecordDao recordDaoMock; + private static MockMvc recordControllerMock; + private static RecordController recordController; + + @BeforeAll + static void setup() { + recordDaoMock = mock(RecordDao.class); + recordController = new RecordController(); + recordControllerMock = MockMvcBuilders.standaloneSetup(recordController) + .build(); + } + + @AfterEach + void cleanUp() { + reset(recordDaoMock); + } + + @Test + void setRecordDaoAndGetRecord() { + Record expectedRecord = getTestRecord(); + + when(recordDaoMock.getRecord("recordId")).thenReturn(expectedRecord); + recordController.setRecordDao(recordDaoMock); + + RecordView recordView = recordController.getRecord("recordId"); + + assertEquals(expectedRecord.getRecordId(), recordView.getRecordId()); + assertEquals(expectedRecord.getEdmRecord(), recordView.getEdmRecord()); + assertEquals(expectedRecord.getDatasetId(), recordView.getDatasetId()); + assertEquals(expectedRecord.isDeleted(), recordView.isMarkedAsDeleted()); + assertEquals(expectedRecord.getDateStamp(), recordView.getDateStamp()); + } + + @Test + void setRecordDaoAndGetRecord_expectException() { + when(recordDaoMock.getRecord("recordId")).thenReturn(null); + recordController.setRecordDao(recordDaoMock); + + RuntimeException expectedException = assertThrows(ResponseStatusException.class, () -> { + recordController.getRecord("recordId"); + }); + + assertEquals("404 NOT_FOUND \"No record found for this identifier.\"", expectedException.getMessage()); + } + + @Test + void getRecordViaController() throws Exception { + Record expectedRecord = getTestRecord(); + + when(recordDaoMock.getRecord("recordId")).thenReturn(expectedRecord); + recordController.setRecordDao(recordDaoMock); + recordControllerMock.perform(get(RestEndpoints.REPOSITORY_RECORDS_RECORD_ID, "recordId") + .content("")) + .andDo(print()) + .andExpect(status().is(200)) + .andExpect(content().string(getXMLTestRecord())); + + verify(recordDaoMock, times(1)).getRecord("recordId"); + } + + @Test + void getRecordViaController_notFound() throws Exception { + when(recordDaoMock.getRecord("recordId")).thenReturn(null); + recordController.setRecordDao(recordDaoMock); + recordControllerMock.perform(get(RestEndpoints.REPOSITORY_RECORDS_RECORD_ID, "recordId") + .content("")) + .andDo(print()) + .andExpect(status().is(404)) + .andExpect(content().string("")); + + verify(recordDaoMock, times(1)).getRecord("recordId"); + } + + @Test + void saveRecord() throws Exception { + when(recordDaoMock.createRecord(any(Record.class))).thenReturn(true); + recordController.setRecordDao(recordDaoMock); + recordControllerMock.perform(post(RestEndpoints.REPOSITORY_RECORDS_RECORD_ID, "recordId") + .contentType(MediaType.APPLICATION_XML) + .param("datasetId", "datasetId") + .param("datestamp", "+1000000000-12-31T23:59:59.999999999Z") + .param("markAsDeleted", "false") + .content("edmRecord")) + .andDo(print()) + .andExpect(status().is(200)) + .andExpect(jsonPath("$.datasetId", is("datasetId"))) + .andExpect(jsonPath("$.dateStamp").exists()) + .andExpect(jsonPath("$.insertedRecords", is(1))) + .andExpect(jsonPath("$.updatedRecords", is(0))) + .andExpect(jsonPath("$.insertedRecordIds").isArray()) + .andExpect(jsonPath("$.insertedRecordIds").isNotEmpty()) + .andExpect(jsonPath("$.updatedRecordIds").isArray()) + .andExpect(jsonPath("$.updatedRecordIds").isEmpty()); + verify(recordDaoMock, times(1)).createRecord(any()); + } + + @Test + void saveRecord_Exception() throws Exception { + when(recordDaoMock.createRecord(any(Record.class))).thenThrow(new RuntimeException("Fail to save record")); + recordController.setRecordDao(recordDaoMock); + recordControllerMock.perform(post(RestEndpoints.REPOSITORY_RECORDS_RECORD_ID, "recordId") + .contentType(MediaType.APPLICATION_XML) + .param("datasetId", "datasetId") + .param("dateStamp", "+1000000000-12-31T23:59:59.999999999Z") + .param("markAsDeleted", "false") + .content("edmRecord")) + .andDo(print()) + .andExpect(status().is(500)); + verify(recordDaoMock, times(1)).createRecord(any()); + } + + @Test + void saveRecords() throws Exception { + InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream("repository-test.zip"); + MockMultipartFile recordsFile = new MockMultipartFile("recordsZipFile", + "repository-test.zip", + "application/zip", + inputStream); + when(recordDaoMock.createRecord(any(Record.class))).thenReturn(true); + recordController.setRecordDao(recordDaoMock); + recordControllerMock.perform(multipart(RestEndpoints.REPOSITORY_RECORDS) + .file(recordsFile) + .param("datasetId", "datasetId") + .param("dateStamp", "+1000000000-12-31T23:59:59.999999999Z") + .contentType(MediaType.MULTIPART_FORM_DATA_VALUE)) + .andDo(print()) + .andExpect(status().is(200)) + .andExpect(jsonPath("$.datasetId", is("datasetId"))) + .andExpect(jsonPath("$.dateStamp").exists()) + .andExpect(jsonPath("$.insertedRecords", is(2))) + .andExpect(jsonPath("$.updatedRecords", is(0))) + .andExpect(jsonPath("$.insertedRecordIds").isArray()) + .andExpect(jsonPath("$.insertedRecordIds").isNotEmpty()) + .andExpect(jsonPath("$.updatedRecordIds").isArray()) + .andExpect(jsonPath("$.updatedRecordIds").isEmpty()); + verify(recordDaoMock, times(2)).createRecord(any()); + } + + @Test + void saveRecords_Exception() throws Exception { + InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream("repository-test-error.zip"); + MockMultipartFile recordsFile = new MockMultipartFile("recordsZipFile", + "repository-test-error.zip", + "application/zip", + inputStream); + when(recordDaoMock.createRecord(any(Record.class))).thenReturn(true); + recordController.setRecordDao(recordDaoMock); + recordControllerMock.perform(multipart(RestEndpoints.REPOSITORY_RECORDS) + .file(recordsFile) + .param("datasetId", "datasetId") + .param("dateStamp", "+1000000000-12-31T23:59:59.999999999Z") + .contentType(MediaType.MULTIPART_FORM_DATA_VALUE)) + .andDo(print()) + .andExpect(status().is(500)); + verify(recordDaoMock, times(0)).createRecord(any()); + } + + @Test + void updateRecordHeader() throws Exception { + when(recordDaoMock.getRecord("recordId")).thenReturn(getTestRecord()); + when(recordDaoMock.createRecord(any(Record.class))).thenReturn(false); + recordController.setRecordDao(recordDaoMock); + recordControllerMock.perform(put(RestEndpoints.REPOSITORY_RECORDS_RECORD_ID_HEADER, "recordId") + .contentType(MediaType.APPLICATION_XML) + .param("datasetId", "datasetId") + .param("datestamp", "+1000000000-12-31T23:59:59.999999999Z") + .param("markAsDeleted", "false") + .content("edmRecord")) + .andDo(print()) + .andExpect(status().is(200)) + .andExpect(jsonPath("$.datasetId", is("datasetId"))) + .andExpect(jsonPath("$.dateStamp").exists()) + .andExpect(jsonPath("$.insertedRecords", is(0))) + .andExpect(jsonPath("$.updatedRecords", is(1))) + .andExpect(jsonPath("$.insertedRecordIds").isArray()) + .andExpect(jsonPath("$.insertedRecordIds").isEmpty()) + .andExpect(jsonPath("$.updatedRecordIds").isArray()) + .andExpect(jsonPath("$.updatedRecordIds").isNotEmpty()); + verify(recordDaoMock, times(1)).getRecord("recordId"); + verify(recordDaoMock, times(1)).createRecord(any()); + } + + @Test + void updateRecordHeader_Exception() throws Exception { + when(recordDaoMock.getRecord("recordId")).thenReturn(null); + + recordController.setRecordDao(recordDaoMock); + recordControllerMock.perform(put(RestEndpoints.REPOSITORY_RECORDS_RECORD_ID_HEADER, "recordId") + .contentType(MediaType.APPLICATION_XML) + .param("datasetId", "datasetId") + .param("datestamp", "+1000000000-12-31T23:59:59.999999999Z") + .param("markAsDeleted", "false") + .content("edmRecord")) + .andDo(print()) + .andExpect(status().is(404)); + verify(recordDaoMock, times(1)).getRecord("recordId"); + } + + @Test + void deleteRecord() throws Exception { + when(recordDaoMock.deleteRecord("recordId")).thenReturn(true); + recordController.setRecordDao(recordDaoMock); + recordControllerMock.perform(delete(RestEndpoints.REPOSITORY_RECORDS_RECORD_ID, "recordId") + .content("")) + .andExpect(status().is(200)) + .andExpect(content().string("")); + } + + @Test + void deleteRecord_notFound() throws Exception { + when(recordDaoMock.deleteRecord("recordId")).thenReturn(false); + recordController.setRecordDao(recordDaoMock); + recordControllerMock.perform(delete(RestEndpoints.REPOSITORY_RECORDS_RECORD_ID, "recordId") + .content("")) + .andExpect(status().is(404)) + .andExpect(content().string("")); + } + + @NotNull + private Record getTestRecord() { + Record testRecord = new Record(); + testRecord.setRecordId("recordId"); + testRecord.setEdmRecord("edmRecord"); + testRecord.setDatasetId("datasetId"); + testRecord.setDeleted(false); + testRecord.setDateStamp(Instant.MAX); + return testRecord; + } + + @NotNull + private String getXMLTestRecord() { + return "" + + "recordId" + + "datasetId" + + "+1000000000-12-31T23:59:59.999999999Z" + + "false" + + "edmRecord" + + ""; + } +} diff --git a/metis-repository/src/test/resources/repository-test-error.zip b/metis-repository/src/test/resources/repository-test-error.zip new file mode 100644 index 0000000000000000000000000000000000000000..81e4d83617f51657e798c0353ba8bee9cd779b4b GIT binary patch literal 3274 zcmbW3c{mjM8plUtNS35&Vyq=(Z5Yc)_DJ@u31yGL*s~;*P%+lAO|r#E_QAmrS&wC! zEK@Xgk}abUS%#Z)Zs(qR@45GR?tS0qxBc_|KJP!D-xt7%sWt=rsn`Jm0AFWsFF!Yb zFW+EEe`i1c3+86507jTkv;+LdIy?h{m;rS33qSzix6N1&NDl@a{n6dtu79ZM9|r)= z{e_w$94;**4VRZhxdxg#$XQ4Qd3yZj&}}B-0Q#YRxcaQG4Ay<9cq+-d4veN%FrulH|G?zud%oYjFZaUBOK*TlnrH9s`H(Upn`dy1#jV zoU6WFXQk_6S4v)TE8_olksds(@$$Rdd_qF{cnYa{AQCPp+XMU4#`!Y}wfEUAcF=@~F6Uy1Ho^ZDZ3zpRXVrGi!;O0|nMIy<^i z1iQ$H@G6D4-15mEzN{!|GLes#U6W}HLEBXja|bZ+Hy;qm$o$1Vu#7P& z|21mNfj97GhHU*AL=-4`a}*h=QlmAYs;pedxo6j}^|-tY<5)NT3z>;DmOT?#Y;Liq5`DH#751Y34YW{Ckl0 zwz1IMx9{IgB^RSW{Ly?vT~E1jM1JGki_a&mdgGqy`j$V;Y)^+1AT;S6DCWVFCi^{i zy3OnnML0|-$4U$q^C@gd_0mlsV{c1k3r!L*#xxGTdxRATg< zut_EadOgFv*8A=4`dT(^JXHWwy)5CZxpG*9nLzLn-RGQ?!4$RWLG%5C@tlKwA?kj7 zcPgF)RyWthx9J+LVa)ZGDF_^o^hMCwq#3ryfXRN7>0V|r1Cxex>$-&;9F5g#29Qcs zuvGC2QrpEP7a3uS6A>MH#eJtXF?7j^fv(C#7;% z-rQ+_p7J6bIw!UtA27)PinYfq%RVYQ>ZwuL|4z7O1KAj*dDnIaLSVC}2Z>MP7=WV8 zsRS>+Uwqk}Lm;K4V0m;m!sm`#ei)vu1Y)6qINBbPXSsCb^yLZ*vq&h?Lwf(nm}i2L z{CQG;K>qqz-6CkIBMDKz3nA1tdDvs;5oX`WVeD5G0S zONM}N#3@HzfjL``ntma{VOJI&#V^5XJMYF|9nHgjo#5=Iv{1nEXaq2x|7(=uxz)lu zE-&?zoIk#103}=swC}nby6m)cB3~^mKu2Xk7St|RJ+iz|e1{FV&EgR)E%-WDPE|X| zy0a)B!xbEAEHw5($y(0#2q9_xV}XlPkaLvZfv3{dU>eZ7j%63Hj2L!4jeZVmn69r zlWgx|g&q0kI2rgdl8(%;UjV(+XWN3fp7$v}rLjbv`WPhsD5`eeys%TwW*|LcGA`WyW5eatphpHhT5{5a(jb%5!d~LN709tNzD`R=LXZLFj+7-pk6t&aRF9xjlU^&M>Am_!={Z{Va;@*g2=g9Y;>9L{s>4%**}ndjcgmUFpk#mL!! zNguwa&6}g%+JEv|zw)`0T1ECmD+EO$l`E}`?i$#Sr$g3&VHXltxfBuQeL03V@!1db zW35zD{9JXx980Yb6o20~v{w?Az`0QYqezxfW1N-@8Kt*0vCRcRO$pO?lxA&?weOxs z_xJ0`J`p~TCFY2@$z(;4QvItRJD3_@?f)=e`+#N_Q-;gpCgC2FGxJu#;M`Z5QMCF_ zVJaRkM)*N~p=dV35AyqKsUQ>;^EmgsN3NiotzF+XZ#J5UAl4b5aK{s7`q<<`pipH> zEbFMV#QP>V2{=AQJx+R8&2Enz#N$(ZQ~MZ{8UK}o4*#eZk6IiyVLXAXaf~7t zZe5Q0PF4Z^S|*{48sG&gzy&hRKuPg16X>#}HB3ui`x2jMX6w>j&(F%#aTQCV;@N_A z3tuA%1d)C7R1!9HccntLqfYfCM^GVzquTP!v&ajlq)=*t5T+vgBK%Z5^Ji^-Set<} z?Sa*Ogk=9U<2E6KM^6(eNiJOwVuI!X3(e$e*s3$#iKJMMH$9NnfL@JH0bbyD_tKD11?0!(*fG0UXZ8hC0&)J|((g z;|E(8z9X*2TrFM?hjG8WQN`0$cl7AW|2 zLTzF_qmsJyY}3{p)JYH6?vuntn{ zHe$Gi*qb@uan$$amz78xnRs9fUeChp32zd2A3aZ}e2eFeG@FNM%4bz)n^YybQ*SH? zuJs^p!S?VxaoQRxE?mvm9>%cCZnHeFxqh;LIC@eYHn9=nAG+u=>UZJQ1KXBUvuY?7 zww0zu(u-PIwOoB#xbtE z7P7GAujSQOjC+|hD#hwqPQ1b$x#=qYAgvk=_KF+`&bID$WJcN2c?j~a*fQStEUau$ zmS-!!z)k=$dByd~QoaTUy(um^%a&Dqf1~GH`>X9uC2L*I;;IUmIo>Cnc^sEY4RJx< zqc#p<41!G@IO`2LSXIv5Jci8R;VmWVAm_!;7=&l2OfosRt-Q$}rG%Gk|H3pxT-o}n zNm_uUw&Z~!`IkM$6i zON0W=MtT6?5Q)ZmbVtBI@NbFxud?^gC{9y9y#4P<{CD-AS^UqKKFs1jtLDDpU#R~- v$^7R$hmME%|MRT>>}bI7@ALfK@qhXZjP*c2lmGzWkJg|A0794#U&sCiqU**| literal 0 HcmV?d00001 diff --git a/metis-repository/src/test/resources/repository-test.zip b/metis-repository/src/test/resources/repository-test.zip new file mode 100644 index 0000000000000000000000000000000000000000..d0658faf2297b2dcb4f24ca80617d58316d5d8c6 GIT binary patch literal 3282 zcmbW3c{mjM8plUtEXR^GO^mgKtPNus$)1vZUqW^@OJg_@bVY)aq{-LU}DS!poQIzvWI`~qw|g*1AvNp0SEy6*z|v>(E;cH-JULg zsA(Su0M7k|nj9Q1At?cu5ktB78re&miu<|W`r*)RENBnqTntkm~bquJZ{AY2(Sns{U{|pFsQZ3*z*eBJyDD zoSysx@9o5awzhnYf|(OT>?WxJly)Kzzei@1pS>HYaz|!B0Oyiw#NaF-H*C!k$56&p z;^|~@MI+Wv74E4G`}LWrZsEmL+eOvD?S|s=xRsuMowAjvuC?2rok+_HxF$uk%3z9Z ztlrttjY8N(S{&QK(^SDkiH*ySKD0fT-yzw&KUy^41h6`$A$>1vESX|7JEO?}k{oQlKKcd1qsa=Rxw zJ*zTqi${L6&Zl{{$`93$)gWYWU2Ssl1PbdG=%5T`qNLU&8v@X_<$XE*82Ib=h)2l0 z#aCcSePZ5A)R;Y&?~Qb+x-*DKP}JrqGF-7*eL_h=p@41AmZbimtQ6x=JN|-9PaMmd z2{5$`OumnvSVwbzii~oHLy!~D`7Bh$V3YTq4}2Se*Wb)_JG}AKh+0KwU89Ja-;Bf-BSAb-+=E?@IdFYE`Z*V$PFnQFJ<;?ki^ynCgA*Y85<5^# z*uzGNaztU+|maJV?I91Upnv@DtcTDA| z2zp(8xg7O=85 zR4MB~DwM$DMbC(B7nhtR1x6hE(1BOoc4`s>mmF!RDs81B3^e5ATFD7}B=bzDC!WRR z!^!aQqC3!n^p5j-LiVj1sRPT~VBa}(p`KGIA=jlPfSeVYSXZz)(j8l=1XmgB7M+lc za8vH89^%;CX@8phEEqZ`d=P(Uk_HrOhgp`2DW!8)tsuP>sNO&}M5^7h-hmL9?WjQ_ z(>NNS5JL*VgZqUSi&Frkqy#L3?nd0c>zWsYXD)`Asv_vx19Hulj-9?-ZfYD3Mc$G) zI5y^4ffL?LRzv-|=R~4Sl)_*TLXguUL|+ z9u-LIn%t7k>lJd!L6djR`c^fGpKr*8kyGxozw*x8F<3|Qkas6It1&eYusj+9jOY0h zDR*wQ;I8v?EqSL8FKIvtmwfHI?gcJ8E}h6z4!WbExF7{;m#!LKUMRZD4BTeC6(zy< zGDlkJil1d?VIGFvKU`Li^Hcai&!RzXaKIkhc!T3W&U_^6LZmL&7-d+MJ-*PY#zz+H zkhJ40o#_*^L#%{^s33u+bFekEN!9+Kj4bDghFdMH!0Exav1zNsg(W#rZ9n-+wMe4coSfekxtnQ~KmTl8aV`Dh?SpLK)x84l8rw+-4PDzT224 zj=yxX{H3Iv)s5DymYzWMrTlp8Eo^^%%X%t;?LecSKT*8ZaiPKEmWo||$H%Xu_`AWt zabpoNqH7V+`W{xmfqRaPhC4m!*!=nh&|7WhEr`qc+eN2Tmnc&o{6u0RYvxTFYO;m* zas1ZV+DbP+9+stwjc!Z!r-e+$1>1e7zntP1qtl}TA)`8STWhp+Nw^5#;}gfBSotf8bw|>3`y1`ZxT0%lv`=qP+0`j(;uhQ5lxz zc{5u~vAVw3jd#Pn2;DIpS}Y#XH`Af2q2)=*5HjTQbhJ(Jn zw`!tq*Xy$D9B(m(d}$h&Gt%$5Q@s^YasZ8c>|NW%`$Q}5wGeiW9Qs6y8O1|A_>hCa z&B}U+zL!<;DacPC{U+JP1JlEgAQ;>1PtF&6Z!NvgmqNu>D#kZ64sN95fY|U{w@&8l z%atp7PCE43@VzTs*($B1lLjP(b0^gc?fNVbBeE&%sijl~U~kS2DIJg}VoMF%obT6|Fnw2k z*6Nq`-ScP?NlWUXz=1>ot>S^3;AqE<9&A0BS3GD>qwSHMk&zun z?`uhO7U4A7SsxMzzSlQwi^?}Q*Uz~(YdL$4{75PB_!Rfl0S0BvbM>&pC-PZLi~S~y z)0ZiZR`7z&<;ZVjMbNLMq6(;fE}$%&H^Uf|6c00mE{j>h)U~f%;ugwiUApJ~Nr5u1 zXx1lpHhbYWp?IC_=QvAC}lnfeW6_;ekz{f z(-j_An~u|dm_;Z;jAWqS#;+6eIB_J&xeL;lpw`d0Z+IQ&y)AHACTt<`s{G0{EwiU!tlyJ=0ISGO)19epXq-W25c^?9avq zI#KyP?sLV)540|PLtKl#R!7z;9EN!6o$@ty95Xz%hB=MpC zLyZbk6x>|Dvia~LRq<4ETOZ#(w&=C^dWNqEpV>=k@&aO1PC|JTMxsWorD8D~aP3HD{+xi~IFdtB6#Evr4m# zN}}B<*BAKKdJs0SJv?XJ6;(xN_GW{KXcnn$#xN`Mlcb@jNfp?{Mu1P?qVuTtg{Cm; zmQ%CJC`RU$#zo?@8jvLh#{AwRzI~A5nwhyjTfz8rYG})2^8TfQ<@q|{MPk{QD9JKC zh$h-0uB-;Ku;ruf@u~>-JZV&%$-S)aDo6OHi%3{%6&maj-tV7f+3mo9vZlJl$FpKh z8|q$AQLiAwTy}wl0HXJZdnGmU#ozCBQSn*k%%aeZp0Djq+ne&1nruat%7sC;NtBU z^n-mXTVIq2NZ+{BV)Bh+L6Skgc5_%AAMmRaJRRGI4QRcoBh*_axo1;K7)<%L8!a9T zKxgkU9l>&mkgwK24FDV=QD2Md7#Iltk+%OTd;iSh`zlAbzn{c^Q~#O8|BUIQEdH~q zLv{Z`{r^ekKj%4eJi`B}EfxasN0Ko4{l?ng|U^sgH1Nphe A{r~^~ literal 0 HcmV?d00001 From eab08601ab8e788d058f675522f45bcd66a0cf5c Mon Sep 17 00:00:00 2001 From: JoanaCMS <70145179+JoanaCMS@users.noreply.github.com> Date: Fri, 20 May 2022 16:38:02 +0200 Subject: [PATCH 38/73] MET-4532 Created new unit tests for RecordDao (#538) --- .../metis/repository/dao/RecordDaoTest.java | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 metis-repository/src/test/java/eu/europeana/metis/repository/dao/RecordDaoTest.java diff --git a/metis-repository/src/test/java/eu/europeana/metis/repository/dao/RecordDaoTest.java b/metis-repository/src/test/java/eu/europeana/metis/repository/dao/RecordDaoTest.java new file mode 100644 index 000000000..1fc17bd4b --- /dev/null +++ b/metis-repository/src/test/java/eu/europeana/metis/repository/dao/RecordDaoTest.java @@ -0,0 +1,116 @@ +package eu.europeana.metis.repository.dao; + +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import eu.europeana.metis.mongo.embedded.EmbeddedLocalhostMongo; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNull; + +class RecordDaoTest { + + private final static String DATABASE_NAME = "dbTest"; + + private RecordDao recordDao; + + private EmbeddedLocalhostMongo embeddedLocalhostMongo; + + @BeforeEach + void setup() { + embeddedLocalhostMongo = new EmbeddedLocalhostMongo(); + embeddedLocalhostMongo.start(); + final String mongoHost = embeddedLocalhostMongo.getMongoHost(); + final int mongoPort = embeddedLocalhostMongo.getMongoPort(); + final MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s", mongoHost, mongoPort)); + recordDao = new RecordDao(mongoClient, DATABASE_NAME); + } + + @AfterEach + void tearDown() { + embeddedLocalhostMongo.stop(); + } + + @Test + void createRecordTest_expectTrue(){ + Instant dateStamp = Instant.now(); + Record recordToTest = new Record("recordId", "datasetId", dateStamp, false, "edmRecord"); + + assertTrue(recordDao.createRecord(recordToTest)); + } + + @Test + void createRecordTest_expectFalse(){ + Instant dateStamp = Instant.now(); + Record recordToTest = new Record("recordId", "datasetId", dateStamp, false, "edmRecord"); + Record otherRecordToTest = new Record("recordId", "datasetId", dateStamp, false, "newEdmRecord"); + recordDao.createRecord(recordToTest); + + assertFalse(recordDao.createRecord(otherRecordToTest)); + } + + @Test + void getAllRecordsFromDatasetTest(){ + Instant dateStamp = Instant.now(); + Record recordToTest = new Record("recordId", "datasetId", dateStamp, false, "edmRecord"); + Record otherRecordToTest = new Record("otherRecordId", "datasetId", dateStamp, false, "otherEdmRecord"); + recordDao.createRecord(recordToTest); + recordDao.createRecord(otherRecordToTest); + + List result = recordDao.getAllRecordsFromDataset("datasetId").collect(Collectors.toUnmodifiableList()); + assertEquals("datasetId", result.get(0).getDatasetId()); + assertEquals("datasetId", result.get(1).getDatasetId()); + assertEquals("recordId", result.get(0).getRecordId()); + assertEquals("otherRecordId", result.get(1).getRecordId()); + assertEquals(dateStamp.toEpochMilli(), result.get(0).getDateStamp().toEpochMilli()); + assertEquals(dateStamp.toEpochMilli(), result.get(1).getDateStamp().toEpochMilli()); + assertFalse(result.get(0).isDeleted()); + assertFalse(result.get(1).isDeleted()); + assertEquals("edmRecord", result.get(0).getEdmRecord()); + assertEquals("otherEdmRecord", result.get(1).getEdmRecord()); + + } + + @Test + void getRecordTest_expectResult(){ + Instant dateStamp = Instant.now(); + Record recordToTest = new Record("recordId", "datasetId", dateStamp, false, "edmRecord"); + recordDao.createRecord(recordToTest); + + Record result = recordDao.getRecord("recordId"); + + assertEquals("recordId", result.getRecordId()); + assertEquals("datasetId", result.getDatasetId()); + assertEquals(dateStamp.toEpochMilli(), result.getDateStamp().toEpochMilli()); + assertFalse(result.isDeleted()); + assertEquals("edmRecord", result.getEdmRecord()); + + } + + @Test + void getRecordTest_expectNull(){ + assertNull(recordDao.getRecord("recordId")); + } + + @Test + void deleteRecord_expectTrue(){ + Instant dateStamp = Instant.now(); + Record recordToTest = new Record("recordId", "datasetId", dateStamp, false, "edmRecord"); + recordDao.createRecord(recordToTest); + + assertTrue(recordDao.deleteRecord("recordId")); + } + + @Test + void deleteRecord_expectFalse(){ + assertFalse(recordDao.deleteRecord("recordId")); + } +} From 096df6b3dac825bb018617b27c1718df42bb4174 Mon Sep 17 00:00:00 2001 From: JoanaCMS <70145179+JoanaCMS@users.noreply.github.com> Date: Fri, 20 May 2022 16:46:44 +0200 Subject: [PATCH 39/73] MET-4534 Created unit tests for InsertionResult (#539) --- .../repository/rest/InsertionResultTest.java | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 metis-repository/src/test/java/eu/europeana/metis/repository/rest/InsertionResultTest.java diff --git a/metis-repository/src/test/java/eu/europeana/metis/repository/rest/InsertionResultTest.java b/metis-repository/src/test/java/eu/europeana/metis/repository/rest/InsertionResultTest.java new file mode 100644 index 000000000..7bcba3aeb --- /dev/null +++ b/metis-repository/src/test/java/eu/europeana/metis/repository/rest/InsertionResultTest.java @@ -0,0 +1,75 @@ +package eu.europeana.metis.repository.rest; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class InsertionResultTest { + + private InsertionResult insertionResultToTest; + private final Instant instantForTest = Instant.now(); + + @BeforeEach + void setUp(){ + insertionResultToTest = new InsertionResult("datasetId", instantForTest); + } + + @Test + void testGetDatasetId(){ + assertEquals("datasetId",insertionResultToTest.getDatasetId()); + } + + @Test + void testGetDateStamp(){ + assertEquals(instantForTest, insertionResultToTest.getDateStamp()); + } + + @Test + void testAddInsertedRecordAndGetInsertedRecords(){ + setInsertedRecords(); + assertEquals(3, insertionResultToTest.getInsertedRecords()); + } + + @Test + void testAddUpdatedRecordAndGetUpdatedRecords(){ + setUpdatedRecords(); + assertEquals(3, insertionResultToTest.getUpdatedRecords()); + } + + @Test + void testGetInsertedRecordIds(){ + setInsertedRecords(); + Set result = insertionResultToTest.getInsertedRecordIds(); + assertTrue(result.contains("recordId1")); + assertTrue(result.contains("recordId2")); + assertTrue(result.contains("recordId3")); + + } + + @Test + void testGetUpdatedRecordIds(){ + setUpdatedRecords(); + Set result = insertionResultToTest.getUpdatedRecordIds(); + assertTrue(result.contains("recordId1")); + assertTrue(result.contains("recordId2")); + assertTrue(result.contains("recordId3")); + + } + + private void setInsertedRecords(){ + insertionResultToTest.addInsertedRecord("recordId1"); + insertionResultToTest.addInsertedRecord("recordId2"); + insertionResultToTest.addInsertedRecord("recordId3"); + } + + private void setUpdatedRecords(){ + insertionResultToTest.addUpdatedRecord("recordId1"); + insertionResultToTest.addUpdatedRecord("recordId2"); + insertionResultToTest.addUpdatedRecord("recordId3"); + } +} From 45d8d020759fb53cefda89238ee0ee90688fe36d Mon Sep 17 00:00:00 2001 From: Jorge Ortiz Date: Fri, 20 May 2022 17:21:51 +0200 Subject: [PATCH 40/73] MET-4375_MET-4537 OaiPmhController unit tests (#540) --- metis-repository/pom.xml | 241 +++++++++-------- .../repository/rest/OaiPmhControllerTest.java | 252 ++++++++++++++++++ .../src/test/resources/record-test.xml | 52 ++++ 3 files changed, 428 insertions(+), 117 deletions(-) create mode 100644 metis-repository/src/test/java/eu/europeana/metis/repository/rest/OaiPmhControllerTest.java create mode 100644 metis-repository/src/test/resources/record-test.xml diff --git a/metis-repository/pom.xml b/metis-repository/pom.xml index 13f332383..998498158 100644 --- a/metis-repository/pom.xml +++ b/metis-repository/pom.xml @@ -1,118 +1,125 @@ - - - metis-framework - eu.europeana.metis - 7-SNAPSHOT - - 4.0.0 - metis-repository - war - - - javax.xml.bind - jaxb-api - - - org.glassfish.jaxb - jaxb-runtime - - - org.apache.logging.log4j - log4j-slf4j-impl - - - org.springframework - spring-core - ${version.spring} - - - org.springframework - spring-webmvc - ${version.spring} - - - javax.servlet - javax.servlet-api - ${version.servlet.api} - provided - - - io.springfox - springfox-swagger2 - ${version.swagger} - - - io.springfox - springfox-swagger-ui - ${version.swagger} - - - com.fasterxml.jackson.core - jackson-annotations - ${version.jackson} - - - com.fasterxml.jackson.dataformat - jackson-dataformat-xml - ${version.jackson} - - - com.fasterxml.jackson.datatype - jackson-datatype-jsr310 - ${version.jackson} - - - eu.europeana.metis - metis-common-utils - ${project.version} - - - eu.europeana.metis - metis-common-mongo - ${project.version} - - - eu.europeana.metis - metis-harvesting - ${project.version} - - - org.junit.jupiter - junit-jupiter - ${version.junit} - test - - - org.mockito - mockito-core - - - com.jayway.jsonpath - json-path - test - - - org.springframework - spring-webmvc - ${version.spring} - - - org.springframework - spring-test - - - - - - org.apache.maven.plugins - maven-war-plugin - ${version.maven.war.plugin} - - false - - - - - \ No newline at end of file + + + metis-framework + eu.europeana.metis + 7-SNAPSHOT + + 4.0.0 + metis-repository + war + + + javax.xml.bind + jaxb-api + + + org.glassfish.jaxb + jaxb-runtime + + + org.apache.logging.log4j + log4j-slf4j-impl + + + org.springframework + spring-core + ${version.spring} + + + org.springframework + spring-webmvc + ${version.spring} + + + javax.servlet + javax.servlet-api + ${version.servlet.api} + provided + + + io.springfox + springfox-swagger2 + ${version.swagger} + + + io.springfox + springfox-swagger-ui + ${version.swagger} + + + com.fasterxml.jackson.core + jackson-annotations + ${version.jackson} + + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + ${version.jackson} + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + ${version.jackson} + + + eu.europeana.metis + metis-common-utils + ${project.version} + + + eu.europeana.metis + metis-common-mongo + ${project.version} + + + eu.europeana.metis + metis-harvesting + ${project.version} + + + org.junit.jupiter + junit-jupiter-api + + + org.junit.jupiter + junit-jupiter-engine + + + org.junit.jupiter + junit-jupiter + ${version.junit} + test + + + org.mockito + mockito-core + + + com.jayway.jsonpath + json-path + test + + + org.springframework + spring-webmvc + ${version.spring} + + + org.springframework + spring-test + + + + + + org.apache.maven.plugins + maven-war-plugin + ${version.maven.war.plugin} + + false + + + + + diff --git a/metis-repository/src/test/java/eu/europeana/metis/repository/rest/OaiPmhControllerTest.java b/metis-repository/src/test/java/eu/europeana/metis/repository/rest/OaiPmhControllerTest.java new file mode 100644 index 000000000..30e714fac --- /dev/null +++ b/metis-repository/src/test/java/eu/europeana/metis/repository/rest/OaiPmhControllerTest.java @@ -0,0 +1,252 @@ +package eu.europeana.metis.repository.rest; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.xpath; + +import eu.europeana.metis.repository.dao.Record; +import eu.europeana.metis.repository.dao.RecordDao; +import eu.europeana.metis.utils.RestEndpoints; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.Instant; +import java.util.stream.Stream; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +/** + * Unit test for {@link OaiPmhController} + */ +class OaiPmhControllerTest { + + private static RecordDao recordDaoMock; + private static MockMvc oaiPmhControllerMock; + private static OaiPmhController oaiPmhController; + + @BeforeAll + static void setup() { + recordDaoMock = mock(RecordDao.class); + oaiPmhController = new OaiPmhController(); + oaiPmhControllerMock = MockMvcBuilders.standaloneSetup(oaiPmhController) + .build(); + } + + @AfterEach + void cleanUp() { + reset(recordDaoMock); + } + + @Test + void oaiPmh_GetRecord() throws Exception { + final Record expectedRecord = getTestRecord("recordId"); + when(recordDaoMock.getRecord("recordId")).thenReturn(expectedRecord); + oaiPmhController.setRecordDao(recordDaoMock); + + oaiPmhControllerMock.perform(get(RestEndpoints.REPOSITORY_OAI_ENDPOINT) + .contentType(MediaType.APPLICATION_XML) + .param("verb", "GetRecord") + .param("set", "oaipmhset") + .param("metadataPrefix", "edm") + .param("identifier", "recordId")) + .andDo(print()) + .andExpect(status().is(200)) + .andExpect(xpath("//OAI-PMH").exists()) + .andExpect(xpath("//OAI-PMH/responseDate").exists()) + .andExpect(xpath("//OAI-PMH/request").exists()) + .andExpect(xpath("//OAI-PMH/GetRecord").exists()) + .andExpect(xpath("//OAI-PMH/GetRecord/record").exists()) + .andExpect(xpath("//OAI-PMH/GetRecord/record/header/identifier").exists()) + .andExpect(xpath("//OAI-PMH/GetRecord/record/header/identifier/text()").string("recordId")) + .andExpect(xpath("//OAI-PMH/GetRecord/record/header/datestamp").exists()) + .andExpect(xpath("//OAI-PMH/GetRecord/record/header/datestamp/text()").string("2022-05-20")) + .andExpect(xpath("//OAI-PMH/GetRecord/record/header/setSpec").exists()) + .andExpect(xpath("//OAI-PMH/GetRecord/record/header/setSpec/text()").string("datasetId")) + .andExpect(xpath("//OAI-PMH/GetRecord/record/metadata").exists()); + + verify(recordDaoMock, times(1)).getRecord("recordId"); + } + + @Test + void oaiPmh_GetRecord_deleted() throws Exception { + final Record expectedRecord = getTestRecordDeleted(); + when(recordDaoMock.getRecord("recordId")).thenReturn(expectedRecord); + oaiPmhController.setRecordDao(recordDaoMock); + + oaiPmhControllerMock.perform(get(RestEndpoints.REPOSITORY_OAI_ENDPOINT) + .contentType(MediaType.APPLICATION_XML) + .param("verb", "GetRecord") + .param("set", "oaipmhset") + .param("metadataPrefix", "edm") + .param("identifier", "recordId")) + .andDo(print()) + .andExpect(status().is(200)) + .andExpect(xpath("//OAI-PMH").exists()) + .andExpect(xpath("//OAI-PMH/responseDate").exists()) + .andExpect(xpath("//OAI-PMH/request").exists()) + .andExpect(xpath("//OAI-PMH/GetRecord").exists()) + .andExpect(xpath("//OAI-PMH/GetRecord/record").exists()) + .andExpect(xpath("//OAI-PMH/GetRecord/record/header/identifier").exists()) + .andExpect(xpath("//OAI-PMH/GetRecord/record/header[@status='deleted']").exists()) + .andExpect(xpath("//OAI-PMH/GetRecord/record/header/identifier/text()").string("recordId")) + .andExpect(xpath("//OAI-PMH/GetRecord/record/header/datestamp").exists()) + .andExpect(xpath("//OAI-PMH/GetRecord/record/header/datestamp/text()").string("2022-05-20")) + .andExpect(xpath("//OAI-PMH/GetRecord/record/header/setSpec").exists()) + .andExpect(xpath("//OAI-PMH/GetRecord/record/header/setSpec/text()").string("datasetId")) + .andExpect(xpath("//OAI-PMH/GetRecord/record/metadata").exists()); + + verify(recordDaoMock, times(1)).getRecord("recordId"); + } + + @Test + void oaiPmh_GetRecord_UnsupportedMetadataPrefix() throws Exception { + final Record expectedRecord = getTestRecord("recordId"); + when(recordDaoMock.getRecord("recordId")).thenReturn(expectedRecord); + oaiPmhController.setRecordDao(recordDaoMock); + + oaiPmhControllerMock.perform(get(RestEndpoints.REPOSITORY_OAI_ENDPOINT) + .contentType(MediaType.APPLICATION_XML) + .param("verb", "GetRecord") + .param("set", "oaipmhset") + .param("metadataPrefix", "other") + .param("identifier", "recordId")) + .andDo(print()) + .andExpect(status().is(400)); + verify(recordDaoMock, times(0)).getRecord("recordId"); + } + + @Test + void oaiPmh_GetRecord_No_Identifier() throws Exception { + final Record expectedRecord = getTestRecord("recordId"); + when(recordDaoMock.getRecord("recordId")).thenReturn(expectedRecord); + oaiPmhController.setRecordDao(recordDaoMock); + + oaiPmhControllerMock.perform(get(RestEndpoints.REPOSITORY_OAI_ENDPOINT) + .contentType(MediaType.APPLICATION_XML) + .param("verb", "GetRecord") + .param("set", "oaipmhset") + .param("metadataPrefix", "edm") + .param("identifier", "")) + .andDo(print()) + .andExpect(status().is(400)); + + verify(recordDaoMock, times(0)).getRecord("recordId"); + } + + @Test + void oaiPmh_GetRecord_NotFound() throws Exception { + when(recordDaoMock.getRecord("recordId")).thenReturn(null); + oaiPmhController.setRecordDao(recordDaoMock); + + oaiPmhControllerMock.perform(get(RestEndpoints.REPOSITORY_OAI_ENDPOINT) + .contentType(MediaType.APPLICATION_XML) + .param("verb", "GetRecord") + .param("set", "oaipmhset") + .param("metadataPrefix", "edm") + .param("identifier", "recordId")) + .andDo(print()) + .andExpect(status().is(404)); + + verify(recordDaoMock, times(1)).getRecord("recordId"); + } + + @Test + void oaiPmh_ListIdentifiers() throws Exception { + when(recordDaoMock.getAllRecordsFromDataset("oaipmhset")) + .thenReturn(Stream.of(getTestRecord("recordId1"), getTestRecord("recordId2"))); + oaiPmhController.setRecordDao(recordDaoMock); + + oaiPmhControllerMock.perform(get(RestEndpoints.REPOSITORY_OAI_ENDPOINT) + .contentType(MediaType.APPLICATION_XML) + .param("verb", "ListIdentifiers") + .param("set", "oaipmhset") + .param("metadataPrefix", "edm") + .param("identifier", "recordId")) + .andDo(print()) + .andExpect(status().is(200)) + .andExpect(xpath("//OAI-PMH").exists()) + .andExpect(xpath("//OAI-PMH/responseDate").exists()) + .andExpect(xpath("//OAI-PMH/request").exists()) + .andExpect(xpath("//OAI-PMH/ListIdentifiers").exists()) + .andExpect(xpath("//OAI-PMH/ListIdentifiers/header/identifier").nodeCount(2)) + .andExpect(xpath("//OAI-PMH/ListIdentifiers/header[1]/identifier[1]/text()").string("recordId1")) + .andExpect(xpath("//OAI-PMH/ListIdentifiers/header[2]/identifier[1]/text()").string("recordId2")) + .andExpect(xpath("//OAI-PMH/ListIdentifiers/header/datestamp").nodeCount(2)) + .andExpect(xpath("//OAI-PMH/ListIdentifiers/header[1]/datestamp[1]/text()").string("2022-05-20")) + .andExpect(xpath("//OAI-PMH/ListIdentifiers/header[2]/datestamp[1]/text()").string("2022-05-20")) + .andExpect(xpath("//OAI-PMH/ListIdentifiers/header/setSpec").nodeCount(2)) + .andExpect(xpath("//OAI-PMH/ListIdentifiers/header[1]/setSpec[1]/text()").string("datasetId")) + .andExpect(xpath("//OAI-PMH/ListIdentifiers/header[2]/setSpec[1]/text()").string("datasetId")); + + verify(recordDaoMock, times(1)).getAllRecordsFromDataset("oaipmhset"); + } + + @Test + void oaiPmh_ListIdentifiers_emptySpec() throws Exception { + when(recordDaoMock.getAllRecordsFromDataset("oaipmhset")) + .thenReturn(Stream.of(getTestRecord("recordId1"), getTestRecord("recordId2"))); + oaiPmhController.setRecordDao(recordDaoMock); + + oaiPmhControllerMock.perform(get(RestEndpoints.REPOSITORY_OAI_ENDPOINT) + .contentType(MediaType.APPLICATION_XML) + .param("verb", "ListIdentifiers") + .param("set", "") + .param("metadataPrefix", "edm") + .param("identifier", "recordId")) + .andDo(print()) + .andExpect(status().is(400)); + + verify(recordDaoMock, times(0)).getAllRecordsFromDataset("oaipmhset"); + } + + @Test + void oaiPmh_ListIdentifiers_emptyHeaders() throws Exception { + when(recordDaoMock.getAllRecordsFromDataset("oaimphset")).thenReturn(null); + oaiPmhController.setRecordDao(recordDaoMock); + + oaiPmhControllerMock.perform(get(RestEndpoints.REPOSITORY_OAI_ENDPOINT, "recordId") + .contentType(MediaType.APPLICATION_XML) + .param("verb", "ListIdentifiers") + .param("set", "oaipmhset") + .param("metadataPrefix", "edm") + .param("identifier", "recordId")) + .andDo(print()) + .andExpect(status().is(404)); + + verify(recordDaoMock, times(1)).getAllRecordsFromDataset("oaipmhset"); + } + + @NotNull + private Record getTestRecord(String recordId) throws IOException, URISyntaxException { + final Record testRecord = new Record(); + final URI pathResourceFile = this.getClass().getClassLoader().getResource("record-test.xml").toURI(); + final String edmRecord = Files.readString(Paths.get(pathResourceFile), Charset.forName("utf-8")); + testRecord.setRecordId(recordId); + testRecord.setEdmRecord(edmRecord); + testRecord.setDatasetId("datasetId"); + testRecord.setDeleted(false); + testRecord.setDateStamp(Instant.parse("2022-05-20T23:59:59.00Z")); + return testRecord; + } + + @NotNull + private Record getTestRecordDeleted() throws IOException, URISyntaxException { + final Record testRecord = getTestRecord("recordId"); + testRecord.setDeleted(true); + return testRecord; + } +} diff --git a/metis-repository/src/test/resources/record-test.xml b/metis-repository/src/test/resources/record-test.xml new file mode 100644 index 000000000..f2e607499 --- /dev/null +++ b/metis-repository/src/test/resources/record-test.xml @@ -0,0 +1,52 @@ + + + + Wosiewicz Leszek + Januszewicz Marek + 1976 + The film is a portrayal of the life of the working-class district residents in Łódź. The individual sequences are observations of one of the working-class courtyards and fields. + video/quicktime + https://etiudy.filmschool.lodz.pl/material/Miejsce + http://media.etiudy.filmschool.lodz.pl/published_img/Miejsce/orig/00:01:33.02.png + pol + eng + Lodz Film School + http://media.etiudy.filmschool.lodz.pl/published/Miejsce.mov + http://rightsstatements.org/vocab/InC/1.0 + Poverty, Scheiblers Factory, Famula, Trade, The Working Class, Ksiezy Mlyn, Postman, Łódź, Post Office, Polish Red Cross, Socjety, fields + Place To live + film + video + VIDEO + + + + + + + Archiwum Filmowe - Szkoła Filmowa w Łodzi + + + + Federacja Bibliotek Cyfrowych + + + From 69cc5eaaccd4eb5d7435960d9d60745b51176bb4 Mon Sep 17 00:00:00 2001 From: JoanaCMS <70145179+JoanaCMS@users.noreply.github.com> Date: Mon, 23 May 2022 09:52:12 +0200 Subject: [PATCH 41/73] MET-4533 Created unit tests for HttpHarvestController class (#541) --- .../rest/HttpHarvestControllerTest.java | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 metis-repository/src/test/java/eu/europeana/metis/repository/rest/HttpHarvestControllerTest.java diff --git a/metis-repository/src/test/java/eu/europeana/metis/repository/rest/HttpHarvestControllerTest.java b/metis-repository/src/test/java/eu/europeana/metis/repository/rest/HttpHarvestControllerTest.java new file mode 100644 index 000000000..df11a48c7 --- /dev/null +++ b/metis-repository/src/test/java/eu/europeana/metis/repository/rest/HttpHarvestControllerTest.java @@ -0,0 +1,152 @@ +package eu.europeana.metis.repository.rest; + +import eu.europeana.metis.repository.dao.Record; +import eu.europeana.metis.repository.dao.RecordDao; +import eu.europeana.metis.utils.RestEndpoints; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.ResponseEntity; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.server.ResponseStatusException; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.time.Instant; +import java.util.Objects; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; + +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +class HttpHarvestControllerTest { + + private RecordDao recordDaoMock; + private MockMvc httpHarvestControllerMock; + private HttpHarvestController httpHarvestController; + + @BeforeEach + void setup() { + recordDaoMock = mock(RecordDao.class); + httpHarvestController = new HttpHarvestController(); + httpHarvestControllerMock = MockMvcBuilders.standaloneSetup(httpHarvestController) + .build(); + } + + @AfterEach + void cleanUp() { + reset(recordDaoMock); + } + + @Test + void testGetDatasetRecords_expectSuccess() throws IOException { + Stream recordsStreamMock = makeStreamRecords(); + when(recordDaoMock.getAllRecordsFromDataset("datasetId")).thenReturn(recordsStreamMock); + httpHarvestController.setRecordDao(recordDaoMock); + + ResponseEntity resultZip = httpHarvestController.getDatasetRecords("datasetId"); + + assertTrue(allContentIsReturned(resultZip.getBody())); + + } + + @Test + void testGetDatasetRecords_expectInternalServerError() { + when(recordDaoMock.getAllRecordsFromDataset("datasetId")).thenThrow(RuntimeException.class); + httpHarvestController.setRecordDao(recordDaoMock); + + RuntimeException expectedException = assertThrows(ResponseStatusException.class, () -> { + httpHarvestController.getDatasetRecords("datasetId"); + }); + + assertEquals("500 INTERNAL_SERVER_ERROR; nested exception is java.lang.RuntimeException", expectedException.getMessage()); + + } + + @Test + void testGetDatasetRecords_expectNotFoundError() { + when(recordDaoMock.getAllRecordsFromDataset("datasetId")).thenReturn(Stream.empty()); + httpHarvestController.setRecordDao(recordDaoMock); + + RuntimeException expectedException = assertThrows(ResponseStatusException.class, () -> { + httpHarvestController.getDatasetRecords("datasetId"); + }); + + assertEquals("404 NOT_FOUND \"No records found for this dataset.\"", expectedException.getMessage()); + + } + + @Test + void getDatasetRecordsViaController() throws Exception { + Stream recordsStreamMock = makeStreamRecords(); + when(recordDaoMock.getAllRecordsFromDataset("datasetId")).thenReturn(recordsStreamMock); + httpHarvestController.setRecordDao(recordDaoMock); + MvcResult resultMvc = httpHarvestControllerMock.perform(get(RestEndpoints.REPOSITORY_HTTP_ENDPOINT_ZIP, "datasetId") + .content("")) + .andDo(print()) + .andExpect(status().is(200)) + .andReturn(); + + assertTrue(allContentIsReturned(resultMvc.getResponse().getContentAsByteArray())); + verify(recordDaoMock, times(1)).getAllRecordsFromDataset("datasetId"); + } + + @Test + void getDatasetRecordsViaController_expectInternalError() throws Exception { + when(recordDaoMock.getAllRecordsFromDataset("datasetId")).thenThrow(RuntimeException.class); + httpHarvestController.setRecordDao(recordDaoMock); + httpHarvestControllerMock.perform(get(RestEndpoints.REPOSITORY_HTTP_ENDPOINT_ZIP, "datasetId") + .content("")) + .andDo(print()) + .andExpect(status().is(500)) + .andExpect(content().string("")); + + verify(recordDaoMock, times(1)).getAllRecordsFromDataset("datasetId"); + } + + @Test + void getDatasetRecordsViaController_expectNotFoundError() throws Exception { + when(recordDaoMock.getAllRecordsFromDataset("datasetId")).thenReturn(Stream.empty()); + httpHarvestController.setRecordDao(recordDaoMock); + httpHarvestControllerMock.perform(get(RestEndpoints.REPOSITORY_HTTP_ENDPOINT_ZIP, "datasetId") + .content("")) + .andDo(print()) + .andExpect(status().is(404)) + .andExpect(content().string("")); + + verify(recordDaoMock, times(1)).getAllRecordsFromDataset("datasetId"); + } + + private Stream makeStreamRecords(){ + Instant instantForTest = Instant.now(); + Record record1 = new Record("recordId1", "datasetId", instantForTest, false, "edmRecord1"); + Record record2 = new Record("recordId2", "datasetId", instantForTest, false, "edmRecord2"); + Record record3 = new Record("recordId3", "datasetId", instantForTest, false, "edmRecord3"); + return Stream.of(record1, record2, record3); + } + + private boolean allContentIsReturned(byte[] result) throws IOException { + ZipEntry zEntry; + boolean areAllRecordsInZip = true; + + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(Objects.requireNonNull(result)); + ZipInputStream zipIs = new ZipInputStream(new BufferedInputStream(byteArrayInputStream)); + while((zEntry = zipIs.getNextEntry()) != null){ + areAllRecordsInZip = areAllRecordsInZip && (zEntry.getName().contains("recordId1") || zEntry.getName().contains("recordId2") || zEntry.getName().contains("recordId3")); + } + zipIs.close(); + + return areAllRecordsInZip; + } + +} From 1478072f90fe23174dc275229cea59421a5e0774 Mon Sep 17 00:00:00 2001 From: Jorge Ortiz Date: Wed, 25 May 2022 17:29:37 +0200 Subject: [PATCH 42/73] MET-4375_MET-4335 InstantSerializer unit tests (#542) --- .../rest/InstantSerializerTest.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 metis-repository/src/test/java/eu/europeana/metis/repository/rest/InstantSerializerTest.java diff --git a/metis-repository/src/test/java/eu/europeana/metis/repository/rest/InstantSerializerTest.java b/metis-repository/src/test/java/eu/europeana/metis/repository/rest/InstantSerializerTest.java new file mode 100644 index 000000000..aaa18f418 --- /dev/null +++ b/metis-repository/src/test/java/eu/europeana/metis/repository/rest/InstantSerializerTest.java @@ -0,0 +1,33 @@ +package eu.europeana.metis.repository.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializerProvider; +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; +import java.time.Instant; +import org.junit.jupiter.api.Test; + +/** + * Unit test for {@link InstantSerializer} class + */ +class InstantSerializerTest { + private InstantSerializer instantSerializer = new InstantSerializer(); + + @Test + void serialize() throws IOException { + final Instant instant = Instant.parse("2020-05-20T17:58:55.00Z"); + final Writer jsonWriter = new StringWriter(); + final JsonGenerator jsonGenerator = new JsonFactory().createGenerator(jsonWriter); + final SerializerProvider serializerProvider = new ObjectMapper().getSerializerProvider(); + + instantSerializer.serialize(instant, jsonGenerator, serializerProvider); + jsonGenerator.flush(); + + assertEquals("\"2020-05-20T17:58:55Z\"", jsonWriter.toString()); + } +} From 8588d7549da93b47b47466c0ef9c82aca0c51563 Mon Sep 17 00:00:00 2001 From: Jorge Ortiz Date: Wed, 25 May 2022 18:24:44 +0200 Subject: [PATCH 43/73] MET-4375 clean up pom file --- metis-repository/pom.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/metis-repository/pom.xml b/metis-repository/pom.xml index 998498158..0bcb0f6d4 100644 --- a/metis-repository/pom.xml +++ b/metis-repository/pom.xml @@ -100,11 +100,6 @@ json-path test - - org.springframework - spring-webmvc - ${version.spring} - org.springframework spring-test From d8fd20d01f591f7570f94526d8a2111a9348bea9 Mon Sep 17 00:00:00 2001 From: Adolfo Peixinho <91942157+adolfopeixinho@users.noreply.github.com> Date: Fri, 27 May 2022 12:01:31 +0200 Subject: [PATCH 44/73] MET-2608 Improve dereference cache. (#528) * MET-2608 improve dereference cache. * MET-2608 improve dereference cache. First Joana Review * MET-2608 improve dereference cache. First Joana Review, forgot the javadoc * MET-2608 improve dereference cache. Sonar still complains about code smell, missing param doc. * MET-2608 add filtering of ProcessedEntities by empty or null XML. change application properties to main App. add controller and DAO endpoints * MET-2608 correct cron expression for springframework. add clearer macros for cron expressions. correct comparing of values in filter * MET-2608 add javadoc in Application.java. Removed this keyword from MongoDereferencingManagementService.java * MET-2608 add more tests for cache purge * MET-2608 add dereferencingcontroller coverage unit tests * MET-2608 add more tests for cache purge empty or null XML * MET-2608 add DeferencingManagementController unit tests * MET-2608 fix name of variables * MET-2608 add unit tests for the DAO * MET-2608 Code cleanup * MET-2608 add MongoDereferencingManagementService unit tests * MET-2608 add more unit tests * MET-2608 remove unnecessary bean annotation, from scheduled method. * MET-2608 Code review. Use of RestEndpoint Constants in tests. Co-authored-by: Jorge Ortiz --- .../europeana/metis/utils/RestEndpoints.java | 3 + .../DereferencingManagementController.java | 35 + .../dereference/rest/config/Application.java | 22 +- .../dereferencing.properties.example | 8 +- .../rest/DereferencingControllerTest.java | 104 +- ...DereferencingManagementControllerTest.java | 144 ++- .../DereferencingManagementService.java | 17 + .../service/MongoDereferenceService.java | 2 +- .../MongoDereferencingManagementService.java | 23 +- .../service/dao/ProcessedEntityDao.java | 67 +- .../service/dao/VocabularyDao.java | 9 + ...ngoDereferencingManagementServiceTest.java | 108 +- .../service/dao/ProcessedEntityDaoTest.java | 119 +++ .../service/dao/VocabularyDaoTest.java | 146 +++ .../src/test/resources/vocabulary-fault.yml | 3 + .../src/test/resources/vocabulary.yml | 3 + .../src/test/resources/vocabulary/voctest.xsl | 960 ++++++++++++++++++ .../src/test/resources/vocabulary/voctest.yml | 18 + 18 files changed, 1705 insertions(+), 86 deletions(-) create mode 100644 metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/dao/ProcessedEntityDaoTest.java create mode 100644 metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/dao/VocabularyDaoTest.java create mode 100644 metis-dereference/metis-dereference-service/src/test/resources/vocabulary-fault.yml create mode 100644 metis-dereference/metis-dereference-service/src/test/resources/vocabulary.yml create mode 100644 metis-dereference/metis-dereference-service/src/test/resources/vocabulary/voctest.xsl create mode 100644 metis-dereference/metis-dereference-service/src/test/resources/vocabulary/voctest.yml diff --git a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/RestEndpoints.java b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/RestEndpoints.java index 1b7d2731b..b3f1dfe16 100644 --- a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/RestEndpoints.java +++ b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/RestEndpoints.java @@ -69,6 +69,9 @@ public final class RestEndpoints { public static final String DEREFERENCE = "/dereference"; public static final String VOCABULARIES = "/vocabularies"; public static final String CACHE_EMPTY = "/cache"; + public static final String CACHE_EMPTY_VOCABULARY = "/cache/vocabulary"; + public static final String CACHE_EMPTY_RESOURCE = "/cache/resource"; + public static final String CACHE_EMPTY_XML = "/cache/emptyxml"; public static final String LOAD_VOCABULARIES = "/load_vocabularies"; /* METIS ENRICHMENT Endpoint */ diff --git a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/DereferencingManagementController.java b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/DereferencingManagementController.java index 43b046ba3..eeade969f 100644 --- a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/DereferencingManagementController.java +++ b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/DereferencingManagementController.java @@ -75,6 +75,41 @@ public void emptyCache() { service.emptyCache(); } + /** + * Empty the cache for all Resources without an XML representation + * */ + @DeleteMapping(value = RestEndpoints.CACHE_EMPTY_XML) + @ResponseBody + @ApiOperation(value = "Empty the cache without XML representations") + public void emptyCacheByEmptyXml() { + service.purgeByNullOrEmptyXml(); + } + + /** + * Empty the cache for a specific resource + * @param resourceId The resourceId to empty the cache for + * */ + @PostMapping(value = RestEndpoints.CACHE_EMPTY_RESOURCE) + @ResponseBody + @ApiOperation(value = "Empty the cache by resource Id") + public void emptyCacheByResourceId( + @ApiParam(value = "Id (URI) of resource to clear cache", required = true) @RequestParam(value = "resourceId") String resourceId) { + service.purgeByResourceId(resourceId); + } + + /** + * Empty the cache for a specific vocabulary, with all associated entities + * @param vocabularyId The vocabularyId to empty the cache for + * */ + @PostMapping(value = RestEndpoints.CACHE_EMPTY_VOCABULARY) + @ResponseBody + @ApiOperation(value = "Empty the cache by vocabulary Id") + public void emptyCacheByVocabularyId( + @ApiParam(value = "Id of vocabulary to clear cache", required = true) @RequestParam(value = "vocabularyId") String vocabularyId) { + service.purgeByVocabularyId(vocabularyId); + } + + /** * Load the vocabularies from an online source. This does NOT purge the cache. * diff --git a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java index d951ae798..653edd953 100644 --- a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java +++ b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java @@ -16,6 +16,8 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.web.servlet.config.annotation.EnableWebMvc; import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; import org.springframework.web.servlet.config.annotation.ViewControllerRegistry; @@ -32,6 +34,7 @@ * Spring configuration class Created by ymamakis on 12-2-16. */ @Configuration +@EnableScheduling @ComponentScan(basePackages = {"eu.europeana.metis.dereference.rest", "eu.europeana.metis.dereference.rest.exceptions"}) @PropertySource("classpath:dereferencing.properties") @@ -70,7 +73,6 @@ public class Application implements WebMvcConfigurer, InitializingBean { //Valid directories list @Value("${allowed.url.domains}") private String[] allowedUrlDomains; - private MongoClient mongoClientEntity; private MongoClient mongoClientVocabulary; @@ -131,6 +133,24 @@ public static PropertySourcesPlaceholderConfigurer propertySourcesPlaceholderCon return new PropertySourcesPlaceholderConfigurer(); } + /** + * Empty Cache with XML entries null or empty. + * This will remove entries with null or empty XML in the cache (Redis). If the same redis instance/cluster is used for multiple + * services then the cache for other services is cleared as well. + * This task is scheduled by a cron expression. + */ + + @Scheduled(cron = "${dereference.purge.emptyxml.frequency}") + public void dereferenceCacheNullOrEmpty(){ getProcessedEntityDao().purgeByNullOrEmptyXml(); } + + /** + * Empty Cache. This will remove ALL entries in the cache (Redis). If the same redis instance/cluster is used for multiple + * services then the cache for other services is cleared as well. + * This task is scheduled by a cron expression. + */ + @Scheduled(cron = "${dereference.purge.all.frequency}") + public void dereferenceCachePurgeAll(){ getProcessedEntityDao().purgeAll(); } + /** * Closes any connections previous acquired. */ diff --git a/metis-dereference/metis-dereference-rest/src/main/resources/dereferencing.properties.example b/metis-dereference/metis-dereference-rest/src/main/resources/dereferencing.properties.example index a7f04f76d..e4be549ec 100644 --- a/metis-dereference/metis-dereference-rest/src/main/resources/dereferencing.properties.example +++ b/metis-dereference/metis-dereference-rest/src/main/resources/dereferencing.properties.example @@ -15,4 +15,10 @@ entity.db= vocabulary.db= #The allowed domains for vocabularies loading without the scheme(always validated against https). e.g. raw.githubusercontent.com -allowed.url.domains= \ No newline at end of file +allowed.url.domains= + +# Dereferencing cache cron expressions, +# refer to Spring Framework CronExpression for documentation +dereference.purge.all.frequency=@monthly +# purge empty xml +dereference.purge.emptyxml.frequency=@daily \ No newline at end of file diff --git a/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingControllerTest.java b/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingControllerTest.java index ec784c9f1..1e3d37c88 100644 --- a/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingControllerTest.java +++ b/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingControllerTest.java @@ -4,6 +4,7 @@ import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.xpath; @@ -12,6 +13,8 @@ import eu.europeana.enrichment.api.external.model.Label; import eu.europeana.metis.dereference.rest.exceptions.RestResponseExceptionHandler; import eu.europeana.metis.dereference.service.DereferenceService; +import eu.europeana.metis.utils.RestEndpoints; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -24,6 +27,9 @@ import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.setup.MockMvcBuilders; +/** + * Unit test {@link DereferencingController} Class + */ class DereferencingControllerTest { private DereferenceService dereferenceServiceMock; @@ -35,62 +41,74 @@ void setUp() { dereferenceServiceMock = mock(DereferenceService.class); namespaceMap = getNamespaceMap(); - DereferencingController dereferenceController = new DereferencingController( - dereferenceServiceMock); + DereferencingController dereferenceController = new DereferencingController(dereferenceServiceMock); dereferencingControllerMock = MockMvcBuilders.standaloneSetup(dereferenceController) - .setControllerAdvice(new RestResponseExceptionHandler()).build(); + .setControllerAdvice(new RestResponseExceptionHandler()).build(); } @Test - void dereferenceGet_outputXML() throws Exception { - when(dereferenceServiceMock.dereference("http://www.example.com")) - .thenReturn(Collections.singletonList(getAgent("http://www.example.com"))); + void dereferenceGet_outputXML_expectSuccess() throws Exception { + when(dereferenceServiceMock.dereference("http://www.example.com")).thenReturn( + Collections.singletonList(getAgent("http://www.example.com"))); dereferencingControllerMock.perform( - get("/dereference/?uri=http://www.example.com").accept(MediaType.APPLICATION_XML_VALUE)) - .andExpect(status().is(200)) - // .andExpect(content().string("")) - .andExpect(xpath("metis:results/metis:result/edm:Agent/@rdf:about", namespaceMap) - .string("http://www.example.com")).andExpect(xpath( - "metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='en']", - namespaceMap).string("labelEn")).andExpect(xpath( - "metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='nl']", - namespaceMap).string("labelNl")).andExpect(xpath( - "metis:results/metis:result/edm:Agent/rdaGr2:dateOfBirth[@xml:lang='en']", - namespaceMap).string("10-10-10")); + get(RestEndpoints.DEREFERENCE + "/?uri=http://www.example.com").accept(MediaType.APPLICATION_XML_VALUE)) + .andExpect(status().is(200)).andExpect( + xpath("metis:results/metis:result/edm:Agent/@rdf:about", namespaceMap).string("http://www.example.com")).andExpect( + xpath("metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='en']", namespaceMap).string("labelEn")).andExpect( + xpath("metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='nl']", namespaceMap).string("labelNl")).andExpect( + xpath("metis:results/metis:result/edm:Agent/rdaGr2:dateOfBirth[@xml:lang='en']", namespaceMap).string("10-10-10")); } @Test - void dereferencePost_outputXML() throws Exception { - when(dereferenceServiceMock.dereference("http://www.example.com")) - .thenReturn(Collections.singletonList(getAgent("http://www.example.com"))); - - dereferencingControllerMock.perform(post("/dereference").accept(MediaType.APPLICATION_XML_VALUE) - .contentType(MediaType.APPLICATION_JSON).content("[ \"http://www.example.com\" ]")) - .andExpect(status().is(200)) - // .andExpect(content().string("")) - .andExpect(xpath("metis:results/metis:result/edm:Agent/@rdf:about", - namespaceMap).string("http://www.example.com")) - .andExpect(xpath( - "metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='en']", - namespaceMap).string("labelEn")) - .andExpect(xpath( - "metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='nl']", - namespaceMap).string("labelNl")) - .andExpect(xpath( - "metis:results/metis:result/edm:Agent/rdaGr2:dateOfBirth[@xml:lang='en']", - namespaceMap).string("10-10-10")); + void dereferenceGet_outputXML_expectInternalServerError() throws Exception { + when(dereferenceServiceMock.dereference("http://www.example.com")).thenThrow( + new URISyntaxException("URI Error", "Error reason")); + + dereferencingControllerMock.perform( + get(RestEndpoints.DEREFERENCE + "/?uri=http://www.example.com").accept(MediaType.APPLICATION_XML_VALUE)).andDo(print()) + .andExpect(status().is(500)).andExpect(xpath("//error").exists()) + .andExpect(xpath("//error/errorMessage").exists()).andExpect(xpath("//error/errorMessage").string( + "Dereferencing failed for uri: http://www.example.com with root cause: Error reason: URI Error")); + } + + @Test + void dereferencePost_outputXML_expectSuccess() throws Exception { + when(dereferenceServiceMock.dereference("http://www.example.com")).thenReturn( + Collections.singletonList(getAgent("http://www.example.com"))); + + dereferencingControllerMock.perform( + post(RestEndpoints.DEREFERENCE).accept(MediaType.APPLICATION_XML_VALUE).contentType(MediaType.APPLICATION_JSON) + .content("[ \"http://www.example.com\" ]")).andDo(print()).andExpect(status().is(200)) + .andExpect(xpath("metis:results/metis:result/edm:Agent/@rdf:about", namespaceMap).string( + "http://www.example.com")).andExpect( + xpath("metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='en']", namespaceMap).string("labelEn")).andExpect( + xpath("metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='nl']", namespaceMap).string("labelNl")).andExpect( + xpath("metis:results/metis:result/edm:Agent/rdaGr2:dateOfBirth[@xml:lang='en']", namespaceMap).string("10-10-10")); + } + + @Test + void dereferencePost_outputXML_expectEmptyList() throws Exception { + when(dereferenceServiceMock.dereference("http://www.example.com")).thenThrow( + new URISyntaxException("URI Error", "Error reason")); + + dereferencingControllerMock.perform( + post(RestEndpoints.DEREFERENCE).accept(MediaType.APPLICATION_XML_VALUE).contentType(MediaType.APPLICATION_JSON) + .content("[ \"http://www.example.com\" ]")).andDo(print()).andExpect(status().is(200)) + .andExpect(xpath("//metis:results", namespaceMap).exists()) + .andExpect(xpath("//metis:results", namespaceMap).nodeCount(1)) + .andExpect(xpath("//metis:results/metis:result", namespaceMap).exists()) + .andExpect(xpath("//metis:results/metis:result[*]", namespaceMap).nodeCount(0)); } @Test void exceptionHandling() throws Exception { - when(dereferenceServiceMock.dereference("http://www.example.com")) - .thenThrow(new TransformerException("myException")); + when(dereferenceServiceMock.dereference("http://www.example.com")).thenThrow(new TransformerException("myException")); dereferencingControllerMock.perform( - post("/dereference").content("[ \"http://www.example.com\" ]") - .accept(MediaType.APPLICATION_JSON).contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().is(500)).andExpect(content().string( - "{\"errorMessage\":\"Dereferencing failed for uri: http://www.example.com with root cause: myException\"}")); + post(RestEndpoints.DEREFERENCE).content("[ \"http://www.example.com\" ]").accept(MediaType.APPLICATION_JSON) + .contentType(MediaType.APPLICATION_JSON)).andExpect(status().is(500)).andExpect( + content().string( + "{\"errorMessage\":\"Dereferencing failed for uri: http://www.example.com with root cause: myException\"}")); } private Agent getAgent(String uri) { @@ -125,4 +143,4 @@ private Map getNamespaceMap() { namespaceMap.put("rdaGr2", "http://rdvocab.info/ElementsGr2/"); return namespaceMap; } -} \ No newline at end of file +} diff --git a/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingManagementControllerTest.java b/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingManagementControllerTest.java index a7e99e76b..8136d7eea 100644 --- a/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingManagementControllerTest.java +++ b/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingManagementControllerTest.java @@ -5,17 +5,24 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import eu.europeana.metis.dereference.Vocabulary; import eu.europeana.metis.dereference.rest.exceptions.RestResponseExceptionHandler; import eu.europeana.metis.dereference.service.DereferencingManagementService; +import eu.europeana.metis.dereference.vocimport.exception.VocabularyImportException; +import eu.europeana.metis.utils.RestEndpoints; import java.net.URL; import java.util.ArrayList; import java.util.Collections; @@ -27,23 +34,24 @@ import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.setup.MockMvcBuilders; +/** + * Unit tests {@link DereferencingController} class + */ class DereferencingManagementControllerTest { - private DereferencingManagementService dereferencingManagementServiceMock; - private MockMvc dereferencingManagementControllerMock; + private DereferencingManagementService deRefManagementServiceMock; + private MockMvc deRefManagementControllerMock; private String testEmptyCacheResult = ""; @BeforeEach void setUp() { - dereferencingManagementServiceMock = mock(DereferencingManagementService.class); + deRefManagementServiceMock = mock(DereferencingManagementService.class); DereferencingManagementController dereferencingManagementController = new DereferencingManagementController( - dereferencingManagementServiceMock, Set.of("valid.domain.com")); + deRefManagementServiceMock, Set.of("valid.domain.com")); - dereferencingManagementControllerMock = MockMvcBuilders - .standaloneSetup(dereferencingManagementController) - .setControllerAdvice(new RestResponseExceptionHandler()) - .build(); + deRefManagementControllerMock = MockMvcBuilders.standaloneSetup(dereferencingManagementController) + .setControllerAdvice(new RestResponseExceptionHandler()).build(); } @Test @@ -62,25 +70,28 @@ void testGetAllVocabularies() throws Exception { dummyVocabList.add(dummyVocab1); dummyVocabList.add(dummyVocab2); - when(dereferencingManagementServiceMock.getAllVocabularies()).thenReturn(dummyVocabList); + when(deRefManagementServiceMock.getAllVocabularies()).thenReturn(dummyVocabList); - dereferencingManagementControllerMock.perform(get("/vocabularies")) - .andExpect(jsonPath("$[0].uris[0]", is("https://dummy1.org/path1"))) - .andExpect(jsonPath("$[1].uris[0]", is("https://dummy2.org/path2"))) - .andExpect(status().is(200)); + deRefManagementControllerMock.perform(get(RestEndpoints.VOCABULARIES)) + .andExpect(jsonPath("$[0].uris[0]", is("https://dummy1.org/path1"))) + .andExpect(jsonPath("$[1].uris[0]", is("https://dummy2.org/path2"))) + .andExpect(status().is(200)); } @Test void testLoadVocabularies_validDomain_expectSuccess() throws Exception { - doNothing().when(dereferencingManagementServiceMock).loadVocabularies(any(URL.class)); - dereferencingManagementControllerMock.perform(post("/load_vocabularies") - .param("directory_url", "https://valid.domain.com/test/call")).andExpect(status().is(200)); + doNothing().when(deRefManagementServiceMock).loadVocabularies(any(URL.class)); + deRefManagementControllerMock.perform(post(RestEndpoints.LOAD_VOCABULARIES) + .param("url", "http://valid.domain.com/path/to/vocab.rdf") + .param("directory_url", "https://valid.domain.com/test/call")) + .andExpect(status().is(200)); } @Test void testLoadVocabularies_invalidDomain_expectFail() throws Exception { - dereferencingManagementControllerMock.perform(post("/load_vocabularies") - .param("directory_url", "https://invalid.domain.com")).andExpect(status().is(400)); + deRefManagementControllerMock.perform(post(RestEndpoints.LOAD_VOCABULARIES) + .param("directory_url", "https://invalid.domain.com")) + .andExpect(status().is(400)); } @Test @@ -88,11 +99,100 @@ void testEmptyCache() throws Exception { doAnswer((Answer) invocationOnMock -> { testEmptyCacheResult = "OK"; return null; - }).when(dereferencingManagementServiceMock).emptyCache(); + }).when(deRefManagementServiceMock).emptyCache(); - dereferencingManagementControllerMock.perform(delete("/cache")) - .andExpect(status().is(200)); + deRefManagementControllerMock.perform(delete(RestEndpoints.CACHE_EMPTY)) + .andExpect(status().is(200)); assertEquals("OK", testEmptyCacheResult); } -} \ No newline at end of file + + @Test + void testEmptyCacheByEmptyXml() throws Exception { + doAnswer((Answer) invocationOnMock -> { + testEmptyCacheResult = "OK"; + return null; + }).when(deRefManagementServiceMock).purgeByNullOrEmptyXml(); + + deRefManagementControllerMock.perform(delete(RestEndpoints.CACHE_EMPTY_XML)) + .andExpect(status().is(200)); + + assertEquals("OK", testEmptyCacheResult); + } + + @Test + void testEmptyNullOrEmptyXML() throws Exception { + doAnswer((Answer) invocationOnMock -> { + testEmptyCacheResult = "OK"; + return null; + }).when(deRefManagementServiceMock).purgeByNullOrEmptyXml(); + + deRefManagementControllerMock.perform(delete(RestEndpoints.CACHE_EMPTY_XML)).andExpect(status().is(200)); + + assertEquals("OK", testEmptyCacheResult); + } + + + @Test + void testEmptyCacheByResourceId() throws Exception { + + doAnswer((Answer) invocationOnMock -> { + testEmptyCacheResult = "OK"; + return null; + }).when(deRefManagementServiceMock).purgeByResourceId(any(String.class)); + + deRefManagementControllerMock.perform(post(RestEndpoints.CACHE_EMPTY_RESOURCE) + .param("resourceId", "resourceId") + .param("resourceId", "12345")) + .andExpect(status().is(200)); + + assertEquals("OK", testEmptyCacheResult); + } + + @Test + void testEmptyCacheByVocabularyId() throws Exception { + + doAnswer((Answer) invocationOnMock -> { + testEmptyCacheResult = "OK"; + return null; + }).when(deRefManagementServiceMock).purgeByVocabularyId(any(String.class)); + + deRefManagementControllerMock.perform(post(RestEndpoints.CACHE_EMPTY_VOCABULARY) + .param("vocabularyId", "12345")) + .andExpect(status().is(200)); + + assertEquals("OK", testEmptyCacheResult); + } + + @Test + void testLoadVocabularies_expectBadRequest() throws Exception { + deRefManagementControllerMock.perform(post(RestEndpoints.LOAD_VOCABULARIES) + .param("directory_url", "directory")) + .andDo(print()) + .andExpect(status().is(400)) + .andExpect(content().string("The url of the directory to import is not valid.")); + } + + @Test + void testLoadVocabularies_expectBadContent() throws Exception { + doThrow(VocabularyImportException.class).when(deRefManagementServiceMock).loadVocabularies(any(URL.class)); + deRefManagementControllerMock.perform(post(RestEndpoints.LOAD_VOCABULARIES) + .param("directory_url", "\\/tttp://test")) + .andDo(print()) + .andExpect(status().is(400)) + .andExpect(content().string("Provided directoryUrl '\\/tttp://test', failed to parse.")); + verify(deRefManagementServiceMock, times(0)).loadVocabularies(any(URL.class)); + } + + @Test + void testLoadVocabularies_expectBadVocabulary() throws Exception { + doThrow(new VocabularyImportException("Cannot load vocabulary")) + .when(deRefManagementServiceMock).loadVocabularies(any(URL.class)); + deRefManagementControllerMock.perform(post(RestEndpoints.LOAD_VOCABULARIES) + .param("directory_url", "https://valid.domain.com/test/call")) + .andDo(print()) + .andExpect(status().is(502)) + .andExpect(content().string("Cannot load vocabulary")); + verify(deRefManagementServiceMock, times(1)).loadVocabularies(any(URL.class)); + } +} diff --git a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/DereferencingManagementService.java b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/DereferencingManagementService.java index 9a057884e..e588df540 100644 --- a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/DereferencingManagementService.java +++ b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/DereferencingManagementService.java @@ -22,6 +22,23 @@ public interface DereferencingManagementService { */ void emptyCache(); + /** + * + * purge all ProcessedEntities with empty XML + */ + void purgeByNullOrEmptyXml(); + + /** + * Empty the cache by resource ID(URI) + * @param resourceId The resourceId (URI) of the resource to be purged from the cache + */ + void purgeByResourceId(String resourceId); + /** + * Empty the cache by vocabulary ID + * @param vocabularyId the vocabulary ID to be purged from the cache, with all associated resources + */ + void purgeByVocabularyId(String vocabularyId); + /** * Load the vocabularies from an online source. This does NOT purge the cache. * diff --git a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferenceService.java b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferenceService.java index 2d0db505f..247646a49 100644 --- a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferenceService.java +++ b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferenceService.java @@ -323,7 +323,7 @@ Pair computeEnrichmentBaseVocabularyPair(String reso throws JAXBException, URISyntaxException { // Try to get the entity and its vocabulary from the cache. - final ProcessedEntity cachedEntity = processedEntityDao.get(resourceId); + final ProcessedEntity cachedEntity = processedEntityDao.getByResourceId(resourceId); final Pair entityVocabularyPair = computeEntityVocabularyPair(resourceId, cachedEntity); diff --git a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementService.java b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementService.java index b6d98362f..824cc4c9f 100644 --- a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementService.java +++ b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementService.java @@ -31,7 +31,7 @@ public class MongoDereferencingManagementService implements DereferencingManagem */ @Autowired public MongoDereferencingManagementService(VocabularyDao vocabularyDao, - ProcessedEntityDao processedEntityDao, VocabularyCollectionImporterFactory vocabularyCollectionImporterFactory) { + ProcessedEntityDao processedEntityDao, VocabularyCollectionImporterFactory vocabularyCollectionImporterFactory) { this.vocabularyDao = vocabularyDao; this.processedEntityDao = processedEntityDao; this.vocabularyCollectionImporterFactory = vocabularyCollectionImporterFactory; @@ -44,7 +44,20 @@ public List getAllVocabularies() { @Override public void emptyCache() { - this.processedEntityDao.purgeAll(); + processedEntityDao.purgeAll(); + } + + @Override + public void purgeByNullOrEmptyXml() { processedEntityDao.purgeByNullOrEmptyXml(); } + + @Override + public void purgeByResourceId(String resourceId) { + processedEntityDao.purgeByResourceId(resourceId); + } + + @Override + public void purgeByVocabularyId(String vocabularyId) { + processedEntityDao.purgeByVocabularyId(vocabularyId); } @Override @@ -59,15 +72,15 @@ public void loadVocabularies(URL directoryUrl) throws VocabularyImportException true, true, true); validator.validateVocabularyOnly(vocabulary -> vocabularies.add(convertVocabulary(vocabulary))); - // All vocabularies are loaded well. Now we replace the vocabularies. - vocabularyDao.replaceAll(vocabularies); + // All vocabularies are loaded well. Now we replace the vocabularies. + vocabularyDao.replaceAll(vocabularies); } catch (VocabularyImportException e) { throw new VocabularyImportException("An error as occurred while loading the vocabularies", e); } } private static Vocabulary convertVocabulary( - eu.europeana.metis.dereference.vocimport.model.Vocabulary input) { + eu.europeana.metis.dereference.vocimport.model.Vocabulary input) { final Vocabulary vocabulary = new Vocabulary(); vocabulary.setName(input.getName()); vocabulary.setUris(input.getPaths()); diff --git a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/dao/ProcessedEntityDao.java b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/dao/ProcessedEntityDao.java index de1fb8aa2..e9a6a6c0c 100644 --- a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/dao/ProcessedEntityDao.java +++ b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/dao/ProcessedEntityDao.java @@ -34,8 +34,8 @@ public class ProcessedEntityDao { */ public ProcessedEntityDao(MongoClient mongo, String databaseName) { final MapperOptions mapperOptions = MapperOptions.builder().discriminatorKey("className") - .discriminator(DiscriminatorFunction.className()) - .collectionNaming(NamingStrategy.identity()).build(); + .discriminator(DiscriminatorFunction.className()) + .collectionNaming(NamingStrategy.identity()).build(); this.datastore = Morphia.createDatastore(mongo, databaseName, mapperOptions); this.datastore.getMapper().map(ProcessedEntity.class); } @@ -46,12 +46,25 @@ public ProcessedEntityDao(MongoClient mongo, String databaseName) { * @param resourceId The resource ID (URI) to retrieve * @return The entity with the given resource ID. */ - public ProcessedEntity get(String resourceId) { + public ProcessedEntity getByResourceId(String resourceId) { return retryableExternalRequestForNetworkExceptions( () -> datastore.find(ProcessedEntity.class).filter(Filters.eq("resourceId", resourceId)) - .first()); + .first()); } + /** + * Get an entity by vocabulary ID. + * + * @param vocabularyId The vocabuylaryDi to retrieve + * @return The entity with the given vocabulary ID. + */ + public ProcessedEntity getByVocabularyId(String vocabularyId) { + return retryableExternalRequestForNetworkExceptions( + () -> datastore.find(ProcessedEntity.class).filter(Filters.eq("vocabularyId", vocabularyId)) + .first()); + } + + /** * Save an entity. * @@ -60,7 +73,7 @@ public ProcessedEntity get(String resourceId) { public void save(ProcessedEntity processedEntity) { try { final ObjectId objectId = Optional.ofNullable(processedEntity.getId()) - .orElseGet(ObjectId::new); + .orElseGet(ObjectId::new); processedEntity.setId(objectId); retryableExternalRequestForNetworkExceptions(() -> datastore.save(processedEntity)); } catch (DuplicateKeyException e) { @@ -70,6 +83,40 @@ public void save(ProcessedEntity processedEntity) { } } + /** + * Delete an entity with no description in XML resources. Empty or Null + **/ + public void purgeByNullOrEmptyXml() { + retryableExternalRequestForNetworkExceptions(() -> + datastore.find(ProcessedEntity.class) + .filter(Filters.eq("xml", null)) + .delete(new DeleteOptions().multi(true))); + } + + /** + * Delete an entity by resource ID. + * + * @param resourceId The resource ID (URI) to delete + **/ + public void purgeByResourceId(String resourceId) { + retryableExternalRequestForNetworkExceptions(() -> + datastore.find(ProcessedEntity.class) + .filter(Filters.eq("resourceId", resourceId)) + .delete(new DeleteOptions())); + } + + /** + * Delete the entity based on its vocabulary ID. + * + * @param vocabularyId The ID of the vocabulary to delete. + **/ + public void purgeByVocabularyId(String vocabularyId) { + retryableExternalRequestForNetworkExceptions(() -> + datastore.find(ProcessedEntity.class) + .filter(Filters.eq("vocabularyId", vocabularyId)) + .delete(new DeleteOptions().multi(true))); + } + /** * Remove all entities. */ @@ -77,4 +124,14 @@ public void purgeAll() { retryableExternalRequestForNetworkExceptions( () -> datastore.find(ProcessedEntity.class).delete(new DeleteOptions().multi(true))); } + + /** + * Size of Processed entities + * + * @return amount of documents in db + */ + protected long size() { + return retryableExternalRequestForNetworkExceptions( + () -> datastore.find(ProcessedEntity.class).stream().count()); + } } diff --git a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/dao/VocabularyDao.java b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/dao/VocabularyDao.java index ade518f21..52bdb7d9e 100644 --- a/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/dao/VocabularyDao.java +++ b/metis-dereference/metis-dereference-service/src/main/java/eu/europeana/metis/dereference/service/dao/VocabularyDao.java @@ -89,4 +89,13 @@ public void replaceAll(List vocabularies) { protected Datastore getDatastore() { return datastore; } + + /** + * Amount of documents + * + * @return amount of documents in db + */ + protected long size() { + return retryableExternalRequestForNetworkExceptions(() -> datastore.find(Vocabulary.class).stream().count()); + } } diff --git a/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementServiceTest.java b/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementServiceTest.java index b6439ee87..0bbf94c13 100644 --- a/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementServiceTest.java +++ b/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/MongoDereferencingManagementServiceTest.java @@ -1,16 +1,30 @@ package eu.europeana.metis.dereference.service; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; import dev.morphia.Datastore; +import eu.europeana.metis.dereference.ProcessedEntity; import eu.europeana.metis.dereference.Vocabulary; import eu.europeana.metis.dereference.service.dao.ProcessedEntityDao; import eu.europeana.metis.dereference.service.dao.VocabularyDao; +import eu.europeana.metis.dereference.vocimport.VocabularyCollectionImporter; import eu.europeana.metis.dereference.vocimport.VocabularyCollectionImporterFactory; +import eu.europeana.metis.dereference.vocimport.exception.VocabularyImportException; import eu.europeana.metis.mongo.embedded.EmbeddedLocalhostMongo; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.Collections; import java.util.List; import org.junit.jupiter.api.AfterEach; @@ -18,16 +32,19 @@ import org.junit.jupiter.api.Test; /** - * Created by ymamakis on 2/22/16. + * Unit tests for {@link MongoDereferencingManagementService} class */ class MongoDereferencingManagementServiceTest { private MongoDereferencingManagementService service; - private EmbeddedLocalhostMongo embeddedLocalhostMongo = new EmbeddedLocalhostMongo(); - private Datastore vocDaoDatastore; + private final EmbeddedLocalhostMongo embeddedLocalhostMongo = new EmbeddedLocalhostMongo(); + private Datastore vocabularyDaoDatastore; + + private VocabularyCollectionImporterFactory vocabularyCollectionImporterFactory; + private final ProcessedEntityDao processedEntityDao = mock(ProcessedEntityDao.class); @BeforeEach - void prepare() { + void prepare() throws IOException { embeddedLocalhostMongo.start(); String mongoHost = embeddedLocalhostMongo.getMongoHost(); int mongoPort = embeddedLocalhostMongo.getMongoPort(); @@ -37,11 +54,11 @@ void prepare() { VocabularyDao vocDao = new VocabularyDao(mongoClient, "voctest") { { - vocDaoDatastore = this.getDatastore(); + vocabularyDaoDatastore = this.getDatastore(); } }; - ProcessedEntityDao processedEntityDao = mock(ProcessedEntityDao.class); - VocabularyCollectionImporterFactory vocabularyCollectionImporterFactory = mock(VocabularyCollectionImporterFactory.class); + vocabularyCollectionImporterFactory = mock(VocabularyCollectionImporterFactory.class); + service = new MongoDereferencingManagementService(vocDao, processedEntityDao, vocabularyCollectionImporterFactory); } @@ -52,11 +69,86 @@ void testGetAllVocabularies() { voc.setName("testName"); voc.setUris(Collections.singleton("http://www.test.uri/")); voc.setXslt("testXSLT"); - vocDaoDatastore.save(voc); + vocabularyDaoDatastore.save(voc); List retVoc = service.getAllVocabularies(); assertEquals(1, retVoc.size()); } + + @Test + void purgeAllCache() { + ProcessedEntity processedEntity = new ProcessedEntity(); + processedEntity.setResourceId("http://www.test.uri/"); + processedEntityDao.save(processedEntity); + service.emptyCache(); + ProcessedEntity ret = processedEntityDao.getByResourceId("http://www.test.uri/"); + assertNull(ret); + } + + @Test + void purgeCacheWithEmptyXML() { + ProcessedEntity processedEntity = new ProcessedEntity(); + processedEntity.setResourceId("http://www.test.uri/"); + processedEntity.setXml(null); + processedEntityDao.save(processedEntity); + service.purgeByNullOrEmptyXml(); + ProcessedEntity ret = processedEntityDao.getByResourceId("http://www.test.uri/"); + assertNull(ret); + } + + + @Test + void purgeCacheByResourceId() { + ProcessedEntity processedEntity = new ProcessedEntity(); + processedEntity.setResourceId("http://www.test.uri/"); + processedEntityDao.save(processedEntity); + service.purgeByResourceId("http://www.test.uri/"); + ProcessedEntity ret = processedEntityDao.getByResourceId("http://www.test.uri/"); + assertNull(ret); + } + + @Test + void purgeCacheByVocabularyId() { + ProcessedEntity processedEntity = new ProcessedEntity(); + processedEntity.setVocabularyId("vocabularyId"); + processedEntityDao.save(processedEntity); + service.purgeByVocabularyId("vocabularyId"); + ProcessedEntity ret = processedEntityDao.getByVocabularyId("vocabularyId"); + assertNull(ret); + } + + @Test + void loadVocabularies_expectSucess() throws VocabularyImportException, URISyntaxException, IOException { + final URL resourceLocation = this.getClass().getClassLoader().getResource("vocabulary.yml"); + final String expectedXslt = Files.readString(Paths.get(getClass() + .getClassLoader() + .getResource("vocabulary/voctest.xsl").toURI())).trim(); + + final VocabularyCollectionImporter importer = new VocabularyCollectionImporterFactory().createImporter(resourceLocation); + doReturn(importer).when(vocabularyCollectionImporterFactory).createImporter(any(URL.class)); + + service.loadVocabularies(resourceLocation); + Vocabulary vocabulary = vocabularyDaoDatastore.find(Vocabulary.class).first(); + + assertEquals("TestWikidata", vocabulary.getName()); + assertEquals(expectedXslt, vocabulary.getXslt()); + assertEquals("http://www.wikidata.org/entity/", vocabulary.getUris().stream().findFirst().get()); + verify(vocabularyCollectionImporterFactory, times(1)).createImporter(resourceLocation); + } + + @Test + void loadVocabularies_expectVocabularyImportError() throws VocabularyImportException, URISyntaxException, IOException { + final URL resourceLocation = this.getClass().getClassLoader().getResource("vocabulary-fault.yml"); + final VocabularyCollectionImporter importer = new VocabularyCollectionImporterFactory().createImporter(resourceLocation); + doReturn(importer).when(vocabularyCollectionImporterFactory).createImporter(any(URL.class)); + + VocabularyImportException expectedException = assertThrows(VocabularyImportException.class, + () -> service.loadVocabularies(resourceLocation)); + + assertEquals("An error as occurred while loading the vocabularies", expectedException.getMessage()); + verify(vocabularyCollectionImporterFactory, times(1)).createImporter(resourceLocation); + } + @AfterEach void destroy() { embeddedLocalhostMongo.stop(); diff --git a/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/dao/ProcessedEntityDaoTest.java b/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/dao/ProcessedEntityDaoTest.java new file mode 100644 index 000000000..0b42c5c44 --- /dev/null +++ b/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/dao/ProcessedEntityDaoTest.java @@ -0,0 +1,119 @@ +package eu.europeana.metis.dereference.service.dao; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import eu.europeana.metis.dereference.ProcessedEntity; +import eu.europeana.metis.dereference.service.dao.ProcessedEntityDao; +import eu.europeana.metis.mongo.embedded.EmbeddedLocalhostMongo; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link ProcessedEntityDao} class + */ +class ProcessedEntityDaoTest { + + private static EmbeddedLocalhostMongo embeddedLocalhostMongo; + + private static ProcessedEntityDao processedEntityDao; + + @BeforeAll + static void prepare() { + embeddedLocalhostMongo = new EmbeddedLocalhostMongo(); + + embeddedLocalhostMongo.start(); + final String mongoHost = embeddedLocalhostMongo.getMongoHost(); + final int mongoPort = embeddedLocalhostMongo.getMongoPort(); + + final MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s", mongoHost, mongoPort)); + processedEntityDao = new ProcessedEntityDao(mongoClient, "metis-dereference"); + } + + @BeforeEach + void setupDb() { + initDatabaseWithEntities(); + } + + @AfterEach + void tearDownDb() { + processedEntityDao.purgeAll(); + } + + @Test + void processDaoPurgeAll() { + assertEquals(5, processedEntityDao.size()); + + processedEntityDao.purgeAll(); + + for (int i = 1; i < 5; i++) { + assertNull(processedEntityDao.getByResourceId("http://www.test" + i + ".uri/")); + } + assertEquals(0, processedEntityDao.size()); + } + + @Test + void processDaoPurgeByNullOrEmptyXML() { + assertEquals(5, processedEntityDao.size()); + + processedEntityDao.purgeByNullOrEmptyXml(); + + assertNull(processedEntityDao.getByResourceId("http://www.test5.uri/")); + for (int i = 1; i < 4; i++) { + assertEquals("http://www.test" + i + ".uri/", + processedEntityDao.getByResourceId("http://www.test" + i + ".uri/").getResourceId()); + } + assertEquals(4, processedEntityDao.size()); + } + + @Test + void processDaoPurgeByResourceId() { + assertEquals(5, processedEntityDao.size()); + + processedEntityDao.purgeByResourceId("http://www.test1.uri/"); + + assertNull(processedEntityDao.getByResourceId("http://www.test1.uri/")); + assertEquals(4, processedEntityDao.size()); + } + + @Test + void processDaoPurgeByVocabulary() { + assertEquals(5, processedEntityDao.size()); + + processedEntityDao.purgeByVocabularyId("vocabularyId1"); + + assertNull(processedEntityDao.getByVocabularyId("vocabularyId1")); + for (int i = 2; i < 5; i++) { + assertEquals("vocabularyId" + i, + processedEntityDao.getByVocabularyId("vocabularyId" + i).getVocabularyId()); + } + assertEquals(4, processedEntityDao.size()); + } + + // side note flapdoodle embedded mongo + // doesn't support DuplicateExceptionKey Mongo. + + void initDatabaseWithEntities() { + for (int i = 1; i <= 5; i++) { + final ProcessedEntity processedEntity = new ProcessedEntity(); + processedEntity.setResourceId("http://www.test" + i + ".uri/"); + processedEntity.setVocabularyId("vocabularyId" + i); + if (i == 5) { + processedEntity.setXml(null); + } else { + processedEntity.setXml("value"); + } + processedEntityDao.save(processedEntity); + } + } + + @AfterAll + static void destroy() { + embeddedLocalhostMongo.stop(); + } +} diff --git a/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/dao/VocabularyDaoTest.java b/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/dao/VocabularyDaoTest.java new file mode 100644 index 000000000..bb1012803 --- /dev/null +++ b/metis-dereference/metis-dereference-service/src/test/java/eu/europeana/metis/dereference/service/dao/VocabularyDaoTest.java @@ -0,0 +1,146 @@ +package eu.europeana.metis.dereference.service.dao; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import dev.morphia.DeleteOptions; +import eu.europeana.metis.dereference.Vocabulary; +import eu.europeana.metis.mongo.embedded.EmbeddedLocalhostMongo; +import java.util.List; +import java.util.Optional; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + + +/** + * Unit tests for {@link VocabularyDao} class + */ +class VocabularyDaoTest { + + private static EmbeddedLocalhostMongo embeddedLocalhostMongo; + + private static VocabularyDao vocabularyDao; + + @BeforeAll + static void prepare() { + embeddedLocalhostMongo = new EmbeddedLocalhostMongo(); + + embeddedLocalhostMongo.start(); + final String mongoHost = embeddedLocalhostMongo.getMongoHost(); + final int mongoPort = embeddedLocalhostMongo.getMongoPort(); + + final MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s", mongoHost, mongoPort)); + vocabularyDao = new VocabularyDao(mongoClient, "metis-dereference"); + } + + @AfterAll + static void destroy() { + embeddedLocalhostMongo.stop(); + } + + @BeforeEach + void setupDb() { + initDatabaseWithEntities(); + } + + @AfterEach + void tearDownDb() { + vocabularyDao.getDatastore().find(Vocabulary.class).delete(new DeleteOptions().multi(true)); + } + + @Test + void getByUriSearch() { + assertEquals(5, vocabularyDao.size()); + + List vocabularies = vocabularyDao.getByUriSearch("http://domain2.uri"); + + assertEquals(2, vocabularies.size()); + assertNotEquals(Optional.empty(), + vocabularies.stream().filter(vocabulary -> vocabulary.getName().equals("vocabulary2")).findFirst()); + assertNotEquals(Optional.empty(), + vocabularies.stream().filter(vocabulary -> vocabulary.getName().equals("vocabulary4")).findFirst()); + assertEquals(Optional.empty(), + vocabularies.stream().filter(vocabulary -> vocabulary.getName().equals("vocabulary1")).findFirst()); + assertEquals(Optional.empty(), + vocabularies.stream().filter(vocabulary -> vocabulary.getName().equals("vocabulary3")).findFirst()); + assertEquals(Optional.empty(), + vocabularies.stream().filter(vocabulary -> vocabulary.getName().equals("vocabulary5")).findFirst()); + } + + @Test + void getAll() { + assertEquals(5, vocabularyDao.size()); + + List vocabularies = vocabularyDao.getAll(); + + assertEquals(5, vocabularies.size()); + assertNotEquals(Optional.empty(), + vocabularies.stream().filter(vocabulary -> vocabulary.getName().equals("vocabulary1")).findFirst()); + assertNotEquals(Optional.empty(), + vocabularies.stream().filter(vocabulary -> vocabulary.getName().equals("vocabulary2")).findFirst()); + assertNotEquals(Optional.empty(), + vocabularies.stream().filter(vocabulary -> vocabulary.getName().equals("vocabulary3")).findFirst()); + assertNotEquals(Optional.empty(), + vocabularies.stream().filter(vocabulary -> vocabulary.getName().equals("vocabulary4")).findFirst()); + assertNotEquals(Optional.empty(), + vocabularies.stream().filter(vocabulary -> vocabulary.getName().equals("vocabulary5")).findFirst()); + } + + @Test + void get() { + assertEquals(5, vocabularyDao.size()); + + Vocabulary expectedVocabulary = vocabularyDao.getDatastore().find(Vocabulary.class).first(); + + Vocabulary vocabulary = vocabularyDao.get(expectedVocabulary.getId().toString()); + + assertEquals(expectedVocabulary.getName(), vocabulary.getName()); + assertEquals(expectedVocabulary.getSuffix(), vocabulary.getSuffix()); + assertEquals(expectedVocabulary.getXslt(), vocabulary.getXslt()); + assertEquals(expectedVocabulary.getUris().stream().findFirst().get(), vocabulary.getUris().stream().findFirst().get()); + assertEquals(expectedVocabulary.getIterations(), vocabulary.getIterations()); + } + + @Test + void replaceAll() { + assertEquals(5, vocabularyDao.size()); + + Vocabulary vocabulary = new Vocabulary(); + vocabulary.setXslt("xlst"); + vocabulary.setSuffix("suffix"); + vocabulary.setUris(List.of("uri")); + vocabulary.setIterations(0); + vocabulary.setName("vocabularyName"); + + vocabularyDao.replaceAll(List.of(vocabulary)); + + assertEquals(1, vocabularyDao.size()); + } + + @Test + void getDatastore() { + assertNotNull(vocabularyDao.getDatastore()); + } + + void initDatabaseWithEntities() { + for (int i = 1; i <= 5; i++) { + Vocabulary vocabulary = new Vocabulary(); + vocabulary.setName("vocabulary" + i); + vocabulary.setSuffix("suffix" + i); + if (i % 2 == 0) { + vocabulary.setUris(List.of("http://domain2.uri")); + } else { + vocabulary.setUris(List.of("http://domain1.uri")); + } + vocabulary.setXslt("xlst" + i); + vocabulary.setIterations(0); + vocabularyDao.getDatastore().save(vocabulary); + } + } +} diff --git a/metis-dereference/metis-dereference-service/src/test/resources/vocabulary-fault.yml b/metis-dereference/metis-dereference-service/src/test/resources/vocabulary-fault.yml new file mode 100644 index 000000000..07d1d8dcb --- /dev/null +++ b/metis-dereference/metis-dereference-service/src/test/resources/vocabulary-fault.yml @@ -0,0 +1,3 @@ +# test +- metadata: vocabulary/voctest1.yml + mapping: vocabulary/voctest1.xsl diff --git a/metis-dereference/metis-dereference-service/src/test/resources/vocabulary.yml b/metis-dereference/metis-dereference-service/src/test/resources/vocabulary.yml new file mode 100644 index 000000000..24bf65236 --- /dev/null +++ b/metis-dereference/metis-dereference-service/src/test/resources/vocabulary.yml @@ -0,0 +1,3 @@ +# test +- metadata: vocabulary/voctest.yml + mapping: vocabulary/voctest.xsl diff --git a/metis-dereference/metis-dereference-service/src/test/resources/vocabulary/voctest.xsl b/metis-dereference/metis-dereference-service/src/test/resources/vocabulary/voctest.xsl new file mode 100644 index 000000000..6042f2139 --- /dev/null +++ b/metis-dereference/metis-dereference-service/src/test/resources/vocabulary/voctest.xsl @@ -0,0 +1,960 @@ + + + + + + + + + + + + + + + + + en,pl,de,nl,fr,it,da,sv,el,fi,hu,cs,sl,et,pt,es,lt,lv,bg,ro,sk,hr,ga,mt,no,ca,ru + + + + http://viaf.org/viaf/$1 + http://d-nb.info/gnd/$1 + http://id.loc.gov/authorities/names/$1 + http://vocab.getty.edu/ulan/$1 + http://data.bnf.fr/ark:/12148/cb$1 + http://www.idref.fr/$1/id + http://id.ndl.go.jp/auth/ndlna/$1 + http://id.nlm.nih.gov/mesh/$1 + http://purl.org/bncf/tid/$1 + https://www.freebase.com$1 + https://g.co/kg$1 + http://openlibrary.org/works/$1 + http://id.nlm.nih.gov/mesh/$1 + http://libris.kb.se/resource/auth/$1 + http://datos.bne.es/resource/$1 + http://data.bibliotheken.nl/id/thes/p$1 + http://vocab.getty.edu/aat/$1 + https://livedata.bibsys.no/authority/$1 + http://dewey.info/class/$1/ + http://iconclass.org/$1 + http://kulturarvsdata.se/$1 + http://ta.sandrart.net/-person-$1 + https://sws.geonames.org/$1/ + http://pleiades.stoa.org/places/$1/rdf + http://vocab.getty.edu/tgn/$1 + urn:uuid:$1 + http://dare.ht.lu.se/places/$1 + http://id.worldcat.org/fast/$1 + http://www.yso.fi/onto/yso/p$1 + http://www.geonames.org/ontology#$1 + http://babelnet.org/rdf/s$1 + http://g.co/kg$1 + http://data.cervantesvirtual.com/person/$1 + http://nomisma.org/id/$1 + http://data.ordnancesurvey.co.uk/id/$1 + http://nlg.okfn.gr/resource/authority/record$1 + http://thesaurus.europeanafashion.eu/thesaurus/$1 + http://zbw.eu/stw/descriptor/$1 + http://vocabularies.unesco.org/thesaurus/$1 + http://data.carnegiehall.org/names/$1 + https://id.erfgoed.net/thesauri/erfgoedtypes/$1 + http://id.loc.gov/authorities/genreForms/$1 + http://lod.nl.go.kr/resource/$1 + http://cv.iptc.org/newscodes/$1 + https://libris.kb.se/$1 + http://uri.gbv.de/terminology/bk/$1 + http://www.yso.fi/onto/ysa/$1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/metis-dereference/metis-dereference-service/src/test/resources/vocabulary/voctest.yml b/metis-dereference/metis-dereference-service/src/test/resources/vocabulary/voctest.yml new file mode 100644 index 000000000..5905cfb3d --- /dev/null +++ b/metis-dereference/metis-dereference-service/src/test/resources/vocabulary/voctest.yml @@ -0,0 +1,18 @@ +name: TestWikidata +types: + - AGENT + - CONCEPT + - PLACE + - TIMESTAMP +paths: + - http://www.wikidata.org/entity/ +examples: + - http://www.wikidata.org/entity/Q3930 + - http://www.wikidata.org/entity/Q604667 + - http://www.wikidata.org/entity/Q79007 + - http://www.wikidata.org/entity/Q1261026 + - http://www.wikidata.org/entity/Q36422 + - http://www.wikidata.org/entity/Q6927 + - http://www.wikidata.org/entity/Q90 + - http://www.wikidata.org/entity/Q187843 + - http://www.wikidata.org/entity/Q145 From 44ea02886de1d7413fdbe00eb966b0b105a27a02 Mon Sep 17 00:00:00 2001 From: jochen_vermeulen Date: Mon, 30 May 2022 11:32:58 +0200 Subject: [PATCH 45/73] MET-4472: Fix bugs with 3D tier calculation. --- .../tiers/media/ThreeDClassifier.java | 29 ++----------- .../tiers/media/ThreeDClassifierTest.java | 41 +++---------------- 2 files changed, 9 insertions(+), 61 deletions(-) diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/tiers/media/ThreeDClassifier.java b/metis-indexing/src/main/java/eu/europeana/indexing/tiers/media/ThreeDClassifier.java index ac827ad73..eb6016660 100644 --- a/metis-indexing/src/main/java/eu/europeana/indexing/tiers/media/ThreeDClassifier.java +++ b/metis-indexing/src/main/java/eu/europeana/indexing/tiers/media/ThreeDClassifier.java @@ -4,12 +4,9 @@ import eu.europeana.indexing.tiers.view.ResolutionTierMetadata; import eu.europeana.indexing.tiers.view.ResolutionTierMetadata.ResolutionTierMetadataBuilder; import eu.europeana.indexing.utils.RdfWrapper; -import eu.europeana.indexing.utils.WebResourceLinkType; import eu.europeana.indexing.utils.WebResourceWrapper; import eu.europeana.metis.schema.model.MediaType; -import java.util.Set; - /** * Classifier for 3D objects. */ @@ -25,25 +22,18 @@ MediaTier preClassifyEntity(RdfWrapper entity) { @Override MediaTier classifyEntityWithoutWebResources(RdfWrapper entity, boolean hasLandingPage) { - // A record without suitable web resources has tier 0. - return MediaTier.T0; + // A record without web resources has tier 1 if there is a landing page, otherwise tier 0. + return hasLandingPage ? MediaTier.T1 : MediaTier.T0; } @Override MediaTier classifyWebResource(WebResourceWrapper webResource, boolean hasLandingPage, boolean hasEmbeddableMedia) { - final MediaTier result; - - if(webResource == null){ - result = MediaTier.T0; - } else if(mimeTypeIsNotImageOrApplicationPdf(webResource) && containsIsShownByOrHasViewWebResource(webResource)){ + if (webResource != null && mimeTypeIsNotImageOrApplicationPdf(webResource)) { result = MediaTier.T4; - } else if(hasLandingPage && onlyContainsShownAtWebResource(webResource)){ - result = MediaTier.T1; } else { result = MediaTier.T0; } - return result; } @@ -61,17 +51,4 @@ private boolean mimeTypeIsNotImageOrApplicationPdf(WebResourceWrapper webResourc String mimeType = webResource.getMimeType(); return mimeType != null && webResource.getMediaType() != MediaType.IMAGE && !mimeType.startsWith("application/pdf"); } - - private boolean containsIsShownByOrHasViewWebResource(WebResourceWrapper webResource){ - Set extractedLinkTypes = webResource.getLinkTypes(); - return extractedLinkTypes != null && (extractedLinkTypes.contains(WebResourceLinkType.IS_SHOWN_BY) || - extractedLinkTypes.contains(WebResourceLinkType.HAS_VIEW)); - } - - private boolean onlyContainsShownAtWebResource(WebResourceWrapper webResource){ - Set linkTypes = webResource.getLinkTypes(); - return linkTypes.contains(WebResourceLinkType.IS_SHOWN_AT) && - !linkTypes.contains(WebResourceLinkType.IS_SHOWN_BY) && - !linkTypes.contains(WebResourceLinkType.HAS_VIEW); - } } diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/media/ThreeDClassifierTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/media/ThreeDClassifierTest.java index 6493c6fbe..5b6552a72 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/media/ThreeDClassifierTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/media/ThreeDClassifierTest.java @@ -3,20 +3,20 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; import eu.europeana.indexing.tiers.model.MediaTier; import eu.europeana.indexing.utils.RdfWrapper; import eu.europeana.indexing.utils.WebResourceLinkType; import eu.europeana.indexing.utils.WebResourceWrapper; import eu.europeana.metis.schema.model.MediaType; - import java.util.Set; -import java.util.stream.Stream; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.provider.Arguments; class ThreeDClassifierTest { @@ -41,28 +41,15 @@ void testPreClassifyEntity() { } /** - * If there are no web resources, the result is always tier 0. + * If there are no web resources, the result is tier 0 or 1 depending on the landing page. */ @Test void testClassifyEntityWithoutWebResources() { final RdfWrapper entity = mock(RdfWrapper.class); - assertEquals(MediaTier.T0, classifier.classifyEntityWithoutWebResources(entity, true)); + assertEquals(MediaTier.T1, classifier.classifyEntityWithoutWebResources(entity, true)); assertEquals(MediaTier.T0, classifier.classifyEntityWithoutWebResources(entity, false)); } - private static Stream testClassifyWebResource() { - return Stream.of( - //If mime type is not blank - Arguments.of(MediaTier.T4, "any mime type"), - //If mime type is blank(only spaces) - Arguments.of(MediaTier.T0, " "), - //If mime type is blank(empty string) - Arguments.of(MediaTier.T0, ""), - //If mime type is blank(null) - Arguments.of(MediaTier.T0, null) - ); - } - @Test void testClassifyWebResource_tier4Result() { final WebResourceWrapper webResource = mock(WebResourceWrapper.class); @@ -72,22 +59,6 @@ void testClassifyWebResource_tier4Result() { assertEquals(MediaTier.T4, classifier.classifyWebResource(webResource, true, false)); } - @Test - void testClassifyWebResource_tier1Result() { - final WebResourceWrapper webResource = mock(WebResourceWrapper.class); - when(webResource.getLinkTypes()).thenReturn(Set.of(WebResourceLinkType.IS_SHOWN_AT)); - when(webResource.getMimeType()).thenReturn("video"); - assertEquals(MediaTier.T1, classifier.classifyWebResource(webResource, true, false)); - } - - @Test - void testClassifyWebResource_tier0Result() { - final WebResourceWrapper webResource = mock(WebResourceWrapper.class); - when(webResource.getLinkTypes()).thenReturn(Set.of()); - when(webResource.getMimeType()).thenReturn("video"); - assertEquals(MediaTier.T0, classifier.classifyWebResource(webResource, false, false)); - } - @Test void testClassifyWebResource_tier0NullValuesResult() { final WebResourceWrapper webResource = mock(WebResourceWrapper.class); From f7583a1df938ff8d61af6a05ff32e6c79e436769 Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Mon, 30 May 2022 17:12:58 +0200 Subject: [PATCH 46/73] MET-4437 Remove metis-schema module (#545) --- metis-common/metis-common-mongo/pom.xml | 1 - .../metis-enrichment-client/pom.xml | 1 - .../metis-enrichment-common/pom.xml | 1 - metis-indexing/pom.xml | 1 - metis-schema/.gitignore | 3 - metis-schema/README.MD | 5 - metis-schema/pom.xml | 204 ---- .../schema/convert/RdfConversionUtils.java | 214 ---- .../convert/SerializationException.java | 19 - .../metis/schema/model/MediaType.java | 116 -- .../metis/schema/model/Orientation.java | 41 - .../resources/rdf_examples/EDM-v52-100730.owl | 743 ------------ .../resources/rdf_examples/EDM-v523-120123.n3 | 576 --------- .../resources/rdf_examples/EDM-v523-120123.nt | 403 ------- .../rdf_examples/EDM-v523-120123.owl | 824 ------------- .../rdf_examples/EDM-v523-120123.ttl | 576 --------- .../resources/rdf_examples/EDM-v524-120820.n3 | 559 --------- .../resources/rdf_examples/EDM-v524-120820.nt | 452 ------- .../rdf_examples/EDM-v524-120820.owl | 893 -------------- .../rdf_examples/EDM-v524-120820.ttl | 654 ---------- .../resources/rdf_examples/edm-v524-130522.n3 | 511 -------- .../resources/rdf_examples/edm-v524-130522.nt | 466 -------- .../rdf_examples/edm-v524-130522.owl | 902 -------------- .../rdf_examples/edm-v524-130522.ttl | 671 ----------- .../src/main/resources/rdf_examples/edm.n3 | 511 -------- .../src/main/resources/rdf_examples/edm.nt | 466 -------- .../src/main/resources/rdf_examples/edm.owl | 902 -------------- .../src/main/resources/rdf_examples/edm.ttl | 671 ----------- .../src/main/resources/schema_xsds/ADMS.xsd | 15 - .../src/main/resources/schema_xsds/CC.xsd | 27 - .../main/resources/schema_xsds/CONTEXTS.xsd | 183 --- .../src/main/resources/schema_xsds/DC.xsd | 169 --- .../src/main/resources/schema_xsds/DCAT.xsd | 51 - .../main/resources/schema_xsds/DCTERMS.xsd | 239 ---- .../src/main/resources/schema_xsds/DOAP.xsd | 14 - .../src/main/resources/schema_xsds/DQV.xsd | 33 - .../main/resources/schema_xsds/EBUCORE.xsd | 108 -- .../resources/schema_xsds/EDM-COMMON-MAIN.xsd | 826 ------------- .../schema_xsds/EDM-EXTERNAL-MAIN.xsd | 192 --- .../schema_xsds/EDM-INTERNAL-MAIN.xsd | 131 -- .../resources/schema_xsds/EDM-INTERNAL.xsd | 17 - .../src/main/resources/schema_xsds/EDM.xsd | 17 - .../main/resources/schema_xsds/ENRICHMENT.xsd | 18 - .../src/main/resources/schema_xsds/FOAF.xsd | 57 - .../src/main/resources/schema_xsds/OA.xsd | 7 - .../src/main/resources/schema_xsds/ODRL.xsd | 13 - .../src/main/resources/schema_xsds/ORE.xsd | 208 ---- .../src/main/resources/schema_xsds/OWL.xsd | 21 - .../src/main/resources/schema_xsds/RDAGR2.xsd | 37 - .../src/main/resources/schema_xsds/RDF.xsd | 183 --- .../src/main/resources/schema_xsds/SKOS.xsd | 62 - .../src/main/resources/schema_xsds/SVCS.xsd | 31 - .../src/main/resources/schema_xsds/WDRS.xsd | 14 - .../src/main/resources/schema_xsds/WGS84.xsd | 18 - .../schematron/schematron-internal.xsl | 790 ------------ .../schema_xsds/schematron/schematron.xsl | 1064 ----------------- .../src/main/resources/schema_xsds/xml.xsd | 287 ----- .../convert/RdfConversionUtilsTest.java | 91 -- .../metis/schema/model/MediaTypeTest.java | 34 - pom.xml | 7 +- 60 files changed, 4 insertions(+), 16346 deletions(-) delete mode 100644 metis-schema/.gitignore delete mode 100644 metis-schema/README.MD delete mode 100644 metis-schema/pom.xml delete mode 100644 metis-schema/src/main/java/eu/europeana/metis/schema/convert/RdfConversionUtils.java delete mode 100644 metis-schema/src/main/java/eu/europeana/metis/schema/convert/SerializationException.java delete mode 100644 metis-schema/src/main/java/eu/europeana/metis/schema/model/MediaType.java delete mode 100644 metis-schema/src/main/java/eu/europeana/metis/schema/model/Orientation.java delete mode 100644 metis-schema/src/main/resources/rdf_examples/EDM-v52-100730.owl delete mode 100644 metis-schema/src/main/resources/rdf_examples/EDM-v523-120123.n3 delete mode 100644 metis-schema/src/main/resources/rdf_examples/EDM-v523-120123.nt delete mode 100644 metis-schema/src/main/resources/rdf_examples/EDM-v523-120123.owl delete mode 100644 metis-schema/src/main/resources/rdf_examples/EDM-v523-120123.ttl delete mode 100644 metis-schema/src/main/resources/rdf_examples/EDM-v524-120820.n3 delete mode 100644 metis-schema/src/main/resources/rdf_examples/EDM-v524-120820.nt delete mode 100644 metis-schema/src/main/resources/rdf_examples/EDM-v524-120820.owl delete mode 100644 metis-schema/src/main/resources/rdf_examples/EDM-v524-120820.ttl delete mode 100644 metis-schema/src/main/resources/rdf_examples/edm-v524-130522.n3 delete mode 100644 metis-schema/src/main/resources/rdf_examples/edm-v524-130522.nt delete mode 100644 metis-schema/src/main/resources/rdf_examples/edm-v524-130522.owl delete mode 100644 metis-schema/src/main/resources/rdf_examples/edm-v524-130522.ttl delete mode 100644 metis-schema/src/main/resources/rdf_examples/edm.n3 delete mode 100644 metis-schema/src/main/resources/rdf_examples/edm.nt delete mode 100644 metis-schema/src/main/resources/rdf_examples/edm.owl delete mode 100644 metis-schema/src/main/resources/rdf_examples/edm.ttl delete mode 100644 metis-schema/src/main/resources/schema_xsds/ADMS.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/CC.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/CONTEXTS.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/DC.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/DCAT.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/DCTERMS.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/DOAP.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/DQV.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/EBUCORE.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/EDM-COMMON-MAIN.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/EDM-EXTERNAL-MAIN.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/EDM-INTERNAL-MAIN.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/EDM-INTERNAL.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/EDM.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/ENRICHMENT.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/FOAF.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/OA.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/ODRL.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/ORE.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/OWL.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/RDAGR2.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/RDF.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/SKOS.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/SVCS.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/WDRS.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/WGS84.xsd delete mode 100644 metis-schema/src/main/resources/schema_xsds/schematron/schematron-internal.xsl delete mode 100644 metis-schema/src/main/resources/schema_xsds/schematron/schematron.xsl delete mode 100644 metis-schema/src/main/resources/schema_xsds/xml.xsd delete mode 100644 metis-schema/src/test/java/eu/europeana/metis/schema/convert/RdfConversionUtilsTest.java delete mode 100644 metis-schema/src/test/java/eu/europeana/metis/schema/model/MediaTypeTest.java diff --git a/metis-common/metis-common-mongo/pom.xml b/metis-common/metis-common-mongo/pom.xml index 180197dbb..234a79441 100644 --- a/metis-common/metis-common-mongo/pom.xml +++ b/metis-common/metis-common-mongo/pom.xml @@ -28,7 +28,6 @@ eu.europeana.metis metis-schema - ${project.version} dev.morphia.morphia diff --git a/metis-enrichment/metis-enrichment-client/pom.xml b/metis-enrichment/metis-enrichment-client/pom.xml index 040678062..3edb4850d 100644 --- a/metis-enrichment/metis-enrichment-client/pom.xml +++ b/metis-enrichment/metis-enrichment-client/pom.xml @@ -42,7 +42,6 @@ eu.europeana.metis metis-schema - ${project.version} com.fasterxml.jackson.core diff --git a/metis-enrichment/metis-enrichment-common/pom.xml b/metis-enrichment/metis-enrichment-common/pom.xml index 2d4736c50..c96a4e597 100644 --- a/metis-enrichment/metis-enrichment-common/pom.xml +++ b/metis-enrichment/metis-enrichment-common/pom.xml @@ -12,7 +12,6 @@ eu.europeana.metis metis-schema - ${project.version} eu.europeana.metis diff --git a/metis-indexing/pom.xml b/metis-indexing/pom.xml index 3c2fd7601..ff23e8558 100644 --- a/metis-indexing/pom.xml +++ b/metis-indexing/pom.xml @@ -26,7 +26,6 @@ eu.europeana.metis metis-schema - ${project.version} eu.europeana.corelib diff --git a/metis-schema/.gitignore b/metis-schema/.gitignore deleted file mode 100644 index 8a61c7c50..000000000 --- a/metis-schema/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -#Those are generated files that we will be skipping until we introduce schema classes instead of jibx -src/main/java/eu/europeana/metis/schema/jibx -src/main/java/*.xml \ No newline at end of file diff --git a/metis-schema/README.MD b/metis-schema/README.MD deleted file mode 100644 index 1a545b9af..000000000 --- a/metis-schema/README.MD +++ /dev/null @@ -1,5 +0,0 @@ -#Metis schema library -This module replaces the previous corelib-edm definitions. -The java classes are auto-generated during mvn install. -The java classes will similarly not be committed on the repository and should be generated during the maven build. -The Schemas are coming from the corelib library but they should eventually be merged with the schemas that exist in the metis-validation library. \ No newline at end of file diff --git a/metis-schema/pom.xml b/metis-schema/pom.xml deleted file mode 100644 index 66d77bdb4..000000000 --- a/metis-schema/pom.xml +++ /dev/null @@ -1,204 +0,0 @@ - - - 4.0.0 - - metis-framework - eu.europeana.metis - 7-SNAPSHOT - - - metis-schema - - - - - - - - - - - - - - - - - - - - - - - ]]> - - ) ]]> - 1.3.3 - EDM-INTERNAL.xsd - 6.5.0 - 1.5.2 - - - - - jackson-annotations - com.fasterxml.jackson.core - ${version.jackson} - - - - org.jibx - jibx-run - ${jibx.version} - - - bcel - bcel - - - - - org.jibx - jibx-extras - ${jibx.version} - - - bcel - bcel - - - - - - org.apache.bcel - bcel - ${bcel.version} - - - - - org.junit.jupiter - junit-jupiter-api - - - org.junit.jupiter - junit-jupiter-engine - - - - - - - - install - - - com.google.code.maven-replacer-plugin - replacer - ${replacer.version} - - - process-sources - - replace - - - - - false - ${project.basedir}/src/main/java/binding.xml - - - ${token.regex} - ${binding.replace} - - - - - - org.jibx - jibx-maven-plugin - ${jibx.version} - - - org.jibx - jibx-bind - ${jibx.version} - - - bcel - bcel - - - - - org.apache.bcel - bcel - ${bcel.version} - - - - - generate-java-code-from-schema - generate-sources - - schema-codegen - - - src/main/resources/schema_xsds - src/main/java - - ${edm.version} - - - eu.europeana.metis.schema.jibx - - - - - compile-binding - process-classes - - bind - - - src/main/java - true - true - false - - - - - - - - maven-resources-plugin - - - - target/test-classes - - - target/classes - false - eu/europeana/metis/schema/jibx/* - - - - - copy-resources - - copy-resources - generate-test-sources - - - ${version.maven.resources.plugin} - - - - - \ No newline at end of file diff --git a/metis-schema/src/main/java/eu/europeana/metis/schema/convert/RdfConversionUtils.java b/metis-schema/src/main/java/eu/europeana/metis/schema/convert/RdfConversionUtils.java deleted file mode 100644 index e15e3788d..000000000 --- a/metis-schema/src/main/java/eu/europeana/metis/schema/convert/RdfConversionUtils.java +++ /dev/null @@ -1,214 +0,0 @@ -package eu.europeana.metis.schema.convert; - -import eu.europeana.metis.schema.jibx.RDF; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.UnsupportedEncodingException; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.IntStream; -import org.jibx.runtime.BindingDirectory; -import org.jibx.runtime.IBindingFactory; -import org.jibx.runtime.IMarshallingContext; -import org.jibx.runtime.IUnmarshallingContext; -import org.jibx.runtime.JiBXException; - -/** - * Utility class for converting {@link RDF} to String and vice versa. - */ -public class RdfConversionUtils { - - private static final int INDENTATION_SPACE = 2; - private static final String UTF8 = StandardCharsets.UTF_8.name(); - @SuppressWarnings("java:S5852") //This regex is safe, and it's only meant for internal use without use input - private static final Pattern complexTypePattern = Pattern.compile("^\\{(.*)}:(.*)$"); - private final IBindingFactory rdfBindingFactory; - private final Map rdfXmlElementMetadataMap; - - /** - * Default constructor - */ - public RdfConversionUtils() { - this(RDF.class); - } - - /** - * Constructor supplying class type for the binding factory. - *

    At the current state this is used for assisting testing

    - * - * @param classType the class object type - * @param the class type - */ - RdfConversionUtils(Class classType) { - try { - rdfBindingFactory = BindingDirectory.getFactory(classType); - rdfXmlElementMetadataMap = initializeRdfXmlElementMetadataMap(); - } catch (JiBXException e) { - throw new IllegalStateException("No binding factory available.", e); - } - } - - /** - * Convert an {@link RDF} to a UTF-8 encoded XML - * - * @param rdf The RDF object to convert - * @return An XML string representation of the RDF object - * @throws SerializationException if during marshalling there is a failure - */ - public byte[] convertRdfToBytes(RDF rdf) throws SerializationException { - try { - IMarshallingContext context = rdfBindingFactory.createMarshallingContext(); - context.setIndent(INDENTATION_SPACE); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - context.marshalDocument(rdf, UTF8, null, out); - return out.toByteArray(); - } catch (JiBXException e) { - throw new SerializationException( - "Something went wrong with converting to or from the RDF format.", e); - } - } - - /** - * Get the xml representation of a class that will contain the namespace prefix and the element name. E.g. dc:subject - *

    This class uses the internal static map that should be generated with regards to the RDF jibx classes

    - * - * @param objectClass the jibx object class to search for - * @return the xml representation - */ - public String getQualifiedElementNameForClass(Class objectClass) { - final RdfXmlElementMetadata rdfXmlElementMetadata = rdfXmlElementMetadataMap.get(objectClass.getCanonicalName()); - Objects.requireNonNull(rdfXmlElementMetadata, - String.format("Element metadata not found for class: %s", objectClass.getCanonicalName())); - return String.format("%s:%s", rdfXmlElementMetadata.getPrefix(), rdfXmlElementMetadata.getName()); - } - - /** - * Convert a UTF-8 encoded XML to {@link RDF} - * - * @param inputStream The xml. The stream is not closed. - * @return the RDF object - * @throws SerializationException if during unmarshalling there is a failure - */ - public RDF convertInputStreamToRdf(InputStream inputStream) throws SerializationException { - try { - final IUnmarshallingContext context = rdfBindingFactory.createUnmarshallingContext(); - return (RDF) context.unmarshalDocument(inputStream, UTF8); - } catch (JiBXException e) { - throw new SerializationException( - "Something went wrong with converting to or from the RDF format.", e); - } - } - - /** - * Convert an {@link RDF} to a UTF-8 encoded XML - * - * @param rdf The RDF object to convert - * @return An XML string representation of the RDF object - * @throws SerializationException if during marshalling there is a failure - */ - public String convertRdfToString(RDF rdf) throws SerializationException { - try { - return new String(convertRdfToBytes(rdf), UTF8); - } catch (UnsupportedEncodingException e) { - throw new IllegalStateException("Unexpected exception - should not occur.", e); - } - } - - static class RdfXmlElementMetadata { - - final String canonicalClassName; - final String prefix; - final String namespace; - final String name; - - public RdfXmlElementMetadata(String canonicalClassName, String prefix, String namespace, String name) { - this.canonicalClassName = canonicalClassName; - this.prefix = prefix; - this.namespace = namespace; - this.name = name; - } - - public String getCanonicalClassName() { - return canonicalClassName; - } - - public String getPrefix() { - return prefix; - } - - public String getNamespace() { - return namespace; - } - - public String getName() { - return name; - } - } - - /** - * Convert a UTF-8 encoded XML to {@link RDF} - * - * @param xml the xml string - * @return the RDF object - * @throws SerializationException if during unmarshalling there is a failure - */ - public RDF convertStringToRdf(String xml) throws SerializationException { - try (final InputStream inputStream = new ByteArrayInputStream( - xml.getBytes(StandardCharsets.UTF_8))) { - return convertInputStreamToRdf(inputStream); - } catch (IOException e) { - throw new SerializationException("Unexpected issue with byte stream.", e); - } - } - - /** - * Collect all information that we can get for jibx classes from the {@link IBindingFactory}. - */ - private Map initializeRdfXmlElementMetadataMap() { - Map elementMetadataMap = new HashMap<>(); - for (int i = 0; i < rdfBindingFactory.getMappedClasses().length; i++) { - final String canonicalName; - final String elementNamespace; - final String elementName; - final Matcher matcher = complexTypePattern.matcher(rdfBindingFactory.getMappedClasses()[i]); - if (matcher.matches()) { - //Complex type search - elementNamespace = matcher.group(1); - elementName = matcher.group(2); - final Pattern canonicalClassNamePattern = Pattern.compile(String.format("^(.*)\\.(%s)$", elementName)); - canonicalName = Arrays.stream(rdfBindingFactory.getAbstractMappings()).flatMap(Arrays::stream) - .filter(Objects::nonNull) - .filter(input -> canonicalClassNamePattern.matcher(input).matches()) - .findFirst().orElse(null); - } else { - //Simple type search - elementNamespace = rdfBindingFactory.getElementNamespaces()[i]; - elementName = rdfBindingFactory.getElementNames()[i]; - canonicalName = rdfBindingFactory.getMappedClasses()[i]; - } - checkAndStoreMetadataInMap(elementMetadataMap, canonicalName, elementNamespace, elementName); - } - return elementMetadataMap; - } - - private void checkAndStoreMetadataInMap(final Map rdfXmlElementMetadataMap, - String canonicalName, String elementNamespace, String elementName) { - //Store only if we could find the canonical name properly - if (canonicalName != null) { - final int namespaceIndex = IntStream.range(0, rdfBindingFactory.getNamespaces().length) - .filter(j -> rdfBindingFactory.getNamespaces()[j].equals(elementNamespace)) - .findFirst().orElseThrow(); - final String prefix = rdfBindingFactory.getPrefixes()[namespaceIndex]; - final RdfXmlElementMetadata rdfXmlElementMetadata = new RdfXmlElementMetadata(canonicalName, prefix, elementNamespace, - elementName); - rdfXmlElementMetadataMap.put(rdfXmlElementMetadata.getCanonicalClassName(), rdfXmlElementMetadata); - } - } -} diff --git a/metis-schema/src/main/java/eu/europeana/metis/schema/convert/SerializationException.java b/metis-schema/src/main/java/eu/europeana/metis/schema/convert/SerializationException.java deleted file mode 100644 index 77efce317..000000000 --- a/metis-schema/src/main/java/eu/europeana/metis/schema/convert/SerializationException.java +++ /dev/null @@ -1,19 +0,0 @@ -package eu.europeana.metis.schema.convert; - -/** - * Exception that marks a failure in serializing or deserializing. - */ -public class SerializationException extends Exception { - - private static final long serialVersionUID = 2094687445199925308L; - - /** - * Constructor. - * - * @param message The message. - * @param cause The cause. - */ - public SerializationException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/metis-schema/src/main/java/eu/europeana/metis/schema/model/MediaType.java b/metis-schema/src/main/java/eu/europeana/metis/schema/model/MediaType.java deleted file mode 100644 index 7a822f510..000000000 --- a/metis-schema/src/main/java/eu/europeana/metis/schema/model/MediaType.java +++ /dev/null @@ -1,116 +0,0 @@ -package eu.europeana.metis.schema.model; - -import com.fasterxml.jackson.annotation.JsonValue; -import java.util.Arrays; - -/** - * This class lists the supported media types. - * - * @author jochen - */ -public enum MediaType { - - /** - * Audio - **/ - AUDIO("AUDIO"), - - /** - * Video - **/ - VIDEO("VIDEO"), - - /** - * Text (including PDFs) - **/ - TEXT("TEXT"), - - /** - * Graphical - **/ - IMAGE("IMAGE"), - - /** - * 3D - */ - THREE_D("3D"), - - /** - * Media that is not any of the other kinds. - **/ - OTHER("OTHER"); - - - private static final String[] SUPPORTED_APPLICATION_TYPES_AS_TEXT = new String[]{ - "application/xml", "application/rtf", "application/epub", "application/pdf", - "application/xhtml+xml"}; - - private final String stringRepresentation; - - MediaType(String stringRepresentation) { - this.stringRepresentation = stringRepresentation; - } - - @Override - @JsonValue - public String toString() { - return stringRepresentation; - } - - /** - * Obtains the media type of a given mime type. This method accepts mime types with subtypes and/or parameters. - * - * @param mimeType The media type. - * @return The media type to which the mime type belongs. - */ - public static MediaType getMediaType(String mimeType) { - final MediaType result; - if (mimeType == null) { - result = OTHER; - } else if (mimeType.startsWith("image/")) { - result = IMAGE; - } else if (mimeType.startsWith("audio/")) { - result = AUDIO; - } else if (mimeType.startsWith("video/") || mimeType.startsWith("application/dash+xml")) { - result = VIDEO; - } else if (mimeType.startsWith("text/") || isApplicationMediaRepresentingText(mimeType)) { - result = TEXT; - } else { - result = OTHER; - } - return result; - } - - /** - * Determines whether the supplied media base type matches the candidate media type. This method - * takes possible subtypes into account: if the candidate has a subtype, the media type is - * required to have it too, but if the candidate does not have a subtype, the media type is - * accepted either with or without subtype. - * - * @param candidateType The candidate type. Does not have parameters. - * @param mediaType The base type. Does not have parameters. - */ - private static boolean mediaTypeMatchesCandidate(String candidateType, String mediaType) { - - // Check for the type itself as well as the addition of a possible subtype. - return mediaType.equals(candidateType) || (!candidateType.contains("+") && mediaType - .startsWith(candidateType + "+")); - } - - /** - * Determines whether the supplied media type is a text media type starting with 'application/'. - * This method takes subtypes and parameters into consideration. - * - * @param mediaType The media type to categorize. - * @return Whether the media type is a text media type starting with 'application/'. - */ - private static boolean isApplicationMediaRepresentingText(String mediaType) { - - // Remove any parameters from the media type. - final String baseType = mediaType.split(";", 2)[0]; - - // Match against all possible application types that are accepted as test types. - return Arrays.stream(SUPPORTED_APPLICATION_TYPES_AS_TEXT) - .anyMatch(candidate -> mediaTypeMatchesCandidate(candidate, baseType)); - } -} diff --git a/metis-schema/src/main/java/eu/europeana/metis/schema/model/Orientation.java b/metis-schema/src/main/java/eu/europeana/metis/schema/model/Orientation.java deleted file mode 100644 index b13b4cc5d..000000000 --- a/metis-schema/src/main/java/eu/europeana/metis/schema/model/Orientation.java +++ /dev/null @@ -1,41 +0,0 @@ -package eu.europeana.metis.schema.model; - -import java.util.Arrays; -import java.util.Locale; - -/** - * Enum for the permissible values of image orientation. - */ -public enum Orientation { - - PORTRAIT, LANDSCAPE; - - /** - * Determines the orientation given a certain height and width. - * - * @param width The width of the image/page. - * @param height The height of the image/page. - * @return The orientation. - */ - public static Orientation calculate(int width, int height) { - return width > height ? Orientation.LANDSCAPE : Orientation.PORTRAIT; - } - - /** - * @return The name of this orientation, converted to lower case. - */ - public String getNameLowercase() { - return name().toLowerCase(Locale.ENGLISH); - } - - /** - * Find the orientation matching the given string. The match will be done ignoring the case. - * - * @param name The name of the orientation to find. - * @return The orientation, or null if no orientation exists with the given name. - */ - public static Orientation getFromNameCaseInsensitive(String name) { - return Arrays.stream(values()).filter(orientation -> orientation.name().equalsIgnoreCase(name)) - .findAny().orElse(null); - } -} diff --git a/metis-schema/src/main/resources/rdf_examples/EDM-v52-100730.owl b/metis-schema/src/main/resources/rdf_examples/EDM-v52-100730.owl deleted file mode 100644 index dcb88de25..000000000 --- a/metis-schema/src/main/resources/rdf_examples/EDM-v52-100730.owl +++ /dev/null @@ -1,743 +0,0 @@ - - - - - - - - - - - - - - - - -]> - - - - - -======= -Changes between http://europeanalabs.eu/attachment/wiki/EDMPrototypingTask11/EDM-v52-100730.owl -and http://europeanalabs.eu/attachment/wiki/EDMPrototypingTask12/EDMOntology.2.owl -======= -1. refined annotations using skos:definition (for "Europeana definition", which was represented using rdfs:comment), skos:example (for "Example"), skos:scopeNote with specific prefixes (for "Rationale" and "Obligation and occurrence"), skos:note (for "Europeana Note") -2. range of dc:coverage, dcterms:created, dc:date, dcterms:issued, dc:publisher, dcterms:spatial and dcterms:temporal were wrong: in the doc they were rather stated as recommendations. I removed from the files the DC elements which were consequently left without any significant specification. -3. fixed error on the spec of hasVersion and hasFormat: it is a subproperty of the inverse of isderivativeOf, not equivalent to that inverse. -4. removed rdfs:seeAlso from isShownAt and isShownBy: they were not really fitting the intent of seeAlso -5. added skos:definition and rdfs:label for europeana:type -6. removed some rdf:typings as owl:ObjectProperties: ens:hasMet, ens:isDerivativeOf, ens:isRelatedTo, ens:isSimilarTo (some sub-properties can be used with literals), ens:country, ens:dataProvider, ens:language, ens:provider, ens:unstored, ens:userTag, ens:year (because they can be directly used with literals) -7. ens:type is now a DatatypeProperty, with a datarange of 4 literals (TEXT, IMAGE, SOUND, VIDEO) -8. removed two namespaces likely never to be used: http://www.w3.org/2004/03/trix/rdfg-1/ and http://dublincore.org/usage/terms/history/ -9. removed all rdfs:seeAlso. It is not valid with a string. Nothing replaces it when the equivalent or sub/super entity is already formally indicated using RDFS/OWL statements, otherwise a rdfs:comment was created. -10. materialized equivalence between ens:TimeSpan and DOLCE TemporalInterval (it is in fact time-interval) -11. cleaned all occurrences of owl:topObjectProperty -12. added labels for europeana:userTag, europeana:year, europeana:country - -======= -Remaining TODOs for http://europeanalabs.eu/attachment/wiki/EDMPrototypingTask11/EDM-v52-100730.owl -======= -- add metadata on the owl:Ontology instance -- check Martin's latest mappings to CRM and FRBR(oo) -- continue adding documentation values (skos:scopeNote, skos:example, etc, according to 1.), starting from ens:InformationResource -- add Europeana examples and rationale notes for non-EDM constructs -- keep specific EDM-doc properties for "rationale" and "obligation and occurrence". Use skos:definition for "Europeana definition", skos:example for "Example", skos:note for "Europeana note" -- try to capture formal cardinality constraints resulting from "Obligation and Occurrence" documentation--if possible -- change FRBR namespace and mappings using http://metadataregistry.org/schema/show/id/5.html - - - - - - - - - agent - - - This class comprises people, either individually or in groups, who have the -potential to perform intentional actions for which they can be held responsible. - Leonardo da Vinci, the British Museum, W3C - Rationale: This class is a domain of ens:wasPresentAt - - - - - - - Europeana aggregation - - - The set of resources related to a single Cultural Heritage Object that -collectively represent that object in Europeana. Such set consists of: all -descriptions about the object that Europeana collects from (possibly different) content providers, including thumbnails and other forms of abstractions, as well as of the description of the object Europeana builds. - An instance of EuropeanaAggregation is created at ingestion time for each different Cultural Heritage Object recognized by Europeana. Such instance is associated to the Cultural Heritage Object that it is about, by the property ens:aggregatedCHO - Obligation and Occurence: The relation between the Cultural Heritage Objects represented in Europeana and the instances of the class EuropeanaAggregation is one-to-one: every Cultural Heritage Object is represented by an instance of EuropeanaAggregation, and every instance of EuropeanaAggregation represent a Cultural Heritage Object. - The painting Mona Lisa is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance - The journal "Le Temps" is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance - The 56th issue of "Le Temps" is a (different) Cultural Heritage Object represented in Europeana by another EuropeanaAggregation instance - Rationale:This class is used in Europeana to gather in a single conceptual unit all the information about a Cultural Heritage Object, necessary for all operations on these objects. - - - - - - Europeana object - - Any object that is the result of Europeana’s activities - Any instance of the class EuropeanaAggregation - An annotation created by a user through the Europeana portal - Any content created by the users through the service made available by Europeana for that purpose - Rationale:This class is used as the range of ens:hasAnnotation - - - - - - event - - - - - An event is a change “of states in cultural, social or physical systems, - regardless of scale, brought about by a series or group of coherent physical, -cultural, technological or legal phenomena” (E5 Event in CIDOC CRM) or a “set of coherent phenomena or cultural manifestations bounded in time and space” (E4 Period in CIDOC CRM) - - Events are identified either by the content provider or by Europeana enrichment at ingestion time - the act of painting Mona Lisa - the 2nd World War - the change of custody of Mona Lisa - Rationale:This class is a domain of ens:happenedAt and the domain of ens:occurredAt - Superclass of Activities (AAT) - - - - - - - information resource - - - - - - - - - - - An information resource is a resource whose essential characteristics can be -conveyed in a single message. It can be associated with a URI, it can have a -representation, for example: a text is an InformationResource. - - - - - - - - non information resource - All resources that are not information resources. - - - - - - physical thing - - - A persistent physical item such as a painting, a building, a book or a stone. -Persons are not items. This class represents Cultural Heritage Objects known -to Europeana to be physical things (such as Mona Lisa) as well as all physical -things Europeana refers to in the descriptions of Cultural Heritage Objects -(such as the Rosetta Stone). - - - - - - - place - - - - - - An "extent in space, in particular on the surface of the earth, in the pure sense of physics: independent from temporal phenomena and matter" (CIDOC CRM) - Equivalent class: Place (TGN) - - - - - - - - - - - time span - The class of “abstract temporal extents, in the sense of Galilean physics, - having a beginning, an end and a duration” (CIDOC CRM) - - - - - - - - web resource - - Information Resources that have at least one Web Representation and at least -a URI. - - - - - - - - - - - aggregated Cultural Heritage Object - This property associates an ORE aggregation with the Cultural Heritage -Object(s) (CHO for short) it is about. - - - - - - - - - - current location - The geographic location and/or name of the repository, building, site, or other entity whose boundaries presently include the resource. - - - - - - - - - - - - happened at - This property associates an event with the place at which the event -happened. - - - - - - - - - - has met - ens:hasMet relates a resource with the objects or phenomena that have happened to or have happened together with the resource under consideration. We can abstractly think of history and the present as a series of “meetings” between people and other things in space-time. Therefore we name this relationship as the things the object “has met” in the course of its existence. These meetings are events in the proper sense, in which other people and things participate in any role. - - - - - - - - - - - - - - - - - - - - has type - This property relates a resource with the concepts it belongs to in a suitable -type system such as MIME or any thesaurus that captures categories of objects in a given field (e.g., the “Objects” facet in Getty’s Art and Architecture Thesaurus). It does not capture aboutness. - - - - - - - - - - - - - - - - - - - - - - has view - This property relates a ORE aggregation about a CHO with a web resource -providing a view of that CHO. Examples of view are: a thumbnail, a textual -abstract and a table of contents. The ORE aggregation may be a Europeana -Aggregation, in which case the view is an object owned by Europeana (i.e., an instance of ens:EuropeanaObject) or an aggregation contributed by a content provider. In order to capture both these cases, the domain of ens:hasView is ore:Aggregation and its range is ens:WebResource - - - - - - - - - incorporates - This property captures the use of some resource to add value to another -resource. Such resources may be nested, such as performing a theater play text, and then recording the performance, or creating an artful edition of a collection of poems or just aggregating various poems in an anthology. There may be no single part that contains ultimately the incorporated object, which may be dispersed in the presentation. Therefore, incorporated resources do in general not form proper parts. Incorporated resources are not part of the same resource, but are taken from other resources, and have an independent history. Therefore ens:incorporates is not a sub-property of dcterm:hasPart. - - - - - - - is annotation of - This property relates an annotation (a Europeana object) with the resource -that it annotates. - - - - - - - - - - - - - - - - - - - - - is derivative of - This property captures a narrower notion of derivation than ens:isSimilarTo, in the sense that it relates a resource to another one, obtained by reworking, reducing, expanding, parts or the whole contents of the former, and possibly adding some minor parts. Versions have an even narrower meaning, in that it requires common identity between the related resources. Translations, summaries, abstractions etc. do not qualify as versions, but do qualify as derivatives. - - - - - - - - - is next in sequence - ens:isNextInSequence relates two resources R and S that are ordered parts of the same resource A, and such that R comes immediately after S in the order created by their being parts of A. - - - - - - - - - is related to - ens:isRelatedTo is the most general contextual property in EDM. Contextual -properties have typically to do either with the things that have happened to or together with the object under consideration, or what the object refers to by its shape, form or features in a figural or encoded form. For sake of simplicity, we include in the contextual relationships also the scholarly classification, which may have either to do with the role and cultural connections of the object in the past, or its kind of structure, substance or contents as it can be verified at present. - - - - - - - - - - - - - - - - - - - is representation of - This property associates an information resource to the resource (if any) that it represents - - - - - - - - is similar to - The most generic derivation property, covering also the case of questionable derivation. Is Similar To asserts that parts of the contents of one resource exhibit common features with respect to ideas, shapes, structures, colors, words, plots, topics with the contents of the related resource. Those common features may be attributed to a common origin or influence (in particular for derivation), but also to more generic cultural or psychological factors. - - - - - - - - - - is successor of - This property captures the relation between the continuation of a resource and that resource. This applies to a story, a serial, a journal etc. No content of the successor resource is identical or has a similar form with that of the precursor. The similarity is only in the context, subjects and figures of a plot. Successors typically form part of a common whole – such as a trilogy, a journal, etc. - - - - - - - - landing page - This property captures the relation between an aggregation representing a -Cultural Heritage Object and the Web Resource representing that Object on -the provider’s web site. - - - - - - - - - occured at - This property associates an event to the smallest known time span that -overlaps with the occurrence of that event - - - - - - - - - - realizes - A relation between an information realization and the information resource it -realizes. - - - - - - - - - - - was present at - This property associates the people, things or information resources with an event at which they were present - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Country - - - - - - - - - - - - - - - - - - - Europeana Data Provider - This element is specifically included to allow the name of the organisation who supplies data to Europeana indirectly via an aggregator to be recorded and displayed in the portal. Aggregator names are recorded in ens:provider. If an organisation provides data directly to Europeana (i.e. not via an aggregator) the values in ens:dataProvider and ens:provider will be the same. Organisation names should be provided as an ordinary text string until the Europeana Authority File for Organisations has been established. At that point providers will be able to send an identifier from the file instead of a text string. The name provided should be the preferred form of the name in the language the provider chooses as the default language for display in the portal. Countries with multiple languages may prefer to concatenate the name in more than one language (See the example below.) Note: Europeana Data Provider is not necessarily the institution where the physical object is located. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - is shown at - An unambiguous URL reference to the digital object on the provider’s web site in its full information context. - - - - - - - - is shown by - An unambiguous URL reference to the digital object on the provider’s web site in the best available resolution/quality. - - - - - - - - - - - - - - - - - - - - Europeana language - A language assigned to the resource with reference to the Provider. - The recommended best practice is to use a controlled vocabulary such as -RFC 4646 (http://www.rfc-archive.org/getrfc.php?rfc=4646) which, in -conjunction with ISO 639, defines two- and three-letter primary language tags. Either a coded value or text string can be represented here. - - - - - - object - The URL of a thumbnail representing the digital object or, if there is no such -thumbnail, the URL of the digital object in the best resolution available on the -web site of the data provider from which a thumbnail could be generated. This will often be the same URL as given in ens:isShownBy. - - - - - - - - Europeana provider - Name of the organization that delivers data to Europeana - - - - - - - - - - - - - - - - - - - - - - - - - - Europeana rights - Information about copyright of the digital object as specified by isShownBy -and isShownAt - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Europeana type - The Europeana material type of the resource - - - - - TEXT - - IMAGE - - SOUND - - VIDEO - - - - - - - - - - - - - - Unstored - This is a container element which includes all relevant information that -otherwise cannot be mapped to another element in the ESE. - - - - - - - Europeana URI - This is a tag created by a user through the Europeana interface. - - - - - - User Tag - This is a tag created by a user through the Europeana interface. - - - - - - - Europeana Year - A point of time associated with an event in the life of the original analog or -born digital object. - - - - \ No newline at end of file diff --git a/metis-schema/src/main/resources/rdf_examples/EDM-v523-120123.n3 b/metis-schema/src/main/resources/rdf_examples/EDM-v523-120123.n3 deleted file mode 100644 index ed67f6343..000000000 --- a/metis-schema/src/main/resources/rdf_examples/EDM-v523-120123.n3 +++ /dev/null @@ -1,576 +0,0 @@ -@prefix dc: . -@prefix DOLCE-Lite: . -@prefix abc: . -@prefix edm: . -@prefix wgs84_pos: . -@prefix foaf: . -@prefix dcmitype: . -@prefix ore: . -@prefix dcterms: . -@prefix rdfs: . -@prefix frbr_core: . -@prefix xsd: . -@prefix owl: . -@prefix rdf: . -@prefix crm: . -@prefix skos: . - - a owl:Ontology ; - dc:title "Europeana Data Model (EDM) vocabulary"@en ; - owl:versionInfo "5.2.3" ; - dc:contributor "Participants of Europeana Version 1.0 Work Package on Further Specification of Functionality and Interoperability aspects of Europeana (WP3)" ; - dc:description "The Europeana Data Model (EDM) is aimed at being an integration medium for collecting, connecting and enriching the descriptions provided by Europeana data providers. The RDF vocabulary for http://www.europeana.eu/schemas/edm/ defines the elements introduced by EDM (as opposed to the ones EDM re-uses from other namespaces)."@en ; - rdfs:comment "The present specification is based on the document \"Definition of the Europeana Data Model elements\", originally edited by Carlo Meghini. It is aligned with the version 5.2.3 of these EDM Definitions."@en ; - dc:creator "Antoine Isaac" ; - dc:contributor "Nasos Drosopoulos" , "Vassilis Tzouvaras" , "Julia Iwanowa" , "Hugo Borbinhas" , "Martin Doerr" ; - rdfs:seeAlso , ; - rdfs:comment """======= -Changes between EDM-v53-120123.owl -and EDM-v52-100730.owl -======= -1. Update of formal specification to v5.2.3 -2. Changed CRM namespace -3. Added or changed formal axioms for Event, aggregatedCHO, currentLocation, hasMet, hasType, isAnnotationOf -4. Property labels upper-cased -5. Added language tags -6. Checked Martin's latest mappings to CRM -7. Added metadata on the owl:Ontology instance -======= -Remaining TODOs for EDM-v53-120123.owl -======= -- change FRBR namespace and mappings using http://metadataregistry.org/schema/show/id/5.html or FRBRoo -- add Martin's mapping to FRBR(oo) for edm:incorporates (equivalent to crm:R14F.incorporates) and edm:WebResource (subclass of F24 Publication Expression) -- try to capture formal cardinality constraints resulting from \"Obligation and Occurrence\" documentation, which should be attached to non-EDM constructs (esp. ore:Aggregation) -- continue adding documentation values (skos:scopeNote, skos:example, etc, according to 1.), starting from edm:InformationResource. Add all Europeana examples and rationale notes for non-EDM constructs -- keep specific EDM-doc properties for \"rationale\" and \"obligation and occurrence\". Use skos:definition for \"Europeana definition\", skos:example for \"Example\", skos:note for \"Europeana note\""""@en . - -edm:Agent a owl:Class ; - rdfs:label "Agent"@en ; - skos:definition """This class comprises people, either individually or in groups, who have the -potential to perform intentional actions for which they can be held responsible.""" ; - skos:example "Leonardo da Vinci, the British Museum, W3C" ; - rdfs:subClassOf edm:NonInformationResource ; - owl:equivalentClass ; - skos:scopeNote "Rationale: This class is a domain of edm:wasPresentAt" . - -edm:EuropeanaAggregation a owl:Class ; - rdfs:label "Europeana Aggregation" ; - skos:definition """The set of resources related to a single Cultural Heritage Object that -collectively represent that object in Europeana. Such set consists of: all -descriptions about the object that Europeana collects from (possibly different) content providers, including thumbnails and other forms of abstractions, as well as of the description of the object Europeana builds.""" ; - rdfs:subClassOf edm:EuropeanaObject , ore:Aggregation ; - owl:equivalentClass _:node16k3ng1n7x1 . - -_:node16k3ng1n7x1 a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty edm:aggregatedCHO . - -edm:EuropeanaAggregation skos:note "An instance of EuropeanaAggregation is created at ingestion time for each different Cultural Heritage Object recognized by Europeana. Such instance is associated to the Cultural Heritage Object that it is about, by the property edm:aggregatedCHO"@en ; - skos:scopeNote "Obligation and Occurence: The relation between the Cultural Heritage Objects represented in Europeana and the instances of the class EuropeanaAggregation is one-to-one, in the data maintained by Europeana: every Cultural Heritage Object is represented by an instance of EuropeanaAggregation, and every instance of EuropeanaAggregation represent a Cultural Heritage Object."@en ; - skos:example "The painting Mona Lisa is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en , "The journal \"Le Temps\" is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en , "The 56th issue of \"Le Temps\" is a (different) Cultural Heritage Object represented in Europeana by another EuropeanaAggregation instance"@en ; - skos:scopeNote "Rationale: This class is used in Europeana to gather in a single conceptual unit all the information about a Cultural Heritage Object, necessary for all operations on these objects."@en . - -edm:EuropeanaObject a owl:Class ; - rdfs:label "Europeana Object"@en ; - skos:definition "Any object that is the result of Europeana’s activities"@en ; - rdfs:subClassOf edm:WebResource ; - skos:example "Any instance of the class EuropeanaAggregation"@en , "An annotation created by a user through the Europeana portal"@en , "Any content created by the users through the service made available by Europeana for that purpose"@en ; - skos:scopeNote "Rationale: This class is used to tag objects that are the result of activity of Europeana, and, as such, objects on which Europeana holds rights"@en . - -edm:Event a owl:Class ; - rdfs:label "Event"@en ; - skos:definition """An event is a change \"of states in cultural, social or physical systems, - regardless of scale, brought about by a series or group of coherent physical, -cultural, technological or legal phenomena\" (E5 Event in CIDOC CRM) or a \"set of coherent phenomena or cultural manifestations bounded in time and space\" (E4 Period in CIDOC CRM)"""@en ; - rdfs:subClassOf edm:NonInformationResource ; - owl:equivalentClass , abc:Temporality , frbr_core:Event ; - rdfs:subClassOf _:node16k3ng1n7x2 . - -_:node16k3ng1n7x2 a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty edm:happenedAt . - -edm:Event skos:note "Events are identified either by the content provider or by Europeana enrichment at ingestion time"@en ; - skos:example "the act of painting Mona Lisa"@en , "the 2nd World War"@en , "the change of custody of Mona Lisa"@en ; - skos:scopeNote "Rationale:This class is a domain of edm:happenedAt and the domain of edm:occurredAt"@en . - -edm:InformationResource a owl:Class ; - rdfs:label "Information Resource"@en ; - skos:definition "An information resource is a resource whose essential characteristics can be conveyed in a single message. It can be associated with a URI, it can have a representation, for example: a text is an InformationResource."@en ; - owl:equivalentClass , _:node16k3ng1n7x3 . - -_:node16k3ng1n7x3 a owl:Class ; - owl:unionOf _:node16k3ng1n7x4 . - -_:node16k3ng1n7x4 rdf:first frbr_core:Work ; - rdf:rest _:node16k3ng1n7x5 . - -_:node16k3ng1n7x5 rdf:first frbr_core:Expression ; - rdf:rest _:node16k3ng1n7x6 . - -_:node16k3ng1n7x6 rdf:first frbr_core:Manifestation ; - rdf:rest rdf:nil . - -edm:NonInformationResource a owl:Class ; - rdfs:label "Non-Information Resource"@en ; - skos:definition "All resources that are not information resources."@en ; - owl:complementOf edm:InformationResource . - -edm:PhysicalThing a owl:Class ; - rdfs:label "Physical Thing"@en ; - skos:definition """A persistent physical item such as a painting, a building, a book or a stone. -Persons are not items. This class represents Cultural Heritage Objects known to Europeana to be physical things (such as Mona Lisa) as well as all physical things Europeana refers to in the descriptions of Cultural Heritage Objects (such as the Rosetta Stone)."""@en ; - rdfs:subClassOf edm:NonInformationResource ; - owl:equivalentClass . - -edm:Place a owl:Class ; - rdfs:label "Place"@en ; - skos:definition "An \"extent in space, in particular on the surface of the earth, in the pure sense of physics: independent from temporal phenomena and matter\" (CIDOC CRM)"@en ; - rdfs:subClassOf edm:NonInformationResource ; - owl:equivalentClass DOLCE-Lite:space-region , abc:Place , , frbr_core:Place . - -edm:ProvidedCHO a owl:Class ; - rdfs:label "Provided CHO"@en ; - skos:definition "This class comprises the Cultural Heritage objects that Europeana collects descriptions about."@en ; - rdfs:subClassOf _:node16k3ng1n7x7 . - -_:node16k3ng1n7x7 a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty _:node16k3ng1n7x8 . - -_:node16k3ng1n7x8 a rdf:Property ; - owl:inverseOf edm:aggregatedCHO . - -edm:ProvidedCHO skos:note "This class has been mostly motivated by the need to assign a type to the “central node” in the EDM pattern, during the ingestion process, related to the XML expression of EDM at that stage. It was especially intended to fit the cases where edm:PhysicalThing cannot be used as the type of the resource standing for the real-world object (independently of any specific data contributor perspective)."@en ; - skos:example "Mona Lisa, Winged Victory of Samothrace"@en ; - skos:scopeNote "Rationale: This class is the range of edm:aggregatedCHO. A resource of type ProvidedCHO can be the subject of statements using edm:isRelatedTo or any more specific property."@en . - -edm:TimeSpan a owl:Class ; - rdfs:label "Time Span"@en ; - skos:definition """The class of \"abstract temporal extents, in the sense of Galilean physics, - having a beginning, an end and a duration\" (CIDOC CRM)"""@en ; - rdfs:subClassOf edm:NonInformationResource , dcterms:PeriodOfTime ; - owl:equivalentClass abc:Time , DOLCE-Lite:time-interval , . - -edm:WebResource a owl:Class ; - rdfs:label "Web Resource"@en ; - skos:definition """Information Resources that have at least one Web Representation and at least -a URI."""@en ; - rdfs:subClassOf edm:InformationResource . - -edm:aggregatedCHO a owl:ObjectProperty ; - rdfs:label "Aggregated Cultural Heritage Object"@en ; - skos:definition """This property associates an ORE aggregation with the Cultural Heritage -Object(s) (CHO for short) it is about."""@en ; - rdfs:subPropertyOf ore:aggregates , dc:subject , ; - rdfs:domain ore:Aggregation ; - rdfs:range edm:ProvidedCHO . - -edm:currentLocation a owl:ObjectProperty ; - rdfs:label "Current Location"@en ; - skos:definition "The geographic location and/or name of the repository, building, site, or other entity whose boundaries presently include the resource."@en ; - rdfs:subPropertyOf dcterms:spatial ; - owl:equivalentProperty wgs84_pos:location , ; - rdfs:domain _:node16k3ng1n7x9 . - -_:node16k3ng1n7x9 a owl:Class ; - owl:unionOf _:node16k3ng1n7x10 . - -_:node16k3ng1n7x10 rdf:first edm:ProvidedCHO ; - rdf:rest _:node16k3ng1n7x12 . - -_:node16k3ng1n7x12 rdf:first _:node16k3ng1n7x11 . - -_:node16k3ng1n7x11 a owl:Class ; - owl:intersectionOf _:node16k3ng1n7x13 . - -_:node16k3ng1n7x13 rdf:first ore:Proxy ; - rdf:rest _:node16k3ng1n7x15 . - -_:node16k3ng1n7x15 rdf:first _:node16k3ng1n7x14 . - -_:node16k3ng1n7x14 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom edm:ProvidedCHO . - -_:node16k3ng1n7x15 rdf:rest rdf:nil . - -_:node16k3ng1n7x12 rdf:rest rdf:nil . - -edm:currentLocation rdfs:range edm:Place . - -edm:happenedAt a owl:ObjectProperty ; - rdfs:label "Happened At"@en ; - skos:definition """This property associates an event with the place at which the event -happened."""@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - rdfs:domain edm:Event ; - rdfs:range edm:Place . - -edm:hasMet a rdf:Property ; - rdfs:label "Has Met"@en ; - skos:definition "edm:hasMet relates a resource with the objects or phenomena that have happened to or have happened together with the resource under consideration. We can abstractly think of history and the present as a series of “meetings” between people and other things in space-time. Therefore we name this relationship as the things the object “has met” in the course of its existence. These meetings are events in the proper sense, in which other people and things participate in any role."@en ; - rdfs:subPropertyOf dc:relation ; - rdfs:domain _:node16k3ng1n7x16 . - -_:node16k3ng1n7x16 a owl:Class ; - owl:unionOf _:node16k3ng1n7x17 . - -_:node16k3ng1n7x17 rdf:first edm:ProvidedCHO ; - rdf:rest _:node16k3ng1n7x19 . - -_:node16k3ng1n7x19 rdf:first _:node16k3ng1n7x18 . - -_:node16k3ng1n7x18 a owl:Class ; - owl:intersectionOf _:node16k3ng1n7x20 . - -_:node16k3ng1n7x20 rdf:first ore:Proxy ; - rdf:rest _:node16k3ng1n7x22 . - -_:node16k3ng1n7x22 rdf:first _:node16k3ng1n7x21 . - -_:node16k3ng1n7x21 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom edm:ProvidedCHO . - -_:node16k3ng1n7x22 rdf:rest rdf:nil . - -_:node16k3ng1n7x19 rdf:rest rdf:nil . - -edm:hasType a owl:ObjectProperty ; - rdfs:label "Has Type"@en ; - skos:definition """This property relates a resource with the concepts it belongs to in a suitable -type system such as MIME or any thesaurus that captures categories of objects in a given field (e.g., the “Objects” facet in Getty’s Art and Architecture Thesaurus). It does not capture aboutness."""@en ; - rdfs:subPropertyOf edm:isRelatedTo ; - owl:equivalentProperty ; - rdfs:domain _:node16k3ng1n7x23 . - -_:node16k3ng1n7x23 a owl:Class ; - owl:unionOf _:node16k3ng1n7x24 . - -_:node16k3ng1n7x24 rdf:first edm:ProvidedCHO ; - rdf:rest _:node16k3ng1n7x26 . - -_:node16k3ng1n7x26 rdf:first _:node16k3ng1n7x25 . - -_:node16k3ng1n7x25 a owl:Class ; - owl:intersectionOf _:node16k3ng1n7x27 . - -_:node16k3ng1n7x27 rdf:first ore:Proxy ; - rdf:rest _:node16k3ng1n7x29 . - -_:node16k3ng1n7x29 rdf:first _:node16k3ng1n7x28 . - -_:node16k3ng1n7x28 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom edm:ProvidedCHO . - -_:node16k3ng1n7x29 rdf:rest rdf:nil . - -_:node16k3ng1n7x26 rdf:rest rdf:nil . - -edm:hasType rdfs:range edm:NonInformationResource . - -edm:hasView a owl:ObjectProperty ; - rdfs:label "Has View"@en ; - skos:definition """This property relates a ORE aggregation about a CHO with a web resource -providing a view of that CHO. Examples of view are: a thumbnail, a textual -abstract and a table of contents. The ORE aggregation may be a Europeana -Aggregation, in which case the view is an object owned by Europeana (i.e., an instance of edm:EuropeanaObject) or an aggregation contributed by a content provider. In order to capture both these cases, the domain of edm:hasView is ore:Aggregation and its range is edm:WebResource"""@en ; - rdfs:subPropertyOf ore:aggregates ; - rdfs:domain ore:Aggregation ; - rdfs:range edm:WebResource . - -edm:incorporates a owl:ObjectProperty ; - rdfs:label "Incorporates"@en ; - skos:definition """This property captures the use of some resource to add value to another -resource. Such resources may be nested, such as performing a theater play text, and then recording the performance, or creating an artful edition of a collection of poems or just aggregating various poems in an anthology. There may be no single part that contains ultimately the incorporated object, which may be dispersed in the presentation. Therefore, incorporated resources do in general not form proper parts. Incorporated resources are not part of the same resource, but are taken from other resources, and have an independent history. Therefore edm:incorporates is not a sub-property of dcterm:hasPart."""@en ; - rdfs:subPropertyOf edm:isSimilarTo . - -edm:isAnnotationOf a owl:ObjectProperty ; - rdfs:label "Is Annotation Of"@en ; - skos:definition """This property relates an annotation (a Europeana object) with the resource -that it annotates."""@en ; - rdfs:subPropertyOf dc:subject , ; - rdfs:domain edm:EuropeanaObject ; - rdfs:range _:node16k3ng1n7x30 . - -_:node16k3ng1n7x30 a owl:Class ; - owl:unionOf _:node16k3ng1n7x31 . - -_:node16k3ng1n7x31 rdf:first edm:ProvidedCHO ; - rdf:rest _:node16k3ng1n7x33 . - -_:node16k3ng1n7x33 rdf:first _:node16k3ng1n7x32 . - -_:node16k3ng1n7x32 a owl:Class ; - owl:intersectionOf _:node16k3ng1n7x34 . - -_:node16k3ng1n7x34 rdf:first ore:Proxy ; - rdf:rest _:node16k3ng1n7x36 . - -_:node16k3ng1n7x36 rdf:first _:node16k3ng1n7x35 . - -_:node16k3ng1n7x35 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom edm:ProvidedCHO . - -_:node16k3ng1n7x36 rdf:rest rdf:nil . - -_:node16k3ng1n7x33 rdf:rest rdf:nil . - -edm:isDerivativeOf a rdf:Property ; - rdfs:label "Is Derivative Of"@en ; - skos:definition "This property captures a narrower notion of derivation than edm:isSimilarTo, in the sense that it relates a resource to another one, obtained by reworking, reducing, expanding, parts or the whole contents of the former, and possibly adding some minor parts. Versions have an even narrower meaning, in that it requires common identity between the related resources. Translations, summaries, abstractions etc. do not qualify as versions, but do qualify as derivatives."@en ; - rdfs:subPropertyOf edm:isSimilarTo . - -edm:isNextInSequence a owl:ObjectProperty ; - rdfs:label "Is Next In Sequence"@en ; - skos:definition "edm:isNextInSequence relates two resources S and R that are ordered parts of the same resource A, and such that R comes immediately after R in the order created by their being parts of S."@en ; - rdfs:subPropertyOf dc:relation . - -edm:isRelatedTo a rdf:Property ; - rdfs:label "Is Related To"@en ; - skos:definition """edm:isRelatedTo is the most general contextual property in EDM. Contextual -properties have typically to do either with the things that have happened to or together with the object under consideration, or what the object refers to by its shape, form or features in a figural or encoded form. For sake of simplicity, we include in the contextual relationships also the scholarly classification, which may have either to do with the role and cultural connections of the object in the past, or its kind of structure, substance or contents as it can be verified at present."""@en ; - rdfs:domain _:node16k3ng1n7x37 . - -_:node16k3ng1n7x37 a owl:Class ; - owl:unionOf _:node16k3ng1n7x38 . - -_:node16k3ng1n7x38 rdf:first edm:ProvidedCHO ; - rdf:rest _:node16k3ng1n7x40 . - -_:node16k3ng1n7x40 rdf:first _:node16k3ng1n7x39 . - -_:node16k3ng1n7x39 a owl:Class ; - owl:intersectionOf _:node16k3ng1n7x41 . - -_:node16k3ng1n7x41 rdf:first ore:Proxy ; - rdf:rest _:node16k3ng1n7x43 . - -_:node16k3ng1n7x43 rdf:first _:node16k3ng1n7x42 . - -_:node16k3ng1n7x42 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom edm:ProvidedCHO . - -_:node16k3ng1n7x43 rdf:rest rdf:nil . - -_:node16k3ng1n7x40 rdf:rest rdf:nil . - -edm:isRepresentationOf a owl:ObjectProperty ; - rdfs:label "Is Representation Of"@en ; - skos:definition "This property associates an information resource to the resource (if any) that it represents"@en ; - rdfs:subPropertyOf dc:subject ; - rdfs:domain edm:InformationResource ; - rdfs:subPropertyOf . - -edm:isSimilarTo a rdf:Property ; - rdfs:label "Is Similar To"@en ; - skos:definition "The most generic derivation property, covering also the case of questionable derivation. Is Similar To asserts that parts of the contents of one resource exhibit common features with respect to ideas, shapes, structures, colors, words, plots, topics with the contents of the related resource. Those common features may be attributed to a common origin or influence (in particular for derivation), but also to more generic cultural or psychological factors."@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty . - -edm:isSuccessorOf a owl:ObjectProperty ; - rdfs:label "Is Successor Of"@en ; - skos:definition "This property captures the relation between the continuation of a resource and that resource. This applies to a story, a serial, a journal etc. No content of the successor resource is identical or has a similar form with that of the precursor. The similarity is only in the context, subjects and figures of a plot. Successors typically form part of a common whole – such as a trilogy, a journal, etc."@en ; - rdfs:subPropertyOf edm:isSimilarTo . - -edm:landingPage a owl:ObjectProperty ; - rdfs:label "Landing Page"@en ; - skos:definition """This property captures the relation between an aggregation representing a -Cultural Heritage Object and the Web Resource representing that Object on -the provider’s web site."""@en ; - rdfs:subPropertyOf ore:aggregates ; - rdfs:range edm:WebResource . - -edm:occurredAt a owl:ObjectProperty ; - rdfs:label "Occured At"@en ; - skos:definition """This property associates an event to the smallest known time span that -overlaps with the occurrence of that event"""@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - rdfs:domain edm:Event ; - rdfs:range edm:TimeSpan . - -edm:realizes a owl:ObjectProperty ; - rdfs:label "Realizes"@en ; - skos:definition "This property describes a relation between a physical thing and the information resource that is contained in it, visible at it or otherwise carried by it, if applicable."@en ; - rdfs:subPropertyOf edm:isRelatedTo ; - owl:equivalentProperty ; - rdfs:domain edm:PhysicalThing ; - rdfs:range edm:InformationResource . - -edm:wasPresentAt a owl:ObjectProperty ; - rdfs:label "Was Present At"@en ; - skos:definition "This property associates the people, things or information resources with an event at which they were present"@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - rdfs:domain _:node16k3ng1n7x44 . - -_:node16k3ng1n7x44 a owl:Class ; - owl:unionOf _:node16k3ng1n7x45 . - -_:node16k3ng1n7x45 rdf:first edm:Agent ; - rdf:rest _:node16k3ng1n7x46 . - -_:node16k3ng1n7x46 rdf:first edm:InformationResource ; - rdf:rest _:node16k3ng1n7x47 . - -_:node16k3ng1n7x47 rdf:first edm:PhysicalThing ; - rdf:rest rdf:nil . - -edm:wasPresentAt rdfs:range edm:Event . - -edm:country a rdf:Property ; - rdfs:label "Country"@en ; - rdfs:subPropertyOf . - -edm:dataProvider a rdf:Property ; - rdfs:label "Europeana Data Provider"@en ; - skos:definition "This element is specifically included to allow the name of the organisation who supplies data to Europeana indirectly via an aggregator to be recorded and displayed in the portal. Aggregator names are recorded in edm:provider. If an organisation provides data directly to Europeana (i.e. not via an aggregator) the values in edm:dataProvider and edm:provider will be the same. Organisation names should be provided as an ordinary text string until the Europeana Authority File for Organisations has been established. At that point providers will be able to send an identifier from the file instead of a text string. The name provided should be the preferred form of the name in the language the provider chooses as the default language for display in the portal. Countries with multiple languages may prefer to concatenate the name in more than one language (See the example below.) Note: Europeana Data Provider is not necessarily the institution where the physical object is located."@en ; - rdfs:subPropertyOf dcterms:provenance ; - rdfs:domain ore:Aggregation ; - rdfs:range edm:Agent . - -edm:isShownAt a owl:ObjectProperty ; - rdfs:label "Is Shown At"@en ; - skos:definition "An unambiguous URL reference to the digital object on the provider’s web site in its full information context."@en ; - rdfs:subPropertyOf edm:landingPage ; - rdfs:range edm:WebResource . - -edm:isShownBy a owl:ObjectProperty ; - rdfs:label "Is Shown By"@en ; - skos:definition "An unambiguous URL reference to the digital object on the provider’s web site in the best available resolution/quality."@en ; - rdfs:subPropertyOf edm:hasView ; - rdfs:range edm:WebResource . - -edm:language a rdf:Property ; - rdfs:label "Europeana Language"@en ; - skos:definition "A language assigned to the resource with reference to the Provider."@en ; - rdfs:comment """The recommended best practice is to use a controlled vocabulary such as -RFC 4646 (http://www.rfc-archive.org/getrfc.php?rfc=4646) which, in -conjunction with ISO 639, defines two- and three-letter primary language tags. Either a coded value or text string can be represented here."""@en . - -edm:object a owl:ObjectProperty ; - rdfs:label "Object"@en ; - skos:definition """The URL of a thumbnail representing the digital object or, if there is no such -thumbnail, the URL of the digital object in the best resolution available on the -web site of the data provider from which a thumbnail could be generated. This will often be the same URL as given in edm:isShownBy."""@en ; - rdfs:subPropertyOf edm:hasView ; - rdfs:range edm:WebResource . - -edm:provider a rdf:Property ; - rdfs:label "Europeana Provider"@en ; - skos:definition "Name of the organization that delivers data to Europeana"@en ; - rdfs:subPropertyOf edm:hasMet ; - rdfs:range edm:Agent . - -edm:rights a owl:ObjectProperty ; - rdfs:label "Europeana Rights"@en ; - skos:definition """Information about copyright of the digital object as specified by isShownBy -and isShownAt"""@en . - -edm:type a owl:DatatypeProperty ; - rdfs:label "Europeana Type"@en ; - skos:definition "The Europeana material type of the resource"@en ; - rdfs:subPropertyOf dc:type ; - rdfs:range _:node16k3ng1n7x48 . - -_:node16k3ng1n7x48 a rdfs:Datatype ; - owl:oneOf _:node16k3ng1n7x49 . - -_:node16k3ng1n7x49 rdf:first "TEXT" ; - rdf:rest _:node16k3ng1n7x50 . - -_:node16k3ng1n7x50 rdf:first "IMAGE" ; - rdf:rest _:node16k3ng1n7x51 . - -_:node16k3ng1n7x51 rdf:first "SOUND" ; - rdf:rest _:node16k3ng1n7x52 . - -_:node16k3ng1n7x52 rdf:first "VIDEO" ; - rdf:rest _:node16k3ng1n7x53 . - -_:node16k3ng1n7x53 rdf:first "3D" ; - rdf:rest rdf:nil . - -edm:ugc a owl:DatatypeProperty ; - rdfs:label "UGC"@en ; - skos:definition "This element is used to identify user generated content (also called user created content). It should be applied to all digitised or born digital content contributed by the general public and collected by Europeana through a crowdsourcing initiative or project."@en ; - rdfs:range _:node16k3ng1n7x54 . - -_:node16k3ng1n7x54 a rdfs:Datatype ; - owl:oneOf _:node16k3ng1n7x55 . - -_:node16k3ng1n7x55 rdf:first "TRUE" ; - rdf:rest rdf:nil . - -edm:unstored a rdf:Property ; - rdfs:label "Unstored"@en ; - skos:definition """This is a container element which includes all relevant information that -otherwise cannot be mapped to another element in the ESE."""@en . - -edm:uri a owl:ObjectProperty ; - rdfs:label "Europeana URI"@en ; - skos:definition "This is a tag created by a user through the Europeana interface."@en . - -edm:userTag a rdf:Property ; - rdfs:label "User Tag"@en ; - skos:definition "This is a tag created by a user through the Europeana interface."@en ; - rdfs:subPropertyOf dc:description . - -edm:year a rdf:Property ; - rdfs:label "Europeana Year"@en ; - skos:definition """A point of time associated with an event in the life of the original analog or -born digital object."""@en ; - rdfs:subPropertyOf dcterms:temporal . - -skos:Concept rdfs:subClassOf edm:NonInformationResource . - -dc:contributor rdfs:subPropertyOf edm:hasMet . - -dc:coverage rdfs:subPropertyOf edm:hasMet . - -dc:creator rdfs:subPropertyOf edm:hasMet . - -dc:date rdfs:subPropertyOf edm:hasMet . - -dc:format rdfs:subPropertyOf edm:hasType . - -dcterms:hasFormat rdfs:subPropertyOf _:node16k3ng1n7x56 . - -_:node16k3ng1n7x56 a rdf:Property ; - owl:inverseOf edm:isDerivativeOf . - -dcterms:hasVersion rdfs:subPropertyOf _:node16k3ng1n7x57 . - -_:node16k3ng1n7x57 a rdf:Property ; - owl:inverseOf edm:isDerivativeOf . - -dcterms:isFormatOf rdfs:subPropertyOf edm:isDerivativeOf . - -dcterms:isReplacedBy rdfs:subPropertyOf _:node16k3ng1n7x58 . - -_:node16k3ng1n7x58 a rdf:Property ; - owl:inverseOf edm:isDerivativeOf . - -dcterms:isVersionOf rdfs:subPropertyOf edm:isDerivativeOf . - -dc:language rdfs:subPropertyOf edm:hasType . - -dc:publisher rdfs:subPropertyOf edm:hasMet . - -dc:relation rdfs:subPropertyOf edm:isRelatedTo . - -dcterms:replaces rdfs:subPropertyOf edm:isDerivativeOf . - -dc:source rdfs:subPropertyOf edm:isDerivativeOf . - -dc:subject rdfs:subPropertyOf edm:isRelatedTo . - -dcterms:tableOfContents rdfs:subPropertyOf edm:hasView . - -dc:type rdfs:subPropertyOf edm:hasType . diff --git a/metis-schema/src/main/resources/rdf_examples/EDM-v523-120123.nt b/metis-schema/src/main/resources/rdf_examples/EDM-v523-120123.nt deleted file mode 100644 index 0e24f34af..000000000 --- a/metis-schema/src/main/resources/rdf_examples/EDM-v523-120123.nt +++ /dev/null @@ -1,403 +0,0 @@ - . - "Europeana Data Model (EDM) vocabulary"@en . - "5.2.3" . - "Participants of Europeana Version 1.0 Work Package on Further Specification of Functionality and Interoperability aspects of Europeana (WP3)" . - "The Europeana Data Model (EDM) is aimed at being an integration medium for collecting, connecting and enriching the descriptions provided by Europeana data providers. The RDF vocabulary for http://www.europeana.eu/schemas/edm/ defines the elements introduced by EDM (as opposed to the ones EDM re-uses from other namespaces)."@en . - "The present specification is based on the document \"Definition of the Europeana Data Model elements\", originally edited by Carlo Meghini. It is aligned with the version 5.2.3 of these EDM Definitions."@en . - "Antoine Isaac" . - "Nasos Drosopoulos" . - "Vassilis Tzouvaras" . - "Julia Iwanowa" . - "Hugo Borbinhas" . - "Martin Doerr" . - . - . - "=======\nChanges between EDM-v53-120123.owl\nand EDM-v52-100730.owl\n=======\n1. Update of formal specification to v5.2.3\n2. Changed CRM namespace \n3. Added or changed formal axioms for Event, aggregatedCHO, currentLocation, hasMet, hasType, isAnnotationOf\n4. Property labels upper-cased\n5. Added language tags\n6. Checked Martin's latest mappings to CRM\n7. Added metadata on the owl:Ontology instance\n=======\nRemaining TODOs for EDM-v53-120123.owl\n=======\n- change FRBR namespace and mappings using http://metadataregistry.org/schema/show/id/5.html or FRBRoo\n- add Martin's mapping to FRBR(oo) for edm:incorporates (equivalent to crm:R14F.incorporates) and edm:WebResource (subclass of F24 Publication Expression)\n- try to capture formal cardinality constraints resulting from \"Obligation and Occurrence\" documentation, which should be attached to non-EDM constructs (esp. ore:Aggregation)\n- continue adding documentation values (skos:scopeNote, skos:example, etc, according to 1.), starting from edm:InformationResource. Add all Europeana examples and rationale notes for non-EDM constructs\n- keep specific EDM-doc properties for \"rationale\" and \"obligation and occurrence\". Use skos:definition for \"Europeana definition\", skos:example for \"Example\", skos:note for \"Europeana note\""@en . - . - "Agent"@en . - "This class comprises people, either individually or in groups, who have the\npotential to perform intentional actions for which they can be held responsible." . - "Leonardo da Vinci, the British Museum, W3C" . - . - . - "Rationale: This class is a domain of edm:wasPresentAt" . - . - "Europeana Aggregation" . - "The set of resources related to a single Cultural Heritage Object that\ncollectively represent that object in Europeana. Such set consists of: all\ndescriptions about the object that Europeana collects from (possibly different) content providers, including thumbnails and other forms of abstractions, as well as of the description of the object Europeana builds." . - . - . - _:node16k3ng1n7x1 . -_:node16k3ng1n7x1 . -_:node16k3ng1n7x1 "1"^^ . -_:node16k3ng1n7x1 . - "An instance of EuropeanaAggregation is created at ingestion time for each different Cultural Heritage Object recognized by Europeana. Such instance is associated to the Cultural Heritage Object that it is about, by the property edm:aggregatedCHO"@en . - "Obligation and Occurence: The relation between the Cultural Heritage Objects represented in Europeana and the instances of the class EuropeanaAggregation is one-to-one, in the data maintained by Europeana: every Cultural Heritage Object is represented by an instance of EuropeanaAggregation, and every instance of EuropeanaAggregation represent a Cultural Heritage Object."@en . - "The painting Mona Lisa is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en . - "The journal \"Le Temps\" is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en . - "The 56th issue of \"Le Temps\" is a (different) Cultural Heritage Object represented in Europeana by another EuropeanaAggregation instance"@en . - "Rationale: This class is used in Europeana to gather in a single conceptual unit all the information about a Cultural Heritage Object, necessary for all operations on these objects."@en . - . - "Europeana Object"@en . - "Any object that is the result of Europeana\u2019s activities"@en . - . - "Any instance of the class EuropeanaAggregation"@en . - "An annotation created by a user through the Europeana portal"@en . - "Any content created by the users through the service made available by Europeana for that purpose"@en . - "Rationale: This class is used to tag objects that are the result of activity of Europeana, and, as such, objects on which Europeana holds rights"@en . - . - "Event"@en . - "An event is a change \"of states in cultural, social or physical systems,\n regardless of scale, brought about by a series or group of coherent physical,\ncultural, technological or legal phenomena\" (E5 Event in CIDOC CRM) or a \"set of coherent phenomena or cultural manifestations bounded in time and space\" (E4 Period in CIDOC CRM)"@en . - . - . - . - . - _:node16k3ng1n7x2 . -_:node16k3ng1n7x2 . -_:node16k3ng1n7x2 "1"^^ . -_:node16k3ng1n7x2 . - "Events are identified either by the content provider or by Europeana enrichment at ingestion time"@en . - "the act of painting Mona Lisa"@en . - "the 2nd World War"@en . - "the change of custody of Mona Lisa"@en . - "Rationale:This class is a domain of edm:happenedAt and the domain of edm:occurredAt"@en . - . - "Information Resource"@en . - "An information resource is a resource whose essential characteristics can be conveyed in a single message. It can be associated with a URI, it can have a representation, for example: a text is an InformationResource."@en . - . - _:node16k3ng1n7x3 . -_:node16k3ng1n7x3 . -_:node16k3ng1n7x3 _:node16k3ng1n7x4 . -_:node16k3ng1n7x4 . -_:node16k3ng1n7x4 _:node16k3ng1n7x5 . -_:node16k3ng1n7x5 . -_:node16k3ng1n7x5 _:node16k3ng1n7x6 . -_:node16k3ng1n7x6 . -_:node16k3ng1n7x6 . - . - "Non-Information Resource"@en . - "All resources that are not information resources."@en . - . - . - "Physical Thing"@en . - "A persistent physical item such as a painting, a building, a book or a stone.\nPersons are not items. This class represents Cultural Heritage Objects known to Europeana to be physical things (such as Mona Lisa) as well as all physical things Europeana refers to in the descriptions of Cultural Heritage Objects (such as the Rosetta Stone)."@en . - . - . - . - "Place"@en . - "An \"extent in space, in particular on the surface of the earth, in the pure sense of physics: independent from temporal phenomena and matter\" (CIDOC CRM)"@en . - . - . - . - . - . - . - "Provided CHO"@en . - "This class comprises the Cultural Heritage objects that Europeana collects descriptions about."@en . - _:node16k3ng1n7x7 . -_:node16k3ng1n7x7 . -_:node16k3ng1n7x7 "1"^^ . -_:node16k3ng1n7x7 _:node16k3ng1n7x8 . -_:node16k3ng1n7x8 . -_:node16k3ng1n7x8 . - "This class has been mostly motivated by the need to assign a type to the \u201Ccentral node\u201D in the EDM pattern, during the ingestion process, related to the XML expression of EDM at that stage. It was especially intended to fit the cases where edm:PhysicalThing cannot be used as the type of the resource standing for the real-world object (independently of any specific data contributor perspective)."@en . - "Mona Lisa, Winged Victory of Samothrace"@en . - "Rationale: This class is the range of edm:aggregatedCHO. A resource of type ProvidedCHO can be the subject of statements using edm:isRelatedTo or any more specific property."@en . - . - "Time Span"@en . - "The class of \"abstract temporal extents, in the sense of Galilean physics,\n having a beginning, an end and a duration\" (CIDOC CRM)"@en . - . - . - . - . - . - . - "Web Resource"@en . - "Information Resources that have at least one Web Representation and at least\na URI."@en . - . - . - "Aggregated Cultural Heritage Object"@en . - "This property associates an ORE aggregation with the Cultural Heritage\nObject(s) (CHO for short) it is about."@en . - . - . - . - . - . - . - "Current Location"@en . - "The geographic location and/or name of the repository, building, site, or other entity whose boundaries presently include the resource."@en . - . - . - . - _:node16k3ng1n7x9 . -_:node16k3ng1n7x9 . -_:node16k3ng1n7x9 _:node16k3ng1n7x10 . -_:node16k3ng1n7x10 . -_:node16k3ng1n7x10 _:node16k3ng1n7x12 . -_:node16k3ng1n7x12 _:node16k3ng1n7x11 . -_:node16k3ng1n7x11 . -_:node16k3ng1n7x11 _:node16k3ng1n7x13 . -_:node16k3ng1n7x13 . -_:node16k3ng1n7x13 _:node16k3ng1n7x15 . -_:node16k3ng1n7x15 _:node16k3ng1n7x14 . -_:node16k3ng1n7x14 . -_:node16k3ng1n7x14 . -_:node16k3ng1n7x14 . -_:node16k3ng1n7x15 . -_:node16k3ng1n7x12 . - . - . - "Happened At"@en . - "This property associates an event with the place at which the event\nhappened."@en . - . - . - . - . - . - "Has Met"@en . - "edm:hasMet relates a resource with the objects or phenomena that have happened to or have happened together with the resource under consideration. We can abstractly think of history and the present as a series of \u201Cmeetings\u201D between people and other things in space-time. Therefore we name this relationship as the things the object \u201Chas met\u201D in the course of its existence. These meetings are events in the proper sense, in which other people and things participate in any role."@en . - . - _:node16k3ng1n7x16 . -_:node16k3ng1n7x16 . -_:node16k3ng1n7x16 _:node16k3ng1n7x17 . -_:node16k3ng1n7x17 . -_:node16k3ng1n7x17 _:node16k3ng1n7x19 . -_:node16k3ng1n7x19 _:node16k3ng1n7x18 . -_:node16k3ng1n7x18 . -_:node16k3ng1n7x18 _:node16k3ng1n7x20 . -_:node16k3ng1n7x20 . -_:node16k3ng1n7x20 _:node16k3ng1n7x22 . -_:node16k3ng1n7x22 _:node16k3ng1n7x21 . -_:node16k3ng1n7x21 . -_:node16k3ng1n7x21 . -_:node16k3ng1n7x21 . -_:node16k3ng1n7x22 . -_:node16k3ng1n7x19 . - . - "Has Type"@en . - "This property relates a resource with the concepts it belongs to in a suitable\ntype system such as MIME or any thesaurus that captures categories of objects in a given field (e.g., the \u201CObjects\u201D facet in Getty\u2019s Art and Architecture Thesaurus). It does not capture aboutness."@en . - . - . - _:node16k3ng1n7x23 . -_:node16k3ng1n7x23 . -_:node16k3ng1n7x23 _:node16k3ng1n7x24 . -_:node16k3ng1n7x24 . -_:node16k3ng1n7x24 _:node16k3ng1n7x26 . -_:node16k3ng1n7x26 _:node16k3ng1n7x25 . -_:node16k3ng1n7x25 . -_:node16k3ng1n7x25 _:node16k3ng1n7x27 . -_:node16k3ng1n7x27 . -_:node16k3ng1n7x27 _:node16k3ng1n7x29 . -_:node16k3ng1n7x29 _:node16k3ng1n7x28 . -_:node16k3ng1n7x28 . -_:node16k3ng1n7x28 . -_:node16k3ng1n7x28 . -_:node16k3ng1n7x29 . -_:node16k3ng1n7x26 . - . - . - "Has View"@en . - "This property relates a ORE aggregation about a CHO with a web resource\nproviding a view of that CHO. Examples of view are: a thumbnail, a textual\nabstract and a table of contents. The ORE aggregation may be a Europeana\nAggregation, in which case the view is an object owned by Europeana (i.e., an instance of edm:EuropeanaObject) or an aggregation contributed by a content provider. In order to capture both these cases, the domain of edm:hasView is ore:Aggregation and its range is edm:WebResource"@en . - . - . - . - . - "Incorporates"@en . - "This property captures the use of some resource to add value to another\nresource. Such resources may be nested, such as performing a theater play text, and then recording the performance, or creating an artful edition of a collection of poems or just aggregating various poems in an anthology. There may be no single part that contains ultimately the incorporated object, which may be dispersed in the presentation. Therefore, incorporated resources do in general not form proper parts. Incorporated resources are not part of the same resource, but are taken from other resources, and have an independent history. Therefore edm:incorporates is not a sub-property of dcterm:hasPart."@en . - . - . - "Is Annotation Of"@en . - "This property relates an annotation (a Europeana object) with the resource\nthat it annotates."@en . - . - . - . - _:node16k3ng1n7x30 . -_:node16k3ng1n7x30 . -_:node16k3ng1n7x30 _:node16k3ng1n7x31 . -_:node16k3ng1n7x31 . -_:node16k3ng1n7x31 _:node16k3ng1n7x33 . -_:node16k3ng1n7x33 _:node16k3ng1n7x32 . -_:node16k3ng1n7x32 . -_:node16k3ng1n7x32 _:node16k3ng1n7x34 . -_:node16k3ng1n7x34 . -_:node16k3ng1n7x34 _:node16k3ng1n7x36 . -_:node16k3ng1n7x36 _:node16k3ng1n7x35 . -_:node16k3ng1n7x35 . -_:node16k3ng1n7x35 . -_:node16k3ng1n7x35 . -_:node16k3ng1n7x36 . -_:node16k3ng1n7x33 . - . - "Is Derivative Of"@en . - "This property captures a narrower notion of derivation than edm:isSimilarTo, in the sense that it relates a resource to another one, obtained by reworking, reducing, expanding, parts or the whole contents of the former, and possibly adding some minor parts. Versions have an even narrower meaning, in that it requires common identity between the related resources. Translations, summaries, abstractions etc. do not qualify as versions, but do qualify as derivatives."@en . - . - . - "Is Next In Sequence"@en . - "edm:isNextInSequence relates two resources S and R that are ordered parts of the same resource A, and such that R comes immediately after R in the order created by their being parts of S."@en . - . - . - "Is Related To"@en . - "edm:isRelatedTo is the most general contextual property in EDM. Contextual\nproperties have typically to do either with the things that have happened to or together with the object under consideration, or what the object refers to by its shape, form or features in a figural or encoded form. For sake of simplicity, we include in the contextual relationships also the scholarly classification, which may have either to do with the role and cultural connections of the object in the past, or its kind of structure, substance or contents as it can be verified at present."@en . - _:node16k3ng1n7x37 . -_:node16k3ng1n7x37 . -_:node16k3ng1n7x37 _:node16k3ng1n7x38 . -_:node16k3ng1n7x38 . -_:node16k3ng1n7x38 _:node16k3ng1n7x40 . -_:node16k3ng1n7x40 _:node16k3ng1n7x39 . -_:node16k3ng1n7x39 . -_:node16k3ng1n7x39 _:node16k3ng1n7x41 . -_:node16k3ng1n7x41 . -_:node16k3ng1n7x41 _:node16k3ng1n7x43 . -_:node16k3ng1n7x43 _:node16k3ng1n7x42 . -_:node16k3ng1n7x42 . -_:node16k3ng1n7x42 . -_:node16k3ng1n7x42 . -_:node16k3ng1n7x43 . -_:node16k3ng1n7x40 . - . - "Is Representation Of"@en . - "This property associates an information resource to the resource (if any) that it represents"@en . - . - . - . - . - "Is Similar To"@en . - "The most generic derivation property, covering also the case of questionable derivation. Is Similar To asserts that parts of the contents of one resource exhibit common features with respect to ideas, shapes, structures, colors, words, plots, topics with the contents of the related resource. Those common features may be attributed to a common origin or influence (in particular for derivation), but also to more generic cultural or psychological factors."@en . - . - . - . - "Is Successor Of"@en . - "This property captures the relation between the continuation of a resource and that resource. This applies to a story, a serial, a journal etc. No content of the successor resource is identical or has a similar form with that of the precursor. The similarity is only in the context, subjects and figures of a plot. Successors typically form part of a common whole \u2013 such as a trilogy, a journal, etc."@en . - . - . - "Landing Page"@en . - "This property captures the relation between an aggregation representing a\nCultural Heritage Object and the Web Resource representing that Object on\nthe provider\u2019s web site."@en . - . - . - . - "Occured At"@en . - "This property associates an event to the smallest known time span that\noverlaps with the occurrence of that event"@en . - . - . - . - . - . - "Realizes"@en . - "This property describes a relation between a physical thing and the information resource that is contained in it, visible at it or otherwise carried by it, if applicable."@en . - . - . - . - . - . - "Was Present At"@en . - "This property associates the people, things or information resources with an event at which they were present"@en . - . - . - _:node16k3ng1n7x44 . -_:node16k3ng1n7x44 . -_:node16k3ng1n7x44 _:node16k3ng1n7x45 . -_:node16k3ng1n7x45 . -_:node16k3ng1n7x45 _:node16k3ng1n7x46 . -_:node16k3ng1n7x46 . -_:node16k3ng1n7x46 _:node16k3ng1n7x47 . -_:node16k3ng1n7x47 . -_:node16k3ng1n7x47 . - . - . - "Country"@en . - . - . - "Europeana Data Provider"@en . - "This element is specifically included to allow the name of the organisation who supplies data to Europeana indirectly via an aggregator to be recorded and displayed in the portal. Aggregator names are recorded in edm:provider. If an organisation provides data directly to Europeana (i.e. not via an aggregator) the values in edm:dataProvider and edm:provider will be the same. Organisation names should be provided as an ordinary text string until the Europeana Authority File for Organisations has been established. At that point providers will be able to send an identifier from the file instead of a text string. The name provided should be the preferred form of the name in the language the provider chooses as the default language for display in the portal. Countries with multiple languages may prefer to concatenate the name in more than one language (See the example below.) Note: Europeana Data Provider is not necessarily the institution where the physical object is located."@en . - . - . - . - . - "Is Shown At"@en . - "An unambiguous URL reference to the digital object on the provider\u2019s web site in its full information context."@en . - . - . - . - "Is Shown By"@en . - "An unambiguous URL reference to the digital object on the provider\u2019s web site in the best available resolution/quality."@en . - . - . - . - "Europeana Language"@en . - "A language assigned to the resource with reference to the Provider."@en . - "The recommended best practice is to use a controlled vocabulary such as\nRFC 4646 (http://www.rfc-archive.org/getrfc.php?rfc=4646) which, in\nconjunction with ISO 639, defines two- and three-letter primary language tags. Either a coded value or text string can be represented here."@en . - . - "Object"@en . - "The URL of a thumbnail representing the digital object or, if there is no such\nthumbnail, the URL of the digital object in the best resolution available on the\nweb site of the data provider from which a thumbnail could be generated. This will often be the same URL as given in edm:isShownBy."@en . - . - . - . - "Europeana Provider"@en . - "Name of the organization that delivers data to Europeana"@en . - . - . - . - "Europeana Rights"@en . - "Information about copyright of the digital object as specified by isShownBy\nand isShownAt"@en . - . - "Europeana Type"@en . - "The Europeana material type of the resource"@en . - . - _:node16k3ng1n7x48 . -_:node16k3ng1n7x48 . -_:node16k3ng1n7x48 _:node16k3ng1n7x49 . -_:node16k3ng1n7x49 "TEXT" . -_:node16k3ng1n7x49 _:node16k3ng1n7x50 . -_:node16k3ng1n7x50 "IMAGE" . -_:node16k3ng1n7x50 _:node16k3ng1n7x51 . -_:node16k3ng1n7x51 "SOUND" . -_:node16k3ng1n7x51 _:node16k3ng1n7x52 . -_:node16k3ng1n7x52 "VIDEO" . -_:node16k3ng1n7x52 _:node16k3ng1n7x53 . -_:node16k3ng1n7x53 "3D" . -_:node16k3ng1n7x53 . - . - "UGC"@en . - "This element is used to identify user generated content (also called user created content). It should be applied to all digitised or born digital content contributed by the general public and collected by Europeana through a crowdsourcing initiative or project."@en . - _:node16k3ng1n7x54 . -_:node16k3ng1n7x54 . -_:node16k3ng1n7x54 _:node16k3ng1n7x55 . -_:node16k3ng1n7x55 "TRUE" . -_:node16k3ng1n7x55 . - . - "Unstored"@en . - "This is a container element which includes all relevant information that\notherwise cannot be mapped to another element in the ESE."@en . - . - "Europeana URI"@en . - "This is a tag created by a user through the Europeana interface."@en . - . - "User Tag"@en . - "This is a tag created by a user through the Europeana interface."@en . - . - . - "Europeana Year"@en . - "A point of time associated with an event in the life of the original analog or\nborn digital object."@en . - . - . - . - . - . - . - . - _:node16k3ng1n7x56 . -_:node16k3ng1n7x56 . -_:node16k3ng1n7x56 . - _:node16k3ng1n7x57 . -_:node16k3ng1n7x57 . -_:node16k3ng1n7x57 . - . - _:node16k3ng1n7x58 . -_:node16k3ng1n7x58 . -_:node16k3ng1n7x58 . - . - . - . - . - . - . - . - . - . diff --git a/metis-schema/src/main/resources/rdf_examples/EDM-v523-120123.owl b/metis-schema/src/main/resources/rdf_examples/EDM-v523-120123.owl deleted file mode 100644 index 491b6e186..000000000 --- a/metis-schema/src/main/resources/rdf_examples/EDM-v523-120123.owl +++ /dev/null @@ -1,824 +0,0 @@ - - - - - - - - - - - - - - - - - -]> - - - - - Europeana Data Model (EDM) vocabulary - 5.2.3 - Participants of Europeana Version 1.0 Work Package on Further Specification of Functionality and Interoperability aspects of Europeana (WP3) - The Europeana Data Model (EDM) is aimed at being an integration medium for collecting, connecting and enriching the descriptions provided by Europeana data providers. The RDF vocabulary for http://www.europeana.eu/schemas/edm/ defines the elements introduced by EDM (as opposed to the ones EDM re-uses from other namespaces). - The present specification is based on the document "Definition of the Europeana Data Model elements", originally edited by Carlo Meghini. It is aligned with the version 5.2.3 of these EDM Definitions. - Antoine Isaac - Nasos Drosopoulos - Vassilis Tzouvaras - Julia Iwanowa - Hugo Borbinhas - Martin Doerr - - - -======= -Changes between EDM-v53-120123.owl -and EDM-v52-100730.owl -======= -1. Update of formal specification to v5.2.3 -2. Changed CRM namespace -3. Added or changed formal axioms for Event, aggregatedCHO, currentLocation, hasMet, hasType, isAnnotationOf -4. Property labels upper-cased -5. Added language tags -6. Checked Martin's latest mappings to CRM -7. Added metadata on the owl:Ontology instance -======= -Remaining TODOs for EDM-v53-120123.owl -======= -- change FRBR namespace and mappings using http://metadataregistry.org/schema/show/id/5.html or FRBRoo -- add Martin's mapping to FRBR(oo) for edm:incorporates (equivalent to crm:R14F.incorporates) and edm:WebResource (subclass of F24 Publication Expression) -- try to capture formal cardinality constraints resulting from "Obligation and Occurrence" documentation, which should be attached to non-EDM constructs (esp. ore:Aggregation) -- continue adding documentation values (skos:scopeNote, skos:example, etc, according to 1.), starting from edm:InformationResource. Add all Europeana examples and rationale notes for non-EDM constructs -- keep specific EDM-doc properties for "rationale" and "obligation and occurrence". Use skos:definition for "Europeana definition", skos:example for "Example", skos:note for "Europeana note" - - - - - - - - - Agent - This class comprises people, either individually or in groups, who have the -potential to perform intentional actions for which they can be held responsible. - Leonardo da Vinci, the British Museum, W3C - - - Rationale: This class is a domain of edm:wasPresentAt - - - - - - - Europeana Aggregation - The set of resources related to a single Cultural Heritage Object that -collectively represent that object in Europeana. Such set consists of: all -descriptions about the object that Europeana collects from (possibly different) content providers, including thumbnails and other forms of abstractions, as well as of the description of the object Europeana builds. - - - - - 1 - - - - An instance of EuropeanaAggregation is created at ingestion time for each different Cultural Heritage Object recognized by Europeana. Such instance is associated to the Cultural Heritage Object that it is about, by the property edm:aggregatedCHO - Obligation and Occurence: The relation between the Cultural Heritage Objects represented in Europeana and the instances of the class EuropeanaAggregation is one-to-one, in the data maintained by Europeana: every Cultural Heritage Object is represented by an instance of EuropeanaAggregation, and every instance of EuropeanaAggregation represent a Cultural Heritage Object. - The painting Mona Lisa is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance - The journal "Le Temps" is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance - The 56th issue of "Le Temps" is a (different) Cultural Heritage Object represented in Europeana by another EuropeanaAggregation instance - Rationale: This class is used in Europeana to gather in a single conceptual unit all the information about a Cultural Heritage Object, necessary for all operations on these objects. - - - - - - Europeana Object - Any object that is the result of Europeana’s activities - - Any instance of the class EuropeanaAggregation - An annotation created by a user through the Europeana portal - Any content created by the users through the service made available by Europeana for that purpose - Rationale: This class is used to tag objects that are the result of activity of Europeana, and, as such, objects on which Europeana holds rights - - - - - - Event - An event is a change "of states in cultural, social or physical systems, - regardless of scale, brought about by a series or group of coherent physical, -cultural, technological or legal phenomena" (E5 Event in CIDOC CRM) or a "set of coherent phenomena or cultural manifestations bounded in time and space" (E4 Period in CIDOC CRM) - - - - - - - - 1 - - - - Events are identified either by the content provider or by Europeana enrichment at ingestion time - the act of painting Mona Lisa - the 2nd World War - the change of custody of Mona Lisa - Rationale:This class is a domain of edm:happenedAt and the domain of edm:occurredAt - - - - - - Information Resource - An information resource is a resource whose essential characteristics can be conveyed in a single message. It can be associated with a URI, it can have a representation, for example: a text is an InformationResource. - - - - - - - - - - - - - - - - Non-Information Resource - All resources that are not information resources. - - - - - - - Physical Thing - A persistent physical item such as a painting, a building, a book or a stone. -Persons are not items. This class represents Cultural Heritage Objects known to Europeana to be physical things (such as Mona Lisa) as well as all physical things Europeana refers to in the descriptions of Cultural Heritage Objects (such as the Rosetta Stone). - - - - - - - - Place - An "extent in space, in particular on the surface of the earth, in the pure sense of physics: independent from temporal phenomena and matter" (CIDOC CRM) - - - - - - - - - - - Provided CHO - This class comprises the Cultural Heritage objects that Europeana collects descriptions about. - - - 1 - - - - - - - - This class has been mostly motivated by the need to assign a type to the “central node” in the EDM pattern, during the ingestion process, related to the XML expression of EDM at that stage. It was especially intended to fit the cases where edm:PhysicalThing cannot be used as the type of the resource standing for the real-world object (independently of any specific data contributor perspective). - Mona Lisa, Winged Victory of Samothrace - Rationale: This class is the range of edm:aggregatedCHO. A resource of type ProvidedCHO can be the subject of statements using edm:isRelatedTo or any more specific property. - - - - - - Time Span - The class of "abstract temporal extents, in the sense of Galilean physics, - having a beginning, an end and a duration" (CIDOC CRM) - - - - - - - - - - - Web Resource - Information Resources that have at least one Web Representation and at least -a URI. - - - - - - - - - - Aggregated Cultural Heritage Object - This property associates an ORE aggregation with the Cultural Heritage -Object(s) (CHO for short) it is about. - - - - - - - - - - - Current Location - The geographic location and/or name of the repository, building, site, or other entity whose boundaries presently include the resource. - - - - - - - - - - - - - - - - - - - - - - - - - - Happened At - This property associates an event with the place at which the event -happened. - - - - - - - - - - Has Met - edm:hasMet relates a resource with the objects or phenomena that have happened to or have happened together with the resource under consideration. We can abstractly think of history and the present as a series of “meetings” between people and other things in space-time. Therefore we name this relationship as the things the object “has met” in the course of its existence. These meetings are events in the proper sense, in which other people and things participate in any role. - - - - - - - - - - - - - - - - - - - - - - - Has Type - This property relates a resource with the concepts it belongs to in a suitable -type system such as MIME or any thesaurus that captures categories of objects in a given field (e.g., the “Objects” facet in Getty’s Art and Architecture Thesaurus). It does not capture aboutness. - - - - - - - - - - - - - - - - - - - - - - - - - Has View - This property relates a ORE aggregation about a CHO with a web resource -providing a view of that CHO. Examples of view are: a thumbnail, a textual -abstract and a table of contents. The ORE aggregation may be a Europeana -Aggregation, in which case the view is an object owned by Europeana (i.e., an instance of edm:EuropeanaObject) or an aggregation contributed by a content provider. In order to capture both these cases, the domain of edm:hasView is ore:Aggregation and its range is edm:WebResource - - - - - - - - - Incorporates - This property captures the use of some resource to add value to another -resource. Such resources may be nested, such as performing a theater play text, and then recording the performance, or creating an artful edition of a collection of poems or just aggregating various poems in an anthology. There may be no single part that contains ultimately the incorporated object, which may be dispersed in the presentation. Therefore, incorporated resources do in general not form proper parts. Incorporated resources are not part of the same resource, but are taken from other resources, and have an independent history. Therefore edm:incorporates is not a sub-property of dcterm:hasPart. - - - - - - - Is Annotation Of - This property relates an annotation (a Europeana object) with the resource -that it annotates. - - - - - - - - - - - - - - - - - - - - - - - - - Is Derivative Of - This property captures a narrower notion of derivation than edm:isSimilarTo, in the sense that it relates a resource to another one, obtained by reworking, reducing, expanding, parts or the whole contents of the former, and possibly adding some minor parts. Versions have an even narrower meaning, in that it requires common identity between the related resources. Translations, summaries, abstractions etc. do not qualify as versions, but do qualify as derivatives. - - - - - - - Is Next In Sequence - edm:isNextInSequence relates two resources S and R that are ordered parts of the same resource A, and such that R comes immediately after R in the order created by their being parts of S. - - - - - - - - Is Related To - edm:isRelatedTo is the most general contextual property in EDM. Contextual -properties have typically to do either with the things that have happened to or together with the object under consideration, or what the object refers to by its shape, form or features in a figural or encoded form. For sake of simplicity, we include in the contextual relationships also the scholarly classification, which may have either to do with the role and cultural connections of the object in the past, or its kind of structure, substance or contents as it can be verified at present. - - - - - - - - - - - - - - - - - - - - - - Is Representation Of - This property associates an information resource to the resource (if any) that it represents - - - - - - - - - Is Similar To - The most generic derivation property, covering also the case of questionable derivation. Is Similar To asserts that parts of the contents of one resource exhibit common features with respect to ideas, shapes, structures, colors, words, plots, topics with the contents of the related resource. Those common features may be attributed to a common origin or influence (in particular for derivation), but also to more generic cultural or psychological factors. - - - - - - - - Is Successor Of - This property captures the relation between the continuation of a resource and that resource. This applies to a story, a serial, a journal etc. No content of the successor resource is identical or has a similar form with that of the precursor. The similarity is only in the context, subjects and figures of a plot. Successors typically form part of a common whole – such as a trilogy, a journal, etc. - - - - - - - - Landing Page - This property captures the relation between an aggregation representing a -Cultural Heritage Object and the Web Resource representing that Object on -the provider’s web site. - - - - - - - - - Occured At - This property associates an event to the smallest known time span that -overlaps with the occurrence of that event - - - - - - - - - - Realizes - This property describes a relation between a physical thing and the information resource that is contained in it, visible at it or otherwise carried by it, if applicable. - - - - - - - - - - - Was Present At - This property associates the people, things or information resources with an event at which they were present - - - - - - - - - - - - - - - - - - - - - - - Country - - - - - - - Europeana Data Provider - This element is specifically included to allow the name of the organisation who supplies data to Europeana indirectly via an aggregator to be recorded and displayed in the portal. Aggregator names are recorded in edm:provider. If an organisation provides data directly to Europeana (i.e. not via an aggregator) the values in edm:dataProvider and edm:provider will be the same. Organisation names should be provided as an ordinary text string until the Europeana Authority File for Organisations has been established. At that point providers will be able to send an identifier from the file instead of a text string. The name provided should be the preferred form of the name in the language the provider chooses as the default language for display in the portal. Countries with multiple languages may prefer to concatenate the name in more than one language (See the example below.) Note: Europeana Data Provider is not necessarily the institution where the physical object is located. - - - - - - - - - Is Shown At - An unambiguous URL reference to the digital object on the provider’s web site in its full information context. - - - - - - - - Is Shown By - An unambiguous URL reference to the digital object on the provider’s web site in the best available resolution/quality. - - - - - - - - Europeana Language - A language assigned to the resource with reference to the Provider. - The recommended best practice is to use a controlled vocabulary such as -RFC 4646 (http://www.rfc-archive.org/getrfc.php?rfc=4646) which, in -conjunction with ISO 639, defines two- and three-letter primary language tags. Either a coded value or text string can be represented here. - - - - - - Object - The URL of a thumbnail representing the digital object or, if there is no such -thumbnail, the URL of the digital object in the best resolution available on the -web site of the data provider from which a thumbnail could be generated. This will often be the same URL as given in edm:isShownBy. - - - - - - - - Europeana Provider - Name of the organization that delivers data to Europeana - - - - - - - - Europeana Rights - Information about copyright of the digital object as specified by isShownBy -and isShownAt - - - - - - Europeana Type - The Europeana material type of the resource - - - - - TEXT - - IMAGE - - SOUND - - VIDEO - - 3D - - - - - - - - - - - - - - UGC - This element is used to identify user generated content (also called user created content). It should be applied to all digitised or born digital content contributed by the general public and collected by Europeana through a crowdsourcing initiative or project. - - - - TRUE - - - - - - - - - - Unstored - This is a container element which includes all relevant information that -otherwise cannot be mapped to another element in the ESE. - - - - - - Europeana URI - This is a tag created by a user through the Europeana interface. - - - - - - User Tag - This is a tag created by a user through the Europeana interface. - - - - - - - Europeana Year - A point of time associated with an event in the life of the original analog or -born digital object. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/metis-schema/src/main/resources/rdf_examples/EDM-v523-120123.ttl b/metis-schema/src/main/resources/rdf_examples/EDM-v523-120123.ttl deleted file mode 100644 index ed67f6343..000000000 --- a/metis-schema/src/main/resources/rdf_examples/EDM-v523-120123.ttl +++ /dev/null @@ -1,576 +0,0 @@ -@prefix dc: . -@prefix DOLCE-Lite: . -@prefix abc: . -@prefix edm: . -@prefix wgs84_pos: . -@prefix foaf: . -@prefix dcmitype: . -@prefix ore: . -@prefix dcterms: . -@prefix rdfs: . -@prefix frbr_core: . -@prefix xsd: . -@prefix owl: . -@prefix rdf: . -@prefix crm: . -@prefix skos: . - - a owl:Ontology ; - dc:title "Europeana Data Model (EDM) vocabulary"@en ; - owl:versionInfo "5.2.3" ; - dc:contributor "Participants of Europeana Version 1.0 Work Package on Further Specification of Functionality and Interoperability aspects of Europeana (WP3)" ; - dc:description "The Europeana Data Model (EDM) is aimed at being an integration medium for collecting, connecting and enriching the descriptions provided by Europeana data providers. The RDF vocabulary for http://www.europeana.eu/schemas/edm/ defines the elements introduced by EDM (as opposed to the ones EDM re-uses from other namespaces)."@en ; - rdfs:comment "The present specification is based on the document \"Definition of the Europeana Data Model elements\", originally edited by Carlo Meghini. It is aligned with the version 5.2.3 of these EDM Definitions."@en ; - dc:creator "Antoine Isaac" ; - dc:contributor "Nasos Drosopoulos" , "Vassilis Tzouvaras" , "Julia Iwanowa" , "Hugo Borbinhas" , "Martin Doerr" ; - rdfs:seeAlso , ; - rdfs:comment """======= -Changes between EDM-v53-120123.owl -and EDM-v52-100730.owl -======= -1. Update of formal specification to v5.2.3 -2. Changed CRM namespace -3. Added or changed formal axioms for Event, aggregatedCHO, currentLocation, hasMet, hasType, isAnnotationOf -4. Property labels upper-cased -5. Added language tags -6. Checked Martin's latest mappings to CRM -7. Added metadata on the owl:Ontology instance -======= -Remaining TODOs for EDM-v53-120123.owl -======= -- change FRBR namespace and mappings using http://metadataregistry.org/schema/show/id/5.html or FRBRoo -- add Martin's mapping to FRBR(oo) for edm:incorporates (equivalent to crm:R14F.incorporates) and edm:WebResource (subclass of F24 Publication Expression) -- try to capture formal cardinality constraints resulting from \"Obligation and Occurrence\" documentation, which should be attached to non-EDM constructs (esp. ore:Aggregation) -- continue adding documentation values (skos:scopeNote, skos:example, etc, according to 1.), starting from edm:InformationResource. Add all Europeana examples and rationale notes for non-EDM constructs -- keep specific EDM-doc properties for \"rationale\" and \"obligation and occurrence\". Use skos:definition for \"Europeana definition\", skos:example for \"Example\", skos:note for \"Europeana note\""""@en . - -edm:Agent a owl:Class ; - rdfs:label "Agent"@en ; - skos:definition """This class comprises people, either individually or in groups, who have the -potential to perform intentional actions for which they can be held responsible.""" ; - skos:example "Leonardo da Vinci, the British Museum, W3C" ; - rdfs:subClassOf edm:NonInformationResource ; - owl:equivalentClass ; - skos:scopeNote "Rationale: This class is a domain of edm:wasPresentAt" . - -edm:EuropeanaAggregation a owl:Class ; - rdfs:label "Europeana Aggregation" ; - skos:definition """The set of resources related to a single Cultural Heritage Object that -collectively represent that object in Europeana. Such set consists of: all -descriptions about the object that Europeana collects from (possibly different) content providers, including thumbnails and other forms of abstractions, as well as of the description of the object Europeana builds.""" ; - rdfs:subClassOf edm:EuropeanaObject , ore:Aggregation ; - owl:equivalentClass _:node16k3ng1n7x1 . - -_:node16k3ng1n7x1 a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty edm:aggregatedCHO . - -edm:EuropeanaAggregation skos:note "An instance of EuropeanaAggregation is created at ingestion time for each different Cultural Heritage Object recognized by Europeana. Such instance is associated to the Cultural Heritage Object that it is about, by the property edm:aggregatedCHO"@en ; - skos:scopeNote "Obligation and Occurence: The relation between the Cultural Heritage Objects represented in Europeana and the instances of the class EuropeanaAggregation is one-to-one, in the data maintained by Europeana: every Cultural Heritage Object is represented by an instance of EuropeanaAggregation, and every instance of EuropeanaAggregation represent a Cultural Heritage Object."@en ; - skos:example "The painting Mona Lisa is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en , "The journal \"Le Temps\" is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en , "The 56th issue of \"Le Temps\" is a (different) Cultural Heritage Object represented in Europeana by another EuropeanaAggregation instance"@en ; - skos:scopeNote "Rationale: This class is used in Europeana to gather in a single conceptual unit all the information about a Cultural Heritage Object, necessary for all operations on these objects."@en . - -edm:EuropeanaObject a owl:Class ; - rdfs:label "Europeana Object"@en ; - skos:definition "Any object that is the result of Europeana’s activities"@en ; - rdfs:subClassOf edm:WebResource ; - skos:example "Any instance of the class EuropeanaAggregation"@en , "An annotation created by a user through the Europeana portal"@en , "Any content created by the users through the service made available by Europeana for that purpose"@en ; - skos:scopeNote "Rationale: This class is used to tag objects that are the result of activity of Europeana, and, as such, objects on which Europeana holds rights"@en . - -edm:Event a owl:Class ; - rdfs:label "Event"@en ; - skos:definition """An event is a change \"of states in cultural, social or physical systems, - regardless of scale, brought about by a series or group of coherent physical, -cultural, technological or legal phenomena\" (E5 Event in CIDOC CRM) or a \"set of coherent phenomena or cultural manifestations bounded in time and space\" (E4 Period in CIDOC CRM)"""@en ; - rdfs:subClassOf edm:NonInformationResource ; - owl:equivalentClass , abc:Temporality , frbr_core:Event ; - rdfs:subClassOf _:node16k3ng1n7x2 . - -_:node16k3ng1n7x2 a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty edm:happenedAt . - -edm:Event skos:note "Events are identified either by the content provider or by Europeana enrichment at ingestion time"@en ; - skos:example "the act of painting Mona Lisa"@en , "the 2nd World War"@en , "the change of custody of Mona Lisa"@en ; - skos:scopeNote "Rationale:This class is a domain of edm:happenedAt and the domain of edm:occurredAt"@en . - -edm:InformationResource a owl:Class ; - rdfs:label "Information Resource"@en ; - skos:definition "An information resource is a resource whose essential characteristics can be conveyed in a single message. It can be associated with a URI, it can have a representation, for example: a text is an InformationResource."@en ; - owl:equivalentClass , _:node16k3ng1n7x3 . - -_:node16k3ng1n7x3 a owl:Class ; - owl:unionOf _:node16k3ng1n7x4 . - -_:node16k3ng1n7x4 rdf:first frbr_core:Work ; - rdf:rest _:node16k3ng1n7x5 . - -_:node16k3ng1n7x5 rdf:first frbr_core:Expression ; - rdf:rest _:node16k3ng1n7x6 . - -_:node16k3ng1n7x6 rdf:first frbr_core:Manifestation ; - rdf:rest rdf:nil . - -edm:NonInformationResource a owl:Class ; - rdfs:label "Non-Information Resource"@en ; - skos:definition "All resources that are not information resources."@en ; - owl:complementOf edm:InformationResource . - -edm:PhysicalThing a owl:Class ; - rdfs:label "Physical Thing"@en ; - skos:definition """A persistent physical item such as a painting, a building, a book or a stone. -Persons are not items. This class represents Cultural Heritage Objects known to Europeana to be physical things (such as Mona Lisa) as well as all physical things Europeana refers to in the descriptions of Cultural Heritage Objects (such as the Rosetta Stone)."""@en ; - rdfs:subClassOf edm:NonInformationResource ; - owl:equivalentClass . - -edm:Place a owl:Class ; - rdfs:label "Place"@en ; - skos:definition "An \"extent in space, in particular on the surface of the earth, in the pure sense of physics: independent from temporal phenomena and matter\" (CIDOC CRM)"@en ; - rdfs:subClassOf edm:NonInformationResource ; - owl:equivalentClass DOLCE-Lite:space-region , abc:Place , , frbr_core:Place . - -edm:ProvidedCHO a owl:Class ; - rdfs:label "Provided CHO"@en ; - skos:definition "This class comprises the Cultural Heritage objects that Europeana collects descriptions about."@en ; - rdfs:subClassOf _:node16k3ng1n7x7 . - -_:node16k3ng1n7x7 a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty _:node16k3ng1n7x8 . - -_:node16k3ng1n7x8 a rdf:Property ; - owl:inverseOf edm:aggregatedCHO . - -edm:ProvidedCHO skos:note "This class has been mostly motivated by the need to assign a type to the “central node” in the EDM pattern, during the ingestion process, related to the XML expression of EDM at that stage. It was especially intended to fit the cases where edm:PhysicalThing cannot be used as the type of the resource standing for the real-world object (independently of any specific data contributor perspective)."@en ; - skos:example "Mona Lisa, Winged Victory of Samothrace"@en ; - skos:scopeNote "Rationale: This class is the range of edm:aggregatedCHO. A resource of type ProvidedCHO can be the subject of statements using edm:isRelatedTo or any more specific property."@en . - -edm:TimeSpan a owl:Class ; - rdfs:label "Time Span"@en ; - skos:definition """The class of \"abstract temporal extents, in the sense of Galilean physics, - having a beginning, an end and a duration\" (CIDOC CRM)"""@en ; - rdfs:subClassOf edm:NonInformationResource , dcterms:PeriodOfTime ; - owl:equivalentClass abc:Time , DOLCE-Lite:time-interval , . - -edm:WebResource a owl:Class ; - rdfs:label "Web Resource"@en ; - skos:definition """Information Resources that have at least one Web Representation and at least -a URI."""@en ; - rdfs:subClassOf edm:InformationResource . - -edm:aggregatedCHO a owl:ObjectProperty ; - rdfs:label "Aggregated Cultural Heritage Object"@en ; - skos:definition """This property associates an ORE aggregation with the Cultural Heritage -Object(s) (CHO for short) it is about."""@en ; - rdfs:subPropertyOf ore:aggregates , dc:subject , ; - rdfs:domain ore:Aggregation ; - rdfs:range edm:ProvidedCHO . - -edm:currentLocation a owl:ObjectProperty ; - rdfs:label "Current Location"@en ; - skos:definition "The geographic location and/or name of the repository, building, site, or other entity whose boundaries presently include the resource."@en ; - rdfs:subPropertyOf dcterms:spatial ; - owl:equivalentProperty wgs84_pos:location , ; - rdfs:domain _:node16k3ng1n7x9 . - -_:node16k3ng1n7x9 a owl:Class ; - owl:unionOf _:node16k3ng1n7x10 . - -_:node16k3ng1n7x10 rdf:first edm:ProvidedCHO ; - rdf:rest _:node16k3ng1n7x12 . - -_:node16k3ng1n7x12 rdf:first _:node16k3ng1n7x11 . - -_:node16k3ng1n7x11 a owl:Class ; - owl:intersectionOf _:node16k3ng1n7x13 . - -_:node16k3ng1n7x13 rdf:first ore:Proxy ; - rdf:rest _:node16k3ng1n7x15 . - -_:node16k3ng1n7x15 rdf:first _:node16k3ng1n7x14 . - -_:node16k3ng1n7x14 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom edm:ProvidedCHO . - -_:node16k3ng1n7x15 rdf:rest rdf:nil . - -_:node16k3ng1n7x12 rdf:rest rdf:nil . - -edm:currentLocation rdfs:range edm:Place . - -edm:happenedAt a owl:ObjectProperty ; - rdfs:label "Happened At"@en ; - skos:definition """This property associates an event with the place at which the event -happened."""@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - rdfs:domain edm:Event ; - rdfs:range edm:Place . - -edm:hasMet a rdf:Property ; - rdfs:label "Has Met"@en ; - skos:definition "edm:hasMet relates a resource with the objects or phenomena that have happened to or have happened together with the resource under consideration. We can abstractly think of history and the present as a series of “meetings” between people and other things in space-time. Therefore we name this relationship as the things the object “has met” in the course of its existence. These meetings are events in the proper sense, in which other people and things participate in any role."@en ; - rdfs:subPropertyOf dc:relation ; - rdfs:domain _:node16k3ng1n7x16 . - -_:node16k3ng1n7x16 a owl:Class ; - owl:unionOf _:node16k3ng1n7x17 . - -_:node16k3ng1n7x17 rdf:first edm:ProvidedCHO ; - rdf:rest _:node16k3ng1n7x19 . - -_:node16k3ng1n7x19 rdf:first _:node16k3ng1n7x18 . - -_:node16k3ng1n7x18 a owl:Class ; - owl:intersectionOf _:node16k3ng1n7x20 . - -_:node16k3ng1n7x20 rdf:first ore:Proxy ; - rdf:rest _:node16k3ng1n7x22 . - -_:node16k3ng1n7x22 rdf:first _:node16k3ng1n7x21 . - -_:node16k3ng1n7x21 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom edm:ProvidedCHO . - -_:node16k3ng1n7x22 rdf:rest rdf:nil . - -_:node16k3ng1n7x19 rdf:rest rdf:nil . - -edm:hasType a owl:ObjectProperty ; - rdfs:label "Has Type"@en ; - skos:definition """This property relates a resource with the concepts it belongs to in a suitable -type system such as MIME or any thesaurus that captures categories of objects in a given field (e.g., the “Objects” facet in Getty’s Art and Architecture Thesaurus). It does not capture aboutness."""@en ; - rdfs:subPropertyOf edm:isRelatedTo ; - owl:equivalentProperty ; - rdfs:domain _:node16k3ng1n7x23 . - -_:node16k3ng1n7x23 a owl:Class ; - owl:unionOf _:node16k3ng1n7x24 . - -_:node16k3ng1n7x24 rdf:first edm:ProvidedCHO ; - rdf:rest _:node16k3ng1n7x26 . - -_:node16k3ng1n7x26 rdf:first _:node16k3ng1n7x25 . - -_:node16k3ng1n7x25 a owl:Class ; - owl:intersectionOf _:node16k3ng1n7x27 . - -_:node16k3ng1n7x27 rdf:first ore:Proxy ; - rdf:rest _:node16k3ng1n7x29 . - -_:node16k3ng1n7x29 rdf:first _:node16k3ng1n7x28 . - -_:node16k3ng1n7x28 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom edm:ProvidedCHO . - -_:node16k3ng1n7x29 rdf:rest rdf:nil . - -_:node16k3ng1n7x26 rdf:rest rdf:nil . - -edm:hasType rdfs:range edm:NonInformationResource . - -edm:hasView a owl:ObjectProperty ; - rdfs:label "Has View"@en ; - skos:definition """This property relates a ORE aggregation about a CHO with a web resource -providing a view of that CHO. Examples of view are: a thumbnail, a textual -abstract and a table of contents. The ORE aggregation may be a Europeana -Aggregation, in which case the view is an object owned by Europeana (i.e., an instance of edm:EuropeanaObject) or an aggregation contributed by a content provider. In order to capture both these cases, the domain of edm:hasView is ore:Aggregation and its range is edm:WebResource"""@en ; - rdfs:subPropertyOf ore:aggregates ; - rdfs:domain ore:Aggregation ; - rdfs:range edm:WebResource . - -edm:incorporates a owl:ObjectProperty ; - rdfs:label "Incorporates"@en ; - skos:definition """This property captures the use of some resource to add value to another -resource. Such resources may be nested, such as performing a theater play text, and then recording the performance, or creating an artful edition of a collection of poems or just aggregating various poems in an anthology. There may be no single part that contains ultimately the incorporated object, which may be dispersed in the presentation. Therefore, incorporated resources do in general not form proper parts. Incorporated resources are not part of the same resource, but are taken from other resources, and have an independent history. Therefore edm:incorporates is not a sub-property of dcterm:hasPart."""@en ; - rdfs:subPropertyOf edm:isSimilarTo . - -edm:isAnnotationOf a owl:ObjectProperty ; - rdfs:label "Is Annotation Of"@en ; - skos:definition """This property relates an annotation (a Europeana object) with the resource -that it annotates."""@en ; - rdfs:subPropertyOf dc:subject , ; - rdfs:domain edm:EuropeanaObject ; - rdfs:range _:node16k3ng1n7x30 . - -_:node16k3ng1n7x30 a owl:Class ; - owl:unionOf _:node16k3ng1n7x31 . - -_:node16k3ng1n7x31 rdf:first edm:ProvidedCHO ; - rdf:rest _:node16k3ng1n7x33 . - -_:node16k3ng1n7x33 rdf:first _:node16k3ng1n7x32 . - -_:node16k3ng1n7x32 a owl:Class ; - owl:intersectionOf _:node16k3ng1n7x34 . - -_:node16k3ng1n7x34 rdf:first ore:Proxy ; - rdf:rest _:node16k3ng1n7x36 . - -_:node16k3ng1n7x36 rdf:first _:node16k3ng1n7x35 . - -_:node16k3ng1n7x35 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom edm:ProvidedCHO . - -_:node16k3ng1n7x36 rdf:rest rdf:nil . - -_:node16k3ng1n7x33 rdf:rest rdf:nil . - -edm:isDerivativeOf a rdf:Property ; - rdfs:label "Is Derivative Of"@en ; - skos:definition "This property captures a narrower notion of derivation than edm:isSimilarTo, in the sense that it relates a resource to another one, obtained by reworking, reducing, expanding, parts or the whole contents of the former, and possibly adding some minor parts. Versions have an even narrower meaning, in that it requires common identity between the related resources. Translations, summaries, abstractions etc. do not qualify as versions, but do qualify as derivatives."@en ; - rdfs:subPropertyOf edm:isSimilarTo . - -edm:isNextInSequence a owl:ObjectProperty ; - rdfs:label "Is Next In Sequence"@en ; - skos:definition "edm:isNextInSequence relates two resources S and R that are ordered parts of the same resource A, and such that R comes immediately after R in the order created by their being parts of S."@en ; - rdfs:subPropertyOf dc:relation . - -edm:isRelatedTo a rdf:Property ; - rdfs:label "Is Related To"@en ; - skos:definition """edm:isRelatedTo is the most general contextual property in EDM. Contextual -properties have typically to do either with the things that have happened to or together with the object under consideration, or what the object refers to by its shape, form or features in a figural or encoded form. For sake of simplicity, we include in the contextual relationships also the scholarly classification, which may have either to do with the role and cultural connections of the object in the past, or its kind of structure, substance or contents as it can be verified at present."""@en ; - rdfs:domain _:node16k3ng1n7x37 . - -_:node16k3ng1n7x37 a owl:Class ; - owl:unionOf _:node16k3ng1n7x38 . - -_:node16k3ng1n7x38 rdf:first edm:ProvidedCHO ; - rdf:rest _:node16k3ng1n7x40 . - -_:node16k3ng1n7x40 rdf:first _:node16k3ng1n7x39 . - -_:node16k3ng1n7x39 a owl:Class ; - owl:intersectionOf _:node16k3ng1n7x41 . - -_:node16k3ng1n7x41 rdf:first ore:Proxy ; - rdf:rest _:node16k3ng1n7x43 . - -_:node16k3ng1n7x43 rdf:first _:node16k3ng1n7x42 . - -_:node16k3ng1n7x42 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom edm:ProvidedCHO . - -_:node16k3ng1n7x43 rdf:rest rdf:nil . - -_:node16k3ng1n7x40 rdf:rest rdf:nil . - -edm:isRepresentationOf a owl:ObjectProperty ; - rdfs:label "Is Representation Of"@en ; - skos:definition "This property associates an information resource to the resource (if any) that it represents"@en ; - rdfs:subPropertyOf dc:subject ; - rdfs:domain edm:InformationResource ; - rdfs:subPropertyOf . - -edm:isSimilarTo a rdf:Property ; - rdfs:label "Is Similar To"@en ; - skos:definition "The most generic derivation property, covering also the case of questionable derivation. Is Similar To asserts that parts of the contents of one resource exhibit common features with respect to ideas, shapes, structures, colors, words, plots, topics with the contents of the related resource. Those common features may be attributed to a common origin or influence (in particular for derivation), but also to more generic cultural or psychological factors."@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty . - -edm:isSuccessorOf a owl:ObjectProperty ; - rdfs:label "Is Successor Of"@en ; - skos:definition "This property captures the relation between the continuation of a resource and that resource. This applies to a story, a serial, a journal etc. No content of the successor resource is identical or has a similar form with that of the precursor. The similarity is only in the context, subjects and figures of a plot. Successors typically form part of a common whole – such as a trilogy, a journal, etc."@en ; - rdfs:subPropertyOf edm:isSimilarTo . - -edm:landingPage a owl:ObjectProperty ; - rdfs:label "Landing Page"@en ; - skos:definition """This property captures the relation between an aggregation representing a -Cultural Heritage Object and the Web Resource representing that Object on -the provider’s web site."""@en ; - rdfs:subPropertyOf ore:aggregates ; - rdfs:range edm:WebResource . - -edm:occurredAt a owl:ObjectProperty ; - rdfs:label "Occured At"@en ; - skos:definition """This property associates an event to the smallest known time span that -overlaps with the occurrence of that event"""@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - rdfs:domain edm:Event ; - rdfs:range edm:TimeSpan . - -edm:realizes a owl:ObjectProperty ; - rdfs:label "Realizes"@en ; - skos:definition "This property describes a relation between a physical thing and the information resource that is contained in it, visible at it or otherwise carried by it, if applicable."@en ; - rdfs:subPropertyOf edm:isRelatedTo ; - owl:equivalentProperty ; - rdfs:domain edm:PhysicalThing ; - rdfs:range edm:InformationResource . - -edm:wasPresentAt a owl:ObjectProperty ; - rdfs:label "Was Present At"@en ; - skos:definition "This property associates the people, things or information resources with an event at which they were present"@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - rdfs:domain _:node16k3ng1n7x44 . - -_:node16k3ng1n7x44 a owl:Class ; - owl:unionOf _:node16k3ng1n7x45 . - -_:node16k3ng1n7x45 rdf:first edm:Agent ; - rdf:rest _:node16k3ng1n7x46 . - -_:node16k3ng1n7x46 rdf:first edm:InformationResource ; - rdf:rest _:node16k3ng1n7x47 . - -_:node16k3ng1n7x47 rdf:first edm:PhysicalThing ; - rdf:rest rdf:nil . - -edm:wasPresentAt rdfs:range edm:Event . - -edm:country a rdf:Property ; - rdfs:label "Country"@en ; - rdfs:subPropertyOf . - -edm:dataProvider a rdf:Property ; - rdfs:label "Europeana Data Provider"@en ; - skos:definition "This element is specifically included to allow the name of the organisation who supplies data to Europeana indirectly via an aggregator to be recorded and displayed in the portal. Aggregator names are recorded in edm:provider. If an organisation provides data directly to Europeana (i.e. not via an aggregator) the values in edm:dataProvider and edm:provider will be the same. Organisation names should be provided as an ordinary text string until the Europeana Authority File for Organisations has been established. At that point providers will be able to send an identifier from the file instead of a text string. The name provided should be the preferred form of the name in the language the provider chooses as the default language for display in the portal. Countries with multiple languages may prefer to concatenate the name in more than one language (See the example below.) Note: Europeana Data Provider is not necessarily the institution where the physical object is located."@en ; - rdfs:subPropertyOf dcterms:provenance ; - rdfs:domain ore:Aggregation ; - rdfs:range edm:Agent . - -edm:isShownAt a owl:ObjectProperty ; - rdfs:label "Is Shown At"@en ; - skos:definition "An unambiguous URL reference to the digital object on the provider’s web site in its full information context."@en ; - rdfs:subPropertyOf edm:landingPage ; - rdfs:range edm:WebResource . - -edm:isShownBy a owl:ObjectProperty ; - rdfs:label "Is Shown By"@en ; - skos:definition "An unambiguous URL reference to the digital object on the provider’s web site in the best available resolution/quality."@en ; - rdfs:subPropertyOf edm:hasView ; - rdfs:range edm:WebResource . - -edm:language a rdf:Property ; - rdfs:label "Europeana Language"@en ; - skos:definition "A language assigned to the resource with reference to the Provider."@en ; - rdfs:comment """The recommended best practice is to use a controlled vocabulary such as -RFC 4646 (http://www.rfc-archive.org/getrfc.php?rfc=4646) which, in -conjunction with ISO 639, defines two- and three-letter primary language tags. Either a coded value or text string can be represented here."""@en . - -edm:object a owl:ObjectProperty ; - rdfs:label "Object"@en ; - skos:definition """The URL of a thumbnail representing the digital object or, if there is no such -thumbnail, the URL of the digital object in the best resolution available on the -web site of the data provider from which a thumbnail could be generated. This will often be the same URL as given in edm:isShownBy."""@en ; - rdfs:subPropertyOf edm:hasView ; - rdfs:range edm:WebResource . - -edm:provider a rdf:Property ; - rdfs:label "Europeana Provider"@en ; - skos:definition "Name of the organization that delivers data to Europeana"@en ; - rdfs:subPropertyOf edm:hasMet ; - rdfs:range edm:Agent . - -edm:rights a owl:ObjectProperty ; - rdfs:label "Europeana Rights"@en ; - skos:definition """Information about copyright of the digital object as specified by isShownBy -and isShownAt"""@en . - -edm:type a owl:DatatypeProperty ; - rdfs:label "Europeana Type"@en ; - skos:definition "The Europeana material type of the resource"@en ; - rdfs:subPropertyOf dc:type ; - rdfs:range _:node16k3ng1n7x48 . - -_:node16k3ng1n7x48 a rdfs:Datatype ; - owl:oneOf _:node16k3ng1n7x49 . - -_:node16k3ng1n7x49 rdf:first "TEXT" ; - rdf:rest _:node16k3ng1n7x50 . - -_:node16k3ng1n7x50 rdf:first "IMAGE" ; - rdf:rest _:node16k3ng1n7x51 . - -_:node16k3ng1n7x51 rdf:first "SOUND" ; - rdf:rest _:node16k3ng1n7x52 . - -_:node16k3ng1n7x52 rdf:first "VIDEO" ; - rdf:rest _:node16k3ng1n7x53 . - -_:node16k3ng1n7x53 rdf:first "3D" ; - rdf:rest rdf:nil . - -edm:ugc a owl:DatatypeProperty ; - rdfs:label "UGC"@en ; - skos:definition "This element is used to identify user generated content (also called user created content). It should be applied to all digitised or born digital content contributed by the general public and collected by Europeana through a crowdsourcing initiative or project."@en ; - rdfs:range _:node16k3ng1n7x54 . - -_:node16k3ng1n7x54 a rdfs:Datatype ; - owl:oneOf _:node16k3ng1n7x55 . - -_:node16k3ng1n7x55 rdf:first "TRUE" ; - rdf:rest rdf:nil . - -edm:unstored a rdf:Property ; - rdfs:label "Unstored"@en ; - skos:definition """This is a container element which includes all relevant information that -otherwise cannot be mapped to another element in the ESE."""@en . - -edm:uri a owl:ObjectProperty ; - rdfs:label "Europeana URI"@en ; - skos:definition "This is a tag created by a user through the Europeana interface."@en . - -edm:userTag a rdf:Property ; - rdfs:label "User Tag"@en ; - skos:definition "This is a tag created by a user through the Europeana interface."@en ; - rdfs:subPropertyOf dc:description . - -edm:year a rdf:Property ; - rdfs:label "Europeana Year"@en ; - skos:definition """A point of time associated with an event in the life of the original analog or -born digital object."""@en ; - rdfs:subPropertyOf dcterms:temporal . - -skos:Concept rdfs:subClassOf edm:NonInformationResource . - -dc:contributor rdfs:subPropertyOf edm:hasMet . - -dc:coverage rdfs:subPropertyOf edm:hasMet . - -dc:creator rdfs:subPropertyOf edm:hasMet . - -dc:date rdfs:subPropertyOf edm:hasMet . - -dc:format rdfs:subPropertyOf edm:hasType . - -dcterms:hasFormat rdfs:subPropertyOf _:node16k3ng1n7x56 . - -_:node16k3ng1n7x56 a rdf:Property ; - owl:inverseOf edm:isDerivativeOf . - -dcterms:hasVersion rdfs:subPropertyOf _:node16k3ng1n7x57 . - -_:node16k3ng1n7x57 a rdf:Property ; - owl:inverseOf edm:isDerivativeOf . - -dcterms:isFormatOf rdfs:subPropertyOf edm:isDerivativeOf . - -dcterms:isReplacedBy rdfs:subPropertyOf _:node16k3ng1n7x58 . - -_:node16k3ng1n7x58 a rdf:Property ; - owl:inverseOf edm:isDerivativeOf . - -dcterms:isVersionOf rdfs:subPropertyOf edm:isDerivativeOf . - -dc:language rdfs:subPropertyOf edm:hasType . - -dc:publisher rdfs:subPropertyOf edm:hasMet . - -dc:relation rdfs:subPropertyOf edm:isRelatedTo . - -dcterms:replaces rdfs:subPropertyOf edm:isDerivativeOf . - -dc:source rdfs:subPropertyOf edm:isDerivativeOf . - -dc:subject rdfs:subPropertyOf edm:isRelatedTo . - -dcterms:tableOfContents rdfs:subPropertyOf edm:hasView . - -dc:type rdfs:subPropertyOf edm:hasType . diff --git a/metis-schema/src/main/resources/rdf_examples/EDM-v524-120820.n3 b/metis-schema/src/main/resources/rdf_examples/EDM-v524-120820.n3 deleted file mode 100644 index d7873e9c9..000000000 --- a/metis-schema/src/main/resources/rdf_examples/EDM-v524-120820.n3 +++ /dev/null @@ -1,559 +0,0 @@ - @prefix : . - @prefix DOLCE-Lite: . - @prefix abc: . - @prefix adms: . - @prefix crm: . - @prefix dc: . - @prefix dcmitype: . - @prefix dcterms: . - @prefix edm: . - @prefix foaf: . - @prefix frbr_core: . - @prefix ore: . - @prefix owl: . - @prefix radion: . - @prefix rdf: . - @prefix skos: . - @prefix vann: . - @prefix voaf: . - @prefix wgs84_pos: . - @prefix xsd: . - - <> a voaf:Vocabulary, - owl:Ontology; - dc:contributor [ - a foaf:Organization; - foaf:name "Participants of Europeana Version 1.0 Work Package on Further Specification of Functionality and Interoperability aspects of Europeana (WP3)" ], - [ - a foaf:Person; - foaf:name "Hugo Manguinhas" ], - , - , - , - ; - dc:creator ; - dc:description "The Europeana Data Model (EDM) is aimed at being an integration medium for collecting, connecting and enriching the descriptions provided by Europeana data providers. The RDF vocabulary for http://www.europeana.eu/schemas/edm/ defines the elements introduced by EDM (as opposed to the ones EDM re-uses from other namespaces)."@en; - dc:issued "2010-03-25"^^xsd:date; - dc:modified "2012-08-20"^^xsd:date; - dc:publisher ; - dc:title "Europeana Data Model (EDM) vocabulary"@en; - vann:changes """ -======= -Changes between EDM-v524-120820.owl -and EDM-v523-120123.owl -======= -1. edm:isShownAt made a sub-property of edm:hasView -2. added edm:begin and edm:end and their mappings to CRM -3. added owl:Class declarations added for compatibility with some OWL-DL reasoners (feedback from Pedro Szekely, ISI) -4. added "of" at the end of the label for edm:isNextInSequence -5. added vocabulary metadata to follow Linked Open Vocabularies (http://lov.okfn.org/) and ADMS (https://joinup.ec.europa.eu/asset/adms/release/100) guidelines -6. removed a domain axiom on edm:hasMet -7. added edm:collectionName and edm:europeanaProxy - """@en; - vann:example , - ; - vann:preferredNamespacePrefix "edm"; - vann:preferredNamespaceUri "http://www.europeana.eu/schemas/edm/"; - voaf:toDoList """ -======= -Remaining TODOs for EDM-v524-120820.owl -======= -- change FRBR namespace and mappings using FRBRoo namespace or http://metadataregistry.org/schema/show/id/5.html -- add Martin's mapping to FRBR(oo) for edm:incorporates (equivalent to crm:R14F.incorporates) and edm:WebResource (subclass of F24 Publication Expression) -- try to capture formal cardinality constraints resulting from "Obligation and Occurrence" documentation, which should be attached to non-EDM constructs (esp. ore:Aggregation) -- continue adding documentation values (skos:scopeNote, skos:example, etc, according to 1.), starting from edm:InformationResource. Add all Europeana examples and rationale notes for non-EDM constructs -- use specific EDM-doc properties for "rationale" and "obligation and occurrence". Use skos:definition for "Europeana definition", skos:example for "Example", skos:note for "Europeana note" - """@en; - owl:versionInfo "5.2.4"; - adms:relatedWebPage ; - radion:versionNotes """The present specification is based on the document "Definition of the Europeana Data Model elements", originally edited by Carlo Meghini. It is aligned with the version 5.2.4 of these EDM Definitions."""@en; - foaf:homePage . - - a foaf:Person; - foaf:name "Antoine Isaac" . - - a foaf:Organization; - foaf:name "Europeana" . - - dc:contributor :subPropertyOf edm:hasMet . - - dc:coverage :subPropertyOf edm:hasMet . - - dc:creator :subPropertyOf edm:hasMet . - - dc:date :subPropertyOf edm:hasMet . - - dc:format :subPropertyOf edm:hasType . - - dc:language :subPropertyOf edm:hasType . - - dc:publisher :subPropertyOf edm:hasMet . - - dc:relation :subPropertyOf edm:isRelatedTo . - - dc:source :subPropertyOf edm:isDerivativeOf . - - dc:subject :subPropertyOf edm:isRelatedTo . - - dc:type :subPropertyOf edm:hasType . - - dcterms:hasFormat :subPropertyOf [ - a rdf:Property; - owl:inverseOf edm:isDerivativeOf ] . - - dcterms:hasVersion :subPropertyOf [ - a rdf:Property; - owl:inverseOf edm:isDerivativeOf ] . - - dcterms:isFormatOf :subPropertyOf edm:isDerivativeOf . - - dcterms:isReplacedBy :subPropertyOf [ - a rdf:Property; - owl:inverseOf edm:isDerivativeOf ] . - - dcterms:isVersionOf :subPropertyOf edm:isDerivativeOf . - - dcterms:replaces :subPropertyOf edm:isDerivativeOf . - - dcterms:tableOfContents :subPropertyOf edm:hasView . - - :subPropertyOf edm:begin . - - :subPropertyOf edm:end . - - edm:Agent a owl:Class; - :label "Agent"@en; - :subClassOf edm:NonInformationResource; - owl:equivalentClass ; - skos:definition """This class comprises people, either individually or in groups, who have the -potential to perform intentional actions for which they can be held responsible."""; - skos:example "Leonardo da Vinci, the British Museum, W3C"; - skos:scopeNote "Rationale: This class is a domain of edm:wasPresentAt" . - - edm:EuropeanaAggregation a owl:Class; - :label "Europeana Aggregation"; - :subClassOf edm:EuropeanaObject, - ore:Aggregation; - owl:equivalentClass [ - a owl:Restriction; - owl:cardinality "1"^^xsd:nonNegativeInteger; - owl:onProperty edm:aggregatedCHO ]; - skos:definition """The set of resources related to a single Cultural Heritage Object that -collectively represent that object in Europeana. Such set consists of: all -descriptions about the object that Europeana collects from (possibly different) content providers, including thumbnails and other forms of abstractions, as well as of the description of the object Europeana builds."""; - skos:example """The 56th issue of "Le Temps" is a (different) Cultural Heritage Object represented in Europeana by another EuropeanaAggregation instance"""@en, - """The journal "Le Temps" is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"""@en, - "The painting Mona Lisa is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en; - skos:note "An instance of EuropeanaAggregation is created at ingestion time for each different Cultural Heritage Object recognized by Europeana. Such instance is associated to the Cultural Heritage Object that it is about, by the property edm:aggregatedCHO"@en; - skos:scopeNote "Obligation and Occurence: The relation between the Cultural Heritage Objects represented in Europeana and the instances of the class EuropeanaAggregation is one-to-one, in the data maintained by Europeana: every Cultural Heritage Object is represented by an instance of EuropeanaAggregation, and every instance of EuropeanaAggregation represent a Cultural Heritage Object."@en, - "Rationale: This class is used in Europeana to gather in a single conceptual unit all the information about a Cultural Heritage Object, necessary for all operations on these objects."@en . - - edm:EuropeanaObject a owl:Class; - :label "Europeana Object"@en; - :subClassOf edm:WebResource; - skos:definition "Any object that is the result of Europeana\u2019s activities"@en; - skos:example "An annotation created by a user through the Europeana portal"@en, - "Any content created by the users through the service made available by Europeana for that purpose"@en, - "Any instance of the class EuropeanaAggregation"@en; - skos:scopeNote "Rationale: This class is used to tag objects that are the result of activity of Europeana, and, as such, objects on which Europeana holds rights"@en . - - edm:Event a owl:Class; - :label "Event"@en; - :subClassOf [ - a owl:Restriction; - owl:cardinality "1"^^xsd:nonNegativeInteger; - owl:onProperty edm:happenedAt ], - edm:NonInformationResource; - owl:equivalentClass abc:Temporality, - frbr_core:Event, - ; - skos:definition """An event is a change "of states in cultural, social or physical systems, - regardless of scale, brought about by a series or group of coherent physical, -cultural, technological or legal phenomena" (E5 Event in CIDOC CRM) or a "set of coherent phenomena or cultural manifestations bounded in time and space" (E4 Period in CIDOC CRM) -"""@en; - skos:example "the 2nd World War"@en, - "the act of painting Mona Lisa"@en, - "the change of custody of Mona Lisa"@en; - skos:note "Events are identified either by the content provider or by Europeana enrichment at ingestion time"@en; - skos:scopeNote "Rationale:This class is a domain of edm:happenedAt and the domain of edm:occurredAt"@en . - - edm:InformationResource a owl:Class; - :label "Information Resource"@en; - owl:equivalentClass [ - a owl:Class; - owl:unionOf ( - frbr_core:Work - frbr_core:Expression - frbr_core:Manifestation ) ], - ; - skos:definition "An information resource is a resource whose essential characteristics can be conveyed in a single message. It can be associated with a URI, it can have a representation, for example: a text is an InformationResource."@en . - - edm:NonInformationResource a owl:Class; - :label "Non-Information Resource"@en; - owl:complementOf edm:InformationResource; - skos:definition "All resources that are not information resources."@en . - - edm:PhysicalThing a owl:Class; - :label "Physical Thing"@en; - :subClassOf edm:NonInformationResource; - owl:equivalentClass ; - skos:definition """A persistent physical item such as a painting, a building, a book or a stone. -Persons are not items. This class represents Cultural Heritage Objects known to Europeana to be physical things (such as Mona Lisa) as well as all physical things Europeana refers to in the descriptions of Cultural Heritage Objects (such as the Rosetta Stone)."""@en . - - edm:Place a owl:Class; - :label "Place"@en; - :subClassOf edm:NonInformationResource; - owl:equivalentClass abc:Place, - frbr_core:Place, - , - DOLCE-Lite:space-region; - skos:definition """An "extent in space, in particular on the surface of the earth, in the pure sense of physics: independent from temporal phenomena and matter" (CIDOC CRM)"""@en . - - edm:ProvidedCHO a owl:Class; - :label "Provided CHO"@en; - :subClassOf [ - a owl:Restriction; - owl:cardinality "1"^^xsd:nonNegativeInteger; - owl:onProperty [ - a rdf:Property; - owl:inverseOf edm:aggregatedCHO ] ]; - skos:definition "This class comprises the Cultural Heritage objects that Europeana collects descriptions about."@en; - skos:example "Mona Lisa, Winged Victory of Samothrace"@en; - skos:note "This class has been mostly motivated by the need to assign a type to the \u201Ccentral node\u201D in the EDM pattern, during the ingestion process, related to the XML expression of EDM at that stage. It was especially intended to fit the cases where edm:PhysicalThing cannot be used as the type of the resource standing for the real-world object (independently of any specific data contributor perspective)."@en; - skos:scopeNote "Rationale: This class is the range of edm:aggregatedCHO. A resource of type ProvidedCHO can be the subject of statements using edm:isRelatedTo or any more specific property."@en . - - edm:TimeSpan a owl:Class; - :label "Time Span"@en; - :subClassOf dcterms:PeriodOfTime, - edm:NonInformationResource; - owl:equivalentClass abc:Time, - , - DOLCE-Lite:time-interval; - skos:definition """The class of "abstract temporal extents, in the sense of Galilean physics, - having a beginning, an end and a duration" (CIDOC CRM)"""@en . - - edm:WebResource a owl:Class; - :label "Web Resource"@en; - :subClassOf edm:InformationResource; - skos:definition """Information Resources that have at least one Web Representation and at least -a URI."""@en . - - edm:aggregatedCHO a owl:ObjectProperty; - :domain ore:Aggregation; - :label "Aggregated Cultural Heritage Object"@en; - :range edm:ProvidedCHO; - :subPropertyOf dc:subject, - , - ore:aggregates; - skos:definition """This property associates an ORE aggregation with the Cultural Heritage -Object(s) (CHO for short) it is about."""@en . - - edm:begin a owl:DatatypeProperty; - :domain [ - a owl:Class; - owl:unionOf ( - edm:Agent - edm:TimeSpan ) ]; - :label "Begin"@en; - :subPropertyOf edm:isRelatedTo; - skos:definition "This property denotes the start date of a period of time."@en . - - edm:collectionName a owl:ObjectProperty; - :label "Collection Name"@en; - skos:definition "This property holds the collection identifier given to the dataset in Europeana."@en; - skos:note "The value of this property is provided by Europeana as part of the ingestion process."@en . - - edm:country a rdf:Property; - :label "Country"@en; - :subPropertyOf . - - edm:currentLocation a owl:ObjectProperty; - :domain [ - a owl:Class; - owl:unionOf ( - edm:ProvidedCHO - [ - owl:intersectionOf ( - ore:Proxy - [ - owl:onProperty ore:proxyFor; - owl:someValuesFrom edm:ProvidedCHO ] ) ] ) ]; - :label "Current Location"@en; - :range edm:Place; - :subPropertyOf dcterms:spatial; - owl:equivalentProperty , - wgs84_pos:location; - skos:definition "The geographic location and/or name of the repository, building, site, or other entity whose boundaries presently include the resource."@en . - - edm:dataProvider a rdf:Property; - :domain ore:Aggregation; - :label "Europeana Data Provider"@en; - :range edm:Agent; - :subPropertyOf dcterms:provenance; - skos:definition "This element is specifically included to allow the name of the organisation who supplies data to Europeana indirectly via an aggregator to be recorded and displayed in the portal. Aggregator names are recorded in edm:provider. If an organisation provides data directly to Europeana (i.e. not via an aggregator) the values in edm:dataProvider and edm:provider will be the same. Organisation names should be provided as an ordinary text string until the Europeana Authority File for Organisations has been established. At that point providers will be able to send an identifier from the file instead of a text string. The name provided should be the preferred form of the name in the language the provider chooses as the default language for display in the portal. Countries with multiple languages may prefer to concatenate the name in more than one language (See the example below.) Note: Europeana Data Provider is not necessarily the institution where the physical object is located."@en . - - edm:end a owl:DatatypeProperty; - :domain [ - a owl:Class; - owl:unionOf ( - edm:Agent - edm:TimeSpan ) ]; - :label "End"@en; - :subPropertyOf edm:isRelatedTo; - skos:definition "This property denotes the end date of a period of time."@en . - - edm:europeanaProxy a owl:ObjectProperty; - :label "Europeana Proxy"@en; - skos:definition "This property serves only as a flag to indicate that a proxy is a Europeana proxy (as opposed to a provider proxy). It is for internal use only."@en; - skos:note "By default, any proxy without this flag can be interpreted as having the value false and is a provider proxy."@en . - - edm:happenedAt a owl:ObjectProperty; - :domain edm:Event; - :label "Happened At"@en; - :range edm:Place; - :subPropertyOf dc:relation; - owl:equivalentProperty ; - skos:definition """This property associates an event with the place at which the event -happened."""@en . - - edm:hasMet a rdf:Property; - :label "Has Met"@en; - :subPropertyOf dc:relation; - skos:definition "edm:hasMet relates a resource with the objects or phenomena that have happened to or have happened together with the resource under consideration. We can abstractly think of history and the present as a series of \u201Cmeetings\u201D between people and other things in space-time. Therefore we name this relationship as the things the object \u201Chas met\u201D in the course of its existence. These meetings are events in the proper sense, in which other people and things participate in any role."@en . - - edm:hasType a owl:ObjectProperty; - :domain [ - a owl:Class; - owl:unionOf ( - edm:ProvidedCHO - [ - owl:intersectionOf ( - ore:Proxy - [ - owl:onProperty ore:proxyFor; - owl:someValuesFrom edm:ProvidedCHO ] ) ] ) ]; - :label "Has Type"@en; - :range edm:NonInformationResource; - :subPropertyOf edm:isRelatedTo; - owl:equivalentProperty ; - skos:definition """This property relates a resource with the concepts it belongs to in a suitable -type system such as MIME or any thesaurus that captures categories of objects in a given field (e.g., the \u201CObjects\u201D facet in Getty\u2019s Art and Architecture Thesaurus). It does not capture aboutness."""@en . - - edm:hasView a owl:ObjectProperty; - :domain ore:Aggregation; - :label "Has View"@en; - :range edm:WebResource; - :subPropertyOf ore:aggregates; - skos:definition """This property relates a ORE aggregation about a CHO with a web resource -providing a view of that CHO. Examples of view are: a thumbnail, a textual -abstract and a table of contents. The ORE aggregation may be a Europeana -Aggregation, in which case the view is an object owned by Europeana (i.e., an instance of edm:EuropeanaObject) or an aggregation contributed by a content provider. In order to capture both these cases, the domain of edm:hasView is ore:Aggregation and its range is edm:WebResource"""@en . - - edm:incorporates a owl:ObjectProperty; - :label "Incorporates"@en; - :subPropertyOf edm:isSimilarTo; - skos:definition """This property captures the use of some resource to add value to another -resource. Such resources may be nested, such as performing a theater play text, and then recording the performance, or creating an artful edition of a collection of poems or just aggregating various poems in an anthology. There may be no single part that contains ultimately the incorporated object, which may be dispersed in the presentation. Therefore, incorporated resources do in general not form proper parts. Incorporated resources are not part of the same resource, but are taken from other resources, and have an independent history. Therefore edm:incorporates is not a sub-property of dcterm:hasPart."""@en . - - edm:isAnnotationOf a owl:ObjectProperty; - :domain edm:EuropeanaObject; - :label "Is Annotation Of"@en; - :range [ - a owl:Class; - owl:unionOf ( - edm:ProvidedCHO - [ - owl:intersectionOf ( - ore:Proxy - [ - owl:onProperty ore:proxyFor; - owl:someValuesFrom edm:ProvidedCHO ] ) ] ) ]; - :subPropertyOf dc:subject, - ; - skos:definition """This property relates an annotation (a Europeana object) with the resource -that it annotates."""@en . - - edm:isDerivativeOf a rdf:Property; - :label "Is Derivative Of"@en; - :subPropertyOf edm:isSimilarTo; - skos:definition "This property captures a narrower notion of derivation than edm:isSimilarTo, in the sense that it relates a resource to another one, obtained by reworking, reducing, expanding, parts or the whole contents of the former, and possibly adding some minor parts. Versions have an even narrower meaning, in that it requires common identity between the related resources. Translations, summaries, abstractions etc. do not qualify as versions, but do qualify as derivatives."@en . - - edm:isNextInSequence a owl:ObjectProperty; - :label "Is Next In Sequence Of"@en; - :subPropertyOf dc:relation; - skos:definition "edm:isNextInSequence relates two resources S and R that are ordered parts of the same resource A, and such that R comes immediately after R in the order created by their being parts of S."@en . - - edm:isRelatedTo a rdf:Property; - :domain [ - a owl:Class; - owl:unionOf ( - edm:ProvidedCHO - [ - owl:intersectionOf ( - ore:Proxy - [ - owl:onProperty ore:proxyFor; - owl:someValuesFrom edm:ProvidedCHO ] ) ] ) ]; - :label "Is Related To"@en; - skos:definition """edm:isRelatedTo is the most general contextual property in EDM. Contextual -properties have typically to do either with the things that have happened to or together with the object under consideration, or what the object refers to by its shape, form or features in a figural or encoded form. For sake of simplicity, we include in the contextual relationships also the scholarly classification, which may have either to do with the role and cultural connections of the object in the past, or its kind of structure, substance or contents as it can be verified at present."""@en . - - edm:isRepresentationOf a owl:ObjectProperty; - :domain edm:InformationResource; - :label "Is Representation Of"@en; - :subPropertyOf dc:subject, - ; - skos:definition "This property associates an information resource to the resource (if any) that it represents"@en . - - edm:isShownAt a owl:ObjectProperty; - :label "Is Shown At"@en; - :range edm:WebResource; - :subPropertyOf edm:hasView; - skos:definition "An unambiguous URL reference to the digital object on the provider\u2019s web site in its full information context."@en . - - edm:isShownBy a owl:ObjectProperty; - :label "Is Shown By"@en; - :range edm:WebResource; - :subPropertyOf edm:hasView; - skos:definition "An unambiguous URL reference to the digital object on the provider\u2019s web site in the best available resolution/quality."@en . - - edm:isSimilarTo a rdf:Property; - :label "Is Similar To"@en; - :subPropertyOf dc:relation; - owl:equivalentProperty ; - skos:definition "The most generic derivation property, covering also the case of questionable derivation. Is Similar To asserts that parts of the contents of one resource exhibit common features with respect to ideas, shapes, structures, colors, words, plots, topics with the contents of the related resource. Those common features may be attributed to a common origin or influence (in particular for derivation), but also to more generic cultural or psychological factors."@en . - - edm:isSuccessorOf a owl:ObjectProperty; - :label "Is Successor Of"@en; - :subPropertyOf edm:isSimilarTo; - skos:definition "This property captures the relation between the continuation of a resource and that resource. This applies to a story, a serial, a journal etc. No content of the successor resource is identical or has a similar form with that of the precursor. The similarity is only in the context, subjects and figures of a plot. Successors typically form part of a common whole \u2013 such as a trilogy, a journal, etc."@en . - - edm:landingPage a owl:ObjectProperty; - :label "Landing Page"@en; - :range edm:WebResource; - :subPropertyOf ore:aggregates; - skos:definition """This property captures the relation between an aggregation representing a -Cultural Heritage Object and the Web Resource representing that Object on -the provider\u2019s web site."""@en . - - edm:language a rdf:Property; - :comment """The recommended best practice is to use a controlled vocabulary such as -RFC 4646 (http://www.rfc-archive.org/getrfc.php?rfc=4646) which, in -conjunction with ISO 639, defines two- and three-letter primary language tags. Either a coded value or text string can be represented here."""@en; - :label "Europeana Language"@en; - skos:definition "A language assigned to the resource with reference to the Provider."@en . - - edm:object a owl:ObjectProperty; - :label "Object"@en; - :range edm:WebResource; - :subPropertyOf edm:hasView; - skos:definition """The URL of a thumbnail representing the digital object or, if there is no such -thumbnail, the URL of the digital object in the best resolution available on the -web site of the data provider from which a thumbnail could be generated. This will often be the same URL as given in edm:isShownBy."""@en . - - edm:occurredAt a owl:ObjectProperty; - :domain edm:Event; - :label "Occured At"@en; - :range edm:TimeSpan; - :subPropertyOf dc:relation; - owl:equivalentProperty ; - skos:definition """This property associates an event to the smallest known time span that -overlaps with the occurrence of that event"""@en . - - edm:preview a owl:ObjectProperty; - :label "Preview"@en; - :range edm:WebResource; - :subPropertyOf edm:hasView; - skos:definition "The URL of a thumbnail representing the digital object, generated by Europeana."@en . - - edm:provider a rdf:Property; - :label "Europeana Provider"@en; - :range edm:Agent; - :subPropertyOf edm:hasMet; - skos:definition "Name of the organization that delivers data to Europeana"@en . - - edm:realizes a owl:ObjectProperty; - :domain edm:PhysicalThing; - :label "Realizes"@en; - :range edm:InformationResource; - :subPropertyOf edm:isRelatedTo; - owl:equivalentProperty ; - skos:definition "This property describes a relation between a physical thing and the information resource that is contained in it, visible at it or otherwise carried by it, if applicable."@en . - - edm:rights a owl:ObjectProperty; - :label "Europeana Rights"@en; - skos:definition """Information about copyright of the digital object as specified by isShownBy -and isShownAt"""@en . - - edm:type a owl:DatatypeProperty; - :label "Europeana Type"@en; - :range [ - a :Datatype; - owl:oneOf ( - "TEXT" - "IMAGE" - "SOUND" - "VIDEO" - "3D" ) ]; - :subPropertyOf dc:type; - skos:definition "The Europeana material type of the resource"@en . - - edm:ugc a owl:DatatypeProperty; - :label "UGC"@en; - :range [ - a :Datatype; - owl:oneOf ( - "TRUE" ) ]; - skos:definition "This element is used to identify user generated content (also called user created content). It should be applied to all digitised or born digital content contributed by the general public and collected by Europeana through a crowdsourcing initiative or project."@en . - - edm:unstored a rdf:Property; - :label "Unstored"@en; - skos:definition """This is a container element which includes all relevant information that -otherwise cannot be mapped to another element in the ESE."""@en . - - edm:uri a owl:ObjectProperty; - :label "Europeana URI"@en; - skos:definition "This is a tag created by a user through the Europeana interface."@en . - - edm:userTag a rdf:Property; - :label "User Tag"@en; - :subPropertyOf dc:description; - skos:definition "This is a tag created by a user through the Europeana interface."@en . - - edm:wasPresentAt a owl:ObjectProperty; - :domain [ - a owl:Class; - owl:unionOf ( - edm:Agent - edm:InformationResource - edm:PhysicalThing ) ]; - :label "Was Present At"@en; - :range edm:Event; - :subPropertyOf dc:relation; - owl:equivalentProperty ; - skos:definition "This property associates the people, things or information resources with an event at which they were present"@en . - - edm:year a rdf:Property; - :label "Europeana Year"@en; - :subPropertyOf dcterms:temporal; - skos:definition """A point of time associated with an event in the life of the original analog or -born digital object."""@en . - - a foaf:Person; - foaf:name "Julia Iwanowa" . - - a foaf:Person; - foaf:name "Martin Doerr" . - - a foaf:Person; - foaf:name "Nasos Drosopoulos" . - - a foaf:Person; - foaf:name "Vassilis Tzouvaras" . - - ore:Aggregation a owl:Class . - - ore:Proxy a owl:Class . - - skos:Concept a owl:Class; - :subClassOf edm:NonInformationResource . \ No newline at end of file diff --git a/metis-schema/src/main/resources/rdf_examples/EDM-v524-120820.nt b/metis-schema/src/main/resources/rdf_examples/EDM-v524-120820.nt deleted file mode 100644 index 158835e5f..000000000 --- a/metis-schema/src/main/resources/rdf_examples/EDM-v524-120820.nt +++ /dev/null @@ -1,452 +0,0 @@ - . - . - "edm" . - "http://www.europeana.eu/schemas/edm/" . - "Europeana Data Model (EDM) vocabulary"@en . - "The Europeana Data Model (EDM) is aimed at being an integration medium for collecting, connecting and enriching the descriptions provided by Europeana data providers. The RDF vocabulary for http://www.europeana.eu/schemas/edm/ defines the elements introduced by EDM (as opposed to the ones EDM re-uses from other namespaces)."@en . - "2010-03-25"^^ . - "2012-08-20"^^ . - "5.2.4" . - "The present specification is based on the document \"Definition of the Europeana Data Model elements\", originally edited by Carlo Meghini. It is aligned with the version 5.2.4 of these EDM Definitions."@en . - . - . - "Antoine Isaac" . - _:node17eprp1n4x131131 . -_:node17eprp1n4x131131 . -_:node17eprp1n4x131131 "Participants of Europeana Version 1.0 Work Package on Further Specification of Functionality and Interoperability aspects of Europeana (WP3)" . - . - . - "Nasos Drosopoulos" . - . - . - "Vassilis Tzouvaras" . - . - . - "Julia Iwanowa" . - _:node17eprp1n4x131132 . -_:node17eprp1n4x131132 . -_:node17eprp1n4x131132 "Hugo Manguinhas" . - . - . - "Martin Doerr" . - . - . - "Europeana" . - . - . - . - . - "=======\nChanges between EDM-v524-120820.owl\nand EDM-v523-120123.owl\n=======\n1. edm:isShownAt made a sub-property of edm:hasView\n2. added edm:begin and edm:end and their mappings to CRM\n3. added owl:Class declarations added for compatibility with some OWL-DL reasoners (feedback from Pedro Szekely, ISI)\n4. added \"of\" at the end of the label for edm:isNextInSequence\n5. added vocabulary metadata to follow Linked Open Vocabularies (http://lov.okfn.org/) and ADMS (https://joinup.ec.europa.eu/asset/adms/release/100) guidelines\n6. removed a domain axiom on edm:hasMet\n7. added edm:collectionName and edm:europeanaProxy"@en . - "=======\nRemaining TODOs for EDM-v524-120820.owl\n=======\n- change FRBR namespace and mappings using FRBRoo namespace or http://metadataregistry.org/schema/show/id/5.html\n- add Martin's mapping to FRBR(oo) for edm:incorporates (equivalent to crm:R14F.incorporates) and edm:WebResource (subclass of F24 Publication Expression)\n- try to capture formal cardinality constraints resulting from \"Obligation and Occurrence\" documentation, which should be attached to non-EDM constructs (esp. ore:Aggregation)\n- continue adding documentation values (skos:scopeNote, skos:example, etc, according to 1.), starting from edm:InformationResource. Add all Europeana examples and rationale notes for non-EDM constructs\n- use specific EDM-doc properties for \"rationale\" and \"obligation and occurrence\". Use skos:definition for \"Europeana definition\", skos:example for \"Example\", skos:note for \"Europeana note\""@en . - . - "Agent"@en . - "This class comprises people, either individually or in groups, who have the\npotential to perform intentional actions for which they can be held responsible." . - "Leonardo da Vinci, the British Museum, W3C" . - . - . - "Rationale: This class is a domain of edm:wasPresentAt" . - . - "Europeana Aggregation" . - "The set of resources related to a single Cultural Heritage Object that\ncollectively represent that object in Europeana. Such set consists of: all\ndescriptions about the object that Europeana collects from (possibly different) content providers, including thumbnails and other forms of abstractions, as well as of the description of the object Europeana builds." . - . - . - _:node17eprp1n4x131133 . -_:node17eprp1n4x131133 . -_:node17eprp1n4x131133 "1"^^ . -_:node17eprp1n4x131133 . - "An instance of EuropeanaAggregation is created at ingestion time for each different Cultural Heritage Object recognized by Europeana. Such instance is associated to the Cultural Heritage Object that it is about, by the property edm:aggregatedCHO"@en . - "Obligation and Occurence: The relation between the Cultural Heritage Objects represented in Europeana and the instances of the class EuropeanaAggregation is one-to-one, in the data maintained by Europeana: every Cultural Heritage Object is represented by an instance of EuropeanaAggregation, and every instance of EuropeanaAggregation represent a Cultural Heritage Object."@en . - "The painting Mona Lisa is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en . - "The journal \"Le Temps\" is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en . - "The 56th issue of \"Le Temps\" is a (different) Cultural Heritage Object represented in Europeana by another EuropeanaAggregation instance"@en . - "Rationale: This class is used in Europeana to gather in a single conceptual unit all the information about a Cultural Heritage Object, necessary for all operations on these objects."@en . - . - "Europeana Object"@en . - "Any object that is the result of Europeana\u0092s activities"@en . - . - "Any instance of the class EuropeanaAggregation"@en . - "An annotation created by a user through the Europeana portal"@en . - "Any content created by the users through the service made available by Europeana for that purpose"@en . - "Rationale: This class is used to tag objects that are the result of activity of Europeana, and, as such, objects on which Europeana holds rights"@en . - . - "Event"@en . - "An event is a change \"of states in cultural, social or physical systems,\n regardless of scale, brought about by a series or group of coherent physical,\ncultural, technological or legal phenomena\" (E5 Event in CIDOC CRM) or a \"set of coherent phenomena or cultural manifestations bounded in time and space\" (E4 Period in CIDOC CRM)"@en . - . - . - . - . - _:node17eprp1n4x131134 . -_:node17eprp1n4x131134 . -_:node17eprp1n4x131134 "1"^^ . -_:node17eprp1n4x131134 . - "Events are identified either by the content provider or by Europeana enrichment at ingestion time"@en . - "the act of painting Mona Lisa"@en . - "the 2nd World War"@en . - "the change of custody of Mona Lisa"@en . - "Rationale:This class is a domain of edm:happenedAt and the domain of edm:occurredAt"@en . - . - "Information Resource"@en . - "An information resource is a resource whose essential characteristics can be conveyed in a single message. It can be associated with a URI, it can have a representation, for example: a text is an InformationResource."@en . - . - _:node17eprp1n4x131135 . -_:node17eprp1n4x131135 . -_:node17eprp1n4x131135 _:node17eprp1n4x131136 . -_:node17eprp1n4x131136 . -_:node17eprp1n4x131136 _:node17eprp1n4x131137 . -_:node17eprp1n4x131137 . -_:node17eprp1n4x131137 _:node17eprp1n4x131138 . -_:node17eprp1n4x131138 . -_:node17eprp1n4x131138 . - . - "Non-Information Resource"@en . - "All resources that are not information resources."@en . - . - . - "Physical Thing"@en . - "A persistent physical item such as a painting, a building, a book or a stone.\nPersons are not items. This class represents Cultural Heritage Objects known to Europeana to be physical things (such as Mona Lisa) as well as all physical things Europeana refers to in the descriptions of Cultural Heritage Objects (such as the Rosetta Stone)."@en . - . - . - . - "Place"@en . - "An \"extent in space, in particular on the surface of the earth, in the pure sense of physics: independent from temporal phenomena and matter\" (CIDOC CRM)"@en . - . - . - . - . - . - . - "Provided CHO"@en . - "This class comprises the Cultural Heritage objects that Europeana collects descriptions about."@en . - _:node17eprp1n4x131139 . -_:node17eprp1n4x131139 . -_:node17eprp1n4x131139 "1"^^ . -_:node17eprp1n4x131139 _:node17eprp1n4x131140 . -_:node17eprp1n4x131140 . -_:node17eprp1n4x131140 . - "This class has been mostly motivated by the need to assign a type to the \u0093central node\u0094 in the EDM pattern, during the ingestion process, related to the XML expression of EDM at that stage. It was especially intended to fit the cases where edm:PhysicalThing cannot be used as the type of the resource standing for the real-world object (independently of any specific data contributor perspective)."@en . - "Mona Lisa, Winged Victory of Samothrace"@en . - "Rationale: This class is the range of edm:aggregatedCHO. A resource of type ProvidedCHO can be the subject of statements using edm:isRelatedTo or any more specific property."@en . - . - "Time Span"@en . - "The class of \"abstract temporal extents, in the sense of Galilean physics,\n having a beginning, an end and a duration\" (CIDOC CRM)"@en . - . - . - . - . - . - . - "Web Resource"@en . - "Information Resources that have at least one Web Representation and at least\na URI."@en . - . - . - "Aggregated Cultural Heritage Object"@en . - "This property associates an ORE aggregation with the Cultural Heritage\nObject(s) (CHO for short) it is about."@en . - . - . - . - . - . - . - "Begin"@en . - "This property denotes the start date of a period of time."@en . - . - _:node17eprp1n4x131141 . -_:node17eprp1n4x131141 . -_:node17eprp1n4x131141 _:node17eprp1n4x131142 . -_:node17eprp1n4x131142 . -_:node17eprp1n4x131142 _:node17eprp1n4x131143 . -_:node17eprp1n4x131143 . -_:node17eprp1n4x131143 . - . - "Collection Name"@en . - "This property holds the collection identifier given to the dataset in Europeana."@en . - "The value of this property is provided by Europeana as part of the ingestion process."@en . - . - "Current Location"@en . - "The geographic location and/or name of the repository, building, site, or other entity whose boundaries presently include the resource."@en . - . - . - . - _:node17eprp1n4x131144 . -_:node17eprp1n4x131144 . -_:node17eprp1n4x131144 _:node17eprp1n4x131145 . -_:node17eprp1n4x131145 . -_:node17eprp1n4x131145 _:node17eprp1n4x131147 . -_:node17eprp1n4x131147 _:node17eprp1n4x131146 . -_:node17eprp1n4x131146 . -_:node17eprp1n4x131146 _:node17eprp1n4x131148 . -_:node17eprp1n4x131148 . -_:node17eprp1n4x131148 _:node17eprp1n4x131150 . -_:node17eprp1n4x131150 _:node17eprp1n4x131149 . -_:node17eprp1n4x131149 . -_:node17eprp1n4x131149 . -_:node17eprp1n4x131149 . -_:node17eprp1n4x131150 . -_:node17eprp1n4x131147 . - . - . - "End"@en . - "This property denotes the end date of a period of time."@en . - . - _:node17eprp1n4x131151 . -_:node17eprp1n4x131151 . -_:node17eprp1n4x131151 _:node17eprp1n4x131152 . -_:node17eprp1n4x131152 . -_:node17eprp1n4x131152 _:node17eprp1n4x131153 . -_:node17eprp1n4x131153 . -_:node17eprp1n4x131153 . - . - "Europeana Proxy"@en . - "This property serves only as a flag to indicate that a proxy is a Europeana proxy (as opposed to a provider proxy). It is for internal use only."@en . - "By default, any proxy without this flag can be interpreted as having the value false and is a provider proxy."@en . - . - "Happened At"@en . - "This property associates an event with the place at which the event\nhappened."@en . - . - . - . - . - . - "Has Met"@en . - "edm:hasMet relates a resource with the objects or phenomena that have happened to or have happened together with the resource under consideration. We can abstractly think of history and the present as a series of \u0093meetings\u0094 between people and other things in space-time. Therefore we name this relationship as the things the object \u0093has met\u0094 in the course of its existence. These meetings are events in the proper sense, in which other people and things participate in any role."@en . - . - . - "Has Type"@en . - "This property relates a resource with the concepts it belongs to in a suitable\ntype system such as MIME or any thesaurus that captures categories of objects in a given field (e.g., the \u0093Objects\u0094 facet in Getty\u0092s Art and Architecture Thesaurus). It does not capture aboutness."@en . - . - . - _:node17eprp1n4x131154 . -_:node17eprp1n4x131154 . -_:node17eprp1n4x131154 _:node17eprp1n4x131155 . -_:node17eprp1n4x131155 . -_:node17eprp1n4x131155 _:node17eprp1n4x131157 . -_:node17eprp1n4x131157 _:node17eprp1n4x131156 . -_:node17eprp1n4x131156 . -_:node17eprp1n4x131156 _:node17eprp1n4x131158 . -_:node17eprp1n4x131158 . -_:node17eprp1n4x131158 _:node17eprp1n4x131160 . -_:node17eprp1n4x131160 _:node17eprp1n4x131159 . -_:node17eprp1n4x131159 . -_:node17eprp1n4x131159 . -_:node17eprp1n4x131159 . -_:node17eprp1n4x131160 . -_:node17eprp1n4x131157 . - . - . - "Has View"@en . - "This property relates a ORE aggregation about a CHO with a web resource\nproviding a view of that CHO. Examples of view are: a thumbnail, a textual\nabstract and a table of contents. The ORE aggregation may be a Europeana\nAggregation, in which case the view is an object owned by Europeana (i.e., an instance of edm:EuropeanaObject) or an aggregation contributed by a content provider. In order to capture both these cases, the domain of edm:hasView is ore:Aggregation and its range is edm:WebResource"@en . - . - . - . - . - "Incorporates"@en . - "This property captures the use of some resource to add value to another\nresource. Such resources may be nested, such as performing a theater play text, and then recording the performance, or creating an artful edition of a collection of poems or just aggregating various poems in an anthology. There may be no single part that contains ultimately the incorporated object, which may be dispersed in the presentation. Therefore, incorporated resources do in general not form proper parts. Incorporated resources are not part of the same resource, but are taken from other resources, and have an independent history. Therefore edm:incorporates is not a sub-property of dcterm:hasPart."@en . - . - . - "Is Annotation Of"@en . - "This property relates an annotation (a Europeana object) with the resource\nthat it annotates."@en . - . - . - . - _:node17eprp1n4x131161 . -_:node17eprp1n4x131161 . -_:node17eprp1n4x131161 _:node17eprp1n4x131162 . -_:node17eprp1n4x131162 . -_:node17eprp1n4x131162 _:node17eprp1n4x131164 . -_:node17eprp1n4x131164 _:node17eprp1n4x131163 . -_:node17eprp1n4x131163 . -_:node17eprp1n4x131163 _:node17eprp1n4x131165 . -_:node17eprp1n4x131165 . -_:node17eprp1n4x131165 _:node17eprp1n4x131167 . -_:node17eprp1n4x131167 _:node17eprp1n4x131166 . -_:node17eprp1n4x131166 . -_:node17eprp1n4x131166 . -_:node17eprp1n4x131166 . -_:node17eprp1n4x131167 . -_:node17eprp1n4x131164 . - . - "Is Derivative Of"@en . - "This property captures a narrower notion of derivation than edm:isSimilarTo, in the sense that it relates a resource to another one, obtained by reworking, reducing, expanding, parts or the whole contents of the former, and possibly adding some minor parts. Versions have an even narrower meaning, in that it requires common identity between the related resources. Translations, summaries, abstractions etc. do not qualify as versions, but do qualify as derivatives."@en . - . - . - "Is Next In Sequence Of"@en . - "edm:isNextInSequence relates two resources S and R that are ordered parts of the same resource A, and such that R comes immediately after R in the order created by their being parts of S."@en . - . - . - "Is Related To"@en . - "edm:isRelatedTo is the most general contextual property in EDM. Contextual\nproperties have typically to do either with the things that have happened to or together with the object under consideration, or what the object refers to by its shape, form or features in a figural or encoded form. For sake of simplicity, we include in the contextual relationships also the scholarly classification, which may have either to do with the role and cultural connections of the object in the past, or its kind of structure, substance or contents as it can be verified at present."@en . - _:node17eprp1n4x131168 . -_:node17eprp1n4x131168 . -_:node17eprp1n4x131168 _:node17eprp1n4x131169 . -_:node17eprp1n4x131169 . -_:node17eprp1n4x131169 _:node17eprp1n4x131171 . -_:node17eprp1n4x131171 _:node17eprp1n4x131170 . -_:node17eprp1n4x131170 . -_:node17eprp1n4x131170 _:node17eprp1n4x131172 . -_:node17eprp1n4x131172 . -_:node17eprp1n4x131172 _:node17eprp1n4x131174 . -_:node17eprp1n4x131174 _:node17eprp1n4x131173 . -_:node17eprp1n4x131173 . -_:node17eprp1n4x131173 . -_:node17eprp1n4x131173 . -_:node17eprp1n4x131174 . -_:node17eprp1n4x131171 . - . - "Is Representation Of"@en . - "This property associates an information resource to the resource (if any) that it represents"@en . - . - . - . - . - "Is Similar To"@en . - "The most generic derivation property, covering also the case of questionable derivation. Is Similar To asserts that parts of the contents of one resource exhibit common features with respect to ideas, shapes, structures, colors, words, plots, topics with the contents of the related resource. Those common features may be attributed to a common origin or influence (in particular for derivation), but also to more generic cultural or psychological factors."@en . - . - . - . - "Is Successor Of"@en . - "This property captures the relation between the continuation of a resource and that resource. This applies to a story, a serial, a journal etc. No content of the successor resource is identical or has a similar form with that of the precursor. The similarity is only in the context, subjects and figures of a plot. Successors typically form part of a common whole \u0096 such as a trilogy, a journal, etc."@en . - . - . - "Landing Page"@en . - "This property captures the relation between an aggregation representing a\nCultural Heritage Object and the Web Resource representing that Object on\nthe provider\u0092s web site."@en . - . - . - . - "Occured At"@en . - "This property associates an event to the smallest known time span that\noverlaps with the occurrence of that event"@en . - . - . - . - . - . - "Preview"@en . - "The URL of a thumbnail representing the digital object, generated by Europeana."@en . - . - . - . - "Realizes"@en . - "This property describes a relation between a physical thing and the information resource that is contained in it, visible at it or otherwise carried by it, if applicable."@en . - . - . - . - . - . - "Was Present At"@en . - "This property associates the people, things or information resources with an event at which they were present"@en . - . - . - _:node17eprp1n4x131175 . -_:node17eprp1n4x131175 . -_:node17eprp1n4x131175 _:node17eprp1n4x131176 . -_:node17eprp1n4x131176 . -_:node17eprp1n4x131176 _:node17eprp1n4x131177 . -_:node17eprp1n4x131177 . -_:node17eprp1n4x131177 _:node17eprp1n4x131178 . -_:node17eprp1n4x131178 . -_:node17eprp1n4x131178 . - . - . - "Country"@en . - . - . - "Europeana Data Provider"@en . - "This element is specifically included to allow the name of the organisation who supplies data to Europeana indirectly via an aggregator to be recorded and displayed in the portal. Aggregator names are recorded in edm:provider. If an organisation provides data directly to Europeana (i.e. not via an aggregator) the values in edm:dataProvider and edm:provider will be the same. Organisation names should be provided as an ordinary text string until the Europeana Authority File for Organisations has been established. At that point providers will be able to send an identifier from the file instead of a text string. The name provided should be the preferred form of the name in the language the provider chooses as the default language for display in the portal. Countries with multiple languages may prefer to concatenate the name in more than one language (See the example below.) Note: Europeana Data Provider is not necessarily the institution where the physical object is located."@en . - . - . - . - . - "Is Shown At"@en . - "An unambiguous URL reference to the digital object on the provider\u0092s web site in its full information context."@en . - . - . - . - "Is Shown By"@en . - "An unambiguous URL reference to the digital object on the provider\u0092s web site in the best available resolution/quality."@en . - . - . - . - "Europeana Language"@en . - "A language assigned to the resource with reference to the Provider."@en . - "The recommended best practice is to use a controlled vocabulary such as\nRFC 4646 (http://www.rfc-archive.org/getrfc.php?rfc=4646) which, in\nconjunction with ISO 639, defines two- and three-letter primary language tags. Either a coded value or text string can be represented here."@en . - . - "Object"@en . - "The URL of a thumbnail representing the digital object or, if there is no such\nthumbnail, the URL of the digital object in the best resolution available on the\nweb site of the data provider from which a thumbnail could be generated. This will often be the same URL as given in edm:isShownBy."@en . - . - . - . - "Europeana Provider"@en . - "Name of the organization that delivers data to Europeana"@en . - . - . - . - "Europeana Rights"@en . - "Information about copyright of the digital object as specified by isShownBy\nand isShownAt"@en . - . - "Europeana Type"@en . - "The Europeana material type of the resource"@en . - . - _:node17eprp1n4x131179 . -_:node17eprp1n4x131179 . -_:node17eprp1n4x131179 _:node17eprp1n4x131180 . -_:node17eprp1n4x131180 "TEXT" . -_:node17eprp1n4x131180 _:node17eprp1n4x131181 . -_:node17eprp1n4x131181 "IMAGE" . -_:node17eprp1n4x131181 _:node17eprp1n4x131182 . -_:node17eprp1n4x131182 "SOUND" . -_:node17eprp1n4x131182 _:node17eprp1n4x131183 . -_:node17eprp1n4x131183 "VIDEO" . -_:node17eprp1n4x131183 _:node17eprp1n4x131184 . -_:node17eprp1n4x131184 "3D" . -_:node17eprp1n4x131184 . - . - "UGC"@en . - "This element is used to identify user generated content (also called user created content). It should be applied to all digitised or born digital content contributed by the general public and collected by Europeana through a crowdsourcing initiative or project."@en . - _:node17eprp1n4x131185 . -_:node17eprp1n4x131185 . -_:node17eprp1n4x131185 _:node17eprp1n4x131186 . -_:node17eprp1n4x131186 "TRUE" . -_:node17eprp1n4x131186 . - . - "Unstored"@en . - "This is a container element which includes all relevant information that\notherwise cannot be mapped to another element in the ESE."@en . - . - "Europeana URI"@en . - "This is a tag created by a user through the Europeana interface."@en . - . - "User Tag"@en . - "This is a tag created by a user through the Europeana interface."@en . - . - . - "Europeana Year"@en . - "A point of time associated with an event in the life of the original analog or\nborn digital object."@en . - . - . - . - . - . - . - . - . - . - _:node17eprp1n4x131187 . -_:node17eprp1n4x131187 . -_:node17eprp1n4x131187 . - _:node17eprp1n4x131188 . -_:node17eprp1n4x131188 . -_:node17eprp1n4x131188 . - . - _:node17eprp1n4x131189 . -_:node17eprp1n4x131189 . -_:node17eprp1n4x131189 . - . - . - . - . - . - . - . - . - . - . - . - . diff --git a/metis-schema/src/main/resources/rdf_examples/EDM-v524-120820.owl b/metis-schema/src/main/resources/rdf_examples/EDM-v524-120820.owl deleted file mode 100644 index 56a216b20..000000000 --- a/metis-schema/src/main/resources/rdf_examples/EDM-v524-120820.owl +++ /dev/null @@ -1,893 +0,0 @@ - - - - - - - - - - - - - - - - - -]> - - - - - - edm - http://www.europeana.eu/schemas/edm/ - Europeana Data Model (EDM) vocabulary - The Europeana Data Model (EDM) is aimed at being an integration medium for collecting, connecting and enriching the descriptions provided by Europeana data providers. The RDF vocabulary for http://www.europeana.eu/schemas/edm/ defines the elements introduced by EDM (as opposed to the ones EDM re-uses from other namespaces). - 2010-03-25 - 2012-08-20 - 5.2.4 - The present specification is based on the document "Definition of the Europeana Data Model elements", originally edited by Carlo Meghini. It is aligned with the version 5.2.4 of these EDM Definitions. - - - - - - - - - - - - - -======= -Changes between EDM-v524-120820.owl -and EDM-v523-120123.owl -======= -1. edm:isShownAt made a sub-property of edm:hasView -2. added edm:begin and edm:end and their mappings to CRM -3. added owl:Class declarations added for compatibility with some OWL-DL reasoners (feedback from Pedro Szekely, ISI) -4. added "of" at the end of the label for edm:isNextInSequence -5. added vocabulary metadata to follow Linked Open Vocabularies (http://lov.okfn.org/) and ADMS (https://joinup.ec.europa.eu/asset/adms/release/100) guidelines -6. removed a domain axiom on edm:hasMet -7. added edm:collectionName and edm:europeanaProxy - - -======= -Remaining TODOs for EDM-v524-120820.owl -======= -- change FRBR namespace and mappings using FRBRoo namespace or http://metadataregistry.org/schema/show/id/5.html -- add Martin's mapping to FRBR(oo) for edm:incorporates (equivalent to crm:R14F.incorporates) and edm:WebResource (subclass of F24 Publication Expression) -- try to capture formal cardinality constraints resulting from "Obligation and Occurrence" documentation, which should be attached to non-EDM constructs (esp. ore:Aggregation) -- continue adding documentation values (skos:scopeNote, skos:example, etc, according to 1.), starting from edm:InformationResource. Add all Europeana examples and rationale notes for non-EDM constructs -- use specific EDM-doc properties for "rationale" and "obligation and occurrence". Use skos:definition for "Europeana definition", skos:example for "Example", skos:note for "Europeana note" - - - - - - - - - Agent - This class comprises people, either individually or in groups, who have the -potential to perform intentional actions for which they can be held responsible. - Leonardo da Vinci, the British Museum, W3C - - - Rationale: This class is a domain of edm:wasPresentAt - - - - - - - Europeana Aggregation - The set of resources related to a single Cultural Heritage Object that -collectively represent that object in Europeana. Such set consists of: all -descriptions about the object that Europeana collects from (possibly different) content providers, including thumbnails and other forms of abstractions, as well as of the description of the object Europeana builds. - - - - - 1 - - - - An instance of EuropeanaAggregation is created at ingestion time for each different Cultural Heritage Object recognized by Europeana. Such instance is associated to the Cultural Heritage Object that it is about, by the property edm:aggregatedCHO - Obligation and Occurence: The relation between the Cultural Heritage Objects represented in Europeana and the instances of the class EuropeanaAggregation is one-to-one, in the data maintained by Europeana: every Cultural Heritage Object is represented by an instance of EuropeanaAggregation, and every instance of EuropeanaAggregation represent a Cultural Heritage Object. - The painting Mona Lisa is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance - The journal "Le Temps" is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance - The 56th issue of "Le Temps" is a (different) Cultural Heritage Object represented in Europeana by another EuropeanaAggregation instance - Rationale: This class is used in Europeana to gather in a single conceptual unit all the information about a Cultural Heritage Object, necessary for all operations on these objects. - - - - - - Europeana Object - Any object that is the result of Europeana’s activities - - Any instance of the class EuropeanaAggregation - An annotation created by a user through the Europeana portal - Any content created by the users through the service made available by Europeana for that purpose - Rationale: This class is used to tag objects that are the result of activity of Europeana, and, as such, objects on which Europeana holds rights - - - - - - Event - An event is a change "of states in cultural, social or physical systems, - regardless of scale, brought about by a series or group of coherent physical, -cultural, technological or legal phenomena" (E5 Event in CIDOC CRM) or a "set of coherent phenomena or cultural manifestations bounded in time and space" (E4 Period in CIDOC CRM) - - - - - - - - 1 - - - - Events are identified either by the content provider or by Europeana enrichment at ingestion time - the act of painting Mona Lisa - the 2nd World War - the change of custody of Mona Lisa - Rationale:This class is a domain of edm:happenedAt and the domain of edm:occurredAt - - - - - - Information Resource - An information resource is a resource whose essential characteristics can be conveyed in a single message. It can be associated with a URI, it can have a representation, for example: a text is an InformationResource. - - - - - - - - - - - - - - - - Non-Information Resource - All resources that are not information resources. - - - - - - - Physical Thing - A persistent physical item such as a painting, a building, a book or a stone. -Persons are not items. This class represents Cultural Heritage Objects known to Europeana to be physical things (such as Mona Lisa) as well as all physical things Europeana refers to in the descriptions of Cultural Heritage Objects (such as the Rosetta Stone). - - - - - - - - Place - An "extent in space, in particular on the surface of the earth, in the pure sense of physics: independent from temporal phenomena and matter" (CIDOC CRM) - - - - - - - - - - - Provided CHO - This class comprises the Cultural Heritage objects that Europeana collects descriptions about. - - - 1 - - - - - - - - This class has been mostly motivated by the need to assign a type to the “central node” in the EDM pattern, during the ingestion process, related to the XML expression of EDM at that stage. It was especially intended to fit the cases where edm:PhysicalThing cannot be used as the type of the resource standing for the real-world object (independently of any specific data contributor perspective). - Mona Lisa, Winged Victory of Samothrace - Rationale: This class is the range of edm:aggregatedCHO. A resource of type ProvidedCHO can be the subject of statements using edm:isRelatedTo or any more specific property. - - - - - - Time Span - The class of "abstract temporal extents, in the sense of Galilean physics, - having a beginning, an end and a duration" (CIDOC CRM) - - - - - - - - - - - Web Resource - Information Resources that have at least one Web Representation and at least -a URI. - - - - - - - - - - Aggregated Cultural Heritage Object - This property associates an ORE aggregation with the Cultural Heritage -Object(s) (CHO for short) it is about. - - - - - - - - - - - Begin - This property denotes the start date of a period of time. - - - - - - - - - - - - - - - Collection Name - This property holds the collection identifier given to the dataset in Europeana. - The value of this property is provided by Europeana as part of the ingestion process. - - - - - - Current Location - The geographic location and/or name of the repository, building, site, or other entity whose boundaries presently include the resource. - - - - - - - - - - - - - - - - - - - - - - - - - - End - This property denotes the end date of a period of time. - - - - - - - - - - - - - - - Europeana Proxy - This property serves only as a flag to indicate that a proxy is a Europeana proxy (as opposed to a provider proxy). It is for internal use only. - By default, any proxy without this flag can be interpreted as having the value false and is a provider proxy. - - - - - - Happened At - This property associates an event with the place at which the event -happened. - - - - - - - - - - Has Met - edm:hasMet relates a resource with the objects or phenomena that have happened to or have happened together with the resource under consideration. We can abstractly think of history and the present as a series of “meetings” between people and other things in space-time. Therefore we name this relationship as the things the object “has met” in the course of its existence. These meetings are events in the proper sense, in which other people and things participate in any role. - - - - - - - Has Type - This property relates a resource with the concepts it belongs to in a suitable -type system such as MIME or any thesaurus that captures categories of objects in a given field (e.g., the “Objects” facet in Getty’s Art and Architecture Thesaurus). It does not capture aboutness. - - - - - - - - - - - - - - - - - - - - - - - - - Has View - This property relates a ORE aggregation about a CHO with a web resource -providing a view of that CHO. Examples of view are: a thumbnail, a textual -abstract and a table of contents. The ORE aggregation may be a Europeana -Aggregation, in which case the view is an object owned by Europeana (i.e., an instance of edm:EuropeanaObject) or an aggregation contributed by a content provider. In order to capture both these cases, the domain of edm:hasView is ore:Aggregation and its range is edm:WebResource - - - - - - - - - Incorporates - This property captures the use of some resource to add value to another -resource. Such resources may be nested, such as performing a theater play text, and then recording the performance, or creating an artful edition of a collection of poems or just aggregating various poems in an anthology. There may be no single part that contains ultimately the incorporated object, which may be dispersed in the presentation. Therefore, incorporated resources do in general not form proper parts. Incorporated resources are not part of the same resource, but are taken from other resources, and have an independent history. Therefore edm:incorporates is not a sub-property of dcterm:hasPart. - - - - - - - Is Annotation Of - This property relates an annotation (a Europeana object) with the resource -that it annotates. - - - - - - - - - - - - - - - - - - - - - - - - - Is Derivative Of - This property captures a narrower notion of derivation than edm:isSimilarTo, in the sense that it relates a resource to another one, obtained by reworking, reducing, expanding, parts or the whole contents of the former, and possibly adding some minor parts. Versions have an even narrower meaning, in that it requires common identity between the related resources. Translations, summaries, abstractions etc. do not qualify as versions, but do qualify as derivatives. - - - - - - - Is Next In Sequence Of - edm:isNextInSequence relates two resources S and R that are ordered parts of the same resource A, and such that R comes immediately after R in the order created by their being parts of S. - - - - - - - - Is Related To - edm:isRelatedTo is the most general contextual property in EDM. Contextual -properties have typically to do either with the things that have happened to or together with the object under consideration, or what the object refers to by its shape, form or features in a figural or encoded form. For sake of simplicity, we include in the contextual relationships also the scholarly classification, which may have either to do with the role and cultural connections of the object in the past, or its kind of structure, substance or contents as it can be verified at present. - - - - - - - - - - - - - - - - - - - - - - Is Representation Of - This property associates an information resource to the resource (if any) that it represents - - - - - - - - - Is Similar To - The most generic derivation property, covering also the case of questionable derivation. Is Similar To asserts that parts of the contents of one resource exhibit common features with respect to ideas, shapes, structures, colors, words, plots, topics with the contents of the related resource. Those common features may be attributed to a common origin or influence (in particular for derivation), but also to more generic cultural or psychological factors. - - - - - - - - Is Successor Of - This property captures the relation between the continuation of a resource and that resource. This applies to a story, a serial, a journal etc. No content of the successor resource is identical or has a similar form with that of the precursor. The similarity is only in the context, subjects and figures of a plot. Successors typically form part of a common whole – such as a trilogy, a journal, etc. - - - - - - - Landing Page - This property captures the relation between an aggregation representing a -Cultural Heritage Object and the Web Resource representing that Object on -the provider’s web site. - - - - - - - - Occured At - This property associates an event to the smallest known time span that -overlaps with the occurrence of that event - - - - - - - - - - Preview - The URL of a thumbnail representing the digital object, generated by Europeana. - - - - - - - - Realizes - This property describes a relation between a physical thing and the information resource that is contained in it, visible at it or otherwise carried by it, if applicable. - - - - - - - - - - - Was Present At - This property associates the people, things or information resources with an event at which they were present - - - - - - - - - - - - - - - - - - - - - - - Country - - - - - - - Europeana Data Provider - This element is specifically included to allow the name of the organisation who supplies data to Europeana indirectly via an aggregator to be recorded and displayed in the portal. Aggregator names are recorded in edm:provider. If an organisation provides data directly to Europeana (i.e. not via an aggregator) the values in edm:dataProvider and edm:provider will be the same. Organisation names should be provided as an ordinary text string until the Europeana Authority File for Organisations has been established. At that point providers will be able to send an identifier from the file instead of a text string. The name provided should be the preferred form of the name in the language the provider chooses as the default language for display in the portal. Countries with multiple languages may prefer to concatenate the name in more than one language (See the example below.) Note: Europeana Data Provider is not necessarily the institution where the physical object is located. - - - - - - - - - Is Shown At - An unambiguous URL reference to the digital object on the provider’s web site in its full information context. - - - - - - - - Is Shown By - An unambiguous URL reference to the digital object on the provider’s web site in the best available resolution/quality. - - - - - - - - Europeana Language - A language assigned to the resource with reference to the Provider. - The recommended best practice is to use a controlled vocabulary such as -RFC 4646 (http://www.rfc-archive.org/getrfc.php?rfc=4646) which, in -conjunction with ISO 639, defines two- and three-letter primary language tags. Either a coded value or text string can be represented here. - - - - - - Object - The URL of a thumbnail representing the digital object or, if there is no such -thumbnail, the URL of the digital object in the best resolution available on the -web site of the data provider from which a thumbnail could be generated. This will often be the same URL as given in edm:isShownBy. - - - - - - - - Europeana Provider - Name of the organization that delivers data to Europeana - - - - - - - - Europeana Rights - Information about copyright of the digital object as specified by isShownBy -and isShownAt - - - - - - Europeana Type - The Europeana material type of the resource - - - - - TEXT - - IMAGE - - SOUND - - VIDEO - - 3D - - - - - - - - - - - - - - UGC - This element is used to identify user generated content (also called user created content). It should be applied to all digitised or born digital content contributed by the general public and collected by Europeana through a crowdsourcing initiative or project. - - - - TRUE - - - - - - - - - - Unstored - This is a container element which includes all relevant information that -otherwise cannot be mapped to another element in the ESE. - - - - - - Europeana URI - This is a tag created by a user through the Europeana interface. - - - - - - User Tag - This is a tag created by a user through the Europeana interface. - - - - - - - Europeana Year - A point of time associated with an event in the life of the original analog or -born digital object. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/metis-schema/src/main/resources/rdf_examples/EDM-v524-120820.ttl b/metis-schema/src/main/resources/rdf_examples/EDM-v524-120820.ttl deleted file mode 100644 index 725436ec5..000000000 --- a/metis-schema/src/main/resources/rdf_examples/EDM-v524-120820.ttl +++ /dev/null @@ -1,654 +0,0 @@ -@prefix : . -@prefix xsd: . -@prefix rdf: . -@prefix rdfs: . -@prefix owl: . -@prefix skos: . -@prefix voaf: . -@prefix vann: . -@prefix adms: . -@prefix radion: . -@prefix dc: . -@prefix dcterms: . -@prefix ore: . -@prefix foaf: . -@prefix wgs84_pos: . -@prefix dcmitype: . -@prefix crm: . -@prefix frbr_core: . -@prefix abc: . -@prefix DOLCE-Lite: . - - a owl:Ontology , voaf:Vocabulary ; - vann:preferredNamespacePrefix "edm" ; - vann:preferredNamespaceUri "http://www.europeana.eu/schemas/edm/" ; - dc:title "Europeana Data Model (EDM) vocabulary"@en ; - dc:description "The Europeana Data Model (EDM) is aimed at being an integration medium for collecting, connecting and enriching the descriptions provided by Europeana data providers. The RDF vocabulary for http://www.europeana.eu/schemas/edm/ defines the elements introduced by EDM (as opposed to the ones EDM re-uses from other namespaces)."@en ; - dc:issued "2010-03-25"^^xsd:date ; - dc:modified "2012-08-20"^^xsd:date ; - owl:versionInfo "5.2.4" ; - radion:versionNotes "The present specification is based on the document \"Definition of the Europeana Data Model elements\", originally edited by Carlo Meghini. It is aligned with the version 5.2.4 of these EDM Definitions."@en ; - dc:creator . - - a foaf:Person ; - foaf:name "Antoine Isaac" . - - dc:contributor _:node17eprgndbx130456 . - -_:node17eprgndbx130456 a foaf:Organization ; - foaf:name "Participants of Europeana Version 1.0 Work Package on Further Specification of Functionality and Interoperability aspects of Europeana (WP3)" . - - dc:contributor . - - a foaf:Person ; - foaf:name "Nasos Drosopoulos" . - - dc:contributor . - - a foaf:Person ; - foaf:name "Vassilis Tzouvaras" . - - dc:contributor . - - a foaf:Person ; - foaf:name "Julia Iwanowa" . - - dc:contributor _:node17eprgndbx130457 . - -_:node17eprgndbx130457 a foaf:Person ; - foaf:name "Hugo Manguinhas" . - - dc:contributor . - - a foaf:Person ; - foaf:name "Martin Doerr" . - - dc:publisher . - - a foaf:Organization ; - foaf:name "Europeana" . - - foaf:homePage ; - adms:relatedWebPage ; - vann:example , ; - vann:changes """======= -Changes between EDM-v524-120820.owl -and EDM-v523-120123.owl -======= -1. edm:isShownAt made a sub-property of edm:hasView -2. added edm:begin and edm:end and their mappings to CRM -3. added owl:Class declarations added for compatibility with some OWL-DL reasoners (feedback from Pedro Szekely, ISI) -4. added \"of\" at the end of the label for edm:isNextInSequence -5. added vocabulary metadata to follow Linked Open Vocabularies (http://lov.okfn.org/) and ADMS (https://joinup.ec.europa.eu/asset/adms/release/100) guidelines -6. removed a domain axiom on edm:hasMet -7. added edm:collectionName and edm:europeanaProxy"""@en ; - voaf:toDoList """======= -Remaining TODOs for EDM-v524-120820.owl -======= -- change FRBR namespace and mappings using FRBRoo namespace or http://metadataregistry.org/schema/show/id/5.html -- add Martin's mapping to FRBR(oo) for edm:incorporates (equivalent to crm:R14F.incorporates) and edm:WebResource (subclass of F24 Publication Expression) -- try to capture formal cardinality constraints resulting from \"Obligation and Occurrence\" documentation, which should be attached to non-EDM constructs (esp. ore:Aggregation) -- continue adding documentation values (skos:scopeNote, skos:example, etc, according to 1.), starting from edm:InformationResource. Add all Europeana examples and rationale notes for non-EDM constructs -- use specific EDM-doc properties for \"rationale\" and \"obligation and occurrence\". Use skos:definition for \"Europeana definition\", skos:example for \"Example\", skos:note for \"Europeana note\""""@en . - -:Agent a owl:Class ; - rdfs:label "Agent"@en ; - skos:definition """This class comprises people, either individually or in groups, who have the -potential to perform intentional actions for which they can be held responsible.""" ; - skos:example "Leonardo da Vinci, the British Museum, W3C" ; - rdfs:subClassOf :NonInformationResource ; - owl:equivalentClass ; - skos:scopeNote "Rationale: This class is a domain of edm:wasPresentAt" . - -:EuropeanaAggregation a owl:Class ; - rdfs:label "Europeana Aggregation" ; - skos:definition """The set of resources related to a single Cultural Heritage Object that -collectively represent that object in Europeana. Such set consists of: all -descriptions about the object that Europeana collects from (possibly different) content providers, including thumbnails and other forms of abstractions, as well as of the description of the object Europeana builds.""" ; - rdfs:subClassOf :EuropeanaObject , ore:Aggregation ; - owl:equivalentClass _:node17eprgndbx130458 . - -_:node17eprgndbx130458 a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty :aggregatedCHO . - -:EuropeanaAggregation skos:note "An instance of EuropeanaAggregation is created at ingestion time for each different Cultural Heritage Object recognized by Europeana. Such instance is associated to the Cultural Heritage Object that it is about, by the property edm:aggregatedCHO"@en ; - skos:scopeNote "Obligation and Occurence: The relation between the Cultural Heritage Objects represented in Europeana and the instances of the class EuropeanaAggregation is one-to-one, in the data maintained by Europeana: every Cultural Heritage Object is represented by an instance of EuropeanaAggregation, and every instance of EuropeanaAggregation represent a Cultural Heritage Object."@en ; - skos:example "The painting Mona Lisa is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en , "The journal \"Le Temps\" is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en , "The 56th issue of \"Le Temps\" is a (different) Cultural Heritage Object represented in Europeana by another EuropeanaAggregation instance"@en ; - skos:scopeNote "Rationale: This class is used in Europeana to gather in a single conceptual unit all the information about a Cultural Heritage Object, necessary for all operations on these objects."@en . - -:EuropeanaObject a owl:Class ; - rdfs:label "Europeana Object"@en ; - skos:definition "Any object that is the result of EuropeanaÂ’s activities"@en ; - rdfs:subClassOf :WebResource ; - skos:example "Any instance of the class EuropeanaAggregation"@en , "An annotation created by a user through the Europeana portal"@en , "Any content created by the users through the service made available by Europeana for that purpose"@en ; - skos:scopeNote "Rationale: This class is used to tag objects that are the result of activity of Europeana, and, as such, objects on which Europeana holds rights"@en . - -:Event a owl:Class ; - rdfs:label "Event"@en ; - skos:definition """An event is a change \"of states in cultural, social or physical systems, - regardless of scale, brought about by a series or group of coherent physical, -cultural, technological or legal phenomena\" (E5 Event in CIDOC CRM) or a \"set of coherent phenomena or cultural manifestations bounded in time and space\" (E4 Period in CIDOC CRM)"""@en ; - rdfs:subClassOf :NonInformationResource ; - owl:equivalentClass , abc:Temporality , frbr_core:Event ; - rdfs:subClassOf _:node17eprgndbx130459 . - -_:node17eprgndbx130459 a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty :happenedAt . - -:Event skos:note "Events are identified either by the content provider or by Europeana enrichment at ingestion time"@en ; - skos:example "the act of painting Mona Lisa"@en , "the 2nd World War"@en , "the change of custody of Mona Lisa"@en ; - skos:scopeNote "Rationale:This class is a domain of edm:happenedAt and the domain of edm:occurredAt"@en . - -:InformationResource a owl:Class ; - rdfs:label "Information Resource"@en ; - skos:definition "An information resource is a resource whose essential characteristics can be conveyed in a single message. It can be associated with a URI, it can have a representation, for example: a text is an InformationResource."@en ; - owl:equivalentClass , _:node17eprgndbx130460 . - -_:node17eprgndbx130460 a owl:Class ; - owl:unionOf _:node17eprgndbx130461 . - -_:node17eprgndbx130461 rdf:first frbr_core:Work ; - rdf:rest _:node17eprgndbx130462 . - -_:node17eprgndbx130462 rdf:first frbr_core:Expression ; - rdf:rest _:node17eprgndbx130463 . - -_:node17eprgndbx130463 rdf:first frbr_core:Manifestation ; - rdf:rest rdf:nil . - -:NonInformationResource a owl:Class ; - rdfs:label "Non-Information Resource"@en ; - skos:definition "All resources that are not information resources."@en ; - owl:complementOf :InformationResource . - -:PhysicalThing a owl:Class ; - rdfs:label "Physical Thing"@en ; - skos:definition """A persistent physical item such as a painting, a building, a book or a stone. -Persons are not items. This class represents Cultural Heritage Objects known to Europeana to be physical things (such as Mona Lisa) as well as all physical things Europeana refers to in the descriptions of Cultural Heritage Objects (such as the Rosetta Stone)."""@en ; - rdfs:subClassOf :NonInformationResource ; - owl:equivalentClass . - -:Place a owl:Class ; - rdfs:label "Place"@en ; - skos:definition "An \"extent in space, in particular on the surface of the earth, in the pure sense of physics: independent from temporal phenomena and matter\" (CIDOC CRM)"@en ; - rdfs:subClassOf :NonInformationResource ; - owl:equivalentClass DOLCE-Lite:space-region , abc:Place , , frbr_core:Place . - -:ProvidedCHO a owl:Class ; - rdfs:label "Provided CHO"@en ; - skos:definition "This class comprises the Cultural Heritage objects that Europeana collects descriptions about."@en ; - rdfs:subClassOf _:node17eprgndbx130464 . - -_:node17eprgndbx130464 a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty _:node17eprgndbx130465 . - -_:node17eprgndbx130465 a rdf:Property ; - owl:inverseOf :aggregatedCHO . - -:ProvidedCHO skos:note "This class has been mostly motivated by the need to assign a type to the “central node” in the EDM pattern, during the ingestion process, related to the XML expression of EDM at that stage. It was especially intended to fit the cases where edm:PhysicalThing cannot be used as the type of the resource standing for the real-world object (independently of any specific data contributor perspective)."@en ; - skos:example "Mona Lisa, Winged Victory of Samothrace"@en ; - skos:scopeNote "Rationale: This class is the range of edm:aggregatedCHO. A resource of type ProvidedCHO can be the subject of statements using edm:isRelatedTo or any more specific property."@en . - -:TimeSpan a owl:Class ; - rdfs:label "Time Span"@en ; - skos:definition """The class of \"abstract temporal extents, in the sense of Galilean physics, - having a beginning, an end and a duration\" (CIDOC CRM)"""@en ; - rdfs:subClassOf :NonInformationResource , dcterms:PeriodOfTime ; - owl:equivalentClass abc:Time , DOLCE-Lite:time-interval , . - -:WebResource a owl:Class ; - rdfs:label "Web Resource"@en ; - skos:definition """Information Resources that have at least one Web Representation and at least -a URI."""@en ; - rdfs:subClassOf :InformationResource . - -:aggregatedCHO a owl:ObjectProperty ; - rdfs:label "Aggregated Cultural Heritage Object"@en ; - skos:definition """This property associates an ORE aggregation with the Cultural Heritage -Object(s) (CHO for short) it is about."""@en ; - rdfs:subPropertyOf ore:aggregates , dc:subject , ; - rdfs:domain ore:Aggregation ; - rdfs:range :ProvidedCHO . - -:begin a owl:DatatypeProperty ; - rdfs:label "Begin"@en ; - skos:definition "This property denotes the start date of a period of time."@en ; - rdfs:subPropertyOf :isRelatedTo ; - rdfs:domain _:node17eprgndbx130466 . - -_:node17eprgndbx130466 a owl:Class ; - owl:unionOf _:node17eprgndbx130467 . - -_:node17eprgndbx130467 rdf:first :Agent ; - rdf:rest _:node17eprgndbx130468 . - -_:node17eprgndbx130468 rdf:first :TimeSpan ; - rdf:rest rdf:nil . - -:collectionName a owl:ObjectProperty ; - rdfs:label "Collection Name"@en ; - skos:definition "This property holds the collection identifier given to the dataset in Europeana."@en ; - skos:note "The value of this property is provided by Europeana as part of the ingestion process."@en . - -:currentLocation a owl:ObjectProperty ; - rdfs:label "Current Location"@en ; - skos:definition "The geographic location and/or name of the repository, building, site, or other entity whose boundaries presently include the resource."@en ; - rdfs:subPropertyOf dcterms:spatial ; - owl:equivalentProperty wgs84_pos:location , ; - rdfs:domain _:node17eprgndbx130469 . - -_:node17eprgndbx130469 a owl:Class ; - owl:unionOf _:node17eprgndbx130470 . - -_:node17eprgndbx130470 rdf:first :ProvidedCHO ; - rdf:rest _:node17eprgndbx130472 . - -_:node17eprgndbx130472 rdf:first _:node17eprgndbx130471 . - -_:node17eprgndbx130471 a owl:Class ; - owl:intersectionOf _:node17eprgndbx130473 . - -_:node17eprgndbx130473 rdf:first ore:Proxy ; - rdf:rest _:node17eprgndbx130475 . - -_:node17eprgndbx130475 rdf:first _:node17eprgndbx130474 . - -_:node17eprgndbx130474 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom :ProvidedCHO . - -_:node17eprgndbx130475 rdf:rest rdf:nil . - -_:node17eprgndbx130472 rdf:rest rdf:nil . - -:currentLocation rdfs:range :Place . - -:end a owl:DatatypeProperty ; - rdfs:label "End"@en ; - skos:definition "This property denotes the end date of a period of time."@en ; - rdfs:subPropertyOf :isRelatedTo ; - rdfs:domain _:node17eprgndbx130476 . - -_:node17eprgndbx130476 a owl:Class ; - owl:unionOf _:node17eprgndbx130477 . - -_:node17eprgndbx130477 rdf:first :Agent ; - rdf:rest _:node17eprgndbx130478 . - -_:node17eprgndbx130478 rdf:first :TimeSpan ; - rdf:rest rdf:nil . - -:europeanaProxy a owl:ObjectProperty ; - rdfs:label "Europeana Proxy"@en ; - skos:definition "This property serves only as a flag to indicate that a proxy is a Europeana proxy (as opposed to a provider proxy). It is for internal use only."@en ; - skos:note "By default, any proxy without this flag can be interpreted as having the value false and is a provider proxy."@en . - -:happenedAt a owl:ObjectProperty ; - rdfs:label "Happened At"@en ; - skos:definition """This property associates an event with the place at which the event -happened."""@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - rdfs:domain :Event ; - rdfs:range :Place . - -:hasMet a rdf:Property ; - rdfs:label "Has Met"@en ; - skos:definition "edm:hasMet relates a resource with the objects or phenomena that have happened to or have happened together with the resource under consideration. We can abstractly think of history and the present as a series of “meetings” between people and other things in space-time. Therefore we name this relationship as the things the object “has met” in the course of its existence. These meetings are events in the proper sense, in which other people and things participate in any role."@en ; - rdfs:subPropertyOf dc:relation . - -:hasType a owl:ObjectProperty ; - rdfs:label "Has Type"@en ; - skos:definition """This property relates a resource with the concepts it belongs to in a suitable -type system such as MIME or any thesaurus that captures categories of objects in a given field (e.g., the “Objects” facet in GettyÂ’s Art and Architecture Thesaurus). It does not capture aboutness."""@en ; - rdfs:subPropertyOf :isRelatedTo ; - owl:equivalentProperty ; - rdfs:domain _:node17eprgndbx130479 . - -_:node17eprgndbx130479 a owl:Class ; - owl:unionOf _:node17eprgndbx130480 . - -_:node17eprgndbx130480 rdf:first :ProvidedCHO ; - rdf:rest _:node17eprgndbx130482 . - -_:node17eprgndbx130482 rdf:first _:node17eprgndbx130481 . - -_:node17eprgndbx130481 a owl:Class ; - owl:intersectionOf _:node17eprgndbx130483 . - -_:node17eprgndbx130483 rdf:first ore:Proxy ; - rdf:rest _:node17eprgndbx130485 . - -_:node17eprgndbx130485 rdf:first _:node17eprgndbx130484 . - -_:node17eprgndbx130484 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom :ProvidedCHO . - -_:node17eprgndbx130485 rdf:rest rdf:nil . - -_:node17eprgndbx130482 rdf:rest rdf:nil . - -:hasType rdfs:range :NonInformationResource . - -:hasView a owl:ObjectProperty ; - rdfs:label "Has View"@en ; - skos:definition """This property relates a ORE aggregation about a CHO with a web resource -providing a view of that CHO. Examples of view are: a thumbnail, a textual -abstract and a table of contents. The ORE aggregation may be a Europeana -Aggregation, in which case the view is an object owned by Europeana (i.e., an instance of edm:EuropeanaObject) or an aggregation contributed by a content provider. In order to capture both these cases, the domain of edm:hasView is ore:Aggregation and its range is edm:WebResource"""@en ; - rdfs:subPropertyOf ore:aggregates ; - rdfs:domain ore:Aggregation ; - rdfs:range :WebResource . - -:incorporates a owl:ObjectProperty ; - rdfs:label "Incorporates"@en ; - skos:definition """This property captures the use of some resource to add value to another -resource. Such resources may be nested, such as performing a theater play text, and then recording the performance, or creating an artful edition of a collection of poems or just aggregating various poems in an anthology. There may be no single part that contains ultimately the incorporated object, which may be dispersed in the presentation. Therefore, incorporated resources do in general not form proper parts. Incorporated resources are not part of the same resource, but are taken from other resources, and have an independent history. Therefore edm:incorporates is not a sub-property of dcterm:hasPart."""@en ; - rdfs:subPropertyOf :isSimilarTo . - -:isAnnotationOf a owl:ObjectProperty ; - rdfs:label "Is Annotation Of"@en ; - skos:definition """This property relates an annotation (a Europeana object) with the resource -that it annotates."""@en ; - rdfs:subPropertyOf dc:subject , ; - rdfs:domain :EuropeanaObject ; - rdfs:range _:node17eprgndbx130486 . - -_:node17eprgndbx130486 a owl:Class ; - owl:unionOf _:node17eprgndbx130487 . - -_:node17eprgndbx130487 rdf:first :ProvidedCHO ; - rdf:rest _:node17eprgndbx130489 . - -_:node17eprgndbx130489 rdf:first _:node17eprgndbx130488 . - -_:node17eprgndbx130488 a owl:Class ; - owl:intersectionOf _:node17eprgndbx130490 . - -_:node17eprgndbx130490 rdf:first ore:Proxy ; - rdf:rest _:node17eprgndbx130492 . - -_:node17eprgndbx130492 rdf:first _:node17eprgndbx130491 . - -_:node17eprgndbx130491 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom :ProvidedCHO . - -_:node17eprgndbx130492 rdf:rest rdf:nil . - -_:node17eprgndbx130489 rdf:rest rdf:nil . - -:isDerivativeOf a rdf:Property ; - rdfs:label "Is Derivative Of"@en ; - skos:definition "This property captures a narrower notion of derivation than edm:isSimilarTo, in the sense that it relates a resource to another one, obtained by reworking, reducing, expanding, parts or the whole contents of the former, and possibly adding some minor parts. Versions have an even narrower meaning, in that it requires common identity between the related resources. Translations, summaries, abstractions etc. do not qualify as versions, but do qualify as derivatives."@en ; - rdfs:subPropertyOf :isSimilarTo . - -:isNextInSequence a owl:ObjectProperty ; - rdfs:label "Is Next In Sequence Of"@en ; - skos:definition "edm:isNextInSequence relates two resources S and R that are ordered parts of the same resource A, and such that R comes immediately after R in the order created by their being parts of S."@en ; - rdfs:subPropertyOf dc:relation . - -:isRelatedTo a rdf:Property ; - rdfs:label "Is Related To"@en ; - skos:definition """edm:isRelatedTo is the most general contextual property in EDM. Contextual -properties have typically to do either with the things that have happened to or together with the object under consideration, or what the object refers to by its shape, form or features in a figural or encoded form. For sake of simplicity, we include in the contextual relationships also the scholarly classification, which may have either to do with the role and cultural connections of the object in the past, or its kind of structure, substance or contents as it can be verified at present."""@en ; - rdfs:domain _:node17eprgndbx130493 . - -_:node17eprgndbx130493 a owl:Class ; - owl:unionOf _:node17eprgndbx130494 . - -_:node17eprgndbx130494 rdf:first :ProvidedCHO ; - rdf:rest _:node17eprgndbx130496 . - -_:node17eprgndbx130496 rdf:first _:node17eprgndbx130495 . - -_:node17eprgndbx130495 a owl:Class ; - owl:intersectionOf _:node17eprgndbx130497 . - -_:node17eprgndbx130497 rdf:first ore:Proxy ; - rdf:rest _:node17eprgndbx130499 . - -_:node17eprgndbx130499 rdf:first _:node17eprgndbx130498 . - -_:node17eprgndbx130498 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom :ProvidedCHO . - -_:node17eprgndbx130499 rdf:rest rdf:nil . - -_:node17eprgndbx130496 rdf:rest rdf:nil . - -:isRepresentationOf a owl:ObjectProperty ; - rdfs:label "Is Representation Of"@en ; - skos:definition "This property associates an information resource to the resource (if any) that it represents"@en ; - rdfs:subPropertyOf dc:subject ; - rdfs:domain :InformationResource ; - rdfs:subPropertyOf . - -:isSimilarTo a rdf:Property ; - rdfs:label "Is Similar To"@en ; - skos:definition "The most generic derivation property, covering also the case of questionable derivation. Is Similar To asserts that parts of the contents of one resource exhibit common features with respect to ideas, shapes, structures, colors, words, plots, topics with the contents of the related resource. Those common features may be attributed to a common origin or influence (in particular for derivation), but also to more generic cultural or psychological factors."@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty . - -:isSuccessorOf a owl:ObjectProperty ; - rdfs:label "Is Successor Of"@en ; - skos:definition "This property captures the relation between the continuation of a resource and that resource. This applies to a story, a serial, a journal etc. No content of the successor resource is identical or has a similar form with that of the precursor. The similarity is only in the context, subjects and figures of a plot. Successors typically form part of a common whole – such as a trilogy, a journal, etc."@en ; - rdfs:subPropertyOf :isSimilarTo . - -:landingPage a owl:ObjectProperty ; - rdfs:label "Landing Page"@en ; - skos:definition """This property captures the relation between an aggregation representing a -Cultural Heritage Object and the Web Resource representing that Object on -the providerÂ’s web site."""@en ; - rdfs:subPropertyOf ore:aggregates ; - rdfs:range :WebResource . - -:occurredAt a owl:ObjectProperty ; - rdfs:label "Occured At"@en ; - skos:definition """This property associates an event to the smallest known time span that -overlaps with the occurrence of that event"""@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - rdfs:domain :Event ; - rdfs:range :TimeSpan . - -:preview a owl:ObjectProperty ; - rdfs:label "Preview"@en ; - skos:definition "The URL of a thumbnail representing the digital object, generated by Europeana."@en ; - rdfs:subPropertyOf :hasView ; - rdfs:range :WebResource . - -:realizes a owl:ObjectProperty ; - rdfs:label "Realizes"@en ; - skos:definition "This property describes a relation between a physical thing and the information resource that is contained in it, visible at it or otherwise carried by it, if applicable."@en ; - rdfs:subPropertyOf :isRelatedTo ; - owl:equivalentProperty ; - rdfs:domain :PhysicalThing ; - rdfs:range :InformationResource . - -:wasPresentAt a owl:ObjectProperty ; - rdfs:label "Was Present At"@en ; - skos:definition "This property associates the people, things or information resources with an event at which they were present"@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - rdfs:domain _:node17eprgndbx130500 . - -_:node17eprgndbx130500 a owl:Class ; - owl:unionOf _:node17eprgndbx130501 . - -_:node17eprgndbx130501 rdf:first :Agent ; - rdf:rest _:node17eprgndbx130502 . - -_:node17eprgndbx130502 rdf:first :InformationResource ; - rdf:rest _:node17eprgndbx130503 . - -_:node17eprgndbx130503 rdf:first :PhysicalThing ; - rdf:rest rdf:nil . - -:wasPresentAt rdfs:range :Event . - -:country a rdf:Property ; - rdfs:label "Country"@en ; - rdfs:subPropertyOf . - -:dataProvider a rdf:Property ; - rdfs:label "Europeana Data Provider"@en ; - skos:definition "This element is specifically included to allow the name of the organisation who supplies data to Europeana indirectly via an aggregator to be recorded and displayed in the portal. Aggregator names are recorded in edm:provider. If an organisation provides data directly to Europeana (i.e. not via an aggregator) the values in edm:dataProvider and edm:provider will be the same. Organisation names should be provided as an ordinary text string until the Europeana Authority File for Organisations has been established. At that point providers will be able to send an identifier from the file instead of a text string. The name provided should be the preferred form of the name in the language the provider chooses as the default language for display in the portal. Countries with multiple languages may prefer to concatenate the name in more than one language (See the example below.) Note: Europeana Data Provider is not necessarily the institution where the physical object is located."@en ; - rdfs:subPropertyOf dcterms:provenance ; - rdfs:domain ore:Aggregation ; - rdfs:range :Agent . - -:isShownAt a owl:ObjectProperty ; - rdfs:label "Is Shown At"@en ; - skos:definition "An unambiguous URL reference to the digital object on the providerÂ’s web site in its full information context."@en ; - rdfs:subPropertyOf :hasView ; - rdfs:range :WebResource . - -:isShownBy a owl:ObjectProperty ; - rdfs:label "Is Shown By"@en ; - skos:definition "An unambiguous URL reference to the digital object on the providerÂ’s web site in the best available resolution/quality."@en ; - rdfs:subPropertyOf :hasView ; - rdfs:range :WebResource . - -:language a rdf:Property ; - rdfs:label "Europeana Language"@en ; - skos:definition "A language assigned to the resource with reference to the Provider."@en ; - rdfs:comment """The recommended best practice is to use a controlled vocabulary such as -RFC 4646 (http://www.rfc-archive.org/getrfc.php?rfc=4646) which, in -conjunction with ISO 639, defines two- and three-letter primary language tags. Either a coded value or text string can be represented here."""@en . - -:object a owl:ObjectProperty ; - rdfs:label "Object"@en ; - skos:definition """The URL of a thumbnail representing the digital object or, if there is no such -thumbnail, the URL of the digital object in the best resolution available on the -web site of the data provider from which a thumbnail could be generated. This will often be the same URL as given in edm:isShownBy."""@en ; - rdfs:subPropertyOf :hasView ; - rdfs:range :WebResource . - -:provider a rdf:Property ; - rdfs:label "Europeana Provider"@en ; - skos:definition "Name of the organization that delivers data to Europeana"@en ; - rdfs:subPropertyOf :hasMet ; - rdfs:range :Agent . - -:rights a owl:ObjectProperty ; - rdfs:label "Europeana Rights"@en ; - skos:definition """Information about copyright of the digital object as specified by isShownBy -and isShownAt"""@en . - -:type a owl:DatatypeProperty ; - rdfs:label "Europeana Type"@en ; - skos:definition "The Europeana material type of the resource"@en ; - rdfs:subPropertyOf dc:type ; - rdfs:range _:node17eprgndbx130504 . - -_:node17eprgndbx130504 a rdfs:Datatype ; - owl:oneOf _:node17eprgndbx130505 . - -_:node17eprgndbx130505 rdf:first "TEXT" ; - rdf:rest _:node17eprgndbx130506 . - -_:node17eprgndbx130506 rdf:first "IMAGE" ; - rdf:rest _:node17eprgndbx130507 . - -_:node17eprgndbx130507 rdf:first "SOUND" ; - rdf:rest _:node17eprgndbx130508 . - -_:node17eprgndbx130508 rdf:first "VIDEO" ; - rdf:rest _:node17eprgndbx130509 . - -_:node17eprgndbx130509 rdf:first "3D" ; - rdf:rest rdf:nil . - -:ugc a owl:DatatypeProperty ; - rdfs:label "UGC"@en ; - skos:definition "This element is used to identify user generated content (also called user created content). It should be applied to all digitised or born digital content contributed by the general public and collected by Europeana through a crowdsourcing initiative or project."@en ; - rdfs:range _:node17eprgndbx130510 . - -_:node17eprgndbx130510 a rdfs:Datatype ; - owl:oneOf _:node17eprgndbx130511 . - -_:node17eprgndbx130511 rdf:first "TRUE" ; - rdf:rest rdf:nil . - -:unstored a rdf:Property ; - rdfs:label "Unstored"@en ; - skos:definition """This is a container element which includes all relevant information that -otherwise cannot be mapped to another element in the ESE."""@en . - -:uri a owl:ObjectProperty ; - rdfs:label "Europeana URI"@en ; - skos:definition "This is a tag created by a user through the Europeana interface."@en . - -:userTag a rdf:Property ; - rdfs:label "User Tag"@en ; - skos:definition "This is a tag created by a user through the Europeana interface."@en ; - rdfs:subPropertyOf dc:description . - -:year a rdf:Property ; - rdfs:label "Europeana Year"@en ; - skos:definition """A point of time associated with an event in the life of the original analog or -born digital object."""@en ; - rdfs:subPropertyOf dcterms:temporal . - -skos:Concept rdfs:subClassOf :NonInformationResource . - - rdfs:subPropertyOf :begin . - - rdfs:subPropertyOf :end . - -dc:contributor rdfs:subPropertyOf :hasMet . - -dc:coverage rdfs:subPropertyOf :hasMet . - -dc:creator rdfs:subPropertyOf :hasMet . - -dc:date rdfs:subPropertyOf :hasMet . - -dc:format rdfs:subPropertyOf :hasType . - -dcterms:hasFormat rdfs:subPropertyOf _:node17eprgndbx130512 . - -_:node17eprgndbx130512 a rdf:Property ; - owl:inverseOf :isDerivativeOf . - -dcterms:hasVersion rdfs:subPropertyOf _:node17eprgndbx130513 . - -_:node17eprgndbx130513 a rdf:Property ; - owl:inverseOf :isDerivativeOf . - -dcterms:isFormatOf rdfs:subPropertyOf :isDerivativeOf . - -dcterms:isReplacedBy rdfs:subPropertyOf _:node17eprgndbx130514 . - -_:node17eprgndbx130514 a rdf:Property ; - owl:inverseOf :isDerivativeOf . - -dcterms:isVersionOf rdfs:subPropertyOf :isDerivativeOf . - -dc:language rdfs:subPropertyOf :hasType . - -dc:publisher rdfs:subPropertyOf :hasMet . - -dc:relation rdfs:subPropertyOf :isRelatedTo . - -dcterms:replaces rdfs:subPropertyOf :isDerivativeOf . - -dc:source rdfs:subPropertyOf :isDerivativeOf . - -dc:subject rdfs:subPropertyOf :isRelatedTo . - -dcterms:tableOfContents rdfs:subPropertyOf :hasView . - -dc:type rdfs:subPropertyOf :hasType . - -skos:Concept a owl:Class . - -ore:Aggregation a owl:Class . - -ore:Proxy a owl:Class . diff --git a/metis-schema/src/main/resources/rdf_examples/edm-v524-130522.n3 b/metis-schema/src/main/resources/rdf_examples/edm-v524-130522.n3 deleted file mode 100644 index 75e63e086..000000000 --- a/metis-schema/src/main/resources/rdf_examples/edm-v524-130522.n3 +++ /dev/null @@ -1,511 +0,0 @@ -@prefix DOLCE-Lite: . -@prefix abc: . -@prefix adms: . -@prefix dc: . -@prefix dcterms: . -@prefix edm: . -@prefix foaf: . -@prefix frbr: . -@prefix frbroo: . -@prefix geo: . -@prefix ore: . -@prefix owl: . -@prefix radion: . -@prefix rdf: . -@prefix rdfs: . -@prefix skos: . -@prefix vann: . -@prefix voaf: . -@prefix xsd: . - -dc:contributor rdfs:subPropertyOf edm:hasMet . - -dc:coverage rdfs:subPropertyOf edm:hasMet . - -dc:creator rdfs:subPropertyOf edm:hasMet . - -dc:date rdfs:subPropertyOf edm:hasMet . - -dc:format rdfs:subPropertyOf edm:hasType . - -dc:language rdfs:subPropertyOf edm:hasType . - -dc:publisher rdfs:subPropertyOf edm:hasMet . - -dc:source rdfs:subPropertyOf edm:isDerivativeOf . - -dcterms:hasFormat rdfs:subPropertyOf [ a rdf:Property ; - owl:inverseOf edm:isDerivativeOf ] . - -dcterms:hasVersion rdfs:subPropertyOf [ a rdf:Property ; - owl:inverseOf edm:isDerivativeOf ] . - -dcterms:isFormatOf rdfs:subPropertyOf edm:isDerivativeOf . - -dcterms:isReplacedBy rdfs:subPropertyOf [ a rdf:Property ; - owl:inverseOf edm:isDerivativeOf ] . - -dcterms:isVersionOf rdfs:subPropertyOf edm:isDerivativeOf . - -dcterms:replaces rdfs:subPropertyOf edm:isDerivativeOf . - -dcterms:tableOfContents rdfs:subPropertyOf edm:hasView . - - rdfs:subPropertyOf edm:begin . - - rdfs:subPropertyOf edm:end . - -edm: a voaf:Vocabulary, - owl:Ontology ; - dc:contributor [ a foaf:Person ; - foaf:name "Hugo Manguinhas" ], - [ a foaf:Organization ; - foaf:name "Participants of Europeana Version 1.0 Work Package on Further Specification of Functionality and Interoperability aspects of Europeana (WP3)" ], - , - , - , - ; - dc:creator ; - dc:description "The Europeana Data Model (EDM) is aimed at being an integration medium for collecting, connecting and enriching the descriptions provided by Europeana data providers. The RDF vocabulary for http://www.europeana.eu/schemas/edm/ defines the elements introduced by EDM (as opposed to the ones EDM re-uses from other namespaces)."@en ; - dc:modified "2013-05-20"^^xsd:date ; - dc:publisher ; - dc:title "Europeana Data Model (EDM) vocabulary"@en ; - dcterms:issued "2010-03-25"^^xsd:date ; - vann:changes """ -======= -Changes between ontology file EDM version 5.2.4 (edm, was once EDM-v524-120820) -and ontology file EDM version 5.2.3 (EDM-v523-120123) -======= -1. edm:isShownAt made a sub-property of edm:hasView -2. added edm:begin and edm:end and their mappings to CRM -3. added owl:Class declarations added for compatibility with some OWL-DL reasoners (feedback from Pedro Szekely, ISI) -4. added "of" at the end of the label for edm:isNextInSequence -5. added vocabulary metadata to follow Linked Open Vocabularies (http://lov.okfn.org/) and ADMS (https://joinup.ec.europa.eu/asset/adms/release/100) guidelines -6. removed a domain axiom on edm:hasMet -7. added edm:collectionName and edm:europeanaProxy -8. removed version number from file name -9. generalisation of Country, DataProvider and Provider -10. updated CRM namespace and CRM class and property identifiers -11. added FRBRoo mappings - """@en ; - vann:example , - ; - vann:preferredNamespacePrefix "edm" ; - vann:preferredNamespaceUri "http://www.europeana.eu/schemas/edm/" ; - voaf:toDoList """ -======= -Remaining TODOs for ontology file EDM version 5.2.4 -======= -- finish and check FRBRoo mappings according to the recommendations of the EDM-FRBRoo task force. Also include implicit mappings and mappings for elements outside the EDM namespace? -- try to capture formal cardinality constraints resulting from "Obligation and Occurrence" documentation, which should be attached to non-EDM constructs (esp. ore:Aggregation) -- continue adding documentation values (skos:scopeNote, skos:example, etc, according to 1.), starting from edm:InformationResource. Add all Europeana examples and rationale notes for non-EDM constructs -- use specific EDM-doc properties for "rationale" and "obligation and occurrence". Use skos:definition for "Europeana definition", skos:example for "Example", skos:note for "Europeana note" - """@en ; - owl:versionInfo "5.2.4" ; - adms:relatedWebPage ; - radion:versionNotes "The present specification is based on the document \"Definition of the Europeana Data Model elements\", originally edited by Carlo Meghini. It is aligned with the version 5.2.4 of these EDM Definitions."@en ; - foaf:homepage . - -edm:EuropeanaAggregation a owl:Class ; - rdfs:label "Europeana Aggregation" ; - rdfs:subClassOf edm:EuropeanaObject, - ore:Aggregation ; - owl:equivalentClass [ a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty edm:aggregatedCHO ] ; - skos:definition """The set of resources related to a single Cultural Heritage Object that -collectively represent that object in Europeana. Such set consists of: all -descriptions about the object that Europeana collects from (possibly different) content providers, including thumbnails and other forms of abstractions, as well as of the description of the object Europeana builds."""; - skos:example "The 56th issue of \"Le Temps\" is a (different) Cultural Heritage Object represented in Europeana by another EuropeanaAggregation instance"@en, - "The journal \"Le Temps\" is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en, - "The painting Mona Lisa is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en ; - skos:note "An instance of EuropeanaAggregation is created at ingestion time for each different Cultural Heritage Object recognized by Europeana. Such instance is associated to the Cultural Heritage Object that it is about, by the property edm:aggregatedCHO"@en ; - skos:scopeNote "Obligation and Occurence: The relation between the Cultural Heritage Objects represented in Europeana and the instances of the class EuropeanaAggregation is one-to-one, in the data maintained by Europeana: every Cultural Heritage Object is represented by an instance of EuropeanaAggregation, and every instance of EuropeanaAggregation represent a Cultural Heritage Object."@en, - "Rationale: This class is used in Europeana to gather in a single conceptual unit all the information about a Cultural Heritage Object, necessary for all operations on these objects."@en . - -edm:collectionName a owl:ObjectProperty ; - rdfs:label "Collection Name"@en ; - skos:definition "This property holds the collection identifier given to the dataset in Europeana."@en ; - skos:note "The value of this property is provided by Europeana as part of the ingestion process."@en . - -edm:country a rdf:Property ; - rdfs:label "Country"@en ; - rdfs:subPropertyOf . - -edm:currentLocation a owl:ObjectProperty ; - rdfs:label "Current Location"@en ; - rdfs:domain [ a owl:Class ; - owl:unionOf ( edm:ProvidedCHO [ a owl:Class ; - owl:intersectionOf ( ore:Proxy [ a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom edm:ProvidedCHO ] ) ] ) ] ; - rdfs:range edm:Place ; - rdfs:subPropertyOf dcterms:spatial ; - owl:equivalentProperty , - geo:location ; - skos:definition "The geographic location and/or name of the repository, building, site, or other entity whose boundaries presently include the resource."@en . - -edm:dataProvider a rdf:Property ; - rdfs:label "Data Provider"@en ; - rdfs:domain ore:Aggregation ; - rdfs:range edm:Agent ; - rdfs:subPropertyOf dcterms:provenance ; - skos:definition "The name or identifier of the organisation who contributes data indirectly to an aggregation service (e.g. Europeana)."@en . - -edm:europeanaProxy a owl:ObjectProperty ; - rdfs:label "Europeana Proxy"@en ; - skos:definition "This property serves only as a flag to indicate that a proxy is a Europeana proxy (as opposed to a provider proxy). It is for internal use only."@en ; - skos:note "By default, any proxy without this flag can be interpreted as having the value false and is a provider proxy."@en . - -edm:incorporates a owl:ObjectProperty ; - rdfs:label "Incorporates"@en ; - rdfs:subPropertyOf edm:isSimilarTo ; - owl:equivalentProperty frbroo:R14_incorporates ; - skos:definition """This property captures the use of some resource to add value to another -resource. Such resources may be nested, such as performing a theater play text, and then recording the performance, or creating an artful edition of a collection of poems or just aggregating various poems in an anthology. There may be no single part that contains ultimately the incorporated object, which may be dispersed in the presentation. Therefore, incorporated resources do in general not form proper parts. Incorporated resources are not part of the same resource, but are taken from other resources, and have an independent history. Therefore edm:incorporates is not a sub-property of dcterm:hasPart."""@en . - -edm:isAnnotationOf a owl:ObjectProperty ; - rdfs:label "Is Annotation Of"@en ; - rdfs:domain edm:EuropeanaObject ; - rdfs:range [ a owl:Class ; - owl:unionOf ( edm:ProvidedCHO [ a owl:Class ; - owl:intersectionOf ( ore:Proxy [ a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom edm:ProvidedCHO ] ) ] ) ] ; - rdfs:subPropertyOf dc:subject, - ; - skos:definition """This property relates an annotation (a Europeana object) with the resource -that it annotates."""@en . - -edm:isNextInSequence a owl:ObjectProperty ; - rdfs:label "Is Next In Sequence Of"@en ; - rdfs:subPropertyOf dc:relation ; - skos:definition "edm:isNextInSequence relates two resources S and R that are ordered parts of the same resource A, and such that S comes immediately after R in the order created by their being parts of A."@en . - -edm:isRepresentationOf a owl:ObjectProperty ; - rdfs:label "Is Representation Of"@en ; - rdfs:domain edm:InformationResource ; - rdfs:subPropertyOf dc:subject, - ; - skos:definition "This property associates an information resource to the resource (if any) that it represents"@en . - -edm:isShownAt a owl:ObjectProperty ; - rdfs:label "Is Shown At"@en ; - rdfs:range edm:WebResource ; - rdfs:subPropertyOf edm:hasView ; - skos:definition "An unambiguous URL reference to the digital object on the provider’s web site in its full information context."@en . - -edm:isShownBy a owl:ObjectProperty ; - rdfs:label "Is Shown By"@en ; - rdfs:range edm:WebResource ; - rdfs:subPropertyOf edm:hasView ; - skos:definition "An unambiguous URL reference to the digital object on the provider’s web site in the best available resolution/quality."@en . - -edm:isSuccessorOf a owl:ObjectProperty ; - rdfs:label "Is Successor Of"@en ; - rdfs:subPropertyOf edm:isSimilarTo ; - skos:definition "This property captures the relation between the continuation of a resource and that resource. This applies to a story, a serial, a journal etc. No content of the successor resource is identical or has a similar form with that of the precursor. The similarity is only in the context, subjects and figures of a plot. Successors typically form part of a common whole – such as a trilogy, a journal, etc."@en . - -edm:landingPage a owl:ObjectProperty ; - rdfs:label "Landing Page"@en ; - rdfs:range edm:WebResource ; - rdfs:subPropertyOf ore:aggregates ; - skos:definition "This property captures the relation between a Europeana aggregation representing a cultural heritage object and a (reference) Web resource giving access to that object. Europeana provides the value for this property."@en . - -edm:language a rdf:Property ; - rdfs:label "Europeana Language"@en ; - skos:definition "The value for this element is added by the Data Ingestion Team as part of the ingestion process, based on the language of the data provider."@en . - -edm:object a owl:ObjectProperty ; - rdfs:label "Object"@en ; - rdfs:range edm:WebResource ; - rdfs:subPropertyOf edm:hasView ; - skos:definition """The URL of a thumbnail representing the digital object or, if there is no such -thumbnail, the URL of the digital object in the best resolution available on the -web site of the data provider from which a thumbnail could be generated. This will often be the same URL as given in edm:isShownBy."""@en . - -edm:occurredAt a owl:ObjectProperty ; - rdfs:label "Occured At"@en ; - rdfs:domain edm:Event ; - rdfs:range edm:TimeSpan ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - skos:definition """This property associates an event to the smallest known time span that -overlaps with the occurrence of that event"""@en . - -edm:preview a owl:ObjectProperty ; - rdfs:label "Preview"@en ; - rdfs:range edm:WebResource ; - rdfs:subPropertyOf edm:hasView ; - skos:definition "The URL of a thumbnail representing the digital object, generated by Europeana."@en . - -edm:provider a rdf:Property ; - rdfs:label "Provider"@en ; - rdfs:range edm:Agent ; - rdfs:subPropertyOf edm:hasMet ; - skos:definition "The name or identifier of the organization who delivers data directly to an aggregation service (e.g. Europeana)"@en . - -edm:realizes a owl:ObjectProperty ; - rdfs:label "Realizes"@en ; - rdfs:domain edm:PhysicalThing ; - rdfs:range edm:InformationResource ; - rdfs:subPropertyOf edm:isRelatedTo ; - owl:equivalentProperty ; - skos:definition "This property describes a relation between a physical thing and the information resource that is contained in it, visible at it or otherwise carried by it, if applicable."@en . - -edm:rights a owl:ObjectProperty ; - rdfs:label "Europeana Rights"@en ; - skos:definition """Information about copyright of the digital object as specified by isShownBy -and isShownAt"""@en . - -edm:type a owl:DatatypeProperty ; - rdfs:label "Europeana Type"@en ; - rdfs:range [ a rdfs:Datatype ; - owl:oneOf ( "TEXT" "IMAGE" "SOUND" "VIDEO" "3D" ) ] ; - rdfs:subPropertyOf dc:type ; - skos:definition "The Europeana material type of the resource"@en . - -edm:ugc a owl:DatatypeProperty ; - rdfs:label "UGC"@en ; - rdfs:range [ a rdfs:Datatype ; - owl:oneOf ( "TRUE" ) ] ; - skos:definition "This element is used to identify user generated content (also called user created content). It should be applied to all digitised or born digital content contributed by the general public and collected by Europeana through a crowdsourcing initiative or project."@en . - -edm:unstored a rdf:Property ; - rdfs:label "Unstored"@en ; - skos:definition """This is a container element which includes all relevant information that -otherwise cannot be mapped to another element in the ESE."""@en . - -edm:uri a owl:ObjectProperty ; - rdfs:label "Europeana URI"@en ; - skos:definition "This is a tag created by a user through the Europeana interface."@en . - -edm:userTag a rdf:Property ; - rdfs:label "User Tag"@en ; - rdfs:subPropertyOf dc:description ; - skos:definition "This is a tag created by a user through the Europeana interface."@en . - -edm:wasPresentAt a owl:ObjectProperty ; - rdfs:label "Was Present At"@en ; - rdfs:domain [ a owl:Class ; - owl:unionOf ( edm:Agent edm:InformationResource edm:PhysicalThing ) ] ; - rdfs:range edm:Event ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - skos:definition "This property associates the people, things or information resources with an event at which they were present"@en . - -edm:year a rdf:Property ; - rdfs:label "Europeana Year"@en ; - rdfs:subPropertyOf dcterms:temporal ; - skos:definition """A point of time associated with an event in the life of the original analog or -born digital object."""@en . - -skos:Concept a owl:Class ; - rdfs:subClassOf edm:NonInformationResource . - - a foaf:Person ; - foaf:name "Antoine Isaac" . - - a foaf:Organization ; - foaf:name "Europeana" . - -dc:type rdfs:subPropertyOf edm:hasType . - -edm:begin a owl:DatatypeProperty ; - rdfs:label "Begin"@en ; - rdfs:domain [ a owl:Class ; - owl:unionOf ( edm:Agent edm:TimeSpan ) ] ; - rdfs:subPropertyOf edm:isRelatedTo ; - skos:definition "This property denotes the start date of a period of time."@en . - -edm:end a owl:DatatypeProperty ; - rdfs:label "End"@en ; - rdfs:domain [ a owl:Class ; - owl:unionOf ( edm:Agent edm:TimeSpan ) ] ; - rdfs:subPropertyOf edm:isRelatedTo ; - skos:definition "This property denotes the end date of a period of time."@en . - -edm:happenedAt a owl:ObjectProperty ; - rdfs:label "Happened At"@en ; - rdfs:domain edm:Event ; - rdfs:range edm:Place ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - skos:definition """This property associates an event with the place at which the event -happened."""@en . - - a foaf:Person ; - foaf:name "Julia Iwanowa" . - - a foaf:Person ; - foaf:name "Martin Doerr" . - - a foaf:Person ; - foaf:name "Nasos Drosopoulos" . - - a foaf:Person ; - foaf:name "Vassilis Tzouvaras" . - -edm:EuropeanaObject a owl:Class ; - rdfs:label "Europeana Object"@en ; - rdfs:subClassOf edm:WebResource ; - skos:definition "Any object that is the result of Europeana’s activities"@en ; - skos:example "An annotation created by a user through the Europeana portal"@en, - "Any content created by the users through the service made available by Europeana for that purpose"@en, - "Any instance of the class EuropeanaAggregation"@en ; - skos:scopeNote "Rationale: This class is used to tag objects that are the result of activity of Europeana, and, as such, objects on which Europeana holds rights"@en . - -edm:PhysicalThing a owl:Class ; - rdfs:label "Physical Thing"@en ; - rdfs:subClassOf edm:NonInformationResource ; - owl:equivalentClass ; - skos:definition """A persistent physical item such as a painting, a building, a book or a stone. -Persons are not items. This class represents Cultural Heritage Objects known to Europeana to be physical things (such as Mona Lisa) as well as all physical things Europeana refers to in the descriptions of Cultural Heritage Objects (such as the Rosetta Stone)."""@en . - -edm:Place a owl:Class ; - rdfs:label "Place"@en ; - rdfs:subClassOf edm:NonInformationResource ; - owl:equivalentClass frbroo:F9_Place, - abc:Place, - frbr:Place, - , - DOLCE-Lite:space-region ; - skos:definition "An \"extent in space, in particular on the surface of the earth, in the pure sense of physics: independent from temporal phenomena and matter\" (CIDOC CRM)"@en . - -edm:aggregatedCHO a owl:ObjectProperty ; - rdfs:label "Aggregated Cultural Heritage Object"@en ; - rdfs:domain ore:Aggregation ; - rdfs:range edm:ProvidedCHO ; - rdfs:subPropertyOf dc:subject, - , - ore:aggregates ; - skos:definition "This property associates an ORE aggregation with the cultural heritage object(s) (CHO for short) it is about."@en . - -dc:subject rdfs:subPropertyOf edm:isRelatedTo . - -edm:Event a owl:Class ; - rdfs:label "Event"@en ; - rdfs:subClassOf [ a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty edm:happenedAt ], - edm:NonInformationResource ; - owl:equivalentClass frbroo:F8_Event, - abc:Temporality, - frbr:Event, - ; - skos:definition """An event is a change "of states in cultural, social or physical systems, - regardless of scale, brought about by a series or group of coherent physical, -cultural, technological or legal phenomena" (E5 Event in CIDOC CRM) or a "set of coherent phenomena or cultural manifestations bounded in time and space" (E4 Period in CIDOC CRM) -"""@en ; - skos:example "the 2nd World War"@en, - "the act of painting Mona Lisa"@en, - "the change of custody of Mona Lisa"@en ; - skos:note "Events are identified either by the content provider or by Europeana enrichment at ingestion time"@en ; - skos:scopeNote "Rationale:This class is a domain of edm:happenedAt and the domain of edm:occurredAt"@en . - -edm:TimeSpan a owl:Class ; - rdfs:label "Time Span"@en ; - rdfs:subClassOf dcterms:PeriodOfTime, - edm:NonInformationResource ; - owl:equivalentClass abc:Time, - , - DOLCE-Lite:time-interval ; - skos:definition """The class of "abstract temporal extents, in the sense of Galilean physics, - having a beginning, an end and a duration" (CIDOC CRM)"""@en . - -edm:hasType a owl:ObjectProperty ; - rdfs:label "Has Type"@en ; - rdfs:domain [ a owl:Class ; - owl:unionOf ( edm:ProvidedCHO [ a owl:Class ; - owl:intersectionOf ( ore:Proxy [ a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom edm:ProvidedCHO ] ) ] ) ] ; - rdfs:range edm:NonInformationResource ; - rdfs:subPropertyOf edm:isRelatedTo ; - owl:equivalentProperty ; - skos:definition """This property relates a resource with the concepts it belongs to in a suitable -type system such as MIME or any thesaurus that captures categories of objects in a given field (e.g., the “Objects” facet in Getty’s Art and Architecture Thesaurus). It does not capture aboutness."""@en . - -edm:isSimilarTo a rdf:Property ; - rdfs:label "Is Similar To"@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - skos:definition "The most generic derivation property, covering also the case of questionable derivation. Is Similar To asserts that parts of the contents of one resource exhibit common features with respect to ideas, shapes, structures, colors, words, plots, topics with the contents of the related resource. Those common features may be attributed to a common origin or influence (in particular for derivation), but also to more generic cultural or psychological factors."@en . - -ore:Aggregation a owl:Class . - -ore:Proxy a owl:Class . - -edm:Agent a owl:Class ; - rdfs:label "Agent"@en ; - rdfs:subClassOf edm:NonInformationResource ; - owl:equivalentClass ; - skos:definition """This class comprises people, either individually or in groups, who have the -potential to perform intentional actions for which they can be held responsible."""; - skos:example "Leonardo da Vinci, the British Museum, W3C" ; - skos:scopeNote "Rationale: This class is a domain of edm:wasPresentAt" . - -edm:InformationResource a owl:Class ; - rdfs:label "Information Resource"@en ; - owl:equivalentClass [ a owl:Class ; - owl:unionOf ( frbr:Work frbr:Expression frbr:Manifestation ) ], - [ a owl:Class ; - owl:unionOf ( frbroo:F1_Work frbroo:F2_Expression frbroo:F3_Manifestation_Product_Type frbroo:F4_Manifestation_Singleton ) ], - ; - skos:definition "An information resource is a resource whose essential characteristics can be conveyed in a single message. It can be associated with a URI, it can have a representation, for example: a text is an InformationResource."@en . - -edm:hasView a owl:ObjectProperty ; - rdfs:label "Has View"@en ; - rdfs:domain ore:Aggregation ; - rdfs:range edm:WebResource ; - rdfs:subPropertyOf ore:aggregates ; - skos:definition """This property relates a ORE aggregation about a CHO with a web resource -providing a view of that CHO. Examples of view are: a thumbnail, a textual -abstract and a table of contents. The ORE aggregation may be a Europeana -Aggregation, in which case the view is an object owned by Europeana (i.e., an instance of edm:EuropeanaObject) or an aggregation contributed by a content provider. In order to capture both these cases, the domain of edm:hasView is ore:Aggregation and its range is edm:WebResource"""@en . - -dc:relation rdfs:subPropertyOf edm:isRelatedTo . - -edm:hasMet a rdf:Property ; - rdfs:label "Has Met"@en ; - rdfs:subPropertyOf dc:relation ; - skos:definition "edm:hasMet relates a resource with the objects or phenomena that have happened to or have happened together with the resource under consideration. We can abstractly think of history and the present as a series of “meetings” between people and other things in space-time. Therefore we name this relationship as the things the object “has met” in the course of its existence. These meetings are events in the proper sense, in which other people and things participate in any role."@en . - -edm:isRelatedTo a rdf:Property ; - rdfs:label "Is Related To"@en ; - rdfs:domain [ a owl:Class ; - owl:unionOf ( edm:ProvidedCHO [ a owl:Class ; - owl:intersectionOf ( ore:Proxy [ a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom edm:ProvidedCHO ] ) ] ) ] ; - skos:definition """edm:isRelatedTo is the most general contextual property in EDM. Contextual -properties have typically to do either with the things that have happened to or together with the object under consideration, or what the object refers to by its shape, form or features in a figural or encoded form. For sake of simplicity, we include in the contextual relationships also the scholarly classification, which may have either to do with the role and cultural connections of the object in the past, or its kind of structure, substance or contents as it can be verified at present."""@en . - -edm:NonInformationResource a owl:Class ; - rdfs:label "Non-Information Resource"@en ; - owl:complementOf edm:InformationResource ; - skos:definition "All resources that are not information resources."@en . - -edm:WebResource a owl:Class ; - rdfs:label "Web Resource"@en ; - rdfs:subClassOf edm:InformationResource ; - skos:definition """Information Resources that have at least one Web Representation and at least -a URI."""@en . - -edm:isDerivativeOf a rdf:Property ; - rdfs:label "Is Derivative Of"@en ; - rdfs:subPropertyOf edm:isSimilarTo ; - owl:equivalentProperty frbroo:R2_is_derivative_of ; - skos:definition "This property captures a narrower notion of derivation than edm:isSimilarTo, in the sense that it relates a resource to another one, obtained by reworking, reducing, expanding, parts or the whole contents of the former, and possibly adding some minor parts. Versions have an even narrower meaning, in that it requires common identity between the related resources. Translations, summaries, abstractions etc. do not qualify as versions, but do qualify as derivatives."@en . - -edm:ProvidedCHO a owl:Class ; - rdfs:label "Provided CHO"@en ; - rdfs:subClassOf [ a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty [ a rdf:Property ; - owl:inverseOf edm:aggregatedCHO ] ] ; - skos:definition "This class comprises the Cultural Heritage objects that Europeana collects descriptions about."@en ; - skos:example "Mona Lisa, Winged Victory of Samothrace"@en ; - skos:note "This class has been mostly motivated by the need to assign a type to the “central node” in the EDM pattern, during the ingestion process, related to the XML expression of EDM at that stage. It was especially intended to fit the cases where edm:PhysicalThing cannot be used as the type of the resource standing for the real-world object (independently of any specific data contributor perspective)."@en ; - skos:scopeNote "Rationale: This class is the range of edm:aggregatedCHO. A resource of type ProvidedCHO can be the subject of statements using edm:isRelatedTo or any more specific property."@en . - diff --git a/metis-schema/src/main/resources/rdf_examples/edm-v524-130522.nt b/metis-schema/src/main/resources/rdf_examples/edm-v524-130522.nt deleted file mode 100644 index 454180579..000000000 --- a/metis-schema/src/main/resources/rdf_examples/edm-v524-130522.nt +++ /dev/null @@ -1,466 +0,0 @@ - . - . - "edm" . - "http://www.europeana.eu/schemas/edm/" . - "Europeana Data Model (EDM) vocabulary"@en . - "The Europeana Data Model (EDM) is aimed at being an integration medium for collecting, connecting and enriching the descriptions provided by Europeana data providers. The RDF vocabulary for http://www.europeana.eu/schemas/edm/ defines the elements introduced by EDM (as opposed to the ones EDM re-uses from other namespaces)."@en . - "2010-03-25"^^ . - "2013-05-20"^^ . - "5.2.4" . - "The present specification is based on the document \"Definition of the Europeana Data Model elements\", originally edited by Carlo Meghini. It is aligned with the version 5.2.4 of these EDM Definitions."@en . - . - . - "Antoine Isaac" . - _:node17eprp1n4x1343398 . -_:node17eprp1n4x1343398 . -_:node17eprp1n4x1343398 "Participants of Europeana Version 1.0 Work Package on Further Specification of Functionality and Interoperability aspects of Europeana (WP3)" . - . - . - "Nasos Drosopoulos" . - . - . - "Vassilis Tzouvaras" . - . - . - "Julia Iwanowa" . - _:node17eprp1n4x1343399 . -_:node17eprp1n4x1343399 . -_:node17eprp1n4x1343399 "Hugo Manguinhas" . - . - . - "Martin Doerr" . - . - . - "Europeana" . - . - . - . - . - "=======\nChanges between ontology file EDM version 5.2.4 (edm, was once EDM-v524-120820)\nand ontology file EDM version 5.2.3 (EDM-v523-120123)\n=======\n1. edm:isShownAt made a sub-property of edm:hasView\n2. added edm:begin and edm:end and their mappings to CRM\n3. added owl:Class declarations added for compatibility with some OWL-DL reasoners (feedback from Pedro Szekely, ISI)\n4. added \"of\" at the end of the label for edm:isNextInSequence\n5. added vocabulary metadata to follow Linked Open Vocabularies (http://lov.okfn.org/) and ADMS (https://joinup.ec.europa.eu/asset/adms/release/100) guidelines\n6. removed a domain axiom on edm:hasMet\n7. added edm:collectionName and edm:europeanaProxy\n8. removed version number from file name\n9. generalisation of Country, DataProvider and Provider\n10. updated CRM namespace and CRM class and property identifiers\n11. added FRBRoo mappings"@en . - "=======\nRemaining TODOs for ontology file EDM version 5.2.4\n=======\n- finish and check FRBRoo mappings according to the recommendations of the EDM-FRBRoo task force. Also include implicit mappings and mappings for elements outside the EDM namespace?\n- try to capture formal cardinality constraints resulting from \"Obligation and Occurrence\" documentation, which should be attached to non-EDM constructs (esp. ore:Aggregation)\n- continue adding documentation values (skos:scopeNote, skos:example, etc, according to 1.), starting from edm:InformationResource. Add all Europeana examples and rationale notes for non-EDM constructs\n- use specific EDM-doc properties for \"rationale\" and \"obligation and occurrence\". Use skos:definition for \"Europeana definition\", skos:example for \"Example\", skos:note for \"Europeana note\""@en . - . - "Agent"@en . - "This class comprises people, either individually or in groups, who have the\npotential to perform intentional actions for which they can be held responsible." . - "Leonardo da Vinci, the British Museum, W3C" . - . - . - "Rationale: This class is a domain of edm:wasPresentAt" . - . - "Europeana Aggregation" . - "The set of resources related to a single Cultural Heritage Object that\ncollectively represent that object in Europeana. Such set consists of: all\ndescriptions about the object that Europeana collects from (possibly different) content providers, including thumbnails and other forms of abstractions, as well as of the description of the object Europeana builds." . - . - . - _:node17eprp1n4x1343400 . -_:node17eprp1n4x1343400 . -_:node17eprp1n4x1343400 "1"^^ . -_:node17eprp1n4x1343400 . - "An instance of EuropeanaAggregation is created at ingestion time for each different Cultural Heritage Object recognized by Europeana. Such instance is associated to the Cultural Heritage Object that it is about, by the property edm:aggregatedCHO"@en . - "Obligation and Occurence: The relation between the Cultural Heritage Objects represented in Europeana and the instances of the class EuropeanaAggregation is one-to-one, in the data maintained by Europeana: every Cultural Heritage Object is represented by an instance of EuropeanaAggregation, and every instance of EuropeanaAggregation represent a Cultural Heritage Object."@en . - "The painting Mona Lisa is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en . - "The journal \"Le Temps\" is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en . - "The 56th issue of \"Le Temps\" is a (different) Cultural Heritage Object represented in Europeana by another EuropeanaAggregation instance"@en . - "Rationale: This class is used in Europeana to gather in a single conceptual unit all the information about a Cultural Heritage Object, necessary for all operations on these objects."@en . - . - "Europeana Object"@en . - "Any object that is the result of Europeana\u0092s activities"@en . - . - "Any instance of the class EuropeanaAggregation"@en . - "An annotation created by a user through the Europeana portal"@en . - "Any content created by the users through the service made available by Europeana for that purpose"@en . - "Rationale: This class is used to tag objects that are the result of activity of Europeana, and, as such, objects on which Europeana holds rights"@en . - . - "Event"@en . - "An event is a change \"of states in cultural, social or physical systems,\n regardless of scale, brought about by a series or group of coherent physical,\ncultural, technological or legal phenomena\" (E5 Event in CIDOC CRM) or a \"set of coherent phenomena or cultural manifestations bounded in time and space\" (E4 Period in CIDOC CRM)"@en . - . - . - . - . - . - _:node17eprp1n4x1343401 . -_:node17eprp1n4x1343401 . -_:node17eprp1n4x1343401 "1"^^ . -_:node17eprp1n4x1343401 . - "Events are identified either by the content provider or by Europeana enrichment at ingestion time"@en . - "the act of painting Mona Lisa"@en . - "the 2nd World War"@en . - "the change of custody of Mona Lisa"@en . - "Rationale:This class is a domain of edm:happenedAt and the domain of edm:occurredAt"@en . - . - "Information Resource"@en . - "An information resource is a resource whose essential characteristics can be conveyed in a single message. It can be associated with a URI, it can have a representation, for example: a text is an InformationResource."@en . - . - _:node17eprp1n4x1343402 . -_:node17eprp1n4x1343402 . -_:node17eprp1n4x1343402 _:node17eprp1n4x1343403 . -_:node17eprp1n4x1343403 . -_:node17eprp1n4x1343403 _:node17eprp1n4x1343404 . -_:node17eprp1n4x1343404 . -_:node17eprp1n4x1343404 _:node17eprp1n4x1343405 . -_:node17eprp1n4x1343405 . -_:node17eprp1n4x1343405 . - _:node17eprp1n4x1343406 . -_:node17eprp1n4x1343406 . -_:node17eprp1n4x1343406 _:node17eprp1n4x1343407 . -_:node17eprp1n4x1343407 . -_:node17eprp1n4x1343407 _:node17eprp1n4x1343408 . -_:node17eprp1n4x1343408 . -_:node17eprp1n4x1343408 _:node17eprp1n4x1343409 . -_:node17eprp1n4x1343409 . -_:node17eprp1n4x1343409 _:node17eprp1n4x1343410 . -_:node17eprp1n4x1343410 . -_:node17eprp1n4x1343410 . - . - "Non-Information Resource"@en . - "All resources that are not information resources."@en . - . - . - "Physical Thing"@en . - "A persistent physical item such as a painting, a building, a book or a stone.\nPersons are not items. This class represents Cultural Heritage Objects known to Europeana to be physical things (such as Mona Lisa) as well as all physical things Europeana refers to in the descriptions of Cultural Heritage Objects (such as the Rosetta Stone)."@en . - . - . - . - "Place"@en . - "An \"extent in space, in particular on the surface of the earth, in the pure sense of physics: independent from temporal phenomena and matter\" (CIDOC CRM)"@en . - . - . - . - . - . - . - . - "Provided CHO"@en . - "This class comprises the Cultural Heritage objects that Europeana collects descriptions about."@en . - _:node17eprp1n4x1343411 . -_:node17eprp1n4x1343411 . -_:node17eprp1n4x1343411 "1"^^ . -_:node17eprp1n4x1343411 _:node17eprp1n4x1343412 . -_:node17eprp1n4x1343412 . -_:node17eprp1n4x1343412 . - "This class has been mostly motivated by the need to assign a type to the \u0093central node\u0094 in the EDM pattern, during the ingestion process, related to the XML expression of EDM at that stage. It was especially intended to fit the cases where edm:PhysicalThing cannot be used as the type of the resource standing for the real-world object (independently of any specific data contributor perspective)."@en . - "Mona Lisa, Winged Victory of Samothrace"@en . - "Rationale: This class is the range of edm:aggregatedCHO. A resource of type ProvidedCHO can be the subject of statements using edm:isRelatedTo or any more specific property."@en . - . - "Time Span"@en . - "The class of \"abstract temporal extents, in the sense of Galilean physics,\n having a beginning, an end and a duration\" (CIDOC CRM)"@en . - . - . - . - . - . - . - "Web Resource"@en . - "Information Resources that have at least one Web Representation and at least\na URI."@en . - . - . - "Aggregated Cultural Heritage Object"@en . - "This property associates an ORE aggregation with the cultural heritage object(s) (CHO for short) it is about."@en . - . - . - . - . - . - . - "Begin"@en . - "This property denotes the start date of a period of time."@en . - . - _:node17eprp1n4x1343413 . -_:node17eprp1n4x1343413 . -_:node17eprp1n4x1343413 _:node17eprp1n4x1343414 . -_:node17eprp1n4x1343414 . -_:node17eprp1n4x1343414 _:node17eprp1n4x1343415 . -_:node17eprp1n4x1343415 . -_:node17eprp1n4x1343415 . - . - "Collection Name"@en . - "This property holds the collection identifier given to the dataset in Europeana."@en . - "The value of this property is provided by Europeana as part of the ingestion process."@en . - . - "Current Location"@en . - "The geographic location and/or name of the repository, building, site, or other entity whose boundaries presently include the resource."@en . - . - . - . - _:node17eprp1n4x1343416 . -_:node17eprp1n4x1343416 . -_:node17eprp1n4x1343416 _:node17eprp1n4x1343417 . -_:node17eprp1n4x1343417 . -_:node17eprp1n4x1343417 _:node17eprp1n4x1343419 . -_:node17eprp1n4x1343419 _:node17eprp1n4x1343418 . -_:node17eprp1n4x1343418 . -_:node17eprp1n4x1343418 _:node17eprp1n4x1343420 . -_:node17eprp1n4x1343420 . -_:node17eprp1n4x1343420 _:node17eprp1n4x1343422 . -_:node17eprp1n4x1343422 _:node17eprp1n4x1343421 . -_:node17eprp1n4x1343421 . -_:node17eprp1n4x1343421 . -_:node17eprp1n4x1343421 . -_:node17eprp1n4x1343422 . -_:node17eprp1n4x1343419 . - . - . - "End"@en . - "This property denotes the end date of a period of time."@en . - . - _:node17eprp1n4x1343423 . -_:node17eprp1n4x1343423 . -_:node17eprp1n4x1343423 _:node17eprp1n4x1343424 . -_:node17eprp1n4x1343424 . -_:node17eprp1n4x1343424 _:node17eprp1n4x1343425 . -_:node17eprp1n4x1343425 . -_:node17eprp1n4x1343425 . - . - "Europeana Proxy"@en . - "This property serves only as a flag to indicate that a proxy is a Europeana proxy (as opposed to a provider proxy). It is for internal use only."@en . - "By default, any proxy without this flag can be interpreted as having the value false and is a provider proxy."@en . - . - "Happened At"@en . - "This property associates an event with the place at which the event\nhappened."@en . - . - . - . - . - . - "Has Met"@en . - "edm:hasMet relates a resource with the objects or phenomena that have happened to or have happened together with the resource under consideration. We can abstractly think of history and the present as a series of \u0093meetings\u0094 between people and other things in space-time. Therefore we name this relationship as the things the object \u0093has met\u0094 in the course of its existence. These meetings are events in the proper sense, in which other people and things participate in any role."@en . - . - . - "Has Type"@en . - "This property relates a resource with the concepts it belongs to in a suitable\ntype system such as MIME or any thesaurus that captures categories of objects in a given field (e.g., the \u0093Objects\u0094 facet in Getty\u0092s Art and Architecture Thesaurus). It does not capture aboutness."@en . - . - . - _:node17eprp1n4x1343426 . -_:node17eprp1n4x1343426 . -_:node17eprp1n4x1343426 _:node17eprp1n4x1343427 . -_:node17eprp1n4x1343427 . -_:node17eprp1n4x1343427 _:node17eprp1n4x1343429 . -_:node17eprp1n4x1343429 _:node17eprp1n4x1343428 . -_:node17eprp1n4x1343428 . -_:node17eprp1n4x1343428 _:node17eprp1n4x1343430 . -_:node17eprp1n4x1343430 . -_:node17eprp1n4x1343430 _:node17eprp1n4x1343432 . -_:node17eprp1n4x1343432 _:node17eprp1n4x1343431 . -_:node17eprp1n4x1343431 . -_:node17eprp1n4x1343431 . -_:node17eprp1n4x1343431 . -_:node17eprp1n4x1343432 . -_:node17eprp1n4x1343429 . - . - . - "Has View"@en . - "This property relates a ORE aggregation about a CHO with a web resource\nproviding a view of that CHO. Examples of view are: a thumbnail, a textual\nabstract and a table of contents. The ORE aggregation may be a Europeana\nAggregation, in which case the view is an object owned by Europeana (i.e., an instance of edm:EuropeanaObject) or an aggregation contributed by a content provider. In order to capture both these cases, the domain of edm:hasView is ore:Aggregation and its range is edm:WebResource"@en . - . - . - . - . - "Incorporates"@en . - "This property captures the use of some resource to add value to another\nresource. Such resources may be nested, such as performing a theater play text, and then recording the performance, or creating an artful edition of a collection of poems or just aggregating various poems in an anthology. There may be no single part that contains ultimately the incorporated object, which may be dispersed in the presentation. Therefore, incorporated resources do in general not form proper parts. Incorporated resources are not part of the same resource, but are taken from other resources, and have an independent history. Therefore edm:incorporates is not a sub-property of dcterm:hasPart."@en . - . - . - . - "Is Annotation Of"@en . - "This property relates an annotation (a Europeana object) with the resource\nthat it annotates."@en . - . - . - . - _:node17eprp1n4x1343433 . -_:node17eprp1n4x1343433 . -_:node17eprp1n4x1343433 _:node17eprp1n4x1343434 . -_:node17eprp1n4x1343434 . -_:node17eprp1n4x1343434 _:node17eprp1n4x1343436 . -_:node17eprp1n4x1343436 _:node17eprp1n4x1343435 . -_:node17eprp1n4x1343435 . -_:node17eprp1n4x1343435 _:node17eprp1n4x1343437 . -_:node17eprp1n4x1343437 . -_:node17eprp1n4x1343437 _:node17eprp1n4x1343439 . -_:node17eprp1n4x1343439 _:node17eprp1n4x1343438 . -_:node17eprp1n4x1343438 . -_:node17eprp1n4x1343438 . -_:node17eprp1n4x1343438 . -_:node17eprp1n4x1343439 . -_:node17eprp1n4x1343436 . - . - "Is Derivative Of"@en . - "This property captures a narrower notion of derivation than edm:isSimilarTo, in the sense that it relates a resource to another one, obtained by reworking, reducing, expanding, parts or the whole contents of the former, and possibly adding some minor parts. Versions have an even narrower meaning, in that it requires common identity between the related resources. Translations, summaries, abstractions etc. do not qualify as versions, but do qualify as derivatives."@en . - . - . - . - "Is Next In Sequence Of"@en . - "edm:isNextInSequence relates two resources S and R that are ordered parts of the same resource A, and such that S comes immediately after R in the order created by their being parts of A."@en . - . - . - "Is Related To"@en . - "edm:isRelatedTo is the most general contextual property in EDM. Contextual\nproperties have typically to do either with the things that have happened to or together with the object under consideration, or what the object refers to by its shape, form or features in a figural or encoded form. For sake of simplicity, we include in the contextual relationships also the scholarly classification, which may have either to do with the role and cultural connections of the object in the past, or its kind of structure, substance or contents as it can be verified at present."@en . - _:node17eprp1n4x1343440 . -_:node17eprp1n4x1343440 . -_:node17eprp1n4x1343440 _:node17eprp1n4x1343441 . -_:node17eprp1n4x1343441 . -_:node17eprp1n4x1343441 _:node17eprp1n4x1343443 . -_:node17eprp1n4x1343443 _:node17eprp1n4x1343442 . -_:node17eprp1n4x1343442 . -_:node17eprp1n4x1343442 _:node17eprp1n4x1343444 . -_:node17eprp1n4x1343444 . -_:node17eprp1n4x1343444 _:node17eprp1n4x1343446 . -_:node17eprp1n4x1343446 _:node17eprp1n4x1343445 . -_:node17eprp1n4x1343445 . -_:node17eprp1n4x1343445 . -_:node17eprp1n4x1343445 . -_:node17eprp1n4x1343446 . -_:node17eprp1n4x1343443 . - . - "Is Representation Of"@en . - "This property associates an information resource to the resource (if any) that it represents"@en . - . - . - . - . - "Is Similar To"@en . - "The most generic derivation property, covering also the case of questionable derivation. Is Similar To asserts that parts of the contents of one resource exhibit common features with respect to ideas, shapes, structures, colors, words, plots, topics with the contents of the related resource. Those common features may be attributed to a common origin or influence (in particular for derivation), but also to more generic cultural or psychological factors."@en . - . - . - . - "Is Successor Of"@en . - "This property captures the relation between the continuation of a resource and that resource. This applies to a story, a serial, a journal etc. No content of the successor resource is identical or has a similar form with that of the precursor. The similarity is only in the context, subjects and figures of a plot. Successors typically form part of a common whole \u0096 such as a trilogy, a journal, etc."@en . - . - . - "Landing Page"@en . - "This property captures the relation between a Europeana aggregation representing a cultural heritage object and a (reference) Web resource giving access to that object. Europeana provides the value for this property."@en . - . - . - . - "Occured At"@en . - "This property associates an event to the smallest known time span that\noverlaps with the occurrence of that event"@en . - . - . - . - . - . - "Preview"@en . - "The URL of a thumbnail representing the digital object, generated by Europeana."@en . - . - . - . - "Realizes"@en . - "This property describes a relation between a physical thing and the information resource that is contained in it, visible at it or otherwise carried by it, if applicable."@en . - . - . - . - . - . - "Was Present At"@en . - "This property associates the people, things or information resources with an event at which they were present"@en . - . - . - _:node17eprp1n4x1343447 . -_:node17eprp1n4x1343447 . -_:node17eprp1n4x1343447 _:node17eprp1n4x1343448 . -_:node17eprp1n4x1343448 . -_:node17eprp1n4x1343448 _:node17eprp1n4x1343449 . -_:node17eprp1n4x1343449 . -_:node17eprp1n4x1343449 _:node17eprp1n4x1343450 . -_:node17eprp1n4x1343450 . -_:node17eprp1n4x1343450 . - . - . - "Country"@en . - . - . - "Data Provider"@en . - "The name or identifier of the organisation who contributes data indirectly to an aggregation service (e.g. Europeana)."@en . - . - . - . - . - "Is Shown At"@en . - "An unambiguous URL reference to the digital object on the provider\u0092s web site in its full information context."@en . - . - . - . - "Is Shown By"@en . - "An unambiguous URL reference to the digital object on the provider\u0092s web site in the best available resolution/quality."@en . - . - . - . - "Europeana Language"@en . - "The value for this element is added by the Data Ingestion Team as part of the ingestion process, based on the language of the data provider."@en . - . - "Object"@en . - "The URL of a thumbnail representing the digital object or, if there is no such\nthumbnail, the URL of the digital object in the best resolution available on the\nweb site of the data provider from which a thumbnail could be generated. This will often be the same URL as given in edm:isShownBy."@en . - . - . - . - "Provider"@en . - "The name or identifier of the organization who delivers data directly to an aggregation service (e.g. Europeana)"@en . - . - . - . - "Europeana Rights"@en . - "Information about copyright of the digital object as specified by isShownBy\nand isShownAt"@en . - . - "Europeana Type"@en . - "The Europeana material type of the resource"@en . - . - _:node17eprp1n4x1343451 . -_:node17eprp1n4x1343451 . -_:node17eprp1n4x1343451 _:node17eprp1n4x1343452 . -_:node17eprp1n4x1343452 "TEXT" . -_:node17eprp1n4x1343452 _:node17eprp1n4x1343453 . -_:node17eprp1n4x1343453 "IMAGE" . -_:node17eprp1n4x1343453 _:node17eprp1n4x1343454 . -_:node17eprp1n4x1343454 "SOUND" . -_:node17eprp1n4x1343454 _:node17eprp1n4x1343455 . -_:node17eprp1n4x1343455 "VIDEO" . -_:node17eprp1n4x1343455 _:node17eprp1n4x1343456 . -_:node17eprp1n4x1343456 "3D" . -_:node17eprp1n4x1343456 . - . - "UGC"@en . - "This element is used to identify user generated content (also called user created content). It should be applied to all digitised or born digital content contributed by the general public and collected by Europeana through a crowdsourcing initiative or project."@en . - _:node17eprp1n4x1343457 . -_:node17eprp1n4x1343457 . -_:node17eprp1n4x1343457 _:node17eprp1n4x1343458 . -_:node17eprp1n4x1343458 "TRUE" . -_:node17eprp1n4x1343458 . - . - "Unstored"@en . - "This is a container element which includes all relevant information that\notherwise cannot be mapped to another element in the ESE."@en . - . - "Europeana URI"@en . - "This is a tag created by a user through the Europeana interface."@en . - . - "User Tag"@en . - "This is a tag created by a user through the Europeana interface."@en . - . - . - "Europeana Year"@en . - "A point of time associated with an event in the life of the original analog or\nborn digital object."@en . - . - . - . - . - . - . - . - . - . - _:node17eprp1n4x1343459 . -_:node17eprp1n4x1343459 . -_:node17eprp1n4x1343459 . - _:node17eprp1n4x1343460 . -_:node17eprp1n4x1343460 . -_:node17eprp1n4x1343460 . - . - _:node17eprp1n4x1343461 . -_:node17eprp1n4x1343461 . -_:node17eprp1n4x1343461 . - . - . - . - . - . - . - . - . - . - . - . - . diff --git a/metis-schema/src/main/resources/rdf_examples/edm-v524-130522.owl b/metis-schema/src/main/resources/rdf_examples/edm-v524-130522.owl deleted file mode 100644 index f92eb32d8..000000000 --- a/metis-schema/src/main/resources/rdf_examples/edm-v524-130522.owl +++ /dev/null @@ -1,902 +0,0 @@ - - - - - - - - - - - - - - - - - - -]> - - - - - - edm - http://www.europeana.eu/schemas/edm/ - Europeana Data Model (EDM) vocabulary - The Europeana Data Model (EDM) is aimed at being an integration medium for collecting, connecting and enriching the descriptions provided by Europeana data providers. The RDF vocabulary for http://www.europeana.eu/schemas/edm/ defines the elements introduced by EDM (as opposed to the ones EDM re-uses from other namespaces). - 2010-03-25 - 2013-05-20 - 5.2.4 - The present specification is based on the document "Definition of the Europeana Data Model elements", originally edited by Carlo Meghini. It is aligned with the version 5.2.4 of these EDM Definitions. - - - - - - - - - - - - - -======= -Changes between ontology file EDM version 5.2.4 (edm, was once EDM-v524-120820) -and ontology file EDM version 5.2.3 (EDM-v523-120123) -======= -1. edm:isShownAt made a sub-property of edm:hasView -2. added edm:begin and edm:end and their mappings to CRM -3. added owl:Class declarations added for compatibility with some OWL-DL reasoners (feedback from Pedro Szekely, ISI) -4. added "of" at the end of the label for edm:isNextInSequence -5. added vocabulary metadata to follow Linked Open Vocabularies (http://lov.okfn.org/) and ADMS (https://joinup.ec.europa.eu/asset/adms/release/100) guidelines -6. removed a domain axiom on edm:hasMet -7. added edm:collectionName and edm:europeanaProxy -8. removed version number from file name -9. generalisation of Country, DataProvider and Provider -10. updated CRM namespace and CRM class and property identifiers -11. added FRBRoo mappings - - -======= -Remaining TODOs for ontology file EDM version 5.2.4 -======= -- finish and check FRBRoo mappings according to the recommendations of the EDM-FRBRoo task force. Also include implicit mappings and mappings for elements outside the EDM namespace? -- try to capture formal cardinality constraints resulting from "Obligation and Occurrence" documentation, which should be attached to non-EDM constructs (esp. ore:Aggregation) -- continue adding documentation values (skos:scopeNote, skos:example, etc, according to 1.), starting from edm:InformationResource. Add all Europeana examples and rationale notes for non-EDM constructs -- use specific EDM-doc properties for "rationale" and "obligation and occurrence". Use skos:definition for "Europeana definition", skos:example for "Example", skos:note for "Europeana note" - - - - - - - - - Agent - This class comprises people, either individually or in groups, who have the -potential to perform intentional actions for which they can be held responsible. - Leonardo da Vinci, the British Museum, W3C - - - Rationale: This class is a domain of edm:wasPresentAt - - - - - - - Europeana Aggregation - The set of resources related to a single Cultural Heritage Object that -collectively represent that object in Europeana. Such set consists of: all -descriptions about the object that Europeana collects from (possibly different) content providers, including thumbnails and other forms of abstractions, as well as of the description of the object Europeana builds. - - - - - 1 - - - - An instance of EuropeanaAggregation is created at ingestion time for each different Cultural Heritage Object recognized by Europeana. Such instance is associated to the Cultural Heritage Object that it is about, by the property edm:aggregatedCHO - Obligation and Occurence: The relation between the Cultural Heritage Objects represented in Europeana and the instances of the class EuropeanaAggregation is one-to-one, in the data maintained by Europeana: every Cultural Heritage Object is represented by an instance of EuropeanaAggregation, and every instance of EuropeanaAggregation represent a Cultural Heritage Object. - The painting Mona Lisa is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance - The journal "Le Temps" is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance - The 56th issue of "Le Temps" is a (different) Cultural Heritage Object represented in Europeana by another EuropeanaAggregation instance - Rationale: This class is used in Europeana to gather in a single conceptual unit all the information about a Cultural Heritage Object, necessary for all operations on these objects. - - - - - - Europeana Object - Any object that is the result of Europeana’s activities - - Any instance of the class EuropeanaAggregation - An annotation created by a user through the Europeana portal - Any content created by the users through the service made available by Europeana for that purpose - Rationale: This class is used to tag objects that are the result of activity of Europeana, and, as such, objects on which Europeana holds rights - - - - - - Event - An event is a change "of states in cultural, social or physical systems, - regardless of scale, brought about by a series or group of coherent physical, -cultural, technological or legal phenomena" (E5 Event in CIDOC CRM) or a "set of coherent phenomena or cultural manifestations bounded in time and space" (E4 Period in CIDOC CRM) - - - - - - - - - 1 - - - - Events are identified either by the content provider or by Europeana enrichment at ingestion time - the act of painting Mona Lisa - the 2nd World War - the change of custody of Mona Lisa - Rationale:This class is a domain of edm:happenedAt and the domain of edm:occurredAt - - - - - - Information Resource - An information resource is a resource whose essential characteristics can be conveyed in a single message. It can be associated with a URI, it can have a representation, for example: a text is an InformationResource. - - - - - - - - - - - - - - - - - - - - - - - - - - Non-Information Resource - All resources that are not information resources. - - - - - - - Physical Thing - A persistent physical item such as a painting, a building, a book or a stone. -Persons are not items. This class represents Cultural Heritage Objects known to Europeana to be physical things (such as Mona Lisa) as well as all physical things Europeana refers to in the descriptions of Cultural Heritage Objects (such as the Rosetta Stone). - - - - - - - - Place - An "extent in space, in particular on the surface of the earth, in the pure sense of physics: independent from temporal phenomena and matter" (CIDOC CRM) - - - - - - - - - - - - Provided CHO - This class comprises the Cultural Heritage objects that Europeana collects descriptions about. - - - 1 - - - - - - - - This class has been mostly motivated by the need to assign a type to the “central node” in the EDM pattern, during the ingestion process, related to the XML expression of EDM at that stage. It was especially intended to fit the cases where edm:PhysicalThing cannot be used as the type of the resource standing for the real-world object (independently of any specific data contributor perspective). - Mona Lisa, Winged Victory of Samothrace - Rationale: This class is the range of edm:aggregatedCHO. A resource of type ProvidedCHO can be the subject of statements using edm:isRelatedTo or any more specific property. - - - - - - Time Span - The class of "abstract temporal extents, in the sense of Galilean physics, - having a beginning, an end and a duration" (CIDOC CRM) - - - - - - - - - - - Web Resource - Information Resources that have at least one Web Representation and at least -a URI. - - - - - - - - - - Aggregated Cultural Heritage Object - This property associates an ORE aggregation with the cultural heritage object(s) (CHO for short) it is about. - - - - - - - - - - - Begin - This property denotes the start date of a period of time. - - - - - - - - - - - - - - - Collection Name - This property holds the collection identifier given to the dataset in Europeana. - The value of this property is provided by Europeana as part of the ingestion process. - - - - - - Current Location - The geographic location and/or name of the repository, building, site, or other entity whose boundaries presently include the resource. - - - - - - - - - - - - - - - - - - - - - - - - - - End - This property denotes the end date of a period of time. - - - - - - - - - - - - - - - Europeana Proxy - This property serves only as a flag to indicate that a proxy is a Europeana proxy (as opposed to a provider proxy). It is for internal use only. - By default, any proxy without this flag can be interpreted as having the value false and is a provider proxy. - - - - - - Happened At - This property associates an event with the place at which the event -happened. - - - - - - - - - - Has Met - edm:hasMet relates a resource with the objects or phenomena that have happened to or have happened together with the resource under consideration. We can abstractly think of history and the present as a series of “meetings” between people and other things in space-time. Therefore we name this relationship as the things the object “has met” in the course of its existence. These meetings are events in the proper sense, in which other people and things participate in any role. - - - - - - - Has Type - This property relates a resource with the concepts it belongs to in a suitable -type system such as MIME or any thesaurus that captures categories of objects in a given field (e.g., the “Objects” facet in Getty’s Art and Architecture Thesaurus). It does not capture aboutness. - - - - - - - - - - - - - - - - - - - - - - - - - Has View - This property relates a ORE aggregation about a CHO with a web resource -providing a view of that CHO. Examples of view are: a thumbnail, a textual -abstract and a table of contents. The ORE aggregation may be a Europeana -Aggregation, in which case the view is an object owned by Europeana (i.e., an instance of edm:EuropeanaObject) or an aggregation contributed by a content provider. In order to capture both these cases, the domain of edm:hasView is ore:Aggregation and its range is edm:WebResource - - - - - - - - - Incorporates - This property captures the use of some resource to add value to another -resource. Such resources may be nested, such as performing a theater play text, and then recording the performance, or creating an artful edition of a collection of poems or just aggregating various poems in an anthology. There may be no single part that contains ultimately the incorporated object, which may be dispersed in the presentation. Therefore, incorporated resources do in general not form proper parts. Incorporated resources are not part of the same resource, but are taken from other resources, and have an independent history. Therefore edm:incorporates is not a sub-property of dcterm:hasPart. - - - - - - - - Is Annotation Of - This property relates an annotation (a Europeana object) with the resource -that it annotates. - - - - - - - - - - - - - - - - - - - - - - - - - Is Derivative Of - This property captures a narrower notion of derivation than edm:isSimilarTo, in the sense that it relates a resource to another one, obtained by reworking, reducing, expanding, parts or the whole contents of the former, and possibly adding some minor parts. Versions have an even narrower meaning, in that it requires common identity between the related resources. Translations, summaries, abstractions etc. do not qualify as versions, but do qualify as derivatives. - - - - - - - - Is Next In Sequence Of - edm:isNextInSequence relates two resources S and R that are ordered parts of the same resource A, and such that S comes immediately after R in the order created by their being parts of A. - - - - - - - - Is Related To - edm:isRelatedTo is the most general contextual property in EDM. Contextual -properties have typically to do either with the things that have happened to or together with the object under consideration, or what the object refers to by its shape, form or features in a figural or encoded form. For sake of simplicity, we include in the contextual relationships also the scholarly classification, which may have either to do with the role and cultural connections of the object in the past, or its kind of structure, substance or contents as it can be verified at present. - - - - - - - - - - - - - - - - - - - - - - Is Representation Of - This property associates an information resource to the resource (if any) that it represents - - - - - - - - - Is Similar To - The most generic derivation property, covering also the case of questionable derivation. Is Similar To asserts that parts of the contents of one resource exhibit common features with respect to ideas, shapes, structures, colors, words, plots, topics with the contents of the related resource. Those common features may be attributed to a common origin or influence (in particular for derivation), but also to more generic cultural or psychological factors. - - - - - - - - Is Successor Of - This property captures the relation between the continuation of a resource and that resource. This applies to a story, a serial, a journal etc. No content of the successor resource is identical or has a similar form with that of the precursor. The similarity is only in the context, subjects and figures of a plot. Successors typically form part of a common whole – such as a trilogy, a journal, etc. - - - - - - - Landing Page - This property captures the relation between a Europeana aggregation representing a cultural heritage object and a (reference) Web resource giving access to that object. Europeana provides the value for this property. - - - - - - - - Occured At - This property associates an event to the smallest known time span that -overlaps with the occurrence of that event - - - - - - - - - - Preview - The URL of a thumbnail representing the digital object, generated by Europeana. - - - - - - - - Realizes - This property describes a relation between a physical thing and the information resource that is contained in it, visible at it or otherwise carried by it, if applicable. - - - - - - - - - - - Was Present At - This property associates the people, things or information resources with an event at which they were present - - - - - - - - - - - - - - - - - - - - - - - Country - - - - - - - Data Provider - The name or identifier of the organisation who contributes data indirectly to an aggregation service (e.g. Europeana). - - - - - - - - - Is Shown At - An unambiguous URL reference to the digital object on the provider’s web site in its full information context. - - - - - - - - Is Shown By - An unambiguous URL reference to the digital object on the provider’s web site in the best available resolution/quality. - - - - - - - - Europeana Language - The value for this element is added by the Data Ingestion Team as part of the ingestion process, based on the language of the data provider. - - - - - - Object - The URL of a thumbnail representing the digital object or, if there is no such -thumbnail, the URL of the digital object in the best resolution available on the -web site of the data provider from which a thumbnail could be generated. This will often be the same URL as given in edm:isShownBy. - - - - - - - - Provider - The name or identifier of the organization who delivers data directly to an aggregation service (e.g. Europeana) - - - - - - - - Europeana Rights - Information about copyright of the digital object as specified by isShownBy -and isShownAt - - - - - - Europeana Type - The Europeana material type of the resource - - - - - TEXT - - IMAGE - - SOUND - - VIDEO - - 3D - - - - - - - - - - - - - - UGC - This element is used to identify user generated content (also called user created content). It should be applied to all digitised or born digital content contributed by the general public and collected by Europeana through a crowdsourcing initiative or project. - - - - TRUE - - - - - - - - - - Unstored - This is a container element which includes all relevant information that -otherwise cannot be mapped to another element in the ESE. - - - - - - Europeana URI - This is a tag created by a user through the Europeana interface. - - - - - - User Tag - This is a tag created by a user through the Europeana interface. - - - - - - - Europeana Year - A point of time associated with an event in the life of the original analog or -born digital object. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/metis-schema/src/main/resources/rdf_examples/edm-v524-130522.ttl b/metis-schema/src/main/resources/rdf_examples/edm-v524-130522.ttl deleted file mode 100644 index d0d19c74f..000000000 --- a/metis-schema/src/main/resources/rdf_examples/edm-v524-130522.ttl +++ /dev/null @@ -1,671 +0,0 @@ -@prefix : . -@prefix xsd: . -@prefix rdf: . -@prefix rdfs: . -@prefix owl: . -@prefix skos: . -@prefix voaf: . -@prefix vann: . -@prefix adms: . -@prefix radion: . -@prefix dc: . -@prefix dcterms: . -@prefix ore: . -@prefix foaf: . -@prefix wgs84_pos: . -@prefix dcmitype: . -@prefix crm: . -@prefix frbr_core: . -@prefix frbroo: . -@prefix abc: . -@prefix DOLCE-Lite: . - - a owl:Ontology , voaf:Vocabulary ; - vann:preferredNamespacePrefix "edm" ; - vann:preferredNamespaceUri "http://www.europeana.eu/schemas/edm/" ; - dc:title "Europeana Data Model (EDM) vocabulary"@en ; - dc:description "The Europeana Data Model (EDM) is aimed at being an integration medium for collecting, connecting and enriching the descriptions provided by Europeana data providers. The RDF vocabulary for http://www.europeana.eu/schemas/edm/ defines the elements introduced by EDM (as opposed to the ones EDM re-uses from other namespaces)."@en ; - dcterms:issued "2010-03-25"^^xsd:date ; - dc:modified "2013-05-20"^^xsd:date ; - owl:versionInfo "5.2.4" ; - radion:versionNotes "The present specification is based on the document \"Definition of the Europeana Data Model elements\", originally edited by Carlo Meghini. It is aligned with the version 5.2.4 of these EDM Definitions."@en ; - dc:creator . - - a foaf:Person ; - foaf:name "Antoine Isaac" . - - dc:contributor _:node17eprp1n4x1343334 . - -_:node17eprp1n4x1343334 a foaf:Organization ; - foaf:name "Participants of Europeana Version 1.0 Work Package on Further Specification of Functionality and Interoperability aspects of Europeana (WP3)" . - - dc:contributor . - - a foaf:Person ; - foaf:name "Nasos Drosopoulos" . - - dc:contributor . - - a foaf:Person ; - foaf:name "Vassilis Tzouvaras" . - - dc:contributor . - - a foaf:Person ; - foaf:name "Julia Iwanowa" . - - dc:contributor _:node17eprp1n4x1343335 . - -_:node17eprp1n4x1343335 a foaf:Person ; - foaf:name "Hugo Manguinhas" . - - dc:contributor . - - a foaf:Person ; - foaf:name "Martin Doerr" . - - dc:publisher . - - a foaf:Organization ; - foaf:name "Europeana" . - - foaf:homepage ; - adms:relatedWebPage ; - vann:example , ; - vann:changes """======= -Changes between ontology file EDM version 5.2.4 (edm, was once EDM-v524-120820) -and ontology file EDM version 5.2.3 (EDM-v523-120123) -======= -1. edm:isShownAt made a sub-property of edm:hasView -2. added edm:begin and edm:end and their mappings to CRM -3. added owl:Class declarations added for compatibility with some OWL-DL reasoners (feedback from Pedro Szekely, ISI) -4. added \"of\" at the end of the label for edm:isNextInSequence -5. added vocabulary metadata to follow Linked Open Vocabularies (http://lov.okfn.org/) and ADMS (https://joinup.ec.europa.eu/asset/adms/release/100) guidelines -6. removed a domain axiom on edm:hasMet -7. added edm:collectionName and edm:europeanaProxy -8. removed version number from file name -9. generalisation of Country, DataProvider and Provider -10. updated CRM namespace and CRM class and property identifiers -11. added FRBRoo mappings"""@en ; - voaf:toDoList """======= -Remaining TODOs for ontology file EDM version 5.2.4 -======= -- finish and check FRBRoo mappings according to the recommendations of the EDM-FRBRoo task force. Also include implicit mappings and mappings for elements outside the EDM namespace? -- try to capture formal cardinality constraints resulting from \"Obligation and Occurrence\" documentation, which should be attached to non-EDM constructs (esp. ore:Aggregation) -- continue adding documentation values (skos:scopeNote, skos:example, etc, according to 1.), starting from edm:InformationResource. Add all Europeana examples and rationale notes for non-EDM constructs -- use specific EDM-doc properties for \"rationale\" and \"obligation and occurrence\". Use skos:definition for \"Europeana definition\", skos:example for \"Example\", skos:note for \"Europeana note\""""@en . - -:Agent a owl:Class ; - rdfs:label "Agent"@en ; - skos:definition """This class comprises people, either individually or in groups, who have the -potential to perform intentional actions for which they can be held responsible.""" ; - skos:example "Leonardo da Vinci, the British Museum, W3C" ; - rdfs:subClassOf :NonInformationResource ; - owl:equivalentClass ; - skos:scopeNote "Rationale: This class is a domain of edm:wasPresentAt" . - -:EuropeanaAggregation a owl:Class ; - rdfs:label "Europeana Aggregation" ; - skos:definition """The set of resources related to a single Cultural Heritage Object that -collectively represent that object in Europeana. Such set consists of: all -descriptions about the object that Europeana collects from (possibly different) content providers, including thumbnails and other forms of abstractions, as well as of the description of the object Europeana builds.""" ; - rdfs:subClassOf :EuropeanaObject , ore:Aggregation ; - owl:equivalentClass _:node17eprp1n4x1343336 . - -_:node17eprp1n4x1343336 a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty :aggregatedCHO . - -:EuropeanaAggregation skos:note "An instance of EuropeanaAggregation is created at ingestion time for each different Cultural Heritage Object recognized by Europeana. Such instance is associated to the Cultural Heritage Object that it is about, by the property edm:aggregatedCHO"@en ; - skos:scopeNote "Obligation and Occurence: The relation between the Cultural Heritage Objects represented in Europeana and the instances of the class EuropeanaAggregation is one-to-one, in the data maintained by Europeana: every Cultural Heritage Object is represented by an instance of EuropeanaAggregation, and every instance of EuropeanaAggregation represent a Cultural Heritage Object."@en ; - skos:example "The painting Mona Lisa is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en , "The journal \"Le Temps\" is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en , "The 56th issue of \"Le Temps\" is a (different) Cultural Heritage Object represented in Europeana by another EuropeanaAggregation instance"@en ; - skos:scopeNote "Rationale: This class is used in Europeana to gather in a single conceptual unit all the information about a Cultural Heritage Object, necessary for all operations on these objects."@en . - -:EuropeanaObject a owl:Class ; - rdfs:label "Europeana Object"@en ; - skos:definition "Any object that is the result of EuropeanaÂ’s activities"@en ; - rdfs:subClassOf :WebResource ; - skos:example "Any instance of the class EuropeanaAggregation"@en , "An annotation created by a user through the Europeana portal"@en , "Any content created by the users through the service made available by Europeana for that purpose"@en ; - skos:scopeNote "Rationale: This class is used to tag objects that are the result of activity of Europeana, and, as such, objects on which Europeana holds rights"@en . - -:Event a owl:Class ; - rdfs:label "Event"@en ; - skos:definition """An event is a change \"of states in cultural, social or physical systems, - regardless of scale, brought about by a series or group of coherent physical, -cultural, technological or legal phenomena\" (E5 Event in CIDOC CRM) or a \"set of coherent phenomena or cultural manifestations bounded in time and space\" (E4 Period in CIDOC CRM)"""@en ; - rdfs:subClassOf :NonInformationResource ; - owl:equivalentClass , abc:Temporality , frbr_core:Event , frbroo:F8_Event ; - rdfs:subClassOf _:node17eprp1n4x1343337 . - -_:node17eprp1n4x1343337 a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty :happenedAt . - -:Event skos:note "Events are identified either by the content provider or by Europeana enrichment at ingestion time"@en ; - skos:example "the act of painting Mona Lisa"@en , "the 2nd World War"@en , "the change of custody of Mona Lisa"@en ; - skos:scopeNote "Rationale:This class is a domain of edm:happenedAt and the domain of edm:occurredAt"@en . - -:InformationResource a owl:Class ; - rdfs:label "Information Resource"@en ; - skos:definition "An information resource is a resource whose essential characteristics can be conveyed in a single message. It can be associated with a URI, it can have a representation, for example: a text is an InformationResource."@en ; - owl:equivalentClass , _:node17eprp1n4x1343338 . - -_:node17eprp1n4x1343338 a owl:Class ; - owl:unionOf _:node17eprp1n4x1343339 . - -_:node17eprp1n4x1343339 rdf:first frbr_core:Work ; - rdf:rest _:node17eprp1n4x1343340 . - -_:node17eprp1n4x1343340 rdf:first frbr_core:Expression ; - rdf:rest _:node17eprp1n4x1343341 . - -_:node17eprp1n4x1343341 rdf:first frbr_core:Manifestation ; - rdf:rest rdf:nil . - -:InformationResource owl:equivalentClass _:node17eprp1n4x1343342 . - -_:node17eprp1n4x1343342 a owl:Class ; - owl:unionOf _:node17eprp1n4x1343343 . - -_:node17eprp1n4x1343343 rdf:first frbroo:F1_Work ; - rdf:rest _:node17eprp1n4x1343344 . - -_:node17eprp1n4x1343344 rdf:first frbroo:F2_Expression ; - rdf:rest _:node17eprp1n4x1343345 . - -_:node17eprp1n4x1343345 rdf:first frbroo:F3_Manifestation_Product_Type ; - rdf:rest _:node17eprp1n4x1343346 . - -_:node17eprp1n4x1343346 rdf:first frbroo:F4_Manifestation_Singleton ; - rdf:rest rdf:nil . - -:NonInformationResource a owl:Class ; - rdfs:label "Non-Information Resource"@en ; - skos:definition "All resources that are not information resources."@en ; - owl:complementOf :InformationResource . - -:PhysicalThing a owl:Class ; - rdfs:label "Physical Thing"@en ; - skos:definition """A persistent physical item such as a painting, a building, a book or a stone. -Persons are not items. This class represents Cultural Heritage Objects known to Europeana to be physical things (such as Mona Lisa) as well as all physical things Europeana refers to in the descriptions of Cultural Heritage Objects (such as the Rosetta Stone)."""@en ; - rdfs:subClassOf :NonInformationResource ; - owl:equivalentClass . - -:Place a owl:Class ; - rdfs:label "Place"@en ; - skos:definition "An \"extent in space, in particular on the surface of the earth, in the pure sense of physics: independent from temporal phenomena and matter\" (CIDOC CRM)"@en ; - rdfs:subClassOf :NonInformationResource ; - owl:equivalentClass DOLCE-Lite:space-region , abc:Place , , frbr_core:Place , frbroo:F9_Place . - -:ProvidedCHO a owl:Class ; - rdfs:label "Provided CHO"@en ; - skos:definition "This class comprises the Cultural Heritage objects that Europeana collects descriptions about."@en ; - rdfs:subClassOf _:node17eprp1n4x1343347 . - -_:node17eprp1n4x1343347 a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty _:node17eprp1n4x1343348 . - -_:node17eprp1n4x1343348 a rdf:Property ; - owl:inverseOf :aggregatedCHO . - -:ProvidedCHO skos:note "This class has been mostly motivated by the need to assign a type to the “central node” in the EDM pattern, during the ingestion process, related to the XML expression of EDM at that stage. It was especially intended to fit the cases where edm:PhysicalThing cannot be used as the type of the resource standing for the real-world object (independently of any specific data contributor perspective)."@en ; - skos:example "Mona Lisa, Winged Victory of Samothrace"@en ; - skos:scopeNote "Rationale: This class is the range of edm:aggregatedCHO. A resource of type ProvidedCHO can be the subject of statements using edm:isRelatedTo or any more specific property."@en . - -:TimeSpan a owl:Class ; - rdfs:label "Time Span"@en ; - skos:definition """The class of \"abstract temporal extents, in the sense of Galilean physics, - having a beginning, an end and a duration\" (CIDOC CRM)"""@en ; - rdfs:subClassOf :NonInformationResource , dcterms:PeriodOfTime ; - owl:equivalentClass abc:Time , DOLCE-Lite:time-interval , . - -:WebResource a owl:Class ; - rdfs:label "Web Resource"@en ; - skos:definition """Information Resources that have at least one Web Representation and at least -a URI."""@en ; - rdfs:subClassOf :InformationResource . - -:aggregatedCHO a owl:ObjectProperty ; - rdfs:label "Aggregated Cultural Heritage Object"@en ; - skos:definition "This property associates an ORE aggregation with the cultural heritage object(s) (CHO for short) it is about."@en ; - rdfs:subPropertyOf ore:aggregates , dc:subject , ; - rdfs:domain ore:Aggregation ; - rdfs:range :ProvidedCHO . - -:begin a owl:DatatypeProperty ; - rdfs:label "Begin"@en ; - skos:definition "This property denotes the start date of a period of time."@en ; - rdfs:subPropertyOf :isRelatedTo ; - rdfs:domain _:node17eprp1n4x1343349 . - -_:node17eprp1n4x1343349 a owl:Class ; - owl:unionOf _:node17eprp1n4x1343350 . - -_:node17eprp1n4x1343350 rdf:first :Agent ; - rdf:rest _:node17eprp1n4x1343351 . - -_:node17eprp1n4x1343351 rdf:first :TimeSpan ; - rdf:rest rdf:nil . - -:collectionName a owl:ObjectProperty ; - rdfs:label "Collection Name"@en ; - skos:definition "This property holds the collection identifier given to the dataset in Europeana."@en ; - skos:note "The value of this property is provided by Europeana as part of the ingestion process."@en . - -:currentLocation a owl:ObjectProperty ; - rdfs:label "Current Location"@en ; - skos:definition "The geographic location and/or name of the repository, building, site, or other entity whose boundaries presently include the resource."@en ; - rdfs:subPropertyOf dcterms:spatial ; - owl:equivalentProperty wgs84_pos:location , ; - rdfs:domain _:node17eprp1n4x1343352 . - -_:node17eprp1n4x1343352 a owl:Class ; - owl:unionOf _:node17eprp1n4x1343353 . - -_:node17eprp1n4x1343353 rdf:first :ProvidedCHO ; - rdf:rest _:node17eprp1n4x1343355 . - -_:node17eprp1n4x1343355 rdf:first _:node17eprp1n4x1343354 . - -_:node17eprp1n4x1343354 a owl:Class ; - owl:intersectionOf _:node17eprp1n4x1343356 . - -_:node17eprp1n4x1343356 rdf:first ore:Proxy ; - rdf:rest _:node17eprp1n4x1343358 . - -_:node17eprp1n4x1343358 rdf:first _:node17eprp1n4x1343357 . - -_:node17eprp1n4x1343357 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom :ProvidedCHO . - -_:node17eprp1n4x1343358 rdf:rest rdf:nil . - -_:node17eprp1n4x1343355 rdf:rest rdf:nil . - -:currentLocation rdfs:range :Place . - -:end a owl:DatatypeProperty ; - rdfs:label "End"@en ; - skos:definition "This property denotes the end date of a period of time."@en ; - rdfs:subPropertyOf :isRelatedTo ; - rdfs:domain _:node17eprp1n4x1343359 . - -_:node17eprp1n4x1343359 a owl:Class ; - owl:unionOf _:node17eprp1n4x1343360 . - -_:node17eprp1n4x1343360 rdf:first :Agent ; - rdf:rest _:node17eprp1n4x1343361 . - -_:node17eprp1n4x1343361 rdf:first :TimeSpan ; - rdf:rest rdf:nil . - -:europeanaProxy a owl:ObjectProperty ; - rdfs:label "Europeana Proxy"@en ; - skos:definition "This property serves only as a flag to indicate that a proxy is a Europeana proxy (as opposed to a provider proxy). It is for internal use only."@en ; - skos:note "By default, any proxy without this flag can be interpreted as having the value false and is a provider proxy."@en . - -:happenedAt a owl:ObjectProperty ; - rdfs:label "Happened At"@en ; - skos:definition """This property associates an event with the place at which the event -happened."""@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - rdfs:domain :Event ; - rdfs:range :Place . - -:hasMet a rdf:Property ; - rdfs:label "Has Met"@en ; - skos:definition "edm:hasMet relates a resource with the objects or phenomena that have happened to or have happened together with the resource under consideration. We can abstractly think of history and the present as a series of “meetings” between people and other things in space-time. Therefore we name this relationship as the things the object “has met” in the course of its existence. These meetings are events in the proper sense, in which other people and things participate in any role."@en ; - rdfs:subPropertyOf dc:relation . - -:hasType a owl:ObjectProperty ; - rdfs:label "Has Type"@en ; - skos:definition """This property relates a resource with the concepts it belongs to in a suitable -type system such as MIME or any thesaurus that captures categories of objects in a given field (e.g., the “Objects” facet in GettyÂ’s Art and Architecture Thesaurus). It does not capture aboutness."""@en ; - rdfs:subPropertyOf :isRelatedTo ; - owl:equivalentProperty ; - rdfs:domain _:node17eprp1n4x1343362 . - -_:node17eprp1n4x1343362 a owl:Class ; - owl:unionOf _:node17eprp1n4x1343363 . - -_:node17eprp1n4x1343363 rdf:first :ProvidedCHO ; - rdf:rest _:node17eprp1n4x1343365 . - -_:node17eprp1n4x1343365 rdf:first _:node17eprp1n4x1343364 . - -_:node17eprp1n4x1343364 a owl:Class ; - owl:intersectionOf _:node17eprp1n4x1343366 . - -_:node17eprp1n4x1343366 rdf:first ore:Proxy ; - rdf:rest _:node17eprp1n4x1343368 . - -_:node17eprp1n4x1343368 rdf:first _:node17eprp1n4x1343367 . - -_:node17eprp1n4x1343367 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom :ProvidedCHO . - -_:node17eprp1n4x1343368 rdf:rest rdf:nil . - -_:node17eprp1n4x1343365 rdf:rest rdf:nil . - -:hasType rdfs:range :NonInformationResource . - -:hasView a owl:ObjectProperty ; - rdfs:label "Has View"@en ; - skos:definition """This property relates a ORE aggregation about a CHO with a web resource -providing a view of that CHO. Examples of view are: a thumbnail, a textual -abstract and a table of contents. The ORE aggregation may be a Europeana -Aggregation, in which case the view is an object owned by Europeana (i.e., an instance of edm:EuropeanaObject) or an aggregation contributed by a content provider. In order to capture both these cases, the domain of edm:hasView is ore:Aggregation and its range is edm:WebResource"""@en ; - rdfs:subPropertyOf ore:aggregates ; - rdfs:domain ore:Aggregation ; - rdfs:range :WebResource . - -:incorporates a owl:ObjectProperty ; - rdfs:label "Incorporates"@en ; - skos:definition """This property captures the use of some resource to add value to another -resource. Such resources may be nested, such as performing a theater play text, and then recording the performance, or creating an artful edition of a collection of poems or just aggregating various poems in an anthology. There may be no single part that contains ultimately the incorporated object, which may be dispersed in the presentation. Therefore, incorporated resources do in general not form proper parts. Incorporated resources are not part of the same resource, but are taken from other resources, and have an independent history. Therefore edm:incorporates is not a sub-property of dcterm:hasPart."""@en ; - rdfs:subPropertyOf :isSimilarTo ; - owl:equivalentProperty frbroo:R14_incorporates . - -:isAnnotationOf a owl:ObjectProperty ; - rdfs:label "Is Annotation Of"@en ; - skos:definition """This property relates an annotation (a Europeana object) with the resource -that it annotates."""@en ; - rdfs:subPropertyOf dc:subject , ; - rdfs:domain :EuropeanaObject ; - rdfs:range _:node17eprp1n4x1343369 . - -_:node17eprp1n4x1343369 a owl:Class ; - owl:unionOf _:node17eprp1n4x1343370 . - -_:node17eprp1n4x1343370 rdf:first :ProvidedCHO ; - rdf:rest _:node17eprp1n4x1343372 . - -_:node17eprp1n4x1343372 rdf:first _:node17eprp1n4x1343371 . - -_:node17eprp1n4x1343371 a owl:Class ; - owl:intersectionOf _:node17eprp1n4x1343373 . - -_:node17eprp1n4x1343373 rdf:first ore:Proxy ; - rdf:rest _:node17eprp1n4x1343375 . - -_:node17eprp1n4x1343375 rdf:first _:node17eprp1n4x1343374 . - -_:node17eprp1n4x1343374 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom :ProvidedCHO . - -_:node17eprp1n4x1343375 rdf:rest rdf:nil . - -_:node17eprp1n4x1343372 rdf:rest rdf:nil . - -:isDerivativeOf a rdf:Property ; - rdfs:label "Is Derivative Of"@en ; - skos:definition "This property captures a narrower notion of derivation than edm:isSimilarTo, in the sense that it relates a resource to another one, obtained by reworking, reducing, expanding, parts or the whole contents of the former, and possibly adding some minor parts. Versions have an even narrower meaning, in that it requires common identity between the related resources. Translations, summaries, abstractions etc. do not qualify as versions, but do qualify as derivatives."@en ; - rdfs:subPropertyOf :isSimilarTo ; - owl:equivalentProperty frbroo:R2_is_derivative_of . - -:isNextInSequence a owl:ObjectProperty ; - rdfs:label "Is Next In Sequence Of"@en ; - skos:definition "edm:isNextInSequence relates two resources S and R that are ordered parts of the same resource A, and such that S comes immediately after R in the order created by their being parts of A."@en ; - rdfs:subPropertyOf dc:relation . - -:isRelatedTo a rdf:Property ; - rdfs:label "Is Related To"@en ; - skos:definition """edm:isRelatedTo is the most general contextual property in EDM. Contextual -properties have typically to do either with the things that have happened to or together with the object under consideration, or what the object refers to by its shape, form or features in a figural or encoded form. For sake of simplicity, we include in the contextual relationships also the scholarly classification, which may have either to do with the role and cultural connections of the object in the past, or its kind of structure, substance or contents as it can be verified at present."""@en ; - rdfs:domain _:node17eprp1n4x1343376 . - -_:node17eprp1n4x1343376 a owl:Class ; - owl:unionOf _:node17eprp1n4x1343377 . - -_:node17eprp1n4x1343377 rdf:first :ProvidedCHO ; - rdf:rest _:node17eprp1n4x1343379 . - -_:node17eprp1n4x1343379 rdf:first _:node17eprp1n4x1343378 . - -_:node17eprp1n4x1343378 a owl:Class ; - owl:intersectionOf _:node17eprp1n4x1343380 . - -_:node17eprp1n4x1343380 rdf:first ore:Proxy ; - rdf:rest _:node17eprp1n4x1343382 . - -_:node17eprp1n4x1343382 rdf:first _:node17eprp1n4x1343381 . - -_:node17eprp1n4x1343381 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom :ProvidedCHO . - -_:node17eprp1n4x1343382 rdf:rest rdf:nil . - -_:node17eprp1n4x1343379 rdf:rest rdf:nil . - -:isRepresentationOf a owl:ObjectProperty ; - rdfs:label "Is Representation Of"@en ; - skos:definition "This property associates an information resource to the resource (if any) that it represents"@en ; - rdfs:subPropertyOf dc:subject ; - rdfs:domain :InformationResource ; - rdfs:subPropertyOf . - -:isSimilarTo a rdf:Property ; - rdfs:label "Is Similar To"@en ; - skos:definition "The most generic derivation property, covering also the case of questionable derivation. Is Similar To asserts that parts of the contents of one resource exhibit common features with respect to ideas, shapes, structures, colors, words, plots, topics with the contents of the related resource. Those common features may be attributed to a common origin or influence (in particular for derivation), but also to more generic cultural or psychological factors."@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty . - -:isSuccessorOf a owl:ObjectProperty ; - rdfs:label "Is Successor Of"@en ; - skos:definition "This property captures the relation between the continuation of a resource and that resource. This applies to a story, a serial, a journal etc. No content of the successor resource is identical or has a similar form with that of the precursor. The similarity is only in the context, subjects and figures of a plot. Successors typically form part of a common whole – such as a trilogy, a journal, etc."@en ; - rdfs:subPropertyOf :isSimilarTo . - -:landingPage a owl:ObjectProperty ; - rdfs:label "Landing Page"@en ; - skos:definition "This property captures the relation between a Europeana aggregation representing a cultural heritage object and a (reference) Web resource giving access to that object. Europeana provides the value for this property."@en ; - rdfs:subPropertyOf ore:aggregates ; - rdfs:range :WebResource . - -:occurredAt a owl:ObjectProperty ; - rdfs:label "Occured At"@en ; - skos:definition """This property associates an event to the smallest known time span that -overlaps with the occurrence of that event"""@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - rdfs:domain :Event ; - rdfs:range :TimeSpan . - -:preview a owl:ObjectProperty ; - rdfs:label "Preview"@en ; - skos:definition "The URL of a thumbnail representing the digital object, generated by Europeana."@en ; - rdfs:subPropertyOf :hasView ; - rdfs:range :WebResource . - -:realizes a owl:ObjectProperty ; - rdfs:label "Realizes"@en ; - skos:definition "This property describes a relation between a physical thing and the information resource that is contained in it, visible at it or otherwise carried by it, if applicable."@en ; - rdfs:subPropertyOf :isRelatedTo ; - owl:equivalentProperty ; - rdfs:domain :PhysicalThing ; - rdfs:range :InformationResource . - -:wasPresentAt a owl:ObjectProperty ; - rdfs:label "Was Present At"@en ; - skos:definition "This property associates the people, things or information resources with an event at which they were present"@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - rdfs:domain _:node17eprp1n4x1343383 . - -_:node17eprp1n4x1343383 a owl:Class ; - owl:unionOf _:node17eprp1n4x1343384 . - -_:node17eprp1n4x1343384 rdf:first :Agent ; - rdf:rest _:node17eprp1n4x1343385 . - -_:node17eprp1n4x1343385 rdf:first :InformationResource ; - rdf:rest _:node17eprp1n4x1343386 . - -_:node17eprp1n4x1343386 rdf:first :PhysicalThing ; - rdf:rest rdf:nil . - -:wasPresentAt rdfs:range :Event . - -:country a rdf:Property ; - rdfs:label "Country"@en ; - rdfs:subPropertyOf . - -:dataProvider a rdf:Property ; - rdfs:label "Data Provider"@en ; - skos:definition "The name or identifier of the organisation who contributes data indirectly to an aggregation service (e.g. Europeana)."@en ; - rdfs:subPropertyOf dcterms:provenance ; - rdfs:domain ore:Aggregation ; - rdfs:range :Agent . - -:isShownAt a owl:ObjectProperty ; - rdfs:label "Is Shown At"@en ; - skos:definition "An unambiguous URL reference to the digital object on the providerÂ’s web site in its full information context."@en ; - rdfs:subPropertyOf :hasView ; - rdfs:range :WebResource . - -:isShownBy a owl:ObjectProperty ; - rdfs:label "Is Shown By"@en ; - skos:definition "An unambiguous URL reference to the digital object on the providerÂ’s web site in the best available resolution/quality."@en ; - rdfs:subPropertyOf :hasView ; - rdfs:range :WebResource . - -:language a rdf:Property ; - rdfs:label "Europeana Language"@en ; - skos:definition "The value for this element is added by the Data Ingestion Team as part of the ingestion process, based on the language of the data provider."@en . - -:object a owl:ObjectProperty ; - rdfs:label "Object"@en ; - skos:definition """The URL of a thumbnail representing the digital object or, if there is no such -thumbnail, the URL of the digital object in the best resolution available on the -web site of the data provider from which a thumbnail could be generated. This will often be the same URL as given in edm:isShownBy."""@en ; - rdfs:subPropertyOf :hasView ; - rdfs:range :WebResource . - -:provider a rdf:Property ; - rdfs:label "Provider"@en ; - skos:definition "The name or identifier of the organization who delivers data directly to an aggregation service (e.g. Europeana)"@en ; - rdfs:subPropertyOf :hasMet ; - rdfs:range :Agent . - -:rights a owl:ObjectProperty ; - rdfs:label "Europeana Rights"@en ; - skos:definition """Information about copyright of the digital object as specified by isShownBy -and isShownAt"""@en . - -:type a owl:DatatypeProperty ; - rdfs:label "Europeana Type"@en ; - skos:definition "The Europeana material type of the resource"@en ; - rdfs:subPropertyOf dc:type ; - rdfs:range _:node17eprp1n4x1343387 . - -_:node17eprp1n4x1343387 a rdfs:Datatype ; - owl:oneOf _:node17eprp1n4x1343388 . - -_:node17eprp1n4x1343388 rdf:first "TEXT" ; - rdf:rest _:node17eprp1n4x1343389 . - -_:node17eprp1n4x1343389 rdf:first "IMAGE" ; - rdf:rest _:node17eprp1n4x1343390 . - -_:node17eprp1n4x1343390 rdf:first "SOUND" ; - rdf:rest _:node17eprp1n4x1343391 . - -_:node17eprp1n4x1343391 rdf:first "VIDEO" ; - rdf:rest _:node17eprp1n4x1343392 . - -_:node17eprp1n4x1343392 rdf:first "3D" ; - rdf:rest rdf:nil . - -:ugc a owl:DatatypeProperty ; - rdfs:label "UGC"@en ; - skos:definition "This element is used to identify user generated content (also called user created content). It should be applied to all digitised or born digital content contributed by the general public and collected by Europeana through a crowdsourcing initiative or project."@en ; - rdfs:range _:node17eprp1n4x1343393 . - -_:node17eprp1n4x1343393 a rdfs:Datatype ; - owl:oneOf _:node17eprp1n4x1343394 . - -_:node17eprp1n4x1343394 rdf:first "TRUE" ; - rdf:rest rdf:nil . - -:unstored a rdf:Property ; - rdfs:label "Unstored"@en ; - skos:definition """This is a container element which includes all relevant information that -otherwise cannot be mapped to another element in the ESE."""@en . - -:uri a owl:ObjectProperty ; - rdfs:label "Europeana URI"@en ; - skos:definition "This is a tag created by a user through the Europeana interface."@en . - -:userTag a rdf:Property ; - rdfs:label "User Tag"@en ; - skos:definition "This is a tag created by a user through the Europeana interface."@en ; - rdfs:subPropertyOf dc:description . - -:year a rdf:Property ; - rdfs:label "Europeana Year"@en ; - skos:definition """A point of time associated with an event in the life of the original analog or -born digital object."""@en ; - rdfs:subPropertyOf dcterms:temporal . - -skos:Concept rdfs:subClassOf :NonInformationResource . - - rdfs:subPropertyOf :begin . - - rdfs:subPropertyOf :end . - -dc:contributor rdfs:subPropertyOf :hasMet . - -dc:coverage rdfs:subPropertyOf :hasMet . - -dc:creator rdfs:subPropertyOf :hasMet . - -dc:date rdfs:subPropertyOf :hasMet . - -dc:format rdfs:subPropertyOf :hasType . - -dcterms:hasFormat rdfs:subPropertyOf _:node17eprp1n4x1343395 . - -_:node17eprp1n4x1343395 a rdf:Property ; - owl:inverseOf :isDerivativeOf . - -dcterms:hasVersion rdfs:subPropertyOf _:node17eprp1n4x1343396 . - -_:node17eprp1n4x1343396 a rdf:Property ; - owl:inverseOf :isDerivativeOf . - -dcterms:isFormatOf rdfs:subPropertyOf :isDerivativeOf . - -dcterms:isReplacedBy rdfs:subPropertyOf _:node17eprp1n4x1343397 . - -_:node17eprp1n4x1343397 a rdf:Property ; - owl:inverseOf :isDerivativeOf . - -dcterms:isVersionOf rdfs:subPropertyOf :isDerivativeOf . - -dc:language rdfs:subPropertyOf :hasType . - -dc:publisher rdfs:subPropertyOf :hasMet . - -dc:relation rdfs:subPropertyOf :isRelatedTo . - -dcterms:replaces rdfs:subPropertyOf :isDerivativeOf . - -dc:source rdfs:subPropertyOf :isDerivativeOf . - -dc:subject rdfs:subPropertyOf :isRelatedTo . - -dcterms:tableOfContents rdfs:subPropertyOf :hasView . - -dc:type rdfs:subPropertyOf :hasType . - -skos:Concept a owl:Class . - -ore:Aggregation a owl:Class . - -ore:Proxy a owl:Class . diff --git a/metis-schema/src/main/resources/rdf_examples/edm.n3 b/metis-schema/src/main/resources/rdf_examples/edm.n3 deleted file mode 100644 index 75e63e086..000000000 --- a/metis-schema/src/main/resources/rdf_examples/edm.n3 +++ /dev/null @@ -1,511 +0,0 @@ -@prefix DOLCE-Lite: . -@prefix abc: . -@prefix adms: . -@prefix dc: . -@prefix dcterms: . -@prefix edm: . -@prefix foaf: . -@prefix frbr: . -@prefix frbroo: . -@prefix geo: . -@prefix ore: . -@prefix owl: . -@prefix radion: . -@prefix rdf: . -@prefix rdfs: . -@prefix skos: . -@prefix vann: . -@prefix voaf: . -@prefix xsd: . - -dc:contributor rdfs:subPropertyOf edm:hasMet . - -dc:coverage rdfs:subPropertyOf edm:hasMet . - -dc:creator rdfs:subPropertyOf edm:hasMet . - -dc:date rdfs:subPropertyOf edm:hasMet . - -dc:format rdfs:subPropertyOf edm:hasType . - -dc:language rdfs:subPropertyOf edm:hasType . - -dc:publisher rdfs:subPropertyOf edm:hasMet . - -dc:source rdfs:subPropertyOf edm:isDerivativeOf . - -dcterms:hasFormat rdfs:subPropertyOf [ a rdf:Property ; - owl:inverseOf edm:isDerivativeOf ] . - -dcterms:hasVersion rdfs:subPropertyOf [ a rdf:Property ; - owl:inverseOf edm:isDerivativeOf ] . - -dcterms:isFormatOf rdfs:subPropertyOf edm:isDerivativeOf . - -dcterms:isReplacedBy rdfs:subPropertyOf [ a rdf:Property ; - owl:inverseOf edm:isDerivativeOf ] . - -dcterms:isVersionOf rdfs:subPropertyOf edm:isDerivativeOf . - -dcterms:replaces rdfs:subPropertyOf edm:isDerivativeOf . - -dcterms:tableOfContents rdfs:subPropertyOf edm:hasView . - - rdfs:subPropertyOf edm:begin . - - rdfs:subPropertyOf edm:end . - -edm: a voaf:Vocabulary, - owl:Ontology ; - dc:contributor [ a foaf:Person ; - foaf:name "Hugo Manguinhas" ], - [ a foaf:Organization ; - foaf:name "Participants of Europeana Version 1.0 Work Package on Further Specification of Functionality and Interoperability aspects of Europeana (WP3)" ], - , - , - , - ; - dc:creator ; - dc:description "The Europeana Data Model (EDM) is aimed at being an integration medium for collecting, connecting and enriching the descriptions provided by Europeana data providers. The RDF vocabulary for http://www.europeana.eu/schemas/edm/ defines the elements introduced by EDM (as opposed to the ones EDM re-uses from other namespaces)."@en ; - dc:modified "2013-05-20"^^xsd:date ; - dc:publisher ; - dc:title "Europeana Data Model (EDM) vocabulary"@en ; - dcterms:issued "2010-03-25"^^xsd:date ; - vann:changes """ -======= -Changes between ontology file EDM version 5.2.4 (edm, was once EDM-v524-120820) -and ontology file EDM version 5.2.3 (EDM-v523-120123) -======= -1. edm:isShownAt made a sub-property of edm:hasView -2. added edm:begin and edm:end and their mappings to CRM -3. added owl:Class declarations added for compatibility with some OWL-DL reasoners (feedback from Pedro Szekely, ISI) -4. added "of" at the end of the label for edm:isNextInSequence -5. added vocabulary metadata to follow Linked Open Vocabularies (http://lov.okfn.org/) and ADMS (https://joinup.ec.europa.eu/asset/adms/release/100) guidelines -6. removed a domain axiom on edm:hasMet -7. added edm:collectionName and edm:europeanaProxy -8. removed version number from file name -9. generalisation of Country, DataProvider and Provider -10. updated CRM namespace and CRM class and property identifiers -11. added FRBRoo mappings - """@en ; - vann:example , - ; - vann:preferredNamespacePrefix "edm" ; - vann:preferredNamespaceUri "http://www.europeana.eu/schemas/edm/" ; - voaf:toDoList """ -======= -Remaining TODOs for ontology file EDM version 5.2.4 -======= -- finish and check FRBRoo mappings according to the recommendations of the EDM-FRBRoo task force. Also include implicit mappings and mappings for elements outside the EDM namespace? -- try to capture formal cardinality constraints resulting from "Obligation and Occurrence" documentation, which should be attached to non-EDM constructs (esp. ore:Aggregation) -- continue adding documentation values (skos:scopeNote, skos:example, etc, according to 1.), starting from edm:InformationResource. Add all Europeana examples and rationale notes for non-EDM constructs -- use specific EDM-doc properties for "rationale" and "obligation and occurrence". Use skos:definition for "Europeana definition", skos:example for "Example", skos:note for "Europeana note" - """@en ; - owl:versionInfo "5.2.4" ; - adms:relatedWebPage ; - radion:versionNotes "The present specification is based on the document \"Definition of the Europeana Data Model elements\", originally edited by Carlo Meghini. It is aligned with the version 5.2.4 of these EDM Definitions."@en ; - foaf:homepage . - -edm:EuropeanaAggregation a owl:Class ; - rdfs:label "Europeana Aggregation" ; - rdfs:subClassOf edm:EuropeanaObject, - ore:Aggregation ; - owl:equivalentClass [ a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty edm:aggregatedCHO ] ; - skos:definition """The set of resources related to a single Cultural Heritage Object that -collectively represent that object in Europeana. Such set consists of: all -descriptions about the object that Europeana collects from (possibly different) content providers, including thumbnails and other forms of abstractions, as well as of the description of the object Europeana builds."""; - skos:example "The 56th issue of \"Le Temps\" is a (different) Cultural Heritage Object represented in Europeana by another EuropeanaAggregation instance"@en, - "The journal \"Le Temps\" is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en, - "The painting Mona Lisa is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en ; - skos:note "An instance of EuropeanaAggregation is created at ingestion time for each different Cultural Heritage Object recognized by Europeana. Such instance is associated to the Cultural Heritage Object that it is about, by the property edm:aggregatedCHO"@en ; - skos:scopeNote "Obligation and Occurence: The relation between the Cultural Heritage Objects represented in Europeana and the instances of the class EuropeanaAggregation is one-to-one, in the data maintained by Europeana: every Cultural Heritage Object is represented by an instance of EuropeanaAggregation, and every instance of EuropeanaAggregation represent a Cultural Heritage Object."@en, - "Rationale: This class is used in Europeana to gather in a single conceptual unit all the information about a Cultural Heritage Object, necessary for all operations on these objects."@en . - -edm:collectionName a owl:ObjectProperty ; - rdfs:label "Collection Name"@en ; - skos:definition "This property holds the collection identifier given to the dataset in Europeana."@en ; - skos:note "The value of this property is provided by Europeana as part of the ingestion process."@en . - -edm:country a rdf:Property ; - rdfs:label "Country"@en ; - rdfs:subPropertyOf . - -edm:currentLocation a owl:ObjectProperty ; - rdfs:label "Current Location"@en ; - rdfs:domain [ a owl:Class ; - owl:unionOf ( edm:ProvidedCHO [ a owl:Class ; - owl:intersectionOf ( ore:Proxy [ a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom edm:ProvidedCHO ] ) ] ) ] ; - rdfs:range edm:Place ; - rdfs:subPropertyOf dcterms:spatial ; - owl:equivalentProperty , - geo:location ; - skos:definition "The geographic location and/or name of the repository, building, site, or other entity whose boundaries presently include the resource."@en . - -edm:dataProvider a rdf:Property ; - rdfs:label "Data Provider"@en ; - rdfs:domain ore:Aggregation ; - rdfs:range edm:Agent ; - rdfs:subPropertyOf dcterms:provenance ; - skos:definition "The name or identifier of the organisation who contributes data indirectly to an aggregation service (e.g. Europeana)."@en . - -edm:europeanaProxy a owl:ObjectProperty ; - rdfs:label "Europeana Proxy"@en ; - skos:definition "This property serves only as a flag to indicate that a proxy is a Europeana proxy (as opposed to a provider proxy). It is for internal use only."@en ; - skos:note "By default, any proxy without this flag can be interpreted as having the value false and is a provider proxy."@en . - -edm:incorporates a owl:ObjectProperty ; - rdfs:label "Incorporates"@en ; - rdfs:subPropertyOf edm:isSimilarTo ; - owl:equivalentProperty frbroo:R14_incorporates ; - skos:definition """This property captures the use of some resource to add value to another -resource. Such resources may be nested, such as performing a theater play text, and then recording the performance, or creating an artful edition of a collection of poems or just aggregating various poems in an anthology. There may be no single part that contains ultimately the incorporated object, which may be dispersed in the presentation. Therefore, incorporated resources do in general not form proper parts. Incorporated resources are not part of the same resource, but are taken from other resources, and have an independent history. Therefore edm:incorporates is not a sub-property of dcterm:hasPart."""@en . - -edm:isAnnotationOf a owl:ObjectProperty ; - rdfs:label "Is Annotation Of"@en ; - rdfs:domain edm:EuropeanaObject ; - rdfs:range [ a owl:Class ; - owl:unionOf ( edm:ProvidedCHO [ a owl:Class ; - owl:intersectionOf ( ore:Proxy [ a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom edm:ProvidedCHO ] ) ] ) ] ; - rdfs:subPropertyOf dc:subject, - ; - skos:definition """This property relates an annotation (a Europeana object) with the resource -that it annotates."""@en . - -edm:isNextInSequence a owl:ObjectProperty ; - rdfs:label "Is Next In Sequence Of"@en ; - rdfs:subPropertyOf dc:relation ; - skos:definition "edm:isNextInSequence relates two resources S and R that are ordered parts of the same resource A, and such that S comes immediately after R in the order created by their being parts of A."@en . - -edm:isRepresentationOf a owl:ObjectProperty ; - rdfs:label "Is Representation Of"@en ; - rdfs:domain edm:InformationResource ; - rdfs:subPropertyOf dc:subject, - ; - skos:definition "This property associates an information resource to the resource (if any) that it represents"@en . - -edm:isShownAt a owl:ObjectProperty ; - rdfs:label "Is Shown At"@en ; - rdfs:range edm:WebResource ; - rdfs:subPropertyOf edm:hasView ; - skos:definition "An unambiguous URL reference to the digital object on the provider’s web site in its full information context."@en . - -edm:isShownBy a owl:ObjectProperty ; - rdfs:label "Is Shown By"@en ; - rdfs:range edm:WebResource ; - rdfs:subPropertyOf edm:hasView ; - skos:definition "An unambiguous URL reference to the digital object on the provider’s web site in the best available resolution/quality."@en . - -edm:isSuccessorOf a owl:ObjectProperty ; - rdfs:label "Is Successor Of"@en ; - rdfs:subPropertyOf edm:isSimilarTo ; - skos:definition "This property captures the relation between the continuation of a resource and that resource. This applies to a story, a serial, a journal etc. No content of the successor resource is identical or has a similar form with that of the precursor. The similarity is only in the context, subjects and figures of a plot. Successors typically form part of a common whole – such as a trilogy, a journal, etc."@en . - -edm:landingPage a owl:ObjectProperty ; - rdfs:label "Landing Page"@en ; - rdfs:range edm:WebResource ; - rdfs:subPropertyOf ore:aggregates ; - skos:definition "This property captures the relation between a Europeana aggregation representing a cultural heritage object and a (reference) Web resource giving access to that object. Europeana provides the value for this property."@en . - -edm:language a rdf:Property ; - rdfs:label "Europeana Language"@en ; - skos:definition "The value for this element is added by the Data Ingestion Team as part of the ingestion process, based on the language of the data provider."@en . - -edm:object a owl:ObjectProperty ; - rdfs:label "Object"@en ; - rdfs:range edm:WebResource ; - rdfs:subPropertyOf edm:hasView ; - skos:definition """The URL of a thumbnail representing the digital object or, if there is no such -thumbnail, the URL of the digital object in the best resolution available on the -web site of the data provider from which a thumbnail could be generated. This will often be the same URL as given in edm:isShownBy."""@en . - -edm:occurredAt a owl:ObjectProperty ; - rdfs:label "Occured At"@en ; - rdfs:domain edm:Event ; - rdfs:range edm:TimeSpan ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - skos:definition """This property associates an event to the smallest known time span that -overlaps with the occurrence of that event"""@en . - -edm:preview a owl:ObjectProperty ; - rdfs:label "Preview"@en ; - rdfs:range edm:WebResource ; - rdfs:subPropertyOf edm:hasView ; - skos:definition "The URL of a thumbnail representing the digital object, generated by Europeana."@en . - -edm:provider a rdf:Property ; - rdfs:label "Provider"@en ; - rdfs:range edm:Agent ; - rdfs:subPropertyOf edm:hasMet ; - skos:definition "The name or identifier of the organization who delivers data directly to an aggregation service (e.g. Europeana)"@en . - -edm:realizes a owl:ObjectProperty ; - rdfs:label "Realizes"@en ; - rdfs:domain edm:PhysicalThing ; - rdfs:range edm:InformationResource ; - rdfs:subPropertyOf edm:isRelatedTo ; - owl:equivalentProperty ; - skos:definition "This property describes a relation between a physical thing and the information resource that is contained in it, visible at it or otherwise carried by it, if applicable."@en . - -edm:rights a owl:ObjectProperty ; - rdfs:label "Europeana Rights"@en ; - skos:definition """Information about copyright of the digital object as specified by isShownBy -and isShownAt"""@en . - -edm:type a owl:DatatypeProperty ; - rdfs:label "Europeana Type"@en ; - rdfs:range [ a rdfs:Datatype ; - owl:oneOf ( "TEXT" "IMAGE" "SOUND" "VIDEO" "3D" ) ] ; - rdfs:subPropertyOf dc:type ; - skos:definition "The Europeana material type of the resource"@en . - -edm:ugc a owl:DatatypeProperty ; - rdfs:label "UGC"@en ; - rdfs:range [ a rdfs:Datatype ; - owl:oneOf ( "TRUE" ) ] ; - skos:definition "This element is used to identify user generated content (also called user created content). It should be applied to all digitised or born digital content contributed by the general public and collected by Europeana through a crowdsourcing initiative or project."@en . - -edm:unstored a rdf:Property ; - rdfs:label "Unstored"@en ; - skos:definition """This is a container element which includes all relevant information that -otherwise cannot be mapped to another element in the ESE."""@en . - -edm:uri a owl:ObjectProperty ; - rdfs:label "Europeana URI"@en ; - skos:definition "This is a tag created by a user through the Europeana interface."@en . - -edm:userTag a rdf:Property ; - rdfs:label "User Tag"@en ; - rdfs:subPropertyOf dc:description ; - skos:definition "This is a tag created by a user through the Europeana interface."@en . - -edm:wasPresentAt a owl:ObjectProperty ; - rdfs:label "Was Present At"@en ; - rdfs:domain [ a owl:Class ; - owl:unionOf ( edm:Agent edm:InformationResource edm:PhysicalThing ) ] ; - rdfs:range edm:Event ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - skos:definition "This property associates the people, things or information resources with an event at which they were present"@en . - -edm:year a rdf:Property ; - rdfs:label "Europeana Year"@en ; - rdfs:subPropertyOf dcterms:temporal ; - skos:definition """A point of time associated with an event in the life of the original analog or -born digital object."""@en . - -skos:Concept a owl:Class ; - rdfs:subClassOf edm:NonInformationResource . - - a foaf:Person ; - foaf:name "Antoine Isaac" . - - a foaf:Organization ; - foaf:name "Europeana" . - -dc:type rdfs:subPropertyOf edm:hasType . - -edm:begin a owl:DatatypeProperty ; - rdfs:label "Begin"@en ; - rdfs:domain [ a owl:Class ; - owl:unionOf ( edm:Agent edm:TimeSpan ) ] ; - rdfs:subPropertyOf edm:isRelatedTo ; - skos:definition "This property denotes the start date of a period of time."@en . - -edm:end a owl:DatatypeProperty ; - rdfs:label "End"@en ; - rdfs:domain [ a owl:Class ; - owl:unionOf ( edm:Agent edm:TimeSpan ) ] ; - rdfs:subPropertyOf edm:isRelatedTo ; - skos:definition "This property denotes the end date of a period of time."@en . - -edm:happenedAt a owl:ObjectProperty ; - rdfs:label "Happened At"@en ; - rdfs:domain edm:Event ; - rdfs:range edm:Place ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - skos:definition """This property associates an event with the place at which the event -happened."""@en . - - a foaf:Person ; - foaf:name "Julia Iwanowa" . - - a foaf:Person ; - foaf:name "Martin Doerr" . - - a foaf:Person ; - foaf:name "Nasos Drosopoulos" . - - a foaf:Person ; - foaf:name "Vassilis Tzouvaras" . - -edm:EuropeanaObject a owl:Class ; - rdfs:label "Europeana Object"@en ; - rdfs:subClassOf edm:WebResource ; - skos:definition "Any object that is the result of Europeana’s activities"@en ; - skos:example "An annotation created by a user through the Europeana portal"@en, - "Any content created by the users through the service made available by Europeana for that purpose"@en, - "Any instance of the class EuropeanaAggregation"@en ; - skos:scopeNote "Rationale: This class is used to tag objects that are the result of activity of Europeana, and, as such, objects on which Europeana holds rights"@en . - -edm:PhysicalThing a owl:Class ; - rdfs:label "Physical Thing"@en ; - rdfs:subClassOf edm:NonInformationResource ; - owl:equivalentClass ; - skos:definition """A persistent physical item such as a painting, a building, a book or a stone. -Persons are not items. This class represents Cultural Heritage Objects known to Europeana to be physical things (such as Mona Lisa) as well as all physical things Europeana refers to in the descriptions of Cultural Heritage Objects (such as the Rosetta Stone)."""@en . - -edm:Place a owl:Class ; - rdfs:label "Place"@en ; - rdfs:subClassOf edm:NonInformationResource ; - owl:equivalentClass frbroo:F9_Place, - abc:Place, - frbr:Place, - , - DOLCE-Lite:space-region ; - skos:definition "An \"extent in space, in particular on the surface of the earth, in the pure sense of physics: independent from temporal phenomena and matter\" (CIDOC CRM)"@en . - -edm:aggregatedCHO a owl:ObjectProperty ; - rdfs:label "Aggregated Cultural Heritage Object"@en ; - rdfs:domain ore:Aggregation ; - rdfs:range edm:ProvidedCHO ; - rdfs:subPropertyOf dc:subject, - , - ore:aggregates ; - skos:definition "This property associates an ORE aggregation with the cultural heritage object(s) (CHO for short) it is about."@en . - -dc:subject rdfs:subPropertyOf edm:isRelatedTo . - -edm:Event a owl:Class ; - rdfs:label "Event"@en ; - rdfs:subClassOf [ a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty edm:happenedAt ], - edm:NonInformationResource ; - owl:equivalentClass frbroo:F8_Event, - abc:Temporality, - frbr:Event, - ; - skos:definition """An event is a change "of states in cultural, social or physical systems, - regardless of scale, brought about by a series or group of coherent physical, -cultural, technological or legal phenomena" (E5 Event in CIDOC CRM) or a "set of coherent phenomena or cultural manifestations bounded in time and space" (E4 Period in CIDOC CRM) -"""@en ; - skos:example "the 2nd World War"@en, - "the act of painting Mona Lisa"@en, - "the change of custody of Mona Lisa"@en ; - skos:note "Events are identified either by the content provider or by Europeana enrichment at ingestion time"@en ; - skos:scopeNote "Rationale:This class is a domain of edm:happenedAt and the domain of edm:occurredAt"@en . - -edm:TimeSpan a owl:Class ; - rdfs:label "Time Span"@en ; - rdfs:subClassOf dcterms:PeriodOfTime, - edm:NonInformationResource ; - owl:equivalentClass abc:Time, - , - DOLCE-Lite:time-interval ; - skos:definition """The class of "abstract temporal extents, in the sense of Galilean physics, - having a beginning, an end and a duration" (CIDOC CRM)"""@en . - -edm:hasType a owl:ObjectProperty ; - rdfs:label "Has Type"@en ; - rdfs:domain [ a owl:Class ; - owl:unionOf ( edm:ProvidedCHO [ a owl:Class ; - owl:intersectionOf ( ore:Proxy [ a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom edm:ProvidedCHO ] ) ] ) ] ; - rdfs:range edm:NonInformationResource ; - rdfs:subPropertyOf edm:isRelatedTo ; - owl:equivalentProperty ; - skos:definition """This property relates a resource with the concepts it belongs to in a suitable -type system such as MIME or any thesaurus that captures categories of objects in a given field (e.g., the “Objects” facet in Getty’s Art and Architecture Thesaurus). It does not capture aboutness."""@en . - -edm:isSimilarTo a rdf:Property ; - rdfs:label "Is Similar To"@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - skos:definition "The most generic derivation property, covering also the case of questionable derivation. Is Similar To asserts that parts of the contents of one resource exhibit common features with respect to ideas, shapes, structures, colors, words, plots, topics with the contents of the related resource. Those common features may be attributed to a common origin or influence (in particular for derivation), but also to more generic cultural or psychological factors."@en . - -ore:Aggregation a owl:Class . - -ore:Proxy a owl:Class . - -edm:Agent a owl:Class ; - rdfs:label "Agent"@en ; - rdfs:subClassOf edm:NonInformationResource ; - owl:equivalentClass ; - skos:definition """This class comprises people, either individually or in groups, who have the -potential to perform intentional actions for which they can be held responsible."""; - skos:example "Leonardo da Vinci, the British Museum, W3C" ; - skos:scopeNote "Rationale: This class is a domain of edm:wasPresentAt" . - -edm:InformationResource a owl:Class ; - rdfs:label "Information Resource"@en ; - owl:equivalentClass [ a owl:Class ; - owl:unionOf ( frbr:Work frbr:Expression frbr:Manifestation ) ], - [ a owl:Class ; - owl:unionOf ( frbroo:F1_Work frbroo:F2_Expression frbroo:F3_Manifestation_Product_Type frbroo:F4_Manifestation_Singleton ) ], - ; - skos:definition "An information resource is a resource whose essential characteristics can be conveyed in a single message. It can be associated with a URI, it can have a representation, for example: a text is an InformationResource."@en . - -edm:hasView a owl:ObjectProperty ; - rdfs:label "Has View"@en ; - rdfs:domain ore:Aggregation ; - rdfs:range edm:WebResource ; - rdfs:subPropertyOf ore:aggregates ; - skos:definition """This property relates a ORE aggregation about a CHO with a web resource -providing a view of that CHO. Examples of view are: a thumbnail, a textual -abstract and a table of contents. The ORE aggregation may be a Europeana -Aggregation, in which case the view is an object owned by Europeana (i.e., an instance of edm:EuropeanaObject) or an aggregation contributed by a content provider. In order to capture both these cases, the domain of edm:hasView is ore:Aggregation and its range is edm:WebResource"""@en . - -dc:relation rdfs:subPropertyOf edm:isRelatedTo . - -edm:hasMet a rdf:Property ; - rdfs:label "Has Met"@en ; - rdfs:subPropertyOf dc:relation ; - skos:definition "edm:hasMet relates a resource with the objects or phenomena that have happened to or have happened together with the resource under consideration. We can abstractly think of history and the present as a series of “meetings” between people and other things in space-time. Therefore we name this relationship as the things the object “has met” in the course of its existence. These meetings are events in the proper sense, in which other people and things participate in any role."@en . - -edm:isRelatedTo a rdf:Property ; - rdfs:label "Is Related To"@en ; - rdfs:domain [ a owl:Class ; - owl:unionOf ( edm:ProvidedCHO [ a owl:Class ; - owl:intersectionOf ( ore:Proxy [ a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom edm:ProvidedCHO ] ) ] ) ] ; - skos:definition """edm:isRelatedTo is the most general contextual property in EDM. Contextual -properties have typically to do either with the things that have happened to or together with the object under consideration, or what the object refers to by its shape, form or features in a figural or encoded form. For sake of simplicity, we include in the contextual relationships also the scholarly classification, which may have either to do with the role and cultural connections of the object in the past, or its kind of structure, substance or contents as it can be verified at present."""@en . - -edm:NonInformationResource a owl:Class ; - rdfs:label "Non-Information Resource"@en ; - owl:complementOf edm:InformationResource ; - skos:definition "All resources that are not information resources."@en . - -edm:WebResource a owl:Class ; - rdfs:label "Web Resource"@en ; - rdfs:subClassOf edm:InformationResource ; - skos:definition """Information Resources that have at least one Web Representation and at least -a URI."""@en . - -edm:isDerivativeOf a rdf:Property ; - rdfs:label "Is Derivative Of"@en ; - rdfs:subPropertyOf edm:isSimilarTo ; - owl:equivalentProperty frbroo:R2_is_derivative_of ; - skos:definition "This property captures a narrower notion of derivation than edm:isSimilarTo, in the sense that it relates a resource to another one, obtained by reworking, reducing, expanding, parts or the whole contents of the former, and possibly adding some minor parts. Versions have an even narrower meaning, in that it requires common identity between the related resources. Translations, summaries, abstractions etc. do not qualify as versions, but do qualify as derivatives."@en . - -edm:ProvidedCHO a owl:Class ; - rdfs:label "Provided CHO"@en ; - rdfs:subClassOf [ a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty [ a rdf:Property ; - owl:inverseOf edm:aggregatedCHO ] ] ; - skos:definition "This class comprises the Cultural Heritage objects that Europeana collects descriptions about."@en ; - skos:example "Mona Lisa, Winged Victory of Samothrace"@en ; - skos:note "This class has been mostly motivated by the need to assign a type to the “central node” in the EDM pattern, during the ingestion process, related to the XML expression of EDM at that stage. It was especially intended to fit the cases where edm:PhysicalThing cannot be used as the type of the resource standing for the real-world object (independently of any specific data contributor perspective)."@en ; - skos:scopeNote "Rationale: This class is the range of edm:aggregatedCHO. A resource of type ProvidedCHO can be the subject of statements using edm:isRelatedTo or any more specific property."@en . - diff --git a/metis-schema/src/main/resources/rdf_examples/edm.nt b/metis-schema/src/main/resources/rdf_examples/edm.nt deleted file mode 100644 index 454180579..000000000 --- a/metis-schema/src/main/resources/rdf_examples/edm.nt +++ /dev/null @@ -1,466 +0,0 @@ - . - . - "edm" . - "http://www.europeana.eu/schemas/edm/" . - "Europeana Data Model (EDM) vocabulary"@en . - "The Europeana Data Model (EDM) is aimed at being an integration medium for collecting, connecting and enriching the descriptions provided by Europeana data providers. The RDF vocabulary for http://www.europeana.eu/schemas/edm/ defines the elements introduced by EDM (as opposed to the ones EDM re-uses from other namespaces)."@en . - "2010-03-25"^^ . - "2013-05-20"^^ . - "5.2.4" . - "The present specification is based on the document \"Definition of the Europeana Data Model elements\", originally edited by Carlo Meghini. It is aligned with the version 5.2.4 of these EDM Definitions."@en . - . - . - "Antoine Isaac" . - _:node17eprp1n4x1343398 . -_:node17eprp1n4x1343398 . -_:node17eprp1n4x1343398 "Participants of Europeana Version 1.0 Work Package on Further Specification of Functionality and Interoperability aspects of Europeana (WP3)" . - . - . - "Nasos Drosopoulos" . - . - . - "Vassilis Tzouvaras" . - . - . - "Julia Iwanowa" . - _:node17eprp1n4x1343399 . -_:node17eprp1n4x1343399 . -_:node17eprp1n4x1343399 "Hugo Manguinhas" . - . - . - "Martin Doerr" . - . - . - "Europeana" . - . - . - . - . - "=======\nChanges between ontology file EDM version 5.2.4 (edm, was once EDM-v524-120820)\nand ontology file EDM version 5.2.3 (EDM-v523-120123)\n=======\n1. edm:isShownAt made a sub-property of edm:hasView\n2. added edm:begin and edm:end and their mappings to CRM\n3. added owl:Class declarations added for compatibility with some OWL-DL reasoners (feedback from Pedro Szekely, ISI)\n4. added \"of\" at the end of the label for edm:isNextInSequence\n5. added vocabulary metadata to follow Linked Open Vocabularies (http://lov.okfn.org/) and ADMS (https://joinup.ec.europa.eu/asset/adms/release/100) guidelines\n6. removed a domain axiom on edm:hasMet\n7. added edm:collectionName and edm:europeanaProxy\n8. removed version number from file name\n9. generalisation of Country, DataProvider and Provider\n10. updated CRM namespace and CRM class and property identifiers\n11. added FRBRoo mappings"@en . - "=======\nRemaining TODOs for ontology file EDM version 5.2.4\n=======\n- finish and check FRBRoo mappings according to the recommendations of the EDM-FRBRoo task force. Also include implicit mappings and mappings for elements outside the EDM namespace?\n- try to capture formal cardinality constraints resulting from \"Obligation and Occurrence\" documentation, which should be attached to non-EDM constructs (esp. ore:Aggregation)\n- continue adding documentation values (skos:scopeNote, skos:example, etc, according to 1.), starting from edm:InformationResource. Add all Europeana examples and rationale notes for non-EDM constructs\n- use specific EDM-doc properties for \"rationale\" and \"obligation and occurrence\". Use skos:definition for \"Europeana definition\", skos:example for \"Example\", skos:note for \"Europeana note\""@en . - . - "Agent"@en . - "This class comprises people, either individually or in groups, who have the\npotential to perform intentional actions for which they can be held responsible." . - "Leonardo da Vinci, the British Museum, W3C" . - . - . - "Rationale: This class is a domain of edm:wasPresentAt" . - . - "Europeana Aggregation" . - "The set of resources related to a single Cultural Heritage Object that\ncollectively represent that object in Europeana. Such set consists of: all\ndescriptions about the object that Europeana collects from (possibly different) content providers, including thumbnails and other forms of abstractions, as well as of the description of the object Europeana builds." . - . - . - _:node17eprp1n4x1343400 . -_:node17eprp1n4x1343400 . -_:node17eprp1n4x1343400 "1"^^ . -_:node17eprp1n4x1343400 . - "An instance of EuropeanaAggregation is created at ingestion time for each different Cultural Heritage Object recognized by Europeana. Such instance is associated to the Cultural Heritage Object that it is about, by the property edm:aggregatedCHO"@en . - "Obligation and Occurence: The relation between the Cultural Heritage Objects represented in Europeana and the instances of the class EuropeanaAggregation is one-to-one, in the data maintained by Europeana: every Cultural Heritage Object is represented by an instance of EuropeanaAggregation, and every instance of EuropeanaAggregation represent a Cultural Heritage Object."@en . - "The painting Mona Lisa is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en . - "The journal \"Le Temps\" is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en . - "The 56th issue of \"Le Temps\" is a (different) Cultural Heritage Object represented in Europeana by another EuropeanaAggregation instance"@en . - "Rationale: This class is used in Europeana to gather in a single conceptual unit all the information about a Cultural Heritage Object, necessary for all operations on these objects."@en . - . - "Europeana Object"@en . - "Any object that is the result of Europeana\u0092s activities"@en . - . - "Any instance of the class EuropeanaAggregation"@en . - "An annotation created by a user through the Europeana portal"@en . - "Any content created by the users through the service made available by Europeana for that purpose"@en . - "Rationale: This class is used to tag objects that are the result of activity of Europeana, and, as such, objects on which Europeana holds rights"@en . - . - "Event"@en . - "An event is a change \"of states in cultural, social or physical systems,\n regardless of scale, brought about by a series or group of coherent physical,\ncultural, technological or legal phenomena\" (E5 Event in CIDOC CRM) or a \"set of coherent phenomena or cultural manifestations bounded in time and space\" (E4 Period in CIDOC CRM)"@en . - . - . - . - . - . - _:node17eprp1n4x1343401 . -_:node17eprp1n4x1343401 . -_:node17eprp1n4x1343401 "1"^^ . -_:node17eprp1n4x1343401 . - "Events are identified either by the content provider or by Europeana enrichment at ingestion time"@en . - "the act of painting Mona Lisa"@en . - "the 2nd World War"@en . - "the change of custody of Mona Lisa"@en . - "Rationale:This class is a domain of edm:happenedAt and the domain of edm:occurredAt"@en . - . - "Information Resource"@en . - "An information resource is a resource whose essential characteristics can be conveyed in a single message. It can be associated with a URI, it can have a representation, for example: a text is an InformationResource."@en . - . - _:node17eprp1n4x1343402 . -_:node17eprp1n4x1343402 . -_:node17eprp1n4x1343402 _:node17eprp1n4x1343403 . -_:node17eprp1n4x1343403 . -_:node17eprp1n4x1343403 _:node17eprp1n4x1343404 . -_:node17eprp1n4x1343404 . -_:node17eprp1n4x1343404 _:node17eprp1n4x1343405 . -_:node17eprp1n4x1343405 . -_:node17eprp1n4x1343405 . - _:node17eprp1n4x1343406 . -_:node17eprp1n4x1343406 . -_:node17eprp1n4x1343406 _:node17eprp1n4x1343407 . -_:node17eprp1n4x1343407 . -_:node17eprp1n4x1343407 _:node17eprp1n4x1343408 . -_:node17eprp1n4x1343408 . -_:node17eprp1n4x1343408 _:node17eprp1n4x1343409 . -_:node17eprp1n4x1343409 . -_:node17eprp1n4x1343409 _:node17eprp1n4x1343410 . -_:node17eprp1n4x1343410 . -_:node17eprp1n4x1343410 . - . - "Non-Information Resource"@en . - "All resources that are not information resources."@en . - . - . - "Physical Thing"@en . - "A persistent physical item such as a painting, a building, a book or a stone.\nPersons are not items. This class represents Cultural Heritage Objects known to Europeana to be physical things (such as Mona Lisa) as well as all physical things Europeana refers to in the descriptions of Cultural Heritage Objects (such as the Rosetta Stone)."@en . - . - . - . - "Place"@en . - "An \"extent in space, in particular on the surface of the earth, in the pure sense of physics: independent from temporal phenomena and matter\" (CIDOC CRM)"@en . - . - . - . - . - . - . - . - "Provided CHO"@en . - "This class comprises the Cultural Heritage objects that Europeana collects descriptions about."@en . - _:node17eprp1n4x1343411 . -_:node17eprp1n4x1343411 . -_:node17eprp1n4x1343411 "1"^^ . -_:node17eprp1n4x1343411 _:node17eprp1n4x1343412 . -_:node17eprp1n4x1343412 . -_:node17eprp1n4x1343412 . - "This class has been mostly motivated by the need to assign a type to the \u0093central node\u0094 in the EDM pattern, during the ingestion process, related to the XML expression of EDM at that stage. It was especially intended to fit the cases where edm:PhysicalThing cannot be used as the type of the resource standing for the real-world object (independently of any specific data contributor perspective)."@en . - "Mona Lisa, Winged Victory of Samothrace"@en . - "Rationale: This class is the range of edm:aggregatedCHO. A resource of type ProvidedCHO can be the subject of statements using edm:isRelatedTo or any more specific property."@en . - . - "Time Span"@en . - "The class of \"abstract temporal extents, in the sense of Galilean physics,\n having a beginning, an end and a duration\" (CIDOC CRM)"@en . - . - . - . - . - . - . - "Web Resource"@en . - "Information Resources that have at least one Web Representation and at least\na URI."@en . - . - . - "Aggregated Cultural Heritage Object"@en . - "This property associates an ORE aggregation with the cultural heritage object(s) (CHO for short) it is about."@en . - . - . - . - . - . - . - "Begin"@en . - "This property denotes the start date of a period of time."@en . - . - _:node17eprp1n4x1343413 . -_:node17eprp1n4x1343413 . -_:node17eprp1n4x1343413 _:node17eprp1n4x1343414 . -_:node17eprp1n4x1343414 . -_:node17eprp1n4x1343414 _:node17eprp1n4x1343415 . -_:node17eprp1n4x1343415 . -_:node17eprp1n4x1343415 . - . - "Collection Name"@en . - "This property holds the collection identifier given to the dataset in Europeana."@en . - "The value of this property is provided by Europeana as part of the ingestion process."@en . - . - "Current Location"@en . - "The geographic location and/or name of the repository, building, site, or other entity whose boundaries presently include the resource."@en . - . - . - . - _:node17eprp1n4x1343416 . -_:node17eprp1n4x1343416 . -_:node17eprp1n4x1343416 _:node17eprp1n4x1343417 . -_:node17eprp1n4x1343417 . -_:node17eprp1n4x1343417 _:node17eprp1n4x1343419 . -_:node17eprp1n4x1343419 _:node17eprp1n4x1343418 . -_:node17eprp1n4x1343418 . -_:node17eprp1n4x1343418 _:node17eprp1n4x1343420 . -_:node17eprp1n4x1343420 . -_:node17eprp1n4x1343420 _:node17eprp1n4x1343422 . -_:node17eprp1n4x1343422 _:node17eprp1n4x1343421 . -_:node17eprp1n4x1343421 . -_:node17eprp1n4x1343421 . -_:node17eprp1n4x1343421 . -_:node17eprp1n4x1343422 . -_:node17eprp1n4x1343419 . - . - . - "End"@en . - "This property denotes the end date of a period of time."@en . - . - _:node17eprp1n4x1343423 . -_:node17eprp1n4x1343423 . -_:node17eprp1n4x1343423 _:node17eprp1n4x1343424 . -_:node17eprp1n4x1343424 . -_:node17eprp1n4x1343424 _:node17eprp1n4x1343425 . -_:node17eprp1n4x1343425 . -_:node17eprp1n4x1343425 . - . - "Europeana Proxy"@en . - "This property serves only as a flag to indicate that a proxy is a Europeana proxy (as opposed to a provider proxy). It is for internal use only."@en . - "By default, any proxy without this flag can be interpreted as having the value false and is a provider proxy."@en . - . - "Happened At"@en . - "This property associates an event with the place at which the event\nhappened."@en . - . - . - . - . - . - "Has Met"@en . - "edm:hasMet relates a resource with the objects or phenomena that have happened to or have happened together with the resource under consideration. We can abstractly think of history and the present as a series of \u0093meetings\u0094 between people and other things in space-time. Therefore we name this relationship as the things the object \u0093has met\u0094 in the course of its existence. These meetings are events in the proper sense, in which other people and things participate in any role."@en . - . - . - "Has Type"@en . - "This property relates a resource with the concepts it belongs to in a suitable\ntype system such as MIME or any thesaurus that captures categories of objects in a given field (e.g., the \u0093Objects\u0094 facet in Getty\u0092s Art and Architecture Thesaurus). It does not capture aboutness."@en . - . - . - _:node17eprp1n4x1343426 . -_:node17eprp1n4x1343426 . -_:node17eprp1n4x1343426 _:node17eprp1n4x1343427 . -_:node17eprp1n4x1343427 . -_:node17eprp1n4x1343427 _:node17eprp1n4x1343429 . -_:node17eprp1n4x1343429 _:node17eprp1n4x1343428 . -_:node17eprp1n4x1343428 . -_:node17eprp1n4x1343428 _:node17eprp1n4x1343430 . -_:node17eprp1n4x1343430 . -_:node17eprp1n4x1343430 _:node17eprp1n4x1343432 . -_:node17eprp1n4x1343432 _:node17eprp1n4x1343431 . -_:node17eprp1n4x1343431 . -_:node17eprp1n4x1343431 . -_:node17eprp1n4x1343431 . -_:node17eprp1n4x1343432 . -_:node17eprp1n4x1343429 . - . - . - "Has View"@en . - "This property relates a ORE aggregation about a CHO with a web resource\nproviding a view of that CHO. Examples of view are: a thumbnail, a textual\nabstract and a table of contents. The ORE aggregation may be a Europeana\nAggregation, in which case the view is an object owned by Europeana (i.e., an instance of edm:EuropeanaObject) or an aggregation contributed by a content provider. In order to capture both these cases, the domain of edm:hasView is ore:Aggregation and its range is edm:WebResource"@en . - . - . - . - . - "Incorporates"@en . - "This property captures the use of some resource to add value to another\nresource. Such resources may be nested, such as performing a theater play text, and then recording the performance, or creating an artful edition of a collection of poems or just aggregating various poems in an anthology. There may be no single part that contains ultimately the incorporated object, which may be dispersed in the presentation. Therefore, incorporated resources do in general not form proper parts. Incorporated resources are not part of the same resource, but are taken from other resources, and have an independent history. Therefore edm:incorporates is not a sub-property of dcterm:hasPart."@en . - . - . - . - "Is Annotation Of"@en . - "This property relates an annotation (a Europeana object) with the resource\nthat it annotates."@en . - . - . - . - _:node17eprp1n4x1343433 . -_:node17eprp1n4x1343433 . -_:node17eprp1n4x1343433 _:node17eprp1n4x1343434 . -_:node17eprp1n4x1343434 . -_:node17eprp1n4x1343434 _:node17eprp1n4x1343436 . -_:node17eprp1n4x1343436 _:node17eprp1n4x1343435 . -_:node17eprp1n4x1343435 . -_:node17eprp1n4x1343435 _:node17eprp1n4x1343437 . -_:node17eprp1n4x1343437 . -_:node17eprp1n4x1343437 _:node17eprp1n4x1343439 . -_:node17eprp1n4x1343439 _:node17eprp1n4x1343438 . -_:node17eprp1n4x1343438 . -_:node17eprp1n4x1343438 . -_:node17eprp1n4x1343438 . -_:node17eprp1n4x1343439 . -_:node17eprp1n4x1343436 . - . - "Is Derivative Of"@en . - "This property captures a narrower notion of derivation than edm:isSimilarTo, in the sense that it relates a resource to another one, obtained by reworking, reducing, expanding, parts or the whole contents of the former, and possibly adding some minor parts. Versions have an even narrower meaning, in that it requires common identity between the related resources. Translations, summaries, abstractions etc. do not qualify as versions, but do qualify as derivatives."@en . - . - . - . - "Is Next In Sequence Of"@en . - "edm:isNextInSequence relates two resources S and R that are ordered parts of the same resource A, and such that S comes immediately after R in the order created by their being parts of A."@en . - . - . - "Is Related To"@en . - "edm:isRelatedTo is the most general contextual property in EDM. Contextual\nproperties have typically to do either with the things that have happened to or together with the object under consideration, or what the object refers to by its shape, form or features in a figural or encoded form. For sake of simplicity, we include in the contextual relationships also the scholarly classification, which may have either to do with the role and cultural connections of the object in the past, or its kind of structure, substance or contents as it can be verified at present."@en . - _:node17eprp1n4x1343440 . -_:node17eprp1n4x1343440 . -_:node17eprp1n4x1343440 _:node17eprp1n4x1343441 . -_:node17eprp1n4x1343441 . -_:node17eprp1n4x1343441 _:node17eprp1n4x1343443 . -_:node17eprp1n4x1343443 _:node17eprp1n4x1343442 . -_:node17eprp1n4x1343442 . -_:node17eprp1n4x1343442 _:node17eprp1n4x1343444 . -_:node17eprp1n4x1343444 . -_:node17eprp1n4x1343444 _:node17eprp1n4x1343446 . -_:node17eprp1n4x1343446 _:node17eprp1n4x1343445 . -_:node17eprp1n4x1343445 . -_:node17eprp1n4x1343445 . -_:node17eprp1n4x1343445 . -_:node17eprp1n4x1343446 . -_:node17eprp1n4x1343443 . - . - "Is Representation Of"@en . - "This property associates an information resource to the resource (if any) that it represents"@en . - . - . - . - . - "Is Similar To"@en . - "The most generic derivation property, covering also the case of questionable derivation. Is Similar To asserts that parts of the contents of one resource exhibit common features with respect to ideas, shapes, structures, colors, words, plots, topics with the contents of the related resource. Those common features may be attributed to a common origin or influence (in particular for derivation), but also to more generic cultural or psychological factors."@en . - . - . - . - "Is Successor Of"@en . - "This property captures the relation between the continuation of a resource and that resource. This applies to a story, a serial, a journal etc. No content of the successor resource is identical or has a similar form with that of the precursor. The similarity is only in the context, subjects and figures of a plot. Successors typically form part of a common whole \u0096 such as a trilogy, a journal, etc."@en . - . - . - "Landing Page"@en . - "This property captures the relation between a Europeana aggregation representing a cultural heritage object and a (reference) Web resource giving access to that object. Europeana provides the value for this property."@en . - . - . - . - "Occured At"@en . - "This property associates an event to the smallest known time span that\noverlaps with the occurrence of that event"@en . - . - . - . - . - . - "Preview"@en . - "The URL of a thumbnail representing the digital object, generated by Europeana."@en . - . - . - . - "Realizes"@en . - "This property describes a relation between a physical thing and the information resource that is contained in it, visible at it or otherwise carried by it, if applicable."@en . - . - . - . - . - . - "Was Present At"@en . - "This property associates the people, things or information resources with an event at which they were present"@en . - . - . - _:node17eprp1n4x1343447 . -_:node17eprp1n4x1343447 . -_:node17eprp1n4x1343447 _:node17eprp1n4x1343448 . -_:node17eprp1n4x1343448 . -_:node17eprp1n4x1343448 _:node17eprp1n4x1343449 . -_:node17eprp1n4x1343449 . -_:node17eprp1n4x1343449 _:node17eprp1n4x1343450 . -_:node17eprp1n4x1343450 . -_:node17eprp1n4x1343450 . - . - . - "Country"@en . - . - . - "Data Provider"@en . - "The name or identifier of the organisation who contributes data indirectly to an aggregation service (e.g. Europeana)."@en . - . - . - . - . - "Is Shown At"@en . - "An unambiguous URL reference to the digital object on the provider\u0092s web site in its full information context."@en . - . - . - . - "Is Shown By"@en . - "An unambiguous URL reference to the digital object on the provider\u0092s web site in the best available resolution/quality."@en . - . - . - . - "Europeana Language"@en . - "The value for this element is added by the Data Ingestion Team as part of the ingestion process, based on the language of the data provider."@en . - . - "Object"@en . - "The URL of a thumbnail representing the digital object or, if there is no such\nthumbnail, the URL of the digital object in the best resolution available on the\nweb site of the data provider from which a thumbnail could be generated. This will often be the same URL as given in edm:isShownBy."@en . - . - . - . - "Provider"@en . - "The name or identifier of the organization who delivers data directly to an aggregation service (e.g. Europeana)"@en . - . - . - . - "Europeana Rights"@en . - "Information about copyright of the digital object as specified by isShownBy\nand isShownAt"@en . - . - "Europeana Type"@en . - "The Europeana material type of the resource"@en . - . - _:node17eprp1n4x1343451 . -_:node17eprp1n4x1343451 . -_:node17eprp1n4x1343451 _:node17eprp1n4x1343452 . -_:node17eprp1n4x1343452 "TEXT" . -_:node17eprp1n4x1343452 _:node17eprp1n4x1343453 . -_:node17eprp1n4x1343453 "IMAGE" . -_:node17eprp1n4x1343453 _:node17eprp1n4x1343454 . -_:node17eprp1n4x1343454 "SOUND" . -_:node17eprp1n4x1343454 _:node17eprp1n4x1343455 . -_:node17eprp1n4x1343455 "VIDEO" . -_:node17eprp1n4x1343455 _:node17eprp1n4x1343456 . -_:node17eprp1n4x1343456 "3D" . -_:node17eprp1n4x1343456 . - . - "UGC"@en . - "This element is used to identify user generated content (also called user created content). It should be applied to all digitised or born digital content contributed by the general public and collected by Europeana through a crowdsourcing initiative or project."@en . - _:node17eprp1n4x1343457 . -_:node17eprp1n4x1343457 . -_:node17eprp1n4x1343457 _:node17eprp1n4x1343458 . -_:node17eprp1n4x1343458 "TRUE" . -_:node17eprp1n4x1343458 . - . - "Unstored"@en . - "This is a container element which includes all relevant information that\notherwise cannot be mapped to another element in the ESE."@en . - . - "Europeana URI"@en . - "This is a tag created by a user through the Europeana interface."@en . - . - "User Tag"@en . - "This is a tag created by a user through the Europeana interface."@en . - . - . - "Europeana Year"@en . - "A point of time associated with an event in the life of the original analog or\nborn digital object."@en . - . - . - . - . - . - . - . - . - . - _:node17eprp1n4x1343459 . -_:node17eprp1n4x1343459 . -_:node17eprp1n4x1343459 . - _:node17eprp1n4x1343460 . -_:node17eprp1n4x1343460 . -_:node17eprp1n4x1343460 . - . - _:node17eprp1n4x1343461 . -_:node17eprp1n4x1343461 . -_:node17eprp1n4x1343461 . - . - . - . - . - . - . - . - . - . - . - . - . diff --git a/metis-schema/src/main/resources/rdf_examples/edm.owl b/metis-schema/src/main/resources/rdf_examples/edm.owl deleted file mode 100644 index f92eb32d8..000000000 --- a/metis-schema/src/main/resources/rdf_examples/edm.owl +++ /dev/null @@ -1,902 +0,0 @@ - - - - - - - - - - - - - - - - - - -]> - - - - - - edm - http://www.europeana.eu/schemas/edm/ - Europeana Data Model (EDM) vocabulary - The Europeana Data Model (EDM) is aimed at being an integration medium for collecting, connecting and enriching the descriptions provided by Europeana data providers. The RDF vocabulary for http://www.europeana.eu/schemas/edm/ defines the elements introduced by EDM (as opposed to the ones EDM re-uses from other namespaces). - 2010-03-25 - 2013-05-20 - 5.2.4 - The present specification is based on the document "Definition of the Europeana Data Model elements", originally edited by Carlo Meghini. It is aligned with the version 5.2.4 of these EDM Definitions. - - - - - - - - - - - - - -======= -Changes between ontology file EDM version 5.2.4 (edm, was once EDM-v524-120820) -and ontology file EDM version 5.2.3 (EDM-v523-120123) -======= -1. edm:isShownAt made a sub-property of edm:hasView -2. added edm:begin and edm:end and their mappings to CRM -3. added owl:Class declarations added for compatibility with some OWL-DL reasoners (feedback from Pedro Szekely, ISI) -4. added "of" at the end of the label for edm:isNextInSequence -5. added vocabulary metadata to follow Linked Open Vocabularies (http://lov.okfn.org/) and ADMS (https://joinup.ec.europa.eu/asset/adms/release/100) guidelines -6. removed a domain axiom on edm:hasMet -7. added edm:collectionName and edm:europeanaProxy -8. removed version number from file name -9. generalisation of Country, DataProvider and Provider -10. updated CRM namespace and CRM class and property identifiers -11. added FRBRoo mappings - - -======= -Remaining TODOs for ontology file EDM version 5.2.4 -======= -- finish and check FRBRoo mappings according to the recommendations of the EDM-FRBRoo task force. Also include implicit mappings and mappings for elements outside the EDM namespace? -- try to capture formal cardinality constraints resulting from "Obligation and Occurrence" documentation, which should be attached to non-EDM constructs (esp. ore:Aggregation) -- continue adding documentation values (skos:scopeNote, skos:example, etc, according to 1.), starting from edm:InformationResource. Add all Europeana examples and rationale notes for non-EDM constructs -- use specific EDM-doc properties for "rationale" and "obligation and occurrence". Use skos:definition for "Europeana definition", skos:example for "Example", skos:note for "Europeana note" - - - - - - - - - Agent - This class comprises people, either individually or in groups, who have the -potential to perform intentional actions for which they can be held responsible. - Leonardo da Vinci, the British Museum, W3C - - - Rationale: This class is a domain of edm:wasPresentAt - - - - - - - Europeana Aggregation - The set of resources related to a single Cultural Heritage Object that -collectively represent that object in Europeana. Such set consists of: all -descriptions about the object that Europeana collects from (possibly different) content providers, including thumbnails and other forms of abstractions, as well as of the description of the object Europeana builds. - - - - - 1 - - - - An instance of EuropeanaAggregation is created at ingestion time for each different Cultural Heritage Object recognized by Europeana. Such instance is associated to the Cultural Heritage Object that it is about, by the property edm:aggregatedCHO - Obligation and Occurence: The relation between the Cultural Heritage Objects represented in Europeana and the instances of the class EuropeanaAggregation is one-to-one, in the data maintained by Europeana: every Cultural Heritage Object is represented by an instance of EuropeanaAggregation, and every instance of EuropeanaAggregation represent a Cultural Heritage Object. - The painting Mona Lisa is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance - The journal "Le Temps" is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance - The 56th issue of "Le Temps" is a (different) Cultural Heritage Object represented in Europeana by another EuropeanaAggregation instance - Rationale: This class is used in Europeana to gather in a single conceptual unit all the information about a Cultural Heritage Object, necessary for all operations on these objects. - - - - - - Europeana Object - Any object that is the result of Europeana’s activities - - Any instance of the class EuropeanaAggregation - An annotation created by a user through the Europeana portal - Any content created by the users through the service made available by Europeana for that purpose - Rationale: This class is used to tag objects that are the result of activity of Europeana, and, as such, objects on which Europeana holds rights - - - - - - Event - An event is a change "of states in cultural, social or physical systems, - regardless of scale, brought about by a series or group of coherent physical, -cultural, technological or legal phenomena" (E5 Event in CIDOC CRM) or a "set of coherent phenomena or cultural manifestations bounded in time and space" (E4 Period in CIDOC CRM) - - - - - - - - - 1 - - - - Events are identified either by the content provider or by Europeana enrichment at ingestion time - the act of painting Mona Lisa - the 2nd World War - the change of custody of Mona Lisa - Rationale:This class is a domain of edm:happenedAt and the domain of edm:occurredAt - - - - - - Information Resource - An information resource is a resource whose essential characteristics can be conveyed in a single message. It can be associated with a URI, it can have a representation, for example: a text is an InformationResource. - - - - - - - - - - - - - - - - - - - - - - - - - - Non-Information Resource - All resources that are not information resources. - - - - - - - Physical Thing - A persistent physical item such as a painting, a building, a book or a stone. -Persons are not items. This class represents Cultural Heritage Objects known to Europeana to be physical things (such as Mona Lisa) as well as all physical things Europeana refers to in the descriptions of Cultural Heritage Objects (such as the Rosetta Stone). - - - - - - - - Place - An "extent in space, in particular on the surface of the earth, in the pure sense of physics: independent from temporal phenomena and matter" (CIDOC CRM) - - - - - - - - - - - - Provided CHO - This class comprises the Cultural Heritage objects that Europeana collects descriptions about. - - - 1 - - - - - - - - This class has been mostly motivated by the need to assign a type to the “central node” in the EDM pattern, during the ingestion process, related to the XML expression of EDM at that stage. It was especially intended to fit the cases where edm:PhysicalThing cannot be used as the type of the resource standing for the real-world object (independently of any specific data contributor perspective). - Mona Lisa, Winged Victory of Samothrace - Rationale: This class is the range of edm:aggregatedCHO. A resource of type ProvidedCHO can be the subject of statements using edm:isRelatedTo or any more specific property. - - - - - - Time Span - The class of "abstract temporal extents, in the sense of Galilean physics, - having a beginning, an end and a duration" (CIDOC CRM) - - - - - - - - - - - Web Resource - Information Resources that have at least one Web Representation and at least -a URI. - - - - - - - - - - Aggregated Cultural Heritage Object - This property associates an ORE aggregation with the cultural heritage object(s) (CHO for short) it is about. - - - - - - - - - - - Begin - This property denotes the start date of a period of time. - - - - - - - - - - - - - - - Collection Name - This property holds the collection identifier given to the dataset in Europeana. - The value of this property is provided by Europeana as part of the ingestion process. - - - - - - Current Location - The geographic location and/or name of the repository, building, site, or other entity whose boundaries presently include the resource. - - - - - - - - - - - - - - - - - - - - - - - - - - End - This property denotes the end date of a period of time. - - - - - - - - - - - - - - - Europeana Proxy - This property serves only as a flag to indicate that a proxy is a Europeana proxy (as opposed to a provider proxy). It is for internal use only. - By default, any proxy without this flag can be interpreted as having the value false and is a provider proxy. - - - - - - Happened At - This property associates an event with the place at which the event -happened. - - - - - - - - - - Has Met - edm:hasMet relates a resource with the objects or phenomena that have happened to or have happened together with the resource under consideration. We can abstractly think of history and the present as a series of “meetings” between people and other things in space-time. Therefore we name this relationship as the things the object “has met” in the course of its existence. These meetings are events in the proper sense, in which other people and things participate in any role. - - - - - - - Has Type - This property relates a resource with the concepts it belongs to in a suitable -type system such as MIME or any thesaurus that captures categories of objects in a given field (e.g., the “Objects” facet in Getty’s Art and Architecture Thesaurus). It does not capture aboutness. - - - - - - - - - - - - - - - - - - - - - - - - - Has View - This property relates a ORE aggregation about a CHO with a web resource -providing a view of that CHO. Examples of view are: a thumbnail, a textual -abstract and a table of contents. The ORE aggregation may be a Europeana -Aggregation, in which case the view is an object owned by Europeana (i.e., an instance of edm:EuropeanaObject) or an aggregation contributed by a content provider. In order to capture both these cases, the domain of edm:hasView is ore:Aggregation and its range is edm:WebResource - - - - - - - - - Incorporates - This property captures the use of some resource to add value to another -resource. Such resources may be nested, such as performing a theater play text, and then recording the performance, or creating an artful edition of a collection of poems or just aggregating various poems in an anthology. There may be no single part that contains ultimately the incorporated object, which may be dispersed in the presentation. Therefore, incorporated resources do in general not form proper parts. Incorporated resources are not part of the same resource, but are taken from other resources, and have an independent history. Therefore edm:incorporates is not a sub-property of dcterm:hasPart. - - - - - - - - Is Annotation Of - This property relates an annotation (a Europeana object) with the resource -that it annotates. - - - - - - - - - - - - - - - - - - - - - - - - - Is Derivative Of - This property captures a narrower notion of derivation than edm:isSimilarTo, in the sense that it relates a resource to another one, obtained by reworking, reducing, expanding, parts or the whole contents of the former, and possibly adding some minor parts. Versions have an even narrower meaning, in that it requires common identity between the related resources. Translations, summaries, abstractions etc. do not qualify as versions, but do qualify as derivatives. - - - - - - - - Is Next In Sequence Of - edm:isNextInSequence relates two resources S and R that are ordered parts of the same resource A, and such that S comes immediately after R in the order created by their being parts of A. - - - - - - - - Is Related To - edm:isRelatedTo is the most general contextual property in EDM. Contextual -properties have typically to do either with the things that have happened to or together with the object under consideration, or what the object refers to by its shape, form or features in a figural or encoded form. For sake of simplicity, we include in the contextual relationships also the scholarly classification, which may have either to do with the role and cultural connections of the object in the past, or its kind of structure, substance or contents as it can be verified at present. - - - - - - - - - - - - - - - - - - - - - - Is Representation Of - This property associates an information resource to the resource (if any) that it represents - - - - - - - - - Is Similar To - The most generic derivation property, covering also the case of questionable derivation. Is Similar To asserts that parts of the contents of one resource exhibit common features with respect to ideas, shapes, structures, colors, words, plots, topics with the contents of the related resource. Those common features may be attributed to a common origin or influence (in particular for derivation), but also to more generic cultural or psychological factors. - - - - - - - - Is Successor Of - This property captures the relation between the continuation of a resource and that resource. This applies to a story, a serial, a journal etc. No content of the successor resource is identical or has a similar form with that of the precursor. The similarity is only in the context, subjects and figures of a plot. Successors typically form part of a common whole – such as a trilogy, a journal, etc. - - - - - - - Landing Page - This property captures the relation between a Europeana aggregation representing a cultural heritage object and a (reference) Web resource giving access to that object. Europeana provides the value for this property. - - - - - - - - Occured At - This property associates an event to the smallest known time span that -overlaps with the occurrence of that event - - - - - - - - - - Preview - The URL of a thumbnail representing the digital object, generated by Europeana. - - - - - - - - Realizes - This property describes a relation between a physical thing and the information resource that is contained in it, visible at it or otherwise carried by it, if applicable. - - - - - - - - - - - Was Present At - This property associates the people, things or information resources with an event at which they were present - - - - - - - - - - - - - - - - - - - - - - - Country - - - - - - - Data Provider - The name or identifier of the organisation who contributes data indirectly to an aggregation service (e.g. Europeana). - - - - - - - - - Is Shown At - An unambiguous URL reference to the digital object on the provider’s web site in its full information context. - - - - - - - - Is Shown By - An unambiguous URL reference to the digital object on the provider’s web site in the best available resolution/quality. - - - - - - - - Europeana Language - The value for this element is added by the Data Ingestion Team as part of the ingestion process, based on the language of the data provider. - - - - - - Object - The URL of a thumbnail representing the digital object or, if there is no such -thumbnail, the URL of the digital object in the best resolution available on the -web site of the data provider from which a thumbnail could be generated. This will often be the same URL as given in edm:isShownBy. - - - - - - - - Provider - The name or identifier of the organization who delivers data directly to an aggregation service (e.g. Europeana) - - - - - - - - Europeana Rights - Information about copyright of the digital object as specified by isShownBy -and isShownAt - - - - - - Europeana Type - The Europeana material type of the resource - - - - - TEXT - - IMAGE - - SOUND - - VIDEO - - 3D - - - - - - - - - - - - - - UGC - This element is used to identify user generated content (also called user created content). It should be applied to all digitised or born digital content contributed by the general public and collected by Europeana through a crowdsourcing initiative or project. - - - - TRUE - - - - - - - - - - Unstored - This is a container element which includes all relevant information that -otherwise cannot be mapped to another element in the ESE. - - - - - - Europeana URI - This is a tag created by a user through the Europeana interface. - - - - - - User Tag - This is a tag created by a user through the Europeana interface. - - - - - - - Europeana Year - A point of time associated with an event in the life of the original analog or -born digital object. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/metis-schema/src/main/resources/rdf_examples/edm.ttl b/metis-schema/src/main/resources/rdf_examples/edm.ttl deleted file mode 100644 index d0d19c74f..000000000 --- a/metis-schema/src/main/resources/rdf_examples/edm.ttl +++ /dev/null @@ -1,671 +0,0 @@ -@prefix : . -@prefix xsd: . -@prefix rdf: . -@prefix rdfs: . -@prefix owl: . -@prefix skos: . -@prefix voaf: . -@prefix vann: . -@prefix adms: . -@prefix radion: . -@prefix dc: . -@prefix dcterms: . -@prefix ore: . -@prefix foaf: . -@prefix wgs84_pos: . -@prefix dcmitype: . -@prefix crm: . -@prefix frbr_core: . -@prefix frbroo: . -@prefix abc: . -@prefix DOLCE-Lite: . - - a owl:Ontology , voaf:Vocabulary ; - vann:preferredNamespacePrefix "edm" ; - vann:preferredNamespaceUri "http://www.europeana.eu/schemas/edm/" ; - dc:title "Europeana Data Model (EDM) vocabulary"@en ; - dc:description "The Europeana Data Model (EDM) is aimed at being an integration medium for collecting, connecting and enriching the descriptions provided by Europeana data providers. The RDF vocabulary for http://www.europeana.eu/schemas/edm/ defines the elements introduced by EDM (as opposed to the ones EDM re-uses from other namespaces)."@en ; - dcterms:issued "2010-03-25"^^xsd:date ; - dc:modified "2013-05-20"^^xsd:date ; - owl:versionInfo "5.2.4" ; - radion:versionNotes "The present specification is based on the document \"Definition of the Europeana Data Model elements\", originally edited by Carlo Meghini. It is aligned with the version 5.2.4 of these EDM Definitions."@en ; - dc:creator . - - a foaf:Person ; - foaf:name "Antoine Isaac" . - - dc:contributor _:node17eprp1n4x1343334 . - -_:node17eprp1n4x1343334 a foaf:Organization ; - foaf:name "Participants of Europeana Version 1.0 Work Package on Further Specification of Functionality and Interoperability aspects of Europeana (WP3)" . - - dc:contributor . - - a foaf:Person ; - foaf:name "Nasos Drosopoulos" . - - dc:contributor . - - a foaf:Person ; - foaf:name "Vassilis Tzouvaras" . - - dc:contributor . - - a foaf:Person ; - foaf:name "Julia Iwanowa" . - - dc:contributor _:node17eprp1n4x1343335 . - -_:node17eprp1n4x1343335 a foaf:Person ; - foaf:name "Hugo Manguinhas" . - - dc:contributor . - - a foaf:Person ; - foaf:name "Martin Doerr" . - - dc:publisher . - - a foaf:Organization ; - foaf:name "Europeana" . - - foaf:homepage ; - adms:relatedWebPage ; - vann:example , ; - vann:changes """======= -Changes between ontology file EDM version 5.2.4 (edm, was once EDM-v524-120820) -and ontology file EDM version 5.2.3 (EDM-v523-120123) -======= -1. edm:isShownAt made a sub-property of edm:hasView -2. added edm:begin and edm:end and their mappings to CRM -3. added owl:Class declarations added for compatibility with some OWL-DL reasoners (feedback from Pedro Szekely, ISI) -4. added \"of\" at the end of the label for edm:isNextInSequence -5. added vocabulary metadata to follow Linked Open Vocabularies (http://lov.okfn.org/) and ADMS (https://joinup.ec.europa.eu/asset/adms/release/100) guidelines -6. removed a domain axiom on edm:hasMet -7. added edm:collectionName and edm:europeanaProxy -8. removed version number from file name -9. generalisation of Country, DataProvider and Provider -10. updated CRM namespace and CRM class and property identifiers -11. added FRBRoo mappings"""@en ; - voaf:toDoList """======= -Remaining TODOs for ontology file EDM version 5.2.4 -======= -- finish and check FRBRoo mappings according to the recommendations of the EDM-FRBRoo task force. Also include implicit mappings and mappings for elements outside the EDM namespace? -- try to capture formal cardinality constraints resulting from \"Obligation and Occurrence\" documentation, which should be attached to non-EDM constructs (esp. ore:Aggregation) -- continue adding documentation values (skos:scopeNote, skos:example, etc, according to 1.), starting from edm:InformationResource. Add all Europeana examples and rationale notes for non-EDM constructs -- use specific EDM-doc properties for \"rationale\" and \"obligation and occurrence\". Use skos:definition for \"Europeana definition\", skos:example for \"Example\", skos:note for \"Europeana note\""""@en . - -:Agent a owl:Class ; - rdfs:label "Agent"@en ; - skos:definition """This class comprises people, either individually or in groups, who have the -potential to perform intentional actions for which they can be held responsible.""" ; - skos:example "Leonardo da Vinci, the British Museum, W3C" ; - rdfs:subClassOf :NonInformationResource ; - owl:equivalentClass ; - skos:scopeNote "Rationale: This class is a domain of edm:wasPresentAt" . - -:EuropeanaAggregation a owl:Class ; - rdfs:label "Europeana Aggregation" ; - skos:definition """The set of resources related to a single Cultural Heritage Object that -collectively represent that object in Europeana. Such set consists of: all -descriptions about the object that Europeana collects from (possibly different) content providers, including thumbnails and other forms of abstractions, as well as of the description of the object Europeana builds.""" ; - rdfs:subClassOf :EuropeanaObject , ore:Aggregation ; - owl:equivalentClass _:node17eprp1n4x1343336 . - -_:node17eprp1n4x1343336 a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty :aggregatedCHO . - -:EuropeanaAggregation skos:note "An instance of EuropeanaAggregation is created at ingestion time for each different Cultural Heritage Object recognized by Europeana. Such instance is associated to the Cultural Heritage Object that it is about, by the property edm:aggregatedCHO"@en ; - skos:scopeNote "Obligation and Occurence: The relation between the Cultural Heritage Objects represented in Europeana and the instances of the class EuropeanaAggregation is one-to-one, in the data maintained by Europeana: every Cultural Heritage Object is represented by an instance of EuropeanaAggregation, and every instance of EuropeanaAggregation represent a Cultural Heritage Object."@en ; - skos:example "The painting Mona Lisa is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en , "The journal \"Le Temps\" is a Cultural Heritage Object represented in Europeana by one EuropeanaAggregation instance"@en , "The 56th issue of \"Le Temps\" is a (different) Cultural Heritage Object represented in Europeana by another EuropeanaAggregation instance"@en ; - skos:scopeNote "Rationale: This class is used in Europeana to gather in a single conceptual unit all the information about a Cultural Heritage Object, necessary for all operations on these objects."@en . - -:EuropeanaObject a owl:Class ; - rdfs:label "Europeana Object"@en ; - skos:definition "Any object that is the result of EuropeanaÂ’s activities"@en ; - rdfs:subClassOf :WebResource ; - skos:example "Any instance of the class EuropeanaAggregation"@en , "An annotation created by a user through the Europeana portal"@en , "Any content created by the users through the service made available by Europeana for that purpose"@en ; - skos:scopeNote "Rationale: This class is used to tag objects that are the result of activity of Europeana, and, as such, objects on which Europeana holds rights"@en . - -:Event a owl:Class ; - rdfs:label "Event"@en ; - skos:definition """An event is a change \"of states in cultural, social or physical systems, - regardless of scale, brought about by a series or group of coherent physical, -cultural, technological or legal phenomena\" (E5 Event in CIDOC CRM) or a \"set of coherent phenomena or cultural manifestations bounded in time and space\" (E4 Period in CIDOC CRM)"""@en ; - rdfs:subClassOf :NonInformationResource ; - owl:equivalentClass , abc:Temporality , frbr_core:Event , frbroo:F8_Event ; - rdfs:subClassOf _:node17eprp1n4x1343337 . - -_:node17eprp1n4x1343337 a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty :happenedAt . - -:Event skos:note "Events are identified either by the content provider or by Europeana enrichment at ingestion time"@en ; - skos:example "the act of painting Mona Lisa"@en , "the 2nd World War"@en , "the change of custody of Mona Lisa"@en ; - skos:scopeNote "Rationale:This class is a domain of edm:happenedAt and the domain of edm:occurredAt"@en . - -:InformationResource a owl:Class ; - rdfs:label "Information Resource"@en ; - skos:definition "An information resource is a resource whose essential characteristics can be conveyed in a single message. It can be associated with a URI, it can have a representation, for example: a text is an InformationResource."@en ; - owl:equivalentClass , _:node17eprp1n4x1343338 . - -_:node17eprp1n4x1343338 a owl:Class ; - owl:unionOf _:node17eprp1n4x1343339 . - -_:node17eprp1n4x1343339 rdf:first frbr_core:Work ; - rdf:rest _:node17eprp1n4x1343340 . - -_:node17eprp1n4x1343340 rdf:first frbr_core:Expression ; - rdf:rest _:node17eprp1n4x1343341 . - -_:node17eprp1n4x1343341 rdf:first frbr_core:Manifestation ; - rdf:rest rdf:nil . - -:InformationResource owl:equivalentClass _:node17eprp1n4x1343342 . - -_:node17eprp1n4x1343342 a owl:Class ; - owl:unionOf _:node17eprp1n4x1343343 . - -_:node17eprp1n4x1343343 rdf:first frbroo:F1_Work ; - rdf:rest _:node17eprp1n4x1343344 . - -_:node17eprp1n4x1343344 rdf:first frbroo:F2_Expression ; - rdf:rest _:node17eprp1n4x1343345 . - -_:node17eprp1n4x1343345 rdf:first frbroo:F3_Manifestation_Product_Type ; - rdf:rest _:node17eprp1n4x1343346 . - -_:node17eprp1n4x1343346 rdf:first frbroo:F4_Manifestation_Singleton ; - rdf:rest rdf:nil . - -:NonInformationResource a owl:Class ; - rdfs:label "Non-Information Resource"@en ; - skos:definition "All resources that are not information resources."@en ; - owl:complementOf :InformationResource . - -:PhysicalThing a owl:Class ; - rdfs:label "Physical Thing"@en ; - skos:definition """A persistent physical item such as a painting, a building, a book or a stone. -Persons are not items. This class represents Cultural Heritage Objects known to Europeana to be physical things (such as Mona Lisa) as well as all physical things Europeana refers to in the descriptions of Cultural Heritage Objects (such as the Rosetta Stone)."""@en ; - rdfs:subClassOf :NonInformationResource ; - owl:equivalentClass . - -:Place a owl:Class ; - rdfs:label "Place"@en ; - skos:definition "An \"extent in space, in particular on the surface of the earth, in the pure sense of physics: independent from temporal phenomena and matter\" (CIDOC CRM)"@en ; - rdfs:subClassOf :NonInformationResource ; - owl:equivalentClass DOLCE-Lite:space-region , abc:Place , , frbr_core:Place , frbroo:F9_Place . - -:ProvidedCHO a owl:Class ; - rdfs:label "Provided CHO"@en ; - skos:definition "This class comprises the Cultural Heritage objects that Europeana collects descriptions about."@en ; - rdfs:subClassOf _:node17eprp1n4x1343347 . - -_:node17eprp1n4x1343347 a owl:Restriction ; - owl:cardinality "1"^^xsd:nonNegativeInteger ; - owl:onProperty _:node17eprp1n4x1343348 . - -_:node17eprp1n4x1343348 a rdf:Property ; - owl:inverseOf :aggregatedCHO . - -:ProvidedCHO skos:note "This class has been mostly motivated by the need to assign a type to the “central node” in the EDM pattern, during the ingestion process, related to the XML expression of EDM at that stage. It was especially intended to fit the cases where edm:PhysicalThing cannot be used as the type of the resource standing for the real-world object (independently of any specific data contributor perspective)."@en ; - skos:example "Mona Lisa, Winged Victory of Samothrace"@en ; - skos:scopeNote "Rationale: This class is the range of edm:aggregatedCHO. A resource of type ProvidedCHO can be the subject of statements using edm:isRelatedTo or any more specific property."@en . - -:TimeSpan a owl:Class ; - rdfs:label "Time Span"@en ; - skos:definition """The class of \"abstract temporal extents, in the sense of Galilean physics, - having a beginning, an end and a duration\" (CIDOC CRM)"""@en ; - rdfs:subClassOf :NonInformationResource , dcterms:PeriodOfTime ; - owl:equivalentClass abc:Time , DOLCE-Lite:time-interval , . - -:WebResource a owl:Class ; - rdfs:label "Web Resource"@en ; - skos:definition """Information Resources that have at least one Web Representation and at least -a URI."""@en ; - rdfs:subClassOf :InformationResource . - -:aggregatedCHO a owl:ObjectProperty ; - rdfs:label "Aggregated Cultural Heritage Object"@en ; - skos:definition "This property associates an ORE aggregation with the cultural heritage object(s) (CHO for short) it is about."@en ; - rdfs:subPropertyOf ore:aggregates , dc:subject , ; - rdfs:domain ore:Aggregation ; - rdfs:range :ProvidedCHO . - -:begin a owl:DatatypeProperty ; - rdfs:label "Begin"@en ; - skos:definition "This property denotes the start date of a period of time."@en ; - rdfs:subPropertyOf :isRelatedTo ; - rdfs:domain _:node17eprp1n4x1343349 . - -_:node17eprp1n4x1343349 a owl:Class ; - owl:unionOf _:node17eprp1n4x1343350 . - -_:node17eprp1n4x1343350 rdf:first :Agent ; - rdf:rest _:node17eprp1n4x1343351 . - -_:node17eprp1n4x1343351 rdf:first :TimeSpan ; - rdf:rest rdf:nil . - -:collectionName a owl:ObjectProperty ; - rdfs:label "Collection Name"@en ; - skos:definition "This property holds the collection identifier given to the dataset in Europeana."@en ; - skos:note "The value of this property is provided by Europeana as part of the ingestion process."@en . - -:currentLocation a owl:ObjectProperty ; - rdfs:label "Current Location"@en ; - skos:definition "The geographic location and/or name of the repository, building, site, or other entity whose boundaries presently include the resource."@en ; - rdfs:subPropertyOf dcterms:spatial ; - owl:equivalentProperty wgs84_pos:location , ; - rdfs:domain _:node17eprp1n4x1343352 . - -_:node17eprp1n4x1343352 a owl:Class ; - owl:unionOf _:node17eprp1n4x1343353 . - -_:node17eprp1n4x1343353 rdf:first :ProvidedCHO ; - rdf:rest _:node17eprp1n4x1343355 . - -_:node17eprp1n4x1343355 rdf:first _:node17eprp1n4x1343354 . - -_:node17eprp1n4x1343354 a owl:Class ; - owl:intersectionOf _:node17eprp1n4x1343356 . - -_:node17eprp1n4x1343356 rdf:first ore:Proxy ; - rdf:rest _:node17eprp1n4x1343358 . - -_:node17eprp1n4x1343358 rdf:first _:node17eprp1n4x1343357 . - -_:node17eprp1n4x1343357 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom :ProvidedCHO . - -_:node17eprp1n4x1343358 rdf:rest rdf:nil . - -_:node17eprp1n4x1343355 rdf:rest rdf:nil . - -:currentLocation rdfs:range :Place . - -:end a owl:DatatypeProperty ; - rdfs:label "End"@en ; - skos:definition "This property denotes the end date of a period of time."@en ; - rdfs:subPropertyOf :isRelatedTo ; - rdfs:domain _:node17eprp1n4x1343359 . - -_:node17eprp1n4x1343359 a owl:Class ; - owl:unionOf _:node17eprp1n4x1343360 . - -_:node17eprp1n4x1343360 rdf:first :Agent ; - rdf:rest _:node17eprp1n4x1343361 . - -_:node17eprp1n4x1343361 rdf:first :TimeSpan ; - rdf:rest rdf:nil . - -:europeanaProxy a owl:ObjectProperty ; - rdfs:label "Europeana Proxy"@en ; - skos:definition "This property serves only as a flag to indicate that a proxy is a Europeana proxy (as opposed to a provider proxy). It is for internal use only."@en ; - skos:note "By default, any proxy without this flag can be interpreted as having the value false and is a provider proxy."@en . - -:happenedAt a owl:ObjectProperty ; - rdfs:label "Happened At"@en ; - skos:definition """This property associates an event with the place at which the event -happened."""@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - rdfs:domain :Event ; - rdfs:range :Place . - -:hasMet a rdf:Property ; - rdfs:label "Has Met"@en ; - skos:definition "edm:hasMet relates a resource with the objects or phenomena that have happened to or have happened together with the resource under consideration. We can abstractly think of history and the present as a series of “meetings” between people and other things in space-time. Therefore we name this relationship as the things the object “has met” in the course of its existence. These meetings are events in the proper sense, in which other people and things participate in any role."@en ; - rdfs:subPropertyOf dc:relation . - -:hasType a owl:ObjectProperty ; - rdfs:label "Has Type"@en ; - skos:definition """This property relates a resource with the concepts it belongs to in a suitable -type system such as MIME or any thesaurus that captures categories of objects in a given field (e.g., the “Objects” facet in GettyÂ’s Art and Architecture Thesaurus). It does not capture aboutness."""@en ; - rdfs:subPropertyOf :isRelatedTo ; - owl:equivalentProperty ; - rdfs:domain _:node17eprp1n4x1343362 . - -_:node17eprp1n4x1343362 a owl:Class ; - owl:unionOf _:node17eprp1n4x1343363 . - -_:node17eprp1n4x1343363 rdf:first :ProvidedCHO ; - rdf:rest _:node17eprp1n4x1343365 . - -_:node17eprp1n4x1343365 rdf:first _:node17eprp1n4x1343364 . - -_:node17eprp1n4x1343364 a owl:Class ; - owl:intersectionOf _:node17eprp1n4x1343366 . - -_:node17eprp1n4x1343366 rdf:first ore:Proxy ; - rdf:rest _:node17eprp1n4x1343368 . - -_:node17eprp1n4x1343368 rdf:first _:node17eprp1n4x1343367 . - -_:node17eprp1n4x1343367 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom :ProvidedCHO . - -_:node17eprp1n4x1343368 rdf:rest rdf:nil . - -_:node17eprp1n4x1343365 rdf:rest rdf:nil . - -:hasType rdfs:range :NonInformationResource . - -:hasView a owl:ObjectProperty ; - rdfs:label "Has View"@en ; - skos:definition """This property relates a ORE aggregation about a CHO with a web resource -providing a view of that CHO. Examples of view are: a thumbnail, a textual -abstract and a table of contents. The ORE aggregation may be a Europeana -Aggregation, in which case the view is an object owned by Europeana (i.e., an instance of edm:EuropeanaObject) or an aggregation contributed by a content provider. In order to capture both these cases, the domain of edm:hasView is ore:Aggregation and its range is edm:WebResource"""@en ; - rdfs:subPropertyOf ore:aggregates ; - rdfs:domain ore:Aggregation ; - rdfs:range :WebResource . - -:incorporates a owl:ObjectProperty ; - rdfs:label "Incorporates"@en ; - skos:definition """This property captures the use of some resource to add value to another -resource. Such resources may be nested, such as performing a theater play text, and then recording the performance, or creating an artful edition of a collection of poems or just aggregating various poems in an anthology. There may be no single part that contains ultimately the incorporated object, which may be dispersed in the presentation. Therefore, incorporated resources do in general not form proper parts. Incorporated resources are not part of the same resource, but are taken from other resources, and have an independent history. Therefore edm:incorporates is not a sub-property of dcterm:hasPart."""@en ; - rdfs:subPropertyOf :isSimilarTo ; - owl:equivalentProperty frbroo:R14_incorporates . - -:isAnnotationOf a owl:ObjectProperty ; - rdfs:label "Is Annotation Of"@en ; - skos:definition """This property relates an annotation (a Europeana object) with the resource -that it annotates."""@en ; - rdfs:subPropertyOf dc:subject , ; - rdfs:domain :EuropeanaObject ; - rdfs:range _:node17eprp1n4x1343369 . - -_:node17eprp1n4x1343369 a owl:Class ; - owl:unionOf _:node17eprp1n4x1343370 . - -_:node17eprp1n4x1343370 rdf:first :ProvidedCHO ; - rdf:rest _:node17eprp1n4x1343372 . - -_:node17eprp1n4x1343372 rdf:first _:node17eprp1n4x1343371 . - -_:node17eprp1n4x1343371 a owl:Class ; - owl:intersectionOf _:node17eprp1n4x1343373 . - -_:node17eprp1n4x1343373 rdf:first ore:Proxy ; - rdf:rest _:node17eprp1n4x1343375 . - -_:node17eprp1n4x1343375 rdf:first _:node17eprp1n4x1343374 . - -_:node17eprp1n4x1343374 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom :ProvidedCHO . - -_:node17eprp1n4x1343375 rdf:rest rdf:nil . - -_:node17eprp1n4x1343372 rdf:rest rdf:nil . - -:isDerivativeOf a rdf:Property ; - rdfs:label "Is Derivative Of"@en ; - skos:definition "This property captures a narrower notion of derivation than edm:isSimilarTo, in the sense that it relates a resource to another one, obtained by reworking, reducing, expanding, parts or the whole contents of the former, and possibly adding some minor parts. Versions have an even narrower meaning, in that it requires common identity between the related resources. Translations, summaries, abstractions etc. do not qualify as versions, but do qualify as derivatives."@en ; - rdfs:subPropertyOf :isSimilarTo ; - owl:equivalentProperty frbroo:R2_is_derivative_of . - -:isNextInSequence a owl:ObjectProperty ; - rdfs:label "Is Next In Sequence Of"@en ; - skos:definition "edm:isNextInSequence relates two resources S and R that are ordered parts of the same resource A, and such that S comes immediately after R in the order created by their being parts of A."@en ; - rdfs:subPropertyOf dc:relation . - -:isRelatedTo a rdf:Property ; - rdfs:label "Is Related To"@en ; - skos:definition """edm:isRelatedTo is the most general contextual property in EDM. Contextual -properties have typically to do either with the things that have happened to or together with the object under consideration, or what the object refers to by its shape, form or features in a figural or encoded form. For sake of simplicity, we include in the contextual relationships also the scholarly classification, which may have either to do with the role and cultural connections of the object in the past, or its kind of structure, substance or contents as it can be verified at present."""@en ; - rdfs:domain _:node17eprp1n4x1343376 . - -_:node17eprp1n4x1343376 a owl:Class ; - owl:unionOf _:node17eprp1n4x1343377 . - -_:node17eprp1n4x1343377 rdf:first :ProvidedCHO ; - rdf:rest _:node17eprp1n4x1343379 . - -_:node17eprp1n4x1343379 rdf:first _:node17eprp1n4x1343378 . - -_:node17eprp1n4x1343378 a owl:Class ; - owl:intersectionOf _:node17eprp1n4x1343380 . - -_:node17eprp1n4x1343380 rdf:first ore:Proxy ; - rdf:rest _:node17eprp1n4x1343382 . - -_:node17eprp1n4x1343382 rdf:first _:node17eprp1n4x1343381 . - -_:node17eprp1n4x1343381 a owl:Restriction ; - owl:onProperty ore:proxyFor ; - owl:someValuesFrom :ProvidedCHO . - -_:node17eprp1n4x1343382 rdf:rest rdf:nil . - -_:node17eprp1n4x1343379 rdf:rest rdf:nil . - -:isRepresentationOf a owl:ObjectProperty ; - rdfs:label "Is Representation Of"@en ; - skos:definition "This property associates an information resource to the resource (if any) that it represents"@en ; - rdfs:subPropertyOf dc:subject ; - rdfs:domain :InformationResource ; - rdfs:subPropertyOf . - -:isSimilarTo a rdf:Property ; - rdfs:label "Is Similar To"@en ; - skos:definition "The most generic derivation property, covering also the case of questionable derivation. Is Similar To asserts that parts of the contents of one resource exhibit common features with respect to ideas, shapes, structures, colors, words, plots, topics with the contents of the related resource. Those common features may be attributed to a common origin or influence (in particular for derivation), but also to more generic cultural or psychological factors."@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty . - -:isSuccessorOf a owl:ObjectProperty ; - rdfs:label "Is Successor Of"@en ; - skos:definition "This property captures the relation between the continuation of a resource and that resource. This applies to a story, a serial, a journal etc. No content of the successor resource is identical or has a similar form with that of the precursor. The similarity is only in the context, subjects and figures of a plot. Successors typically form part of a common whole – such as a trilogy, a journal, etc."@en ; - rdfs:subPropertyOf :isSimilarTo . - -:landingPage a owl:ObjectProperty ; - rdfs:label "Landing Page"@en ; - skos:definition "This property captures the relation between a Europeana aggregation representing a cultural heritage object and a (reference) Web resource giving access to that object. Europeana provides the value for this property."@en ; - rdfs:subPropertyOf ore:aggregates ; - rdfs:range :WebResource . - -:occurredAt a owl:ObjectProperty ; - rdfs:label "Occured At"@en ; - skos:definition """This property associates an event to the smallest known time span that -overlaps with the occurrence of that event"""@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - rdfs:domain :Event ; - rdfs:range :TimeSpan . - -:preview a owl:ObjectProperty ; - rdfs:label "Preview"@en ; - skos:definition "The URL of a thumbnail representing the digital object, generated by Europeana."@en ; - rdfs:subPropertyOf :hasView ; - rdfs:range :WebResource . - -:realizes a owl:ObjectProperty ; - rdfs:label "Realizes"@en ; - skos:definition "This property describes a relation between a physical thing and the information resource that is contained in it, visible at it or otherwise carried by it, if applicable."@en ; - rdfs:subPropertyOf :isRelatedTo ; - owl:equivalentProperty ; - rdfs:domain :PhysicalThing ; - rdfs:range :InformationResource . - -:wasPresentAt a owl:ObjectProperty ; - rdfs:label "Was Present At"@en ; - skos:definition "This property associates the people, things or information resources with an event at which they were present"@en ; - rdfs:subPropertyOf dc:relation ; - owl:equivalentProperty ; - rdfs:domain _:node17eprp1n4x1343383 . - -_:node17eprp1n4x1343383 a owl:Class ; - owl:unionOf _:node17eprp1n4x1343384 . - -_:node17eprp1n4x1343384 rdf:first :Agent ; - rdf:rest _:node17eprp1n4x1343385 . - -_:node17eprp1n4x1343385 rdf:first :InformationResource ; - rdf:rest _:node17eprp1n4x1343386 . - -_:node17eprp1n4x1343386 rdf:first :PhysicalThing ; - rdf:rest rdf:nil . - -:wasPresentAt rdfs:range :Event . - -:country a rdf:Property ; - rdfs:label "Country"@en ; - rdfs:subPropertyOf . - -:dataProvider a rdf:Property ; - rdfs:label "Data Provider"@en ; - skos:definition "The name or identifier of the organisation who contributes data indirectly to an aggregation service (e.g. Europeana)."@en ; - rdfs:subPropertyOf dcterms:provenance ; - rdfs:domain ore:Aggregation ; - rdfs:range :Agent . - -:isShownAt a owl:ObjectProperty ; - rdfs:label "Is Shown At"@en ; - skos:definition "An unambiguous URL reference to the digital object on the providerÂ’s web site in its full information context."@en ; - rdfs:subPropertyOf :hasView ; - rdfs:range :WebResource . - -:isShownBy a owl:ObjectProperty ; - rdfs:label "Is Shown By"@en ; - skos:definition "An unambiguous URL reference to the digital object on the providerÂ’s web site in the best available resolution/quality."@en ; - rdfs:subPropertyOf :hasView ; - rdfs:range :WebResource . - -:language a rdf:Property ; - rdfs:label "Europeana Language"@en ; - skos:definition "The value for this element is added by the Data Ingestion Team as part of the ingestion process, based on the language of the data provider."@en . - -:object a owl:ObjectProperty ; - rdfs:label "Object"@en ; - skos:definition """The URL of a thumbnail representing the digital object or, if there is no such -thumbnail, the URL of the digital object in the best resolution available on the -web site of the data provider from which a thumbnail could be generated. This will often be the same URL as given in edm:isShownBy."""@en ; - rdfs:subPropertyOf :hasView ; - rdfs:range :WebResource . - -:provider a rdf:Property ; - rdfs:label "Provider"@en ; - skos:definition "The name or identifier of the organization who delivers data directly to an aggregation service (e.g. Europeana)"@en ; - rdfs:subPropertyOf :hasMet ; - rdfs:range :Agent . - -:rights a owl:ObjectProperty ; - rdfs:label "Europeana Rights"@en ; - skos:definition """Information about copyright of the digital object as specified by isShownBy -and isShownAt"""@en . - -:type a owl:DatatypeProperty ; - rdfs:label "Europeana Type"@en ; - skos:definition "The Europeana material type of the resource"@en ; - rdfs:subPropertyOf dc:type ; - rdfs:range _:node17eprp1n4x1343387 . - -_:node17eprp1n4x1343387 a rdfs:Datatype ; - owl:oneOf _:node17eprp1n4x1343388 . - -_:node17eprp1n4x1343388 rdf:first "TEXT" ; - rdf:rest _:node17eprp1n4x1343389 . - -_:node17eprp1n4x1343389 rdf:first "IMAGE" ; - rdf:rest _:node17eprp1n4x1343390 . - -_:node17eprp1n4x1343390 rdf:first "SOUND" ; - rdf:rest _:node17eprp1n4x1343391 . - -_:node17eprp1n4x1343391 rdf:first "VIDEO" ; - rdf:rest _:node17eprp1n4x1343392 . - -_:node17eprp1n4x1343392 rdf:first "3D" ; - rdf:rest rdf:nil . - -:ugc a owl:DatatypeProperty ; - rdfs:label "UGC"@en ; - skos:definition "This element is used to identify user generated content (also called user created content). It should be applied to all digitised or born digital content contributed by the general public and collected by Europeana through a crowdsourcing initiative or project."@en ; - rdfs:range _:node17eprp1n4x1343393 . - -_:node17eprp1n4x1343393 a rdfs:Datatype ; - owl:oneOf _:node17eprp1n4x1343394 . - -_:node17eprp1n4x1343394 rdf:first "TRUE" ; - rdf:rest rdf:nil . - -:unstored a rdf:Property ; - rdfs:label "Unstored"@en ; - skos:definition """This is a container element which includes all relevant information that -otherwise cannot be mapped to another element in the ESE."""@en . - -:uri a owl:ObjectProperty ; - rdfs:label "Europeana URI"@en ; - skos:definition "This is a tag created by a user through the Europeana interface."@en . - -:userTag a rdf:Property ; - rdfs:label "User Tag"@en ; - skos:definition "This is a tag created by a user through the Europeana interface."@en ; - rdfs:subPropertyOf dc:description . - -:year a rdf:Property ; - rdfs:label "Europeana Year"@en ; - skos:definition """A point of time associated with an event in the life of the original analog or -born digital object."""@en ; - rdfs:subPropertyOf dcterms:temporal . - -skos:Concept rdfs:subClassOf :NonInformationResource . - - rdfs:subPropertyOf :begin . - - rdfs:subPropertyOf :end . - -dc:contributor rdfs:subPropertyOf :hasMet . - -dc:coverage rdfs:subPropertyOf :hasMet . - -dc:creator rdfs:subPropertyOf :hasMet . - -dc:date rdfs:subPropertyOf :hasMet . - -dc:format rdfs:subPropertyOf :hasType . - -dcterms:hasFormat rdfs:subPropertyOf _:node17eprp1n4x1343395 . - -_:node17eprp1n4x1343395 a rdf:Property ; - owl:inverseOf :isDerivativeOf . - -dcterms:hasVersion rdfs:subPropertyOf _:node17eprp1n4x1343396 . - -_:node17eprp1n4x1343396 a rdf:Property ; - owl:inverseOf :isDerivativeOf . - -dcterms:isFormatOf rdfs:subPropertyOf :isDerivativeOf . - -dcterms:isReplacedBy rdfs:subPropertyOf _:node17eprp1n4x1343397 . - -_:node17eprp1n4x1343397 a rdf:Property ; - owl:inverseOf :isDerivativeOf . - -dcterms:isVersionOf rdfs:subPropertyOf :isDerivativeOf . - -dc:language rdfs:subPropertyOf :hasType . - -dc:publisher rdfs:subPropertyOf :hasMet . - -dc:relation rdfs:subPropertyOf :isRelatedTo . - -dcterms:replaces rdfs:subPropertyOf :isDerivativeOf . - -dc:source rdfs:subPropertyOf :isDerivativeOf . - -dc:subject rdfs:subPropertyOf :isRelatedTo . - -dcterms:tableOfContents rdfs:subPropertyOf :hasView . - -dc:type rdfs:subPropertyOf :hasType . - -skos:Concept a owl:Class . - -ore:Aggregation a owl:Class . - -ore:Proxy a owl:Class . diff --git a/metis-schema/src/main/resources/schema_xsds/ADMS.xsd b/metis-schema/src/main/resources/schema_xsds/ADMS.xsd deleted file mode 100644 index 1434be6d4..000000000 --- a/metis-schema/src/main/resources/schema_xsds/ADMS.xsd +++ /dev/null @@ -1,15 +0,0 @@ - - - - - Europeana representation of Asset Description Metadata Schema (ADMS) elements - - - - - - - - \ No newline at end of file diff --git a/metis-schema/src/main/resources/schema_xsds/CC.xsd b/metis-schema/src/main/resources/schema_xsds/CC.xsd deleted file mode 100644 index 8ded5f191..000000000 --- a/metis-schema/src/main/resources/schema_xsds/CC.xsd +++ /dev/null @@ -1,27 +0,0 @@ - - - - - EDM First Implementation Schema: CC - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/metis-schema/src/main/resources/schema_xsds/CONTEXTS.xsd b/metis-schema/src/main/resources/schema_xsds/CONTEXTS.xsd deleted file mode 100644 index 53e738750..000000000 --- a/metis-schema/src/main/resources/schema_xsds/CONTEXTS.xsd +++ /dev/null @@ -1,183 +0,0 @@ - - - - - - EDM First Implementation Schema: Contextual elements (vocabulary terms) - - - - - - - - - - - - - - - - - - - This class comprises people, either individually or in groups, who have - the potential to perform intentional actions for which they can be held responsible. - Example:Leonardo da Vinci, the British Museum, W3C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - An "extent in space, in particular on the surface of the earth, in the - pure sense of physics: independent from temporal phenomena and matter" (CIDOC CRM) - Example:the region of space occupied by Rome today, the region of space occupied by - the United Kingdom today, the region of space occupied by the Republic of Crimea in - 1945 - - - - - - - - - - - - - - - - - - - - - - - The class of "abstract temporal extents, in the sense of Galilean - physics, having a beginning, an end and a duration" (CIDOC CRM) Example:2001-12-31, - 01.01.01 - 02.02.02, 1503 - 1506 (the time span of the creation of Mona Lisa) - - - - - - - - - - - - - - - - - - - - - - Base class for WebResource implementations - - - - The element dcterms:isPartOf should not have a literal value in the edm:WebResource context - with id: . Use an rdf:resource instead. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/metis-schema/src/main/resources/schema_xsds/DC.xsd b/metis-schema/src/main/resources/schema_xsds/DC.xsd deleted file mode 100644 index 580f31ca0..000000000 --- a/metis-schema/src/main/resources/schema_xsds/DC.xsd +++ /dev/null @@ -1,169 +0,0 @@ - - - - - EDM First Implementation Schema: DC - - - - - - - - - Information about copyright of the original object. Example: - Creative Commons Attribution 3.0 License - Type: String - - - - - - - - An entity responsible for making contributions to the resource. Example: - Maria Callas - Type: String - - - - - - - - The spatial or temporal topic of the resource, the spatial applicability of the resource, or the jurisdiction under which the resource is relevant. This may be a named place, a - location, a spatial coordinate, a period, date, date range or a named administrative entity. Example: - 1995-1996 - Boston, MA - Type: String - - - - - - - - An entity primarily responsible for making the resource. This may be a person, organisation or a service. Example: - Shakespeare, William - Type: String - - - - - - - - A point or period of time associated with an event in the lifecycle of the resource. Example: - 17th century - (For example the date when an object was repaired) Type: String - - - - - - - - An account of the resource. Example: - - Illustrated guide to airport markings and lighting signals, with particular reference to SMGCS (Surface Movement Guidance and Control System) for airports with low - visibility - conditions. - - Type: Stringdescription - - - - - - - - - identifier - - - - - - - - The file format, physical medium or dimensions of the resource. Example: - image/jpeg - Type: String - - - - - - - - A language of the resource. Example: - it - Type: String - - - - - - - - An entity responsible for making the resource available. Examples of a publisher include a person, an organisation and a service. Example: - Oxford University Press - Type: String - - - - - - - - A related resource. The recommended best practice is to identify the resource using a formal identification scheme. Example: - maps.crace.1/33 - (This is the shelf mark for a map held in the British Library's Crace Collection). Type: String - - - - - - - - A related resource from which the described resource is derived in whole or in part. Example: - Security Magazine pp 3-12 - BAM portal - Type: String - - - - - - - - The topic of the resource. Example: - submarine - Type: String - - - - - - - - A name given to the resource. Typically, a Title will be a name by which the resource is formally known. Example: - Taal vitaal - Type: String - - - - - - - - The nature or genre of the resource. Type includes terms describing general categories, functions, genres, or aggregation levels for content. Example: - painting - photograph - coin - Type: String - - - - \ No newline at end of file diff --git a/metis-schema/src/main/resources/schema_xsds/DCAT.xsd b/metis-schema/src/main/resources/schema_xsds/DCAT.xsd deleted file mode 100644 index a1c7587af..000000000 --- a/metis-schema/src/main/resources/schema_xsds/DCAT.xsd +++ /dev/null @@ -1,51 +0,0 @@ - - - - - Europeana representation of Data Catalog Vocabulary (DCAT) elements - - - - - - - - - - - - The EDM representation of Dataset - consisting of records ingested in Europeana - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/metis-schema/src/main/resources/schema_xsds/DCTERMS.xsd b/metis-schema/src/main/resources/schema_xsds/DCTERMS.xsd deleted file mode 100644 index 788ee6d0f..000000000 --- a/metis-schema/src/main/resources/schema_xsds/DCTERMS.xsd +++ /dev/null @@ -1,239 +0,0 @@ - - - - - EDM First Implementation Schema: DC Terms - - - - - - - - - - - A related resource that references, cites, or otherwise points to the described resource. Example: - Till, Nicholas (1994) Mozart and the Enlightenment: Truth, Virtue and Beauty in Mozart's Operas, W. W. Norton & - Company - Type: String - - - - - - - - Temporal characteristics of the resource Example: - Roman - Type: String - - - - - - - - A list of subunits of the resource. Example: - Chapter 1. Introduction, Chapter 2. History - Type: String - - - - - - - - Spatial characteristics of the resource. Example: - Portugal - Type: String - - - - - - - - A related resource that is required by the described resource to support its function, delivery or coherence. Example: - http://ads.ahds.ac.uk/project/userinfo/css/oldbrowsers.css - where the resource described is a HTML file at http://ads.ahds.ac.uk/project/userinfo/digitalTextArchiving.html Type: String - - - - - - - - A related resource that is supplanted, displaced, or superseded by the described resource. Example: - http://dublincore.org/about/2006/01/01/bylaws/ - where the resource described is a newer version (http://dublincore.org/about/2009/01/05/bylaws/) Type: String - - - - - - - - A related resource that is referenced, cited, or otherwise pointed to by the described resource Example: - Honderd jaar Noorse schilderkunst - Type: String - - - - - - - - A statement of any changes in ownership and custody of the resource since its creation that are significant for its authenticity, - integrity and interpretation. This may include a - description of any changes successive custodians made to the resource. Example: - Donated by The National Library in 1965 - Type: String - - - - - - - - The material or physical carrier of the resource. Example: - metal - Type: String - - - - - - - - A related resource of which the described resource is a version, edition, or adaptation. Changes in version imply substantive changes in - content rather than differences in format - Example: - ESE Version 0.5 - Type: String - - - - - - - - Date of formal issuance (e.g., publication) of the resource. Example: - 1993 - Type: String - - - - - - - - A related resource that requires the described resource to support its function, delivery or coherence. Example: - http://www.myslides.com/myslideshow.ppt - where the image being described is required for an online slideshow. Type: String - - - - - - - - A related resource that supplants, displaces, or supersedes the described resource. Example: - http://dublincore.org/about/2009/01/05/bylaws/ - where the resource described is an older version (http://dublincore.org/about/2006/01/01/bylaws/) Type: String - - - - - - - - A related resource that is substantially the same as the described resource, but in another format. Example: - Europeana_logo.tiff - where the resource being described is a png image file. Type: String - - - - - - - - A related resource that is a version, edition, or adaptation of the described resource. Changes in version imply substantive changes in - content rather than differences in format. - Example: - The Sorcerer's Apprentice (translation by Edwin Zeydel, 1955) - . In this example the 1955 translation is a version of the described resource. Type: String - - - - - - - - A related resource that is substantially the same as the pre-existing described resource, but in another format. Example: - http://upload.wikimedia.org/wikipedia/en/f/f3/Europeana _logo.png - where the resource being described is a tiff image file. Type: String - - - - - - - - The size or duration of the resource. Example: - 13 cm - (the width of an original object). - 34 minutes - (the length of an audio file). Type: String - - - - - - - - Date of creation of the resource Example: - 1564 - Iron Age - Type: String - - - - - - - - Modified Date - - - - - - - - An established standard to which the described resource conforms. Example: - W3C WCAG 2.0 - (for an HTML document that conforms to web content accessibility guidelines). Type: String - - - - - - - - An alternative name for the resource. This can be any form of the title that is used as a substitute or an alternative to the formal - title of the resource including abbreviations or - translations of the title. Example: - Ocho semanas - (When - Eight weeks - ) Type: String - - - - - - - - \ No newline at end of file diff --git a/metis-schema/src/main/resources/schema_xsds/DOAP.xsd b/metis-schema/src/main/resources/schema_xsds/DOAP.xsd deleted file mode 100644 index c3f655284..000000000 --- a/metis-schema/src/main/resources/schema_xsds/DOAP.xsd +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - A specification that a project implements. Could be a standard, API or legally defined level of conformance. - In IIIF doap:implements refers to the the general protocol without specific version. - <doap:implements rdf:resource=”http://iiif.io/api/image” /> - - - - - \ No newline at end of file diff --git a/metis-schema/src/main/resources/schema_xsds/DQV.xsd b/metis-schema/src/main/resources/schema_xsds/DQV.xsd deleted file mode 100644 index cbd4676aa..000000000 --- a/metis-schema/src/main/resources/schema_xsds/DQV.xsd +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - - - The identifier of the Quality annotation. Example: - <dqv:hasQualityAnnotation rdf:resource="#metadataTier"/> - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/metis-schema/src/main/resources/schema_xsds/EBUCORE.xsd b/metis-schema/src/main/resources/schema_xsds/EBUCORE.xsd deleted file mode 100644 index 298181b02..000000000 --- a/metis-schema/src/main/resources/schema_xsds/EBUCORE.xsd +++ /dev/null @@ -1,108 +0,0 @@ - - - - - - EDM First Implementation Schema: EBUCORE - - - - - - The size of a Media Resource expressed in bytes. Example: - 39508 - - - - - - - The main MIME types as defined by IANA: e.g. audio, video, text, application, or a container MIME type. Example: - video/mp4 - - - - - - - - The duration of a track or a signal expressed in ms. Example: - 270000 - - - - - - - The width of e.g. a video frame typically expressed as a number of pixels. Example: - 1024 - - - - - - - The height of e.g. a video frame typically expressed as a number of pixels. Example: - 768 - - - - - - - The size of an audio sample in bits. Also called bit depth. Example: - 16 - - - - - - - The frequency at which an audio is sampled per second. Also called sampling rate. Example: - 44100 - - - - - - - To provide the bitrate at which the Media Resource can be played in bits per second. Example: - 128000 - - - - - - - The frame rate of the video signal in frames per second. Example: - 300 - - - - - - - The orientation of a document or an image. Example: - landscape - - - - - - - The total number of audio channels contained in the Media Resource. Example: - 2 - - - - - - - - - - - - - \ No newline at end of file diff --git a/metis-schema/src/main/resources/schema_xsds/EDM-COMMON-MAIN.xsd b/metis-schema/src/main/resources/schema_xsds/EDM-COMMON-MAIN.xsd deleted file mode 100644 index 31c9cf21f..000000000 --- a/metis-schema/src/main/resources/schema_xsds/EDM-COMMON-MAIN.xsd +++ /dev/null @@ -1,826 +0,0 @@ - - - - - - - - - - - - - - EDM First Implementation Schema: Main schema in the EDM namespace, to be - wrapped up in RDF - - - - - - - - Base class for ProvidedCHO implementations - - - - - - - - - - - - - - - - - - - - - - - - EuropeanaType contains the DC & DCTERMS elements. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Acronym for an Organization - Example: BNF - - - - - - This property associates an ORE aggregation with the cultural heritage - object(s) (CHO for short) it is about. In Europeana, an aggregation aggregates at - least one CHO. Typically in an aggregation there will be exactly one aggregated - object, but some aggregations, e.g. those representing archive finding aids, may - refer to more than one object. Conversely, a CHO may be aggregated by several - aggregations. Typically, in the data maintained by Europeana, a CHO would be - aggregated by one EuropeanaAggregation, and at least one provider Aggregation. - Example:The aggregation of Mona Lisa edm:aggregatedCHO Mona Lisa. - - - - - - The beginning of a temporal period - - - - - - This is the name of the country in which the Provider is based or - "Europe" in the case of Europe-wide projects. Example: - <edm:country>AL</edm:country> - - - - - - Deprecated:A literal indicating the name of the collection - Use instead - - - - - - - The geographic location and/or name of the repository, building, site, or - other entity whose boundaries presently include the resource. - - - - - - The name or identifier of the organisation that contributes data to - Europeana. This element is specifically included to allow the name of the - organisation who supplies data to Europeana indirectly via an aggregator to be - recorded and displayed in the portal. Aggregator names are recorded in edm:provider. - If an organisation provides data directly to Europeana (i.e. not via an aggregator) - the values in edm:dataProvider and edm:provider will be the same. Although the range - of this property is given as edm:Agent organisation names should be provided as an - ordinary text string until a Europeana authority file for organisations has been - established. At that point providers will be able to send an identifier from the - file instead of a text string. The name provided should be the preferred form of the - name in the language the provider chooses as the default language for display in the - portal. Countries with multiple languages may prefer to concatenate the name in more - than one language (See the example below.) Note: Europeana Data Provider is not - necessarily the institution where the physical object is located. Example: The - current <edm:dataProvider>Palais des Beaux Arts de - Lille</edm:dataProvider> could become <edm:dataProvider>http:// - www.pba-lille.fr/</edm:dataProvider> - - - - - - - The name of a dataset provided by an Organization - to Europeana - - - - - - - The ending of a temporal period - - - - - - Role of an Organization according to its relationship to Europeana - Example: Content Provider/Data Aggregator - - - - - - Geographic Level of an Organization - Example: Regional/National/European/Worldwide - - - - - - The EDM Content Type - - - - - - edm:hasMet relates a resource with the objects or phenomena that have - happened to or have happened together with the resource under consideration. We can - abstractly think of history and the present as a series of "meetings" between people - and other things in space-time. Therefore we name this relationship as the things - the object "has met" in the course of its existence. These meetings are events in - the proper sense, in which other people and things participate in any role. - Example:The location of an object may be due to a transport, move to a place, or - because it has been created at that spot. - - - - - - This property relates a resource with the concepts it belongs to in a - suitable type system such as MIME or any thesaurus that captures categories of - objects in a given field (e.g., the "Objects" facet in Getty's Art and Architecture - Thesaurus). It does not capture aboutness. Example:The type of Mona Lisa is (AAT) - Painting. The type of a digital image of Mona Lisa may be JPEG. - - - - - - This property relates a ORE aggregation about a CHO with a web resource - providing a view of that CHO. Examples of view are: a thumbnail, a textual abstract - and a table of contents. The ORE aggregation may be a Europeana aggregation, in - which case the view is an object owned by Europeana (i.e., an instance of - edm:EuropeanaObject) or an aggregation contributed by a content provider. In order - to capture both these cases, the domain of edm:hasView is ore:Aggregation and its - range is edm:WebResource Example: An ore:Aggregation of Mona Lisa contributed by - Louvre may have as view a low resolution digital image of Mona Lisa. The issue - number 56 of "Le Temps" contributed by BNF may have as view a text of some parts of - the issue - - - - - - This property captures the use of some resource to add value to another - resource. Such resources may be nested, such as performing a theater play text, and - then recording the performance, or creating an artful edition of a collection of - poems or just aggregating various poems in an anthology. There may be no single part - that contains ultimately the incorporated object, which may be dispersed in the - presentation. Therefore, incorporated resources do in general not form proper parts. - Incorporated resources are not part of the same resource, but are taken from other - resources, and have an independent history. Therefore edm:incorporates is not a - sub-property of dcterm:hasPart. Example:The movie "A Clockwork Orange" incorporates - Rossini's symphony from "La Gazza Ladra" in its original soundtrack. "E.A.Poe, The - Raven (poem)" is incorporated in "Emerson Lake & Palmers Tales of Mystery - (music)" which is incorporated in "Concert Recording 1973 (vinyl)". - - - - - - This property captures a narrower notion of derivation than - edm:isSimilarTo, in the sense that it relates a resource to another one, obtained by - reworking, reducing, expanding, parts or the whole contents of the former, and - possibly adding some minor parts. Versions have an even narrower meaning, in that it - requires common identity between the related resources. Translations, summaries, - abstractions etc. do not qualify as versions, but do qualify as derivatives. - Example:The Italian translation of Moby Dick is a derivation of the original work. - - - - - - - edm:isNextInSequence relates two resources S and R that are ordered - parts of the same resource A, and such that S comes immediately after R in the order - created by their being parts of A. Example: Page 34 of the Gutenberg Bible is next - in sequence to page 33 of the same title. - - - - - - edm:isRelatedTo is the most general contextual property in EDM. - Contextual properties have typically to do either with the things that have happened - to or together with the object under consideration, or what the object refers to by - its shape, form or features in a figural or encoded form. For sake of simplicity, we - include in the contextual relationships also the scholarly classification, which may - have either to do with the role and cultural connections of the object in the past, - or its kind of structure, substance or contents as it can be verified at present. - Example:Moby Dick is related to XIX century literature. Mona Lisa is related to - Renaissance Art. - - - - - - This property associates an information resource to the resource (if any) - that it represents. Example:A high resolution image created by the Multimedia Louvre - Lab by digitizing Mona Lisa is a representation of Mona Lisa - - - - - - An unambiguous URL reference to the digital object on the provider's web - site in its full information context. See also edm:isShownBy.This is a URL that will - be active in the Europeana interface. It will lead users to the digital object - displayed on the provider's web site in its full information context. Use - edm:isShownAt if you display the digital object with extra information (such as - header, banner etc). Example: - <edm:isShownAt>http://www.photo.rmn.fr/cf/htm/CPICZ.aspx?E=2C6NU0VFLVNY</edm:isShownAt> - - - - - - - An unambiguous URL reference to the digital object on the provider's web - site in the best available resolution/quality. See also edm:isShownAt. This is a URL - that will be active in the Europeana interface. It will lead users to the digital - object on the provider's website where they can view or play it. The digital object - needs to be directly accessible by the URL and reasonably independent at that - location. If the URL includes short copyright information with the pointer to the - object it can be entered in edm:isShownBy. Use edm:isShownAt for digital objects - embedded in HTML pages (even where the page is extremely simple). Example: - <edm:isShownBy>http://resolver.kb.nl/resolve?urn=urn:gvn:RA01:30051001524450</edm:isShownBy> - - - - - - - Definition The most generic derivation property, covering also the case - of questionable derivation. Is Similar To asserts that parts of the contents of one - resource exhibit common features with respect to ideas, shapes, structures, colors, - words, plots, topics with the contents of the related resource. Those common - features may be attributed to a common origin or influence (in particular for - derivation), but also to more generic cultural or psychological factors. - - - - - - - This property captures the relation between the continuation of a - resource and that resource. This applies to a story, a serial, a journal etc. No - content of the successor resource is identical or has a similar form with that of - the precursor. The similarity is only in the context, subjects and figures of a - plot. Successors typically form part of a common whole - such as a trilogy, a - journal, etc. Example: "The Two Towers" is a successor of "Fellowship of the Ring". - The issue 57 of "Le Temps" is a successor of issue 56 of the Le Temps. - - - - - - - This property captures the relation between an aggregation representing a - cultural heritage object and the Web resource representing that object on the - provider's web site. Example: Mona Lisa, represented by the Europeana aggregation - europeana:ea-monalisa, has landing page - http://www.culture.gouv.fr/public/mistral/joconde_fr?ACTION=CHERCHER&FIELD_1=REF&VALUE_1=000PE025604 - - - - - - A language assigned to the resource with reference to the Provider. - Language Codes: - bg:Bulgarian - ca:Catalan - cs:Czech - da:Danish - de:German - el:Greek - en:English - es:Spanish - et:Estonian - eu:Basque - fi:Finnish - fr:French - ga:Irish - gd:Gaelic (Scottish) - he:Hebrew - hr:Croatian (hrvatski jezik) - hu:Hungarian - ie:Interlingue - is:Icelandic - it:Italian - ka:Georgian - lt:Lithuanian - lv:Latvian (Lettish) - mk:Macedonian - mt:Maltese - mul:Multilingual Content - nl:Netherlands - no:Norway - pl:Polish - pt:Portugese - ro:Romanian - ru:Russian - sk:Slovak - sl:Slovenian - sr:Serbian - sv:Swedish - tr:Turkish - uk:Ukrainian - yi:Yiddish - cy:Welsh - sq:Albanian - hy:Armenian - az:Azerbaijani - be:Belarusian - bs:Bosnian - gl:Galician - ja:Japanese - ar:Arabic - ko:Korean - zh:Chinese - hi:Hindi - - - - - - - The URL of a thumbnail representing the digital object or, if there is no - such thumbnail, the URL of the digital object in the best resolution available on - the web site of the data provider from which a thumbnail could be generated. This - will often be the same URL as given in edm:isShownBy. - Example:<edm:object>http://upload.wikimedia.org/wikipedia/en/f/f3/Europeana_logo.png</edm:object> - - - - - - - Scope of an Organization - Example: Cross/Single/Thematic/Individual - - - - - - - Domain of an Organization - Example: Library/Archive/Museum - - - - - - Sector of an Organization - Example: Private/Public/Goverment Department - - - - - - - A reference to the Europeana cached thumbnail. - - - - - - - Name of the organization that delivers data to Europeana. The - edm:provider is the organization that sends the data to Europeana, and this is not - necessarily the institution that holds or owns the original or digitised object. - Where data is being supplied by an aggregator or project edm:provider is the name of - aggregator/project. The name of the content holder can be recorded in - edm:dataProvider. If the content holder supplies data directly to Europeana then the - name should also appear in this element. Although the range of this property is - given as edm:Agent, organisation names should be provided as an ordinary text string - until a Europeana authority file for organisations has been established. At that - point providers will be able to send an identifier from the file instead of a text - string. The name should be in the original language(s). Example: The current - <edm:provider>Geheugen van Nederland</edm:provider> could become - <edm:provider>http://www.geheugenvannederland.nl/</edm:provider> - - - - - - - This property describes a relation between a physical thing and the - information resource that is contained in it, visible at it or otherwise carried by - it, if applicable. Example: An item of the Gutenberg's edition realizes the Bible - - - - - - - Information about copyright of the digital object as specified by - isShownBy and isShownAt.The value in this element is a URL constructed according to - the specifications in the "Specifications of the controlled values for edm:rights". - The URLs are constructed by adding a code indicating the copyright status of an - object to the domain name where that status is defined. The domain will be either - the europeana.eu domain or the creativecommons.org domain. For users of Europeana.eu - this copyright information also applies to the preview specified in edm:object. - - - - - - Not valid Rights Statement - - - - - - - - - - The Completeness of the record - - - - - - This element is used to identify user generated content (also called user - created content). It should be applied to all digitised or born digital content - contributed by the general public and collected by Europeana through a crowdsourcing - initiative or project. The only value this element can take is "TRUE" to indicate - that the object is user generated. It should be entered in uppercase. If the content - is not user generated then the element should not be provided. - Example:<edm:UGC>TRUE<edm:UGC> - - - - - - This is a tag created by a user through the Europeana interface. - - - - - - - A point of time associated with an event in the life of the original - analog or born digital object. Example:<edm:year >1523</edm:year> - - - - - - - A flag indicating that the specific Proxy can be used as a europeanaProxy - - - - - - The name of a device or computer program capable of encoding or decoding a digital data stream or signal (i.e. coder-decoder). Example: - mp4v - - - - - - The spatial resolution of a media resource expressed in DPIs. Example: - 300 - - - - - - The color space of an image resource i.e. 'grayscale' or 'sRGB'. Example: - grayscale - - - - - - A significant color present in an image. The colors must be taken from the CSS3 standard color palette - and are expressed as a hexadecimal binary value. Example: - <spatialResolution>FF0000</spatialResolution> - - - - - - - - The name or identifier of the intermediate organization that selects, collates, or curates data - from a Data Provider that is then aggregated by a Provider from which Europeana harvests. - The Intermedatiate Provider must be distinct from both the Data Provider and the Provider - in the data supply chain. - Example: - <edm:intermediateProvider< Erfgoedplus.be</edm:intermediateProvider< - <edm:intermediateProvider rdf:resource=”www.erfgoedplus.be/”/< - - - - - - - - - Denotes whether the metadata statment was generated by a person or a software agent (e.g. enrichment or translation tool). - See documentation at: https://docs.google.com/document/d/1AFxfr7BESYGd_1sFOLWBWs7C6uGXcAchom8TE2ClvTM/ - Example: - - - - - - - - - - - - - - A confidence level represented as a floating number between [0,1] to be indicated at the level of each metadata statement, - reflecting a measure of condidence associated with it. In the case where the metadata statement was created by a person, - the confidence level can be omitted. - See documentation at: https://docs.google.com/document/d/1AFxfr7BESYGd_1sFOLWBWs7C6uGXcAchom8TE2ClvTM/ - Example: - - - - - - - - - - - diff --git a/metis-schema/src/main/resources/schema_xsds/EDM-EXTERNAL-MAIN.xsd b/metis-schema/src/main/resources/schema_xsds/EDM-EXTERNAL-MAIN.xsd deleted file mode 100644 index f16ea1c4c..000000000 --- a/metis-schema/src/main/resources/schema_xsds/EDM-EXTERNAL-MAIN.xsd +++ /dev/null @@ -1,192 +0,0 @@ - - - - - - - - - - - - - - - - - The RDF root element declaration - - - Schematron validation - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - This class comprises the Cultural Heritage objects that Europeana - collects descriptions about. - - - - - - id: - - - A ProvidedCHO must have a dc:subject or dc:type or dct:temporal or dct:spatial. - - - A Proxy must have a non empty - dc:subject or dc:type or dct:temporal or - dct:spatial. - - - A Proxy must have a non empty dc:title or a non empty - dc:description - - - id: - - - Within a Proxy - context, dc:language is mandatory when dc:language has the value - 'TEXT'. - - - - - - - - - - - - - - diff --git a/metis-schema/src/main/resources/schema_xsds/EDM-INTERNAL-MAIN.xsd b/metis-schema/src/main/resources/schema_xsds/EDM-INTERNAL-MAIN.xsd deleted file mode 100644 index 08470edd6..000000000 --- a/metis-schema/src/main/resources/schema_xsds/EDM-INTERNAL-MAIN.xsd +++ /dev/null @@ -1,131 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - The RDF root element declaration - - - Schematron validation - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - The set of resources related to a single cultural heritage object that - collectively represent that object in Europeana. Such set consists of: all - descriptions about the object that Europeana collects from (possibly different) - content providers, including thumbnails and other forms of abstractions, as well as - of the description of the object Europeana builds. - - - - - Empty rdf:about values are not allowed - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - This class comprises the Cultural Heritage objects that - Europeana collects descriptions about. - - - - - - - - - diff --git a/metis-schema/src/main/resources/schema_xsds/EDM-INTERNAL.xsd b/metis-schema/src/main/resources/schema_xsds/EDM-INTERNAL.xsd deleted file mode 100644 index 330857d02..000000000 --- a/metis-schema/src/main/resources/schema_xsds/EDM-INTERNAL.xsd +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - EDM First Implementation Schema: wrapped up as RDF/XML (For Europeana Internal Use Only) - - - - - - - - - \ No newline at end of file diff --git a/metis-schema/src/main/resources/schema_xsds/EDM.xsd b/metis-schema/src/main/resources/schema_xsds/EDM.xsd deleted file mode 100644 index 18af8b039..000000000 --- a/metis-schema/src/main/resources/schema_xsds/EDM.xsd +++ /dev/null @@ -1,17 +0,0 @@ - - - - - EDM First Implementation Schema: wrapped up as RDF/XML - - - - - - diff --git a/metis-schema/src/main/resources/schema_xsds/ENRICHMENT.xsd b/metis-schema/src/main/resources/schema_xsds/ENRICHMENT.xsd deleted file mode 100644 index 1516e4394..000000000 --- a/metis-schema/src/main/resources/schema_xsds/ENRICHMENT.xsd +++ /dev/null @@ -1,18 +0,0 @@ - - - - - EDM First Implementation Schema: Who-What-When-Where enrichments - - - - - - - - - - - - \ No newline at end of file diff --git a/metis-schema/src/main/resources/schema_xsds/FOAF.xsd b/metis-schema/src/main/resources/schema_xsds/FOAF.xsd deleted file mode 100644 index 4e614fc0c..000000000 --- a/metis-schema/src/main/resources/schema_xsds/FOAF.xsd +++ /dev/null @@ -1,57 +0,0 @@ - - - - - Europeana representation of Foaf elements - - - - - - - - - - - - - - - - - - - - - - The EDM representation of an Organization - providing data to Europeana - - - - - - - - - - - - - - - - - - - - - - - diff --git a/metis-schema/src/main/resources/schema_xsds/OA.xsd b/metis-schema/src/main/resources/schema_xsds/OA.xsd deleted file mode 100644 index 4908e3716..000000000 --- a/metis-schema/src/main/resources/schema_xsds/OA.xsd +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/metis-schema/src/main/resources/schema_xsds/ODRL.xsd b/metis-schema/src/main/resources/schema_xsds/ODRL.xsd deleted file mode 100644 index 288221ff3..000000000 --- a/metis-schema/src/main/resources/schema_xsds/ODRL.xsd +++ /dev/null @@ -1,13 +0,0 @@ - - - - - EDM First Implementation Schema: ODRL - - - - - - - \ No newline at end of file diff --git a/metis-schema/src/main/resources/schema_xsds/ORE.xsd b/metis-schema/src/main/resources/schema_xsds/ORE.xsd deleted file mode 100644 index 5b937c114..000000000 --- a/metis-schema/src/main/resources/schema_xsds/ORE.xsd +++ /dev/null @@ -1,208 +0,0 @@ - - - - - EDM First Implementation Schema: Aggregations - - - - - - - - - - - - - A set of related resources (Aggregated Resources), - grouped together - such that the set can be treated as a single resource. This is the - entity - described within the ORE interoperability framework by a Resource Map. - - - - - - id: - - An ore:Aggregation must have either - edm:isShownAt or edm:isShownBy - - - - - - - id: - - An ore:Aggregation must have at least one instance of - edm:dataProvider - - - - - - - - id: - - An ore:Aggregation must have at least one instance of - edm:provider - - - - - - - id: - - An ore:Aggregation must have at least one instance of - edm:rights - - - - - - An ore:Aggregation must have a non - empty edm:provider - - - - - An ore:Aggregation must have a non - empty edm:dataProvider - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - A proxy is a resource that stands for an aggregated - resource A in - the context of a specific aggregation. The URI of a proxy then can be - used - in assertions specific to the aggregated resource A in the context of - that - aggregation (http://www.openarchives.org/ore/1.0/primer.html). - - - - - - id: - - - A Proxy must have a - dc:subject or dc:type or dct:temporal or - dct:spatial. - - - A Proxy must have a non empty - dc:subject or dc:type or dc:coverage or dct:temporal or - dct:spatial. - - A Proxy must have a non empty dc:title or a non empty - dc:description - - id: - - - Within a Proxy - context, dc:language is mandatory when dc:language has the value - 'TEXT'. - - - id: - - edm:type should be present in an ore:Proxy context. - - - id: - - edm:type should not be present in an Europeana Proxy context - (when the edm:europeanaProxy value is present). - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/metis-schema/src/main/resources/schema_xsds/OWL.xsd b/metis-schema/src/main/resources/schema_xsds/OWL.xsd deleted file mode 100644 index 97981f0dd..000000000 --- a/metis-schema/src/main/resources/schema_xsds/OWL.xsd +++ /dev/null @@ -1,21 +0,0 @@ - - - - - EDM First Implementation Schema: OWL - - - - - - - - The built-in OWL property owl:sameAs links an individual - to an individual. Such an owl:sameAs statement indicates that two - URI references actually refer to the same thing: the individuals - have the same "identity". - - - - \ No newline at end of file diff --git a/metis-schema/src/main/resources/schema_xsds/RDAGR2.xsd b/metis-schema/src/main/resources/schema_xsds/RDAGR2.xsd deleted file mode 100644 index 30a8f0e9a..000000000 --- a/metis-schema/src/main/resources/schema_xsds/RDAGR2.xsd +++ /dev/null @@ -1,37 +0,0 @@ - - - - - Europeana representation of the RDA Group 2 Element Vocabulary - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/metis-schema/src/main/resources/schema_xsds/RDF.xsd b/metis-schema/src/main/resources/schema_xsds/RDF.xsd deleted file mode 100644 index 2e0e6877d..000000000 --- a/metis-schema/src/main/resources/schema_xsds/RDF.xsd +++ /dev/null @@ -1,183 +0,0 @@ - - - - - EDM First Implementation Schema: RDF resources and - literals - - - - - - - - - - - To indicate the type of a resource. Currently not used - - - - - - - - - - Empty rdf:resource attribute is not allowed for - - element. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Empty rdf:about attribute is not allowed for - - element. - - - - - - - - - - - - - - - Empty xml:lang attribute is not allowed for - - element. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Element - - should not have both rdf:resource attribute and text value - populated. - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/metis-schema/src/main/resources/schema_xsds/SKOS.xsd b/metis-schema/src/main/resources/schema_xsds/SKOS.xsd deleted file mode 100644 index 8cdd1e17d..000000000 --- a/metis-schema/src/main/resources/schema_xsds/SKOS.xsd +++ /dev/null @@ -1,62 +0,0 @@ - - - - - - EDM First Implementation Schema: SKOS - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/metis-schema/src/main/resources/schema_xsds/SVCS.xsd b/metis-schema/src/main/resources/schema_xsds/SVCS.xsd deleted file mode 100644 index 1f23c4af8..000000000 --- a/metis-schema/src/main/resources/schema_xsds/SVCS.xsd +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - - - The identifier of the Service require to consume the WebResource. - <svcs:has_service rdf:resource=“http://www.example.org/Service/IIIF”> - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/metis-schema/src/main/resources/schema_xsds/WDRS.xsd b/metis-schema/src/main/resources/schema_xsds/WDRS.xsd deleted file mode 100644 index b8b67c083..000000000 --- a/metis-schema/src/main/resources/schema_xsds/WDRS.xsd +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - A specification that a project implements. Could be a standard, API or legally defined level of conformance. - In IIIF doap:implements refers to the the general protocol without specific version. - <doap:implements rdf:resource=”http://iiif.io/api/image” /> - - - - - diff --git a/metis-schema/src/main/resources/schema_xsds/WGS84.xsd b/metis-schema/src/main/resources/schema_xsds/WGS84.xsd deleted file mode 100644 index 08fb24b3d..000000000 --- a/metis-schema/src/main/resources/schema_xsds/WGS84.xsd +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - EDM First Implementation Schema: WGS84 coordinates - - - - - - - - - - \ No newline at end of file diff --git a/metis-schema/src/main/resources/schema_xsds/schematron/schematron-internal.xsl b/metis-schema/src/main/resources/schema_xsds/schematron/schematron-internal.xsl deleted file mode 100644 index df87dce7a..000000000 --- a/metis-schema/src/main/resources/schema_xsds/schematron/schematron-internal.xsl +++ /dev/null @@ -1,790 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - / - - - - - [] - - - - *[local-name()=' - - '] - - - [] - - - - - - / - - @ - - - @*[local-name()=' - - ' and namespace-uri()=' - - '] - - - - - - - - - / - - - [ - - ] - - - - /@ - - - - - - - / - - - [ - - ] - - - - /@ - - - - - - - - - - - - - - - - - - - - - - - - . - - - - - U - - U - - - - U. - - n - - - - U. - - _ - - _ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Element should not have both rdf:resource attribute and text value populated. - - - - - - - - - - - - - - - - - - - - - - - - - - - - Empty rdf:resource attribute is not allowed for element. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - An ore:Aggregation must have either edm:isShownAt or edm:isShownBy - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - An ore:Aggregation must have at least one instance of edm:dataProvider - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - An ore:Aggregation must have at least one instance of edm:provider - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - An ore:Aggregation must have at least one instance of edm:rights - - - - - - - - - - - - - - - - - - - - - - - - - - - An ore:Aggregation must have a non empty edm:provider - - - - - - - - - - - - - - - - - - - - - - - - - - An ore:Aggregation must have a non empty edm:dataProvider - - - - - - - - - - - - - - - - - - - - - - - - - - - Empty rdf:about values are not allowed - - - - - - - - - - - - - - - - - - - - - - - - - - - - Empty xml:lang attribute is not allowed for element. - - - - - - - - - - - Schematron validation - - - - - - - - - - - - - - - - - - - - - - - - - A Provider Proxy must have a dc:subject or dc:type or dct:temporal or dct:spatial. - - - - - - - - - - - - - - A Provider Proxy must have at least one of dc:subject or dc:type or dct:temporal or dct:spatial, that has an rdf:resource attribute or have non empty text value. - - - - - - - - - - - - - - - - A Proxy must have a non empty dc:title or a non empty dc:description - - - - - - - - - - - - - - - edm:type should be present in an ore:Proxy context. - - - - - - - - - - - - - - - - Within a Proxy context, dc:language is mandatory when edm:type has the value 'TEXT'. - - - - - - - - - - - - - - - - - - - - A Europeana Proxy must NOT have a dc:subject or dc:type or dct:temporal or dct:spatial. - - - - - - - - - - - - - - - edm:type should not be present in an Europeana Proxy context (when the edm:europeanaProxy value is present). - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - The element dcterms:isPartOf should not have a literal value in the edm:WebResource context with this id. Use an rdf:resource instead. - - - - - - - - - - - - diff --git a/metis-schema/src/main/resources/schema_xsds/schematron/schematron.xsl b/metis-schema/src/main/resources/schema_xsds/schematron/schematron.xsl deleted file mode 100644 index d87db43e7..000000000 --- a/metis-schema/src/main/resources/schema_xsds/schematron/schematron.xsl +++ /dev/null @@ -1,1064 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - / - - - - - [] - - - - *[local-name()=' - - '] - - - [] - - - - - - / - - @ - - - @*[local-name()=' - - ' and namespace-uri()=' - - '] - - - - - - - - - / - - - [ - - ] - - - - /@ - - - - - - - / - - - [ - - ] - - - - /@ - - - - - - - - - - - - - - - - - - - - - - - - . - - - - - U - - U - - - - U. - - n - - - - U. - - _ - - _ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Empty rdf:resource attribute is not allowed for element. - - - - - - - - - - - - - - - - - - - - - - - - - - - - Element should not have both rdf:resource attribute and text value populated. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - A Proxy must have a dc:subject or dc:type or dct:temporal or dct:spatial. - - - - - - - - - - - - - - - - - - A Proxy must have a non empty dc:subject or dc:type or dct:temporal or dct:spatial. - - - - - - - - - - - - - - - - - - A Proxy must have a dc:title or dc:description. - - - - - - - - - - - - - - - - - - A Proxy must have a non empty dc:title or a non empty dc:description - - - - - - - - - - - - - - - - - - Within a Proxy context, dc:language is mandatory when dc:language has the value 'TEXT'. - - - - - - - - - - - - - - - - - - edm:type should be present in an ore:Proxy context. - - - - - - - - - - - - - - - - - - edm:type should not be present in an Europeana Proxy context (when the - edm:europeanaProxy value is present). - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - An ore:Aggregation must have either edm:isShownAt or edm:isShownBy - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - An ore:Aggregation must have at least one instance of edm:dataProvider - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - An ore:Aggregation must have at least one instance of edm:provider - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - An ore:Aggregation must have at least one instance of edm:rights - - - - - - - - - - - - - - - - - - - - - - - - - - - An ore:Aggregation must have a non empty edm:provider - - - - - - - - - - - - - - - - - - - - - - - - - - An ore:Aggregation must have a non empty edm:dataProvider - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Invalid Rights Statements - - - - - - - - - - - Schematron validation - - - - - - - - - - - - - - - - - - Empty xml:lang attribute is not allowed for - element. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Invalid Rights Statements - - - - - - - - - - - Schematron validation - - - - - - - - - - - - - - - - - - The element dcterms:isPartOf should not have a literal value in the edm:WebResource context with this id. Use an rdf:resource instead. - - - - - - - - - - - Schematron validation - - - - - - - - - - - - - - - - - - - - - A ProvidedCHO must have a dc:subject or dc:type or dct:temporal or dct:spatial. - - - - - - - - - - - - - - - A ProvidedCHO must have at least one of dc:subject or dc:type or dct:temporal or dct:spatial, that has an rdf:resource attribute or have non empty text value. - - - - - - - - - - - - - - - - - - A ProvidedCHO must have a non empty dc:title or a non empty dc:description - - - - - - - - - - - - - - - - - - Within a ProvidedCHO context, dc:language is mandatory when edm:type has the value 'TEXT'. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Within a RDF context, there must be exactly one edm:ProvidedCHO. - - - - - - - - - - - - - - - - Within a RDF context, there must be exactly one ore:Aggregation. - - - - - - - - - - - - - diff --git a/metis-schema/src/main/resources/schema_xsds/xml.xsd b/metis-schema/src/main/resources/schema_xsds/xml.xsd deleted file mode 100644 index aea7d0db0..000000000 --- a/metis-schema/src/main/resources/schema_xsds/xml.xsd +++ /dev/null @@ -1,287 +0,0 @@ - - - - - - -
    -

    About the XML namespace

    - -
    -

    - This schema document describes the XML namespace, in a form - suitable for import by other schema documents. -

    -

    - See - http://www.w3.org/XML/1998/namespace.html and - - http://www.w3.org/TR/REC-xml for information - about this namespace. -

    -

    - Note that local names in this namespace are intended to be - defined only by the World Wide Web Consortium or its subgroups. - The names currently defined in this namespace are listed below. - They should not be used with conflicting semantics by any Working - Group, specification, or document instance. -

    -

    - See further below in this document for more information about how to refer to this schema document from your own - XSD schema documents and about the - namespace-versioning policy governing this schema document. -

    -
    -
    -     -     - - - - -
    - -

    lang (as an attribute name)

    -

    - denotes an attribute whose value - is a language code for the natural language of the content of - any element; its value is inherited. This name is reserved - by virtue of its definition in the XML specification.

    - -
    -
    -

    Notes

    -

    - Attempting to install the relevant ISO 2- and 3-letter - codes as the enumerated possible values is probably never - going to be a realistic possibility. -

    -

    - See BCP 47 at - http://www.rfc-editor.org/rfc/bcp/bcp47.txt - and the IANA language subtag registry at - - http://www.iana.org/assignments/language-subtag-registry - for further information. -

    -

    - The union allows for the 'un-declaration' of xml:lang with - the empty string. -

    -
    -     -     - - - - - - - - - -     - - - - -
    - -

    space (as an attribute name)

    -

    - denotes an attribute whose - value is a keyword indicating what whitespace processing - discipline is intended for the content of the element; its - value is inherited. This name is reserved by virtue of its - definition in the XML specification.

    - -
    -     -     - - - - - - -     - - - -
    - -

    base (as an attribute name)

    -

    - denotes an attribute whose value - provides a URI to be used as the base for interpreting any - relative URIs in the scope of the element on which it - appears; its value is inherited. This name is reserved - by virtue of its definition in the XML Base specification.

    - -

    - See http://www.w3.org/TR/xmlbase/ - for information about this attribute. -

    -
    -     -     -     - - - - -
    - -

    id (as an attribute name)

    -

    - denotes an attribute whose value - should be interpreted as if declared to be of type ID. - This name is reserved by virtue of its definition in the - xml:id specification.

    - -

    - See http://www.w3.org/TR/xml-id/ - for information about this attribute. -

    -
    -     -     -     - - - - - - - - - - -
    - -

    Father (in any context at all)

    - -
    -

    - denotes Jon Bosak, the chair of - the original XML Working Group. This name is reserved by - the following decision of the W3C XML Plenary and - XML Coordination groups: -

    -
    -

    - In appreciation for his vision, leadership and - dedication the W3C XML Plenary on this 10th day of - February, 2000, reserves for Jon Bosak in perpetuity - the XML name "xml:Father". -

    -
    -
    -
    -     -     - - - -
    -

    About this schema document

    - -
    -

    - This schema defines attributes and an attribute group suitable - for use by schemas wishing to allow xml:base, - xml:lang, xml:space or - xml:id attributes on elements they define. -

    -

    - To enable this, such a schema must import this schema for - the XML namespace, e.g. as follows: -

    - 
    -          <schema . . .>
-           . . .
-           <import namespace="http://www.w3.org/XML/1998/namespace"
-                      schemaLocation="http://www.w3.org/2001/xml.xsd"/>
-
    -

    - or -

    - 
    -           <import namespace="http://www.w3.org/XML/1998/namespace"
-                      schemaLocation="http://www.w3.org/2009/01/xml.xsd"/>
-
    -

    - Subsequently, qualified reference to any of the attributes or the - group defined below will have the desired effect, e.g. -

    - 
    -          <type . . .>
-           . . .
-           <attributeGroup ref="xml:specialAttrs"/>
-
    -

    - will define a type which will schema-validate an instance element - with any of those attributes. -

    -
    -
    -     -     - - - -
    -

    Versioning policy for this schema document

    -
    -

    - In keeping with the XML Schema WG's standard versioning - policy, this schema document will persist at - - http://www.w3.org/2009/01/xml.xsd. -

    -

    - At the date of issue it can also be found at - - http://www.w3.org/2001/xml.xsd. -

    -

    - The schema document at that URI may however change in the future, - in order to remain compatible with the latest version of XML - Schema itself, or with the XML namespace itself. In other words, - if the XML Schema or XML namespaces change, the version of this - document at - http://www.w3.org/2001/xml.xsd - - will change accordingly; the version at - - http://www.w3.org/2009/01/xml.xsd - - will not change. -

    -

    - Previous dated (and unchanging) versions of this schema - document are at: -

    - -
    -
    -     -     - -     - diff --git a/metis-schema/src/test/java/eu/europeana/metis/schema/convert/RdfConversionUtilsTest.java b/metis-schema/src/test/java/eu/europeana/metis/schema/convert/RdfConversionUtilsTest.java deleted file mode 100644 index b6e574f6d..000000000 --- a/metis-schema/src/test/java/eu/europeana/metis/schema/convert/RdfConversionUtilsTest.java +++ /dev/null @@ -1,91 +0,0 @@ -package eu.europeana.metis.schema.convert; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import eu.europeana.metis.schema.jibx.AgentType; -import eu.europeana.metis.schema.jibx.Alternative; -import eu.europeana.metis.schema.jibx.Concept; -import eu.europeana.metis.schema.jibx.Coverage; -import eu.europeana.metis.schema.jibx.CurrentLocation; -import eu.europeana.metis.schema.jibx.Description; -import eu.europeana.metis.schema.jibx.Format; -import eu.europeana.metis.schema.jibx.HasPart; -import eu.europeana.metis.schema.jibx.HasType; -import eu.europeana.metis.schema.jibx.IsPartOf; -import eu.europeana.metis.schema.jibx.IsReferencedBy; -import eu.europeana.metis.schema.jibx.IsRelatedTo; -import eu.europeana.metis.schema.jibx.Medium; -import eu.europeana.metis.schema.jibx.PlaceType; -import eu.europeana.metis.schema.jibx.Provenance; -import eu.europeana.metis.schema.jibx.References; -import eu.europeana.metis.schema.jibx.Relation; -import eu.europeana.metis.schema.jibx.Rights; -import eu.europeana.metis.schema.jibx.Source; -import eu.europeana.metis.schema.jibx.Spatial; -import eu.europeana.metis.schema.jibx.Subject; -import eu.europeana.metis.schema.jibx.TableOfContents; -import eu.europeana.metis.schema.jibx.Temporal; -import eu.europeana.metis.schema.jibx.TimeSpanType; -import eu.europeana.metis.schema.jibx.Title; -import eu.europeana.metis.schema.jibx.Type; -import org.junit.jupiter.api.Test; - -class RdfConversionUtilsTest { - - @Test - void failRdfConversionUtilsInitialization() { - //Force failure - assertThrows(IllegalStateException.class, () -> new RdfConversionUtils(RdfConversionUtils.class)); - } - - @Test - void getQualifiedElementNameForClass_ContextualClasses() { - //Check contextual classes - final RdfConversionUtils rdfConversionUtils = new RdfConversionUtils(); - assertEquals("edm:AgentType", rdfConversionUtils.getQualifiedElementNameForClass(AgentType.class)); - assertEquals("edm:TimeSpanType", rdfConversionUtils.getQualifiedElementNameForClass(TimeSpanType.class)); - assertEquals("edm:PlaceType", rdfConversionUtils.getQualifiedElementNameForClass(PlaceType.class)); - assertEquals("skos:Concept", rdfConversionUtils.getQualifiedElementNameForClass(Concept.class)); - } - - @Test - void getQualifiedElementNameForClass_Dc() { - //Check dc elements - final RdfConversionUtils rdfConversionUtils = new RdfConversionUtils(); - assertEquals("dc:coverage", rdfConversionUtils.getQualifiedElementNameForClass(Coverage.class)); - assertEquals("dc:description", rdfConversionUtils.getQualifiedElementNameForClass(Description.class)); - assertEquals("dc:format", rdfConversionUtils.getQualifiedElementNameForClass(Format.class)); - assertEquals("dc:relation", rdfConversionUtils.getQualifiedElementNameForClass(Relation.class)); - assertEquals("dc:rights", rdfConversionUtils.getQualifiedElementNameForClass(Rights.class)); - assertEquals("dc:source", rdfConversionUtils.getQualifiedElementNameForClass(Source.class)); - assertEquals("dc:subject", rdfConversionUtils.getQualifiedElementNameForClass(Subject.class)); - assertEquals("dc:title", rdfConversionUtils.getQualifiedElementNameForClass(Title.class)); - assertEquals("dc:type", rdfConversionUtils.getQualifiedElementNameForClass(Type.class)); - } - - @Test - void getQualifiedElementNameForClass_Dcterms() { - //Check dcterms elements - final RdfConversionUtils rdfConversionUtils = new RdfConversionUtils(); - assertEquals("dcterms:alternative", rdfConversionUtils.getQualifiedElementNameForClass(Alternative.class)); - assertEquals("dcterms:hasPart", rdfConversionUtils.getQualifiedElementNameForClass(HasPart.class)); - assertEquals("dcterms:isPartOf", rdfConversionUtils.getQualifiedElementNameForClass(IsPartOf.class)); - assertEquals("dcterms:isReferencedBy", rdfConversionUtils.getQualifiedElementNameForClass(IsReferencedBy.class)); - assertEquals("dcterms:medium", rdfConversionUtils.getQualifiedElementNameForClass(Medium.class)); - assertEquals("dcterms:provenance", rdfConversionUtils.getQualifiedElementNameForClass(Provenance.class)); - assertEquals("dcterms:references", rdfConversionUtils.getQualifiedElementNameForClass(References.class)); - assertEquals("dcterms:spatial", rdfConversionUtils.getQualifiedElementNameForClass(Spatial.class)); - assertEquals("dcterms:tableOfContents", rdfConversionUtils.getQualifiedElementNameForClass(TableOfContents.class)); - assertEquals("dcterms:temporal", rdfConversionUtils.getQualifiedElementNameForClass(Temporal.class)); - } - - @Test - void getQualifiedElementNameForClass_Edm() { - //Check edm elements - final RdfConversionUtils rdfConversionUtils = new RdfConversionUtils(); - assertEquals("edm:currentLocation", rdfConversionUtils.getQualifiedElementNameForClass(CurrentLocation.class)); - assertEquals("edm:hasType", rdfConversionUtils.getQualifiedElementNameForClass(HasType.class)); - assertEquals("edm:isRelatedTo", rdfConversionUtils.getQualifiedElementNameForClass(IsRelatedTo.class)); - } -} \ No newline at end of file diff --git a/metis-schema/src/test/java/eu/europeana/metis/schema/model/MediaTypeTest.java b/metis-schema/src/test/java/eu/europeana/metis/schema/model/MediaTypeTest.java deleted file mode 100644 index 2cc47e28c..000000000 --- a/metis-schema/src/test/java/eu/europeana/metis/schema/model/MediaTypeTest.java +++ /dev/null @@ -1,34 +0,0 @@ -package eu.europeana.metis.schema.model; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.Test; - -class MediaTypeTest { - - @Test - void testGetMediaType() { - assertEquals(MediaType.IMAGE, MediaType.getMediaType("image/unknown_type")); - assertEquals(MediaType.AUDIO, MediaType.getMediaType("audio/unknown_type")); - assertEquals(MediaType.VIDEO, MediaType.getMediaType("video/unknown_type")); - assertEquals(MediaType.TEXT, MediaType.getMediaType("text/unknown_type")); - assertEquals(MediaType.TEXT, MediaType.getMediaType("application/xml")); - assertEquals(MediaType.TEXT, MediaType.getMediaType("application/rtf")); - assertEquals(MediaType.TEXT, MediaType.getMediaType("application/epub")); - assertEquals(MediaType.TEXT, MediaType.getMediaType("application/pdf")); - assertEquals(MediaType.TEXT, MediaType.getMediaType("application/pdf+xml")); - assertEquals(MediaType.TEXT, MediaType.getMediaType("application/pdf+xml; key=value")); - assertEquals(MediaType.TEXT, MediaType.getMediaType("application/xhtml+xml")); - assertEquals(MediaType.OTHER, MediaType.getMediaType("application/xhtml")); - assertEquals(MediaType.OTHER, MediaType.getMediaType("unknown_type")); - assertEquals(MediaType.OTHER, MediaType.getMediaType(null)); - - assertEquals("AUDIO", MediaType.AUDIO.toString()); - assertEquals("VIDEO", MediaType.VIDEO.toString()); - assertEquals("TEXT", MediaType.TEXT.toString()); - assertEquals("IMAGE", MediaType.IMAGE.toString()); - assertEquals("3D", MediaType.THREE_D.toString()); - assertEquals("OTHER", MediaType.OTHER.toString()); - } -} - diff --git a/pom.xml b/pom.xml index d4aedc5ed..c716efe26 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,6 @@ pom - metis-schema metis-common metis-core metis-indexing @@ -122,6 +121,9 @@ UTF-8 + 7-SNAPSHOT + 7-SNAPSHOT + 2.9.0 5.1 1.7.0 @@ -140,7 +142,6 @@ 1.0 4.2.0 - 7-SNAPSHOT 3.0.0 1.3 @@ -440,7 +441,7 @@ eu.europeana.metis metis-schema - ${project.version} + ${version.metis-schema} xml-apis From 3c64db123d40d905fd14370bd9883b628c0406a6 Mon Sep 17 00:00:00 2001 From: Adolfo Peixinho <91942157+adolfopeixinho@users.noreply.github.com> Date: Thu, 2 Jun 2022 15:30:44 +0200 Subject: [PATCH 47/73] MET-4572 Added test for AgentSolrCreator (#550) * MET-4568 add AgentSolrCreator unit tests. * MET-4572 removed public method access from main class, because of code smells --- .../solr/property/AgentSolrCreatorTest.java | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 metis-indexing/src/test/java/eu/europeana/indexing/solr/property/AgentSolrCreatorTest.java diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/AgentSolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/AgentSolrCreatorTest.java new file mode 100644 index 000000000..eca466b90 --- /dev/null +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/AgentSolrCreatorTest.java @@ -0,0 +1,76 @@ +package eu.europeana.indexing.solr.property; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import eu.europeana.corelib.definitions.edm.entity.Agent; +import eu.europeana.corelib.solr.entity.AgentImpl; +import eu.europeana.indexing.solr.EdmLabel; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.apache.solr.common.SolrInputDocument; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class AgentSolrCreatorTest { + + private SolrInputDocument solrInputDocument; + private AgentSolrCreator agentSolrCreator; + + + @BeforeEach + void setup() { + solrInputDocument = new SolrInputDocument(); + agentSolrCreator = new AgentSolrCreator(); + } + + + @Test + void agentSolrCreatorAddToSolrDocument() { + + Agent agent = new AgentImpl(); + agent.setAbout("About Agent"); + agent.setPrefLabel(Map.of("pref_label", List.of("val1", "val2"))); + agent.setAltLabel(Map.of("alt_label", List.of("val1", "val2"))); + agent.setFoafName(Map.of("foaf_name", List.of("val1", "val2"))); + agent.setRdaGr2DateOfBirth(Map.of("date_of_birth_label", List.of("some_date"))); + agent.setRdaGr2DateOfDeath(Map.of("date_of_death_label", List.of("other_date"))); + agent.setRdaGr2PlaceOfBirth(Map.of("place_of_birth_label", List.of("some_place"))); + agent.setRdaGr2PlaceOfDeath(Map.of("place_of_death_label", List.of("other_place"))); + agent.setRdaGr2ProfessionOrOccupation(Map.of("profession_label", List.of("lawyer", "chef"))); + + // the actual method to test + agentSolrCreator.addToDocument(solrInputDocument, agent); + + // assertions + assertTrue(solrInputDocument.containsKey(EdmLabel.EDM_AGENT.toString())); + assertTrue(solrInputDocument.containsKey(EdmLabel.AG_SKOS_PREF_LABEL + ".pref_label")); + assertTrue(solrInputDocument.containsKey(EdmLabel.AG_SKOS_ALT_LABEL + ".alt_label")); + assertTrue(solrInputDocument.containsKey(EdmLabel.AG_FOAF_NAME + ".foaf_name")); + assertTrue(solrInputDocument.containsKey(EdmLabel.AG_RDAGR2_DATEOFBIRTH + ".date_of_birth_label")); + assertTrue(solrInputDocument.containsKey(EdmLabel.AG_RDAGR2_DATEOFDEATH + ".date_of_death_label")); + assertTrue(solrInputDocument.containsKey(EdmLabel.AG_RDAGR2_PLACEOFBIRTH + ".place_of_birth_label")); + assertTrue(solrInputDocument.containsKey(EdmLabel.AG_RDAGR2_PLACEOFDEATH + ".place_of_death_label")); + assertTrue(solrInputDocument.containsKey(EdmLabel.AG_RDAGR2_PROFESSIONOROCCUPATION + ".profession_label")); + + assertEquals("About Agent", solrInputDocument.getFieldValue(EdmLabel.EDM_AGENT.toString())); + assertEquals(Arrays.asList("val1", "val2"), solrInputDocument.getFieldValues(EdmLabel.AG_SKOS_PREF_LABEL + ".pref_label")); + assertEquals(Arrays.asList("val1", "val2"), solrInputDocument.getFieldValues(EdmLabel.AG_SKOS_ALT_LABEL + ".alt_label")); + assertEquals(Arrays.asList("val1", "val2"), solrInputDocument.getFieldValues(EdmLabel.AG_FOAF_NAME + ".foaf_name")); + assertEquals(Arrays.asList("some_date"), + solrInputDocument.getFieldValues(EdmLabel.AG_RDAGR2_DATEOFBIRTH + ".date_of_birth_label")); + assertEquals(Arrays.asList("other_date"), + solrInputDocument.getFieldValues(EdmLabel.AG_RDAGR2_DATEOFDEATH + ".date_of_death_label")); + assertEquals(Arrays.asList("some_place"), + solrInputDocument.getFieldValues(EdmLabel.AG_RDAGR2_PLACEOFBIRTH + ".place_of_birth_label")); + assertEquals(Arrays.asList("other_place"), + solrInputDocument.getFieldValues(EdmLabel.AG_RDAGR2_PLACEOFDEATH + ".place_of_death_label")); + assertEquals(Arrays.asList("lawyer", "chef"), + solrInputDocument.getFieldValues(EdmLabel.AG_RDAGR2_PROFESSIONOROCCUPATION + ".profession_label")); + + assertEquals(9, solrInputDocument.size()); + + } + +} From ea976236c0cc072a5450c314b831fb6e6623d1d0 Mon Sep 17 00:00:00 2001 From: JoanaCMS <70145179+JoanaCMS@users.noreply.github.com> Date: Thu, 2 Jun 2022 15:53:55 +0200 Subject: [PATCH 48/73] MET-4473 Support 3d tier calculation for sandbox (#549) * MET-4473 Implemented changes to include new fields for content tier breakdown * MET-4581 Created builder for ContentTierBreakdown * MET-4581 Fixed unit tests * MET-4581 Removed Code smells --- .../tiers/media/AbstractMediaClassifier.java | 18 +- .../indexing/tiers/media/MediaClassifier.java | 5 +- .../tiers/view/ContentTierBreakdown.java | 213 ++++++++++++------ .../tiers/view/ContentTierBreakdownTest.java | 16 +- .../view/RecordTierCalculationViewTest.java | 13 +- 5 files changed, 179 insertions(+), 86 deletions(-) diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/tiers/media/AbstractMediaClassifier.java b/metis-indexing/src/main/java/eu/europeana/indexing/tiers/media/AbstractMediaClassifier.java index 7f278981b..c9192cd81 100644 --- a/metis-indexing/src/main/java/eu/europeana/indexing/tiers/media/AbstractMediaClassifier.java +++ b/metis-indexing/src/main/java/eu/europeana/indexing/tiers/media/AbstractMediaClassifier.java @@ -33,8 +33,9 @@ public final TierClassification classify(RdfWra // Look at the entity as a whole: we may classify without considering the web resources. final MediaTier entityTier = preClassifyEntity(entity); if (entityTier != null) { - return new TierClassification<>(entityTier, new ContentTierBreakdown(null, null, false, - false, false, Collections.emptyList())); + return new TierClassification<>(entityTier, new ContentTierBreakdown.Builder() + .setMediaResourceTechnicalMetadataList(Collections.emptyList()) + .build()); } // Find candidate web resources @@ -62,9 +63,18 @@ public final TierClassification classify(RdfWra .orElse(MediaTier.T0); mediaResourceTechnicalMetadataList = descendingMediaResourceTechnicalMetadata; } + MediaType mediaTypeResult = getMediaType(); + boolean hasMediaResource3DAvailable = mediaTypeResult == MediaType.THREE_D && (mediaTier != MediaTier.T0 && mediaTier != MediaTier.T1); + + final ContentTierBreakdown contentTierBreakdown = new ContentTierBreakdown.Builder() + .setRecordType(mediaTypeResult) + .setLicenseType(entityLicenseType) + .setThumbnailAvailable(hasThumbnails) + .setLandingPageAvailable(hasLandingPage) + .setMediaResource3DAvailable(hasMediaResource3DAvailable) + .setEmbeddableMediaAvailable(hasEmbeddableMedia) + .setMediaResourceTechnicalMetadataList(mediaResourceTechnicalMetadataList).build(); - final ContentTierBreakdown contentTierBreakdown = new ContentTierBreakdown(getMediaType(), entityLicenseType, hasThumbnails, - hasLandingPage, hasEmbeddableMedia, mediaResourceTechnicalMetadataList); return new TierClassification<>(mediaTier, contentTierBreakdown); } diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/tiers/media/MediaClassifier.java b/metis-indexing/src/main/java/eu/europeana/indexing/tiers/media/MediaClassifier.java index 6030a181f..3d2a950f7 100644 --- a/metis-indexing/src/main/java/eu/europeana/indexing/tiers/media/MediaClassifier.java +++ b/metis-indexing/src/main/java/eu/europeana/indexing/tiers/media/MediaClassifier.java @@ -50,8 +50,9 @@ public MediaClassifier() { public TierClassification classify(RdfWrapper entity) { final TierClassifier deferredClassifier = getDeferredClassifier(entity.getEdmType()); if (deferredClassifier == null) { - return new TierClassification<>(MediaTier.T0, new ContentTierBreakdown(null, null, false, - false, false, Collections.emptyList())); + return new TierClassification<>(MediaTier.T0, new ContentTierBreakdown.Builder() + .setMediaResourceTechnicalMetadataList(Collections.emptyList()) + .build()); } return deferredClassifier.classify(entity); } diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/tiers/view/ContentTierBreakdown.java b/metis-indexing/src/main/java/eu/europeana/indexing/tiers/view/ContentTierBreakdown.java index ea4f74bda..3e780a265 100644 --- a/metis-indexing/src/main/java/eu/europeana/indexing/tiers/view/ContentTierBreakdown.java +++ b/metis-indexing/src/main/java/eu/europeana/indexing/tiers/view/ContentTierBreakdown.java @@ -2,6 +2,7 @@ import eu.europeana.indexing.utils.LicenseType; import eu.europeana.metis.schema.model.MediaType; + import java.util.ArrayList; import java.util.List; @@ -10,83 +11,147 @@ */ public class ContentTierBreakdown { - private final MediaType recordType; - private final LicenseType licenseType; - private final boolean thumbnailAvailable; - private final boolean landingPageAvailable; - private final boolean embeddableMediaAvailable; - private final List mediaResourceTechnicalMetadataList; - private final List processingErrorsList; + private final MediaType recordType; + private final LicenseType licenseType; + private final boolean thumbnailAvailable; + private final boolean landingPageAvailable; + private final boolean mediaResource3DAvailable; + private final boolean embeddableMediaAvailable; + private final List mediaResourceTechnicalMetadataList; + private final List processingErrorsList; - /** - * Constructor with required parameters. - * - * @param recordType the record media type - * @param licenseType the license type - * @param thumbnailAvailable the flag indicating if a thumbnails is available - * @param landingPageAvailable the flag indicating if a page is available - * @param embeddableMediaAvailable the flag indicating if embeddable media are available - * @param mediaResourceTechnicalMetadataList the list of media resource technical metadata - */ - public ContentTierBreakdown(MediaType recordType, LicenseType licenseType, boolean thumbnailAvailable, - boolean landingPageAvailable, boolean embeddableMediaAvailable, - List mediaResourceTechnicalMetadataList) { - this(recordType, licenseType, thumbnailAvailable, landingPageAvailable, embeddableMediaAvailable, - mediaResourceTechnicalMetadataList, null); - } + /** + * Constructor with required parameters. + *

    It creates a copy of the content tier breakdown extended with the processing errors list.

    + * + * @param contentTierBreakdown the content tier breakdown + * @param processingErrorsList the processing errors list + */ + public ContentTierBreakdown(ContentTierBreakdown contentTierBreakdown, List processingErrorsList) { + this(new Builder() + .setRecordType(contentTierBreakdown.getRecordType()) + .setLicenseType(contentTierBreakdown.getLicenseType()) + .setThumbnailAvailable(contentTierBreakdown.isThumbnailAvailable()) + .setLandingPageAvailable(contentTierBreakdown.isLandingPageAvailable()) + .setMediaResource3DAvailable(contentTierBreakdown.isMediaResource3DAvailable()) + .setEmbeddableMediaAvailable(contentTierBreakdown.isEmbeddableMediaAvailable()) + .setMediaResourceTechnicalMetadataList(contentTierBreakdown.mediaResourceTechnicalMetadataList) + .setProcessingErrorsList(processingErrorsList)); + } + + private ContentTierBreakdown(Builder builder) { + this.recordType = builder.recordType; + this.licenseType = builder.licenseType; + this.thumbnailAvailable = builder.thumbnailAvailable; + this.landingPageAvailable = builder.landingPageAvailable; + this.mediaResource3DAvailable = builder.mediaResource3DAvailable; + this.embeddableMediaAvailable = builder.embeddableMediaAvailable; + this.mediaResourceTechnicalMetadataList = + builder.mediaResourceTechnicalMetadataList == null ? new ArrayList<>() : new ArrayList<>(builder.mediaResourceTechnicalMetadataList); + this.processingErrorsList = builder.processingErrorsList == null ? new ArrayList<>() : new ArrayList<>(builder.processingErrorsList); + } + + public MediaType getRecordType() { + return recordType; + } + + public LicenseType getLicenseType() { + return licenseType; + } + + public boolean isThumbnailAvailable() { + return thumbnailAvailable; + } + + public boolean isLandingPageAvailable() { + return landingPageAvailable; + } + + public boolean isMediaResource3DAvailable() { + return mediaResource3DAvailable; + } + + public boolean isEmbeddableMediaAvailable() { + return embeddableMediaAvailable; + } + + public List getMediaResourceTechnicalMetadataList() { + return new ArrayList<>(mediaResourceTechnicalMetadataList); + } + + public List getProcessingErrorsList() { + return new ArrayList<>(processingErrorsList); + } /** - * Constructor with required parameters. - *

    It creates a copy of the content tier breakdown extended with the processing errors list.

    - * - * @param contentTierBreakdown the content tier breakdown - * @param processingErrorsList the processing errors list + * Builder class of ContentTierBreakdown. Used to make the build of ContentTierBreakdown easier */ - public ContentTierBreakdown(ContentTierBreakdown contentTierBreakdown, List processingErrorsList) { - this(contentTierBreakdown.getRecordType(), contentTierBreakdown.getLicenseType(), contentTierBreakdown.isThumbnailAvailable(), - contentTierBreakdown.isLandingPageAvailable(), contentTierBreakdown.isEmbeddableMediaAvailable(), - contentTierBreakdown.mediaResourceTechnicalMetadataList, processingErrorsList); - } - - private ContentTierBreakdown(MediaType recordType, LicenseType licenseType, boolean thumbnailAvailable, - boolean landingPageAvailable, boolean embeddableMediaAvailable, - List mediaResourceTechnicalMetadataList, List processingErrorsList) { - this.recordType = recordType; - this.licenseType = licenseType; - this.thumbnailAvailable = thumbnailAvailable; - this.landingPageAvailable = landingPageAvailable; - this.embeddableMediaAvailable = embeddableMediaAvailable; - this.mediaResourceTechnicalMetadataList = - mediaResourceTechnicalMetadataList == null ? new ArrayList<>() : new ArrayList<>(mediaResourceTechnicalMetadataList); - this.processingErrorsList = processingErrorsList == null ? new ArrayList<>() : new ArrayList<>(processingErrorsList); - } - - public MediaType getRecordType() { - return recordType; - } - - public LicenseType getLicenseType() { - return licenseType; - } - - public boolean isThumbnailAvailable() { - return thumbnailAvailable; - } - - public boolean isLandingPageAvailable() { - return landingPageAvailable; - } - - public boolean isEmbeddableMediaAvailable() { - return embeddableMediaAvailable; - } - - public List getMediaResourceTechnicalMetadataList() { - return new ArrayList<>(mediaResourceTechnicalMetadataList); - } - - public List getProcessingErrorsList() { - return new ArrayList<>(processingErrorsList); - } + public static class Builder { + private MediaType recordType; + private LicenseType licenseType; + private boolean thumbnailAvailable; + private boolean landingPageAvailable; + private boolean mediaResource3DAvailable; + private boolean embeddableMediaAvailable; + private List mediaResourceTechnicalMetadataList; + private List processingErrorsList; + + /** + * Constructor of the object's builder + */ + public Builder() { + //There are no mandatory values + } + + public Builder setRecordType(MediaType recordType) { + this.recordType = recordType; + return this; + } + + public Builder setLicenseType(LicenseType licenseType) { + this.licenseType = licenseType; + return this; + } + + public Builder setThumbnailAvailable(boolean thumbnailAvailable) { + this.thumbnailAvailable = thumbnailAvailable; + return this; + } + + public Builder setLandingPageAvailable(boolean landingPageAvailable) { + this.landingPageAvailable = landingPageAvailable; + return this; + } + + public Builder setMediaResource3DAvailable(boolean mediaResource3DAvailable) { + this.mediaResource3DAvailable = mediaResource3DAvailable; + return this; + } + + public Builder setEmbeddableMediaAvailable(boolean embeddableMediaAvailable) { + this.embeddableMediaAvailable = embeddableMediaAvailable; + return this; + } + + public Builder setMediaResourceTechnicalMetadataList(List mediaResourceTechnicalMetadataList) { + this.mediaResourceTechnicalMetadataList = mediaResourceTechnicalMetadataList == null ? new ArrayList<>() : + new ArrayList<>(mediaResourceTechnicalMetadataList); + return this; + } + + public Builder setProcessingErrorsList(List processingErrorsList) { + this.processingErrorsList = processingErrorsList == null ? new ArrayList<>() : new ArrayList<>(processingErrorsList); + return this; + } + + /** + * Creates a new ContentTierBreakdown object based on the values that were set from the builder + * + * @return A new instance of ContentTierBreakdown object + */ + public ContentTierBreakdown build() { + return new ContentTierBreakdown(this); + } + } } diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/ContentTierBreakdownTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/ContentTierBreakdownTest.java index 400b12946..4378062b2 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/ContentTierBreakdownTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/ContentTierBreakdownTest.java @@ -22,6 +22,7 @@ void objectCreationTest_Without_Processing_Errors_Parameter() { final LicenseType licenseType = LicenseType.OPEN; final boolean thumbnailAvailable = true; final boolean landingPageAvailable = true; + final boolean mediaResource3DAvailable = true; final boolean embeddableMediaAvailable = true; final MediaResourceTechnicalMetadata mediaResourceTechnicalMetadata1 = new MediaResourceTechnicalMetadataBuilder( new ResolutionTierMetadataBuilder().build()).setResourceUrl("resourceUrl").setMediaType(MediaType.IMAGE) @@ -36,13 +37,21 @@ void objectCreationTest_Without_Processing_Errors_Parameter() { final List mediaResourceTechnicalMetadataList = List.of( mediaResourceTechnicalMetadata1, mediaResourceTechnicalMetadata2); - final ContentTierBreakdown contentTierBreakdown = new ContentTierBreakdown(recordType, licenseType, thumbnailAvailable, - landingPageAvailable, embeddableMediaAvailable, mediaResourceTechnicalMetadataList); + final ContentTierBreakdown contentTierBreakdown = new ContentTierBreakdown.Builder() + .setRecordType(recordType) + .setLicenseType(licenseType) + .setThumbnailAvailable(thumbnailAvailable) + .setLandingPageAvailable(landingPageAvailable) + .setMediaResource3DAvailable(mediaResource3DAvailable) + .setEmbeddableMediaAvailable(embeddableMediaAvailable) + .setMediaResourceTechnicalMetadataList(mediaResourceTechnicalMetadataList) + .build(); assertEquals(recordType, contentTierBreakdown.getRecordType()); assertEquals(licenseType, contentTierBreakdown.getLicenseType()); assertEquals(thumbnailAvailable, contentTierBreakdown.isThumbnailAvailable()); assertEquals(landingPageAvailable, contentTierBreakdown.isLandingPageAvailable()); + assertEquals(mediaResource3DAvailable, contentTierBreakdown.isMediaResource3DAvailable()); assertEquals(embeddableMediaAvailable, contentTierBreakdown.isEmbeddableMediaAvailable()); assertNotSame(mediaResourceTechnicalMetadataList, contentTierBreakdown.getMediaResourceTechnicalMetadataList()); assertTrue(CollectionUtils.isEqualCollection(mediaResourceTechnicalMetadataList, @@ -61,10 +70,11 @@ void objectCreationTest_Without_Processing_Errors_Parameter() { assertEquals(licenseType, contentTierBreakdown.getLicenseType()); assertEquals(thumbnailAvailable, contentTierBreakdown.isThumbnailAvailable()); assertEquals(landingPageAvailable, contentTierBreakdown.isLandingPageAvailable()); + assertEquals(mediaResource3DAvailable, contentTierBreakdown.isMediaResource3DAvailable()); assertEquals(embeddableMediaAvailable, contentTierBreakdown.isEmbeddableMediaAvailable()); assertNotSame(mediaResourceTechnicalMetadataList, contentTierBreakdown.getMediaResourceTechnicalMetadataList()); assertTrue(CollectionUtils.isEqualCollection(mediaResourceTechnicalMetadataList, contentTierBreakdown.getMediaResourceTechnicalMetadataList())); } -} \ No newline at end of file +} diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/RecordTierCalculationViewTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/RecordTierCalculationViewTest.java index 8ba2f794c..4df07b540 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/RecordTierCalculationViewTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/RecordTierCalculationViewTest.java @@ -44,8 +44,15 @@ void objectCreationTest() { .build(); final RecordTierCalculationView recordTierCalculationView = new RecordTierCalculationView(new RecordTierCalculationSummary(), - new ContentTierBreakdown(MediaType.AUDIO, LicenseType.OPEN, true, - true, true, Collections.singletonList(mediaResourceTechnicalMetadata)), + new ContentTierBreakdown.Builder() + .setRecordType(MediaType.AUDIO) + .setLicenseType(LicenseType.OPEN) + .setThumbnailAvailable(true) + .setLandingPageAvailable(true) + .setMediaResource3DAvailable(true) + .setEmbeddableMediaAvailable(true) + .setMediaResourceTechnicalMetadataList(Collections.singletonList(mediaResourceTechnicalMetadata)) + .build(), new MetadataTierBreakdown(languageBreakdown, enablingElementsBreakdown, contextualClassesBreakdown)); @@ -54,4 +61,4 @@ void objectCreationTest() { assertNotNull(recordTierCalculationView.getMetadataTierBreakdown()); } -} \ No newline at end of file +} From 7bf643683caf0ccab50e8fb2f5dd0133069e7e97 Mon Sep 17 00:00:00 2001 From: Adolfo Peixinho <91942157+adolfopeixinho@users.noreply.github.com> Date: Tue, 7 Jun 2022 18:19:22 +0200 Subject: [PATCH 49/73] MET-4573 unit tests for PlaceSolrCreator (#552) * MET-4573 unit tests for PlaceSolrCreator * MET-4573 cleanup unnecessary initializations. First code review, added more test for WGS84 missing latitude, longitude --- .../solr/property/PlaceSolrCreatorTest.java | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 metis-indexing/src/test/java/eu/europeana/indexing/solr/property/PlaceSolrCreatorTest.java diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/PlaceSolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/PlaceSolrCreatorTest.java new file mode 100644 index 000000000..56d685f6b --- /dev/null +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/PlaceSolrCreatorTest.java @@ -0,0 +1,154 @@ +package eu.europeana.indexing.solr.property; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import eu.europeana.corelib.definitions.edm.entity.Place; +import eu.europeana.corelib.solr.entity.PlaceImpl; +import eu.europeana.indexing.solr.EdmLabel; +import java.util.List; +import java.util.Map; +import org.apache.solr.common.SolrInputDocument; +import org.bson.types.ObjectId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class PlaceSolrCreatorTest { + + private SolrInputDocument solrInputDocument; + private PlaceSolrCreator placeSolrCreator; + private Place place; + + @BeforeEach + void setup() { + solrInputDocument = new SolrInputDocument(); + placeSolrCreator = new PlaceSolrCreator(); + place = new PlaceImpl(); + } + + @Test + void addToDocumentWithPlaceSolrCreator() { + + place.setId(new ObjectId(String.valueOf(ObjectId.get()))); + place.setAbout("Netherlands"); + place.setPrefLabel(Map.of("Municipalities", List.of("place", "another place", "really nice place"))); + place.setAltLabel(Map.of("Region", List.of("altLabel1", "altLabel2"))); + place.setAltitude(22.5f); + place.setLatitude(44.5f); + place.setLongitude(55.5f); + + // the method to test + placeSolrCreator.addToDocument(solrInputDocument, place); + + // assertions + assertTrue(solrInputDocument.containsKey(EdmLabel.EDM_PLACE.toString())); + assertTrue(solrInputDocument.containsKey(EdmLabel.PL_SKOS_PREF_LABEL + ".Municipalities")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PL_SKOS_ALT_LABEL + ".Region")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PL_WGS84_POS_LAT.toString())); + assertTrue(solrInputDocument.containsKey(EdmLabel.PL_WGS84_POS_LONG.toString())); + assertTrue(solrInputDocument.containsKey(EdmLabel.PL_WGS84_POS_ALT.toString())); + + assertEquals("Netherlands", solrInputDocument.getFieldValue(EdmLabel.EDM_PLACE.toString())); + assertEquals("place", solrInputDocument.getFieldValue(EdmLabel.PL_SKOS_PREF_LABEL + ".Municipalities")); + assertEquals("altLabel1", solrInputDocument.getFieldValue(EdmLabel.PL_SKOS_ALT_LABEL + ".Region")); + assertEquals(44.5f, solrInputDocument.getFieldValue(EdmLabel.PL_WGS84_POS_LAT.toString())); + assertEquals(55.5f, solrInputDocument.getFieldValue(EdmLabel.PL_WGS84_POS_LONG.toString())); + assertEquals(22.5f, solrInputDocument.getFieldValue(EdmLabel.PL_WGS84_POS_ALT.toString())); + + } + + @Test + void addToDocumentWithPlaceSolrCreator_withEmptyAltitude() { + + place.setId(new ObjectId(String.valueOf(ObjectId.get()))); + place.setAbout("Netherlands"); + place.setPrefLabel(Map.of("Municipalities", List.of("place", "another place", "really nice place"))); + place.setAltLabel(Map.of("Region", List.of("altLabel1", "altLabel2"))); + place.setLatitude(44.5f); + place.setLongitude(55.5f); + + // the method to test + placeSolrCreator.addToDocument(solrInputDocument, place); + + // assertions + assertTrue(solrInputDocument.containsKey(EdmLabel.EDM_PLACE.toString())); + assertTrue(solrInputDocument.containsKey(EdmLabel.PL_SKOS_PREF_LABEL + ".Municipalities")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PL_SKOS_ALT_LABEL + ".Region")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PL_WGS84_POS_LAT.toString())); + assertTrue(solrInputDocument.containsKey(EdmLabel.PL_WGS84_POS_LONG.toString())); + assertFalse(solrInputDocument.containsKey(EdmLabel.PL_WGS84_POS_ALT.toString())); + + assertEquals("Netherlands", solrInputDocument.getFieldValue(EdmLabel.EDM_PLACE.toString())); + assertEquals("place", solrInputDocument.getFieldValue(EdmLabel.PL_SKOS_PREF_LABEL + ".Municipalities")); + assertEquals("altLabel1", solrInputDocument.getFieldValue(EdmLabel.PL_SKOS_ALT_LABEL + ".Region")); + assertEquals(44.5f, solrInputDocument.getFieldValue(EdmLabel.PL_WGS84_POS_LAT.toString())); + assertEquals(55.5f, solrInputDocument.getFieldValue(EdmLabel.PL_WGS84_POS_LONG.toString())); + assertNull(solrInputDocument.getFieldValue(EdmLabel.PL_WGS84_POS_ALT.toString())); + + } + + @Test + void addToDocumentWithPlaceSolrCreator_withEmptyLatitude() { + // according to the specification of the World Geodetic System 1984, the latitude and longitude are mandatory + // so if not present the labels are not added to the solr document, even if they have valid values + place.setId(new ObjectId(String.valueOf(ObjectId.get()))); + place.setAbout("Netherlands"); + place.setPrefLabel(Map.of("Municipalities", List.of("place", "another place", "really nice place"))); + place.setAltLabel(Map.of("Region", List.of("altLabel1", "altLabel2"))); + place.setAltitude(22.5f); + place.setLongitude(55.5f); + + // the method to test + placeSolrCreator.addToDocument(solrInputDocument, place); + + // assertions + assertTrue(solrInputDocument.containsKey(EdmLabel.EDM_PLACE.toString())); + assertTrue(solrInputDocument.containsKey(EdmLabel.PL_SKOS_PREF_LABEL + ".Municipalities")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PL_SKOS_ALT_LABEL + ".Region")); + assertFalse(solrInputDocument.containsKey(EdmLabel.PL_WGS84_POS_LAT.toString())); + assertFalse(solrInputDocument.containsKey(EdmLabel.PL_WGS84_POS_LONG.toString())); + assertFalse(solrInputDocument.containsKey(EdmLabel.PL_WGS84_POS_ALT.toString())); + + assertEquals("Netherlands", solrInputDocument.getFieldValue(EdmLabel.EDM_PLACE.toString())); + assertEquals("place", solrInputDocument.getFieldValue(EdmLabel.PL_SKOS_PREF_LABEL + ".Municipalities")); + assertEquals("altLabel1", solrInputDocument.getFieldValue(EdmLabel.PL_SKOS_ALT_LABEL + ".Region")); + assertNull(solrInputDocument.getFieldValue(EdmLabel.PL_WGS84_POS_LAT.toString())); + assertNull(solrInputDocument.getFieldValue(EdmLabel.PL_WGS84_POS_LONG.toString())); + assertNull(solrInputDocument.getFieldValue(EdmLabel.PL_WGS84_POS_ALT.toString())); + + } + + @Test + void addToDocumentWithPlaceSolrCreator_withEmptyLongitude() { + // according to the specification of the World Geodetic System 1984, the latitude and longitude are mandatory + // so if not present the labels are not added to the solr document, even if they have valid values + place.setId(new ObjectId(String.valueOf(ObjectId.get()))); + place.setAbout("Netherlands"); + place.setPrefLabel(Map.of("Municipalities", List.of("place", "another place", "really nice place"))); + place.setAltLabel(Map.of("Region", List.of("altLabel1", "altLabel2"))); + place.setAltitude(22.5f); + place.setLatitude(44.5f); + + // the method to test + placeSolrCreator.addToDocument(solrInputDocument, place); + + // assertions + assertTrue(solrInputDocument.containsKey(EdmLabel.EDM_PLACE.toString())); + assertTrue(solrInputDocument.containsKey(EdmLabel.PL_SKOS_PREF_LABEL + ".Municipalities")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PL_SKOS_ALT_LABEL + ".Region")); + assertFalse(solrInputDocument.containsKey(EdmLabel.PL_WGS84_POS_LAT.toString())); + assertFalse(solrInputDocument.containsKey(EdmLabel.PL_WGS84_POS_LONG.toString())); + assertFalse(solrInputDocument.containsKey(EdmLabel.PL_WGS84_POS_ALT.toString())); + + assertEquals("Netherlands", solrInputDocument.getFieldValue(EdmLabel.EDM_PLACE.toString())); + assertEquals("place", solrInputDocument.getFieldValue(EdmLabel.PL_SKOS_PREF_LABEL + ".Municipalities")); + assertEquals("altLabel1", solrInputDocument.getFieldValue(EdmLabel.PL_SKOS_ALT_LABEL + ".Region")); + assertNull(solrInputDocument.getFieldValue(EdmLabel.PL_WGS84_POS_LAT.toString())); + assertNull(solrInputDocument.getFieldValue(EdmLabel.PL_WGS84_POS_LONG.toString())); + assertNull(solrInputDocument.getFieldValue(EdmLabel.PL_WGS84_POS_ALT.toString())); + + } + +} From 3e47ef071ff22e60f6672099cabc24d5c53ae052 Mon Sep 17 00:00:00 2001 From: Jorge Ortiz Date: Tue, 7 Jun 2022 18:20:26 +0200 Subject: [PATCH 50/73] MET-4440_MET-4567 unit tests ProvidedChoSolrCreator (#543) --- .../property/ProvidedChoSolrCreatorTest.java | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ProvidedChoSolrCreatorTest.java diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ProvidedChoSolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ProvidedChoSolrCreatorTest.java new file mode 100644 index 000000000..2f4488502 --- /dev/null +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ProvidedChoSolrCreatorTest.java @@ -0,0 +1,55 @@ +package eu.europeana.indexing.solr.property; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import eu.europeana.corelib.solr.entity.ProvidedCHOImpl; +import eu.europeana.indexing.solr.EdmLabel; +import org.apache.solr.common.SolrInputDocument; +import org.bson.types.ObjectId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit test for {@link ProvidedChoSolrCreator} class + */ +class ProvidedChoSolrCreatorTest { + + private ProvidedChoSolrCreator providedChoSolrCreator; + private SolrInputDocument solrInputDocument; + private ProvidedCHOImpl providedCHO; + + @BeforeEach + void setup() { + solrInputDocument = new SolrInputDocument(); + providedCHO = new ProvidedCHOImpl(); + providedChoSolrCreator = new ProvidedChoSolrCreator(); + } + + @Test + void addProvidedChoToDocument_withAbout() { + providedCHO.setId(new ObjectId(String.valueOf(ObjectId.get()))); + providedCHO.setAbout("about"); + providedCHO.setOwlSameAs(new String[]{"data1", "data2"}); + + providedChoSolrCreator.addToDocument(solrInputDocument, providedCHO); + + assertTrue(solrInputDocument.containsKey(EdmLabel.EUROPEANA_ID.toString())); + assertEquals("about", solrInputDocument.getFieldValue(EdmLabel.EUROPEANA_ID.toString())); + assertEquals(1, solrInputDocument.size()); + } + + @Test + void addProvidedChoToDocument_withoutAbout() { + providedCHO.setId(new ObjectId(String.valueOf(ObjectId.get()))); + providedCHO.setOwlSameAs(new String[]{"data1", "data2"}); + + providedChoSolrCreator.addToDocument(solrInputDocument, providedCHO); + + assertFalse(solrInputDocument.containsKey(EdmLabel.EUROPEANA_ID.toString())); + assertNull(solrInputDocument.getFieldValue(EdmLabel.EUROPEANA_ID.toString())); + assertEquals(0, solrInputDocument.size()); + } +} From afa70c15ffc21f822a94a0f29046406980fd6947 Mon Sep 17 00:00:00 2001 From: Jorge Ortiz Date: Tue, 7 Jun 2022 18:20:42 +0200 Subject: [PATCH 51/73] MET-4440_MET-4574 add ServiceSolrCreator unit tests (#544) * MET-4440_MET-4574 unit tests ServiceSolrCreator * MET-4440_MET-4574 include edmlabels keys check --- .../solr/property/ServiceSolrCreatorTest.java | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ServiceSolrCreatorTest.java diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ServiceSolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ServiceSolrCreatorTest.java new file mode 100644 index 000000000..794463109 --- /dev/null +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ServiceSolrCreatorTest.java @@ -0,0 +1,87 @@ +package eu.europeana.indexing.solr.property; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import eu.europeana.corelib.solr.entity.ServiceImpl; +import eu.europeana.indexing.solr.EdmLabel; +import java.util.List; +import org.apache.solr.common.SolrInputDocument; +import org.bson.types.ObjectId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit test for {@link ServiceSolrCreator} class + */ +class ServiceSolrCreatorTest { + + private ServiceSolrCreator serviceSolrCreator; + private SolrInputDocument solrInputDocument; + private ServiceImpl service; + + @BeforeEach + void setup() { + solrInputDocument = new SolrInputDocument(); + serviceSolrCreator = new ServiceSolrCreator(); + service = new ServiceImpl(); + } + + @Test + void addToDocument_withServiceAndWithDcTerms() { + service.setId(new ObjectId("6294c725de3fe70c48362a88")); + service.setAbout("service"); + service.setDcTermsConformsTo(new String[]{"data1", "data2"}); + + serviceSolrCreator.addToDocument(solrInputDocument, service); + + assertTrue(solrInputDocument.containsKey(EdmLabel.SV_SERVICE.toString()) && + solrInputDocument.containsKey(EdmLabel.SV_DCTERMS_CONFORMS_TO.toString())); + assertEquals("service", solrInputDocument.getFieldValue(EdmLabel.SV_SERVICE.toString())); + assertEquals(List.of("data1", "data2"), solrInputDocument.getFieldValues(EdmLabel.SV_DCTERMS_CONFORMS_TO.toString())); + assertEquals(2, solrInputDocument.size()); + } + + @Test + void addToDocument_withServiceAndWithoutDcTerms() { + service.setId(new ObjectId("6294c725de3fe70c48362a88")); + service.setAbout("service"); + + serviceSolrCreator.addToDocument(solrInputDocument, service); + + assertTrue(solrInputDocument.containsKey(EdmLabel.SV_SERVICE.toString())); + assertFalse(solrInputDocument.containsKey(EdmLabel.SV_DCTERMS_CONFORMS_TO.toString())); + assertEquals("service", solrInputDocument.getFieldValue(EdmLabel.SV_SERVICE.toString())); + assertNull(solrInputDocument.getFieldValues(EdmLabel.SV_DCTERMS_CONFORMS_TO.toString())); + assertEquals(1, solrInputDocument.size()); + } + + @Test + void addToDocument_withoutServiceAndWithDcTerms() { + service.setId(new ObjectId("6294c725de3fe70c48362a88")); + service.setDcTermsConformsTo(new String[]{"data1", "data2"}); + + serviceSolrCreator.addToDocument(solrInputDocument, service); + + assertFalse(solrInputDocument.containsKey(EdmLabel.SV_SERVICE.toString())); + assertTrue(solrInputDocument.containsKey(EdmLabel.SV_DCTERMS_CONFORMS_TO.toString())); + assertNull(solrInputDocument.getFieldValue(EdmLabel.SV_SERVICE.toString())); + assertEquals(List.of("data1", "data2"), solrInputDocument.getFieldValues(EdmLabel.SV_DCTERMS_CONFORMS_TO.toString())); + assertEquals(1, solrInputDocument.size()); + } + + @Test + void addToDocument_withoutServiceAndWithoutDcTerms() { + service.setId(new ObjectId("6294c725de3fe70c48362a88")); + + serviceSolrCreator.addToDocument(solrInputDocument, service); + + assertFalse(solrInputDocument.containsKey(EdmLabel.SV_SERVICE.toString())); + assertFalse(solrInputDocument.containsKey(EdmLabel.SV_DCTERMS_CONFORMS_TO.toString())); + assertNull(solrInputDocument.getFieldValue(EdmLabel.SV_SERVICE.toString())); + assertNull(solrInputDocument.getFieldValues(EdmLabel.SV_DCTERMS_CONFORMS_TO.toString())); + assertEquals(0, solrInputDocument.size()); + } +} From adf719bbbb2d5820fc16b3584f92f2c5ae2a0343 Mon Sep 17 00:00:00 2001 From: SrishtiSingh-eu <52709142+SrishtiSingh-eu@users.noreply.github.com> Date: Wed, 8 Jun 2022 14:55:28 +0200 Subject: [PATCH 52/73] Ea 2890 entity client implmentation (#522) * EA-2891 added spring aop profiling for enrichment * Ea-2891 remove annotations * Ea-2891 removing the test * Ea-2890 new implmentation * Ea-2890 small tweak * Ea-2890 Moved the EntityClientRequest in metis * Revert "Ea 2891 profing for enrichment" * Ea-2890 few tweaks, resolved dependencies * Ea-2890 added null check and invalid value tests * Ea-2890 fix organisation mapping * EA-2890 added rest endpoints for entity client enrichment * EA-2890 removing api-commons constants * EA-2890 removing api-commons constants * EA-2890 fix swagger dependencies, add seperate swaggerConfig class to resolve the issue * EA-2890 add parent entities logic * EA-2893 add new enrich method changes * EA-2890 replace current Impl in controller with new Resolver * EA-2890 add junit and check for text and uri search for parent entity * EA-2890 FIX jackson jabx error * EA-2983 fix error response of text search * EA-2984 Review suggestion - refactoring * EA-2891 Fix poms and log4j2.xml * EA-2891 Fix EnrichmentService, make it pluggable, fix tests * EA-2891 Remove organization implementation * EA-2891 Rename ClientEntityResolverITTest * EA-2891 Move EntityResolverUtilsTest * EA-2891 Cleanup * EA-2891 Add batch size default * EA-2891 Remove EnrichmentQuery * EA-2891 Reusable language code converter * EA-2891 Refactor parent searching * EA-2891 Fix resolve references for id and uri searching * EA-2891 Use regex mapping for europeana entity ids * EA-2891 Cleanup * EA-2891 Make values converter re-usable * EA-2891 Move batches splitting method to interface default * EA-2891 Use apache partitioning for batches * EA-2891 Cleanup * EA-2891 Disable swagger temporarily * EA-2891 Fix commons-collections references * EA-2891 Fix swagger * EA-2891 Write tests for resolveByText * EA-2891 Write tests for resolveByUri * EA-2891 Divide tests * EA-2891 Write test for resolveById * EA-2891 Add test for multiple entities results * EA-2891 Cleanup * EA-2891 Use one property file * EA-2891 Add sanity check on bean creation Co-authored-by: Simon Tzanakis --- .../metis-enrichment-client/.gitignore | 2 - .../dereference/DereferencerProvider.java | 2 +- .../rest/client/enrichment/EnricherImpl.java | 19 +- .../client/enrichment/EnricherProvider.java | 1 + .../dereference/DereferencerImplTest.java | 2 +- .../client/enrichment/EnricherImplTest.java | 1 + .../enrichment/RemoteEntityResolverTest.java | 1 + .../metis-enrichment-common/pom.xml | 42 +- .../external/impl/ClientEntityResolver.java | 242 +++++++++++ .../external/impl}/RemoteEntityResolver.java | 59 ++- .../enrichment/api/external/model/Agent.java | 10 + .../api/external/model/AgentBase.java | 43 ++ .../api/external/model/Concept.java | 24 ++ .../api/external/model/EnrichmentBase.java | 12 + .../api/external/model/Organization.java | 60 ++- .../enrichment/api/external/model/Place.java | 26 ++ .../api/external/model/TimeSpan.java | 29 ++ .../external/model/WikidataOrganization.java | 31 -- .../api/internal/EntityResolver.java | 7 +- .../api/internal/ReferenceTermImpl.java | 6 + .../api/internal/SearchTermImpl.java | 7 + .../utils/EnrichmentBaseConverter.java | 53 ++- .../utils/EntityValuesConverter.java | 118 ++++++ .../utils/LanguageCodeConverter.java | 58 +++ .../impl/ClientEntityResolverTest.java | 397 ++++++++++++++++++ .../utils/LanguageCodeConverterTest.java | 18 + .../metis-enrichment-rest/.gitignore | 4 +- .../metis-enrichment-rest/pom.xml | 21 +- .../enrichment/rest/EnrichmentController.java | 5 +- .../enrichment/rest/config/Application.java | 104 +++-- .../rest/config/EntityResolverType.java | 8 + .../enrichment/rest/config/SwaggerConfig.java | 65 +++ .../resources/enrichment.properties.example | 8 +- .../metis-enrichment-service/.gitignore | 1 - .../enrichment/service/EnrichmentService.java | 97 +---- .../service/PersistentEntityResolver.java | 129 +----- ...chmentTermsToEnrichmentBaseConverter.java} | 151 ++----- .../service/EnrichmentServiceTest.java | 115 +---- .../service/PersistentEntityResolverTest.java | 89 ---- ...ntTermsToEnrichmentBaseConverterTest.java} | 31 +- metis-indexing/pom.xml | 4 + .../solr/property/AggregationSolrCreator.java | 4 +- .../europeana/indexing/utils/RdfWrapper.java | 2 +- ...ablingElementsBreakdownClassifierTest.java | 2 +- .../tiers/view/ContentTierBreakdownTest.java | 2 +- .../view/ContextualClassesBreakdownTest.java | 2 +- .../view/EnablingElementsBreakdownTest.java | 2 +- ...aResourceTechnicalMetadataBuilderTest.java | 2 +- pom.xml | 17 +- 49 files changed, 1455 insertions(+), 680 deletions(-) delete mode 100644 metis-enrichment/metis-enrichment-client/.gitignore create mode 100644 metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/ClientEntityResolver.java rename metis-enrichment/{metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment => metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl}/RemoteEntityResolver.java (80%) delete mode 100644 metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/WikidataOrganization.java create mode 100644 metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/utils/EntityValuesConverter.java create mode 100644 metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/utils/LanguageCodeConverter.java create mode 100644 metis-enrichment/metis-enrichment-common/src/test/java/eu/europeana/enrichment/api/external/impl/ClientEntityResolverTest.java create mode 100644 metis-enrichment/metis-enrichment-common/src/test/java/eu/europeana/enrichment/utils/LanguageCodeConverterTest.java create mode 100644 metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/EntityResolverType.java create mode 100644 metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/SwaggerConfig.java delete mode 100644 metis-enrichment/metis-enrichment-service/.gitignore rename metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/{Converter.java => utils/EnrichmentTermsToEnrichmentBaseConverter.java} (66%) rename metis-enrichment/metis-enrichment-service/src/test/java/eu/europeana/enrichment/service/{ConverterTest.java => utils/EnrichmentTermsToEnrichmentBaseConverterTest.java} (91%) diff --git a/metis-enrichment/metis-enrichment-client/.gitignore b/metis-enrichment/metis-enrichment-client/.gitignore deleted file mode 100644 index e8b90183b..000000000 --- a/metis-enrichment/metis-enrichment-client/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -##Add to ignore to not commit by mistake - diff --git a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/dereference/DereferencerProvider.java b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/dereference/DereferencerProvider.java index 4f1c6f82c..22677591b 100644 --- a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/dereference/DereferencerProvider.java +++ b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/dereference/DereferencerProvider.java @@ -1,7 +1,7 @@ package eu.europeana.enrichment.rest.client.dereference; +import eu.europeana.enrichment.api.external.impl.RemoteEntityResolver; import eu.europeana.enrichment.rest.client.ConnectionProvider; -import eu.europeana.enrichment.rest.client.enrichment.RemoteEntityResolver; import eu.europeana.enrichment.rest.client.exceptions.DereferenceException; import eu.europeana.enrichment.utils.EntityMergeEngine; import java.net.MalformedURLException; diff --git a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/EnricherImpl.java b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/EnricherImpl.java index c84c0abb3..b94b4e8ef 100644 --- a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/EnricherImpl.java +++ b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/EnricherImpl.java @@ -1,5 +1,7 @@ package eu.europeana.enrichment.rest.client.enrichment; +import static eu.europeana.enrichment.api.internal.EntityResolver.europeanaLinkPattern; + import eu.europeana.enrichment.api.external.model.EnrichmentBase; import eu.europeana.enrichment.api.internal.EntityResolver; import eu.europeana.enrichment.api.internal.ProxyFieldType; @@ -19,7 +21,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; @@ -31,13 +32,17 @@ public class EnricherImpl implements Enricher { private static final Logger LOGGER = LoggerFactory.getLogger(EnricherImpl.class); - private static final Pattern europeanaLinkPattern = Pattern - .compile("^https?://data.europeana.eu.*$"); - private final RecordParser recordParser; private final EntityResolver entityResolver; private final EntityMergeEngine entityMergeEngine; + /** + * Constructor with required parameters. + * + * @param recordParser the record parser + * @param entityResolver the entity resolver + * @param entityMergeEngine the entity merge engine + */ public EnricherImpl(RecordParser recordParser, EntityResolver entityResolver, EntityMergeEngine entityMergeEngine) { this.recordParser = recordParser; @@ -113,9 +118,9 @@ public void cleanupPreviousEnrichmentEntities(RDF rdf) { final ProxyType europeanaProxy = RdfEntityUtils.getEuropeanaProxy(rdf); //Find the correct links final Set europeanaLinks = Arrays.stream(ProxyFieldType.values()) - .map(proxyFieldType -> proxyFieldType.extractFieldLinksForEnrichment(europeanaProxy)) - .flatMap(Collection::stream).filter(europeanaLinkPattern.asPredicate()) - .collect(Collectors.toSet()); + .map(proxyFieldType -> proxyFieldType.extractFieldLinksForEnrichment(europeanaProxy)) + .flatMap(Collection::stream).filter(europeanaLinkPattern.asPredicate()) + .collect(Collectors.toSet()); RdfEntityUtils.removeMatchingEntities(rdf, europeanaLinks); } } diff --git a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/EnricherProvider.java b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/EnricherProvider.java index f0aff04aa..ed1c82318 100644 --- a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/EnricherProvider.java +++ b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/EnricherProvider.java @@ -1,5 +1,6 @@ package eu.europeana.enrichment.rest.client.enrichment; +import eu.europeana.enrichment.api.external.impl.RemoteEntityResolver; import eu.europeana.enrichment.api.internal.EntityResolver; import eu.europeana.enrichment.api.internal.RecordParser; import eu.europeana.enrichment.rest.client.ConnectionProvider; diff --git a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/dereference/DereferencerImplTest.java b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/dereference/DereferencerImplTest.java index 332bed182..76abf28c0 100644 --- a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/dereference/DereferencerImplTest.java +++ b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/dereference/DereferencerImplTest.java @@ -10,6 +10,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import eu.europeana.enrichment.api.external.impl.RemoteEntityResolver; import eu.europeana.enrichment.api.external.model.Agent; import eu.europeana.enrichment.api.external.model.EnrichmentBase; import eu.europeana.enrichment.api.external.model.EnrichmentResultBaseWrapper; @@ -18,7 +19,6 @@ import eu.europeana.enrichment.api.external.model.TimeSpan; import eu.europeana.enrichment.api.internal.SearchTerm; import eu.europeana.enrichment.api.internal.SearchTermImpl; -import eu.europeana.enrichment.rest.client.enrichment.RemoteEntityResolver; import eu.europeana.enrichment.rest.client.exceptions.DereferenceException; import eu.europeana.enrichment.utils.EntityMergeEngine; import eu.europeana.enrichment.utils.EntityType; diff --git a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/EnricherImplTest.java b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/EnricherImplTest.java index b90fc5ec3..7c25212b3 100644 --- a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/EnricherImplTest.java +++ b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/EnricherImplTest.java @@ -11,6 +11,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import eu.europeana.enrichment.api.external.impl.RemoteEntityResolver; import eu.europeana.enrichment.api.external.model.EnrichmentBase; import eu.europeana.enrichment.api.external.model.Place; import eu.europeana.enrichment.api.internal.ProxyFieldType; diff --git a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/RemoteEntityResolverTest.java b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/RemoteEntityResolverTest.java index b3be97e28..510285f29 100644 --- a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/RemoteEntityResolverTest.java +++ b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/rest/client/enrichment/RemoteEntityResolverTest.java @@ -14,6 +14,7 @@ import static org.mockito.Mockito.verify; import eu.europeana.enrichment.api.exceptions.UnknownException; +import eu.europeana.enrichment.api.external.impl.RemoteEntityResolver; import eu.europeana.enrichment.api.external.model.Agent; import eu.europeana.enrichment.api.external.model.EnrichmentBase; import eu.europeana.enrichment.api.external.model.EnrichmentResultBaseWrapper; diff --git a/metis-enrichment/metis-enrichment-common/pom.xml b/metis-enrichment/metis-enrichment-common/pom.xml index c96a4e597..7006a252c 100644 --- a/metis-enrichment/metis-enrichment-common/pom.xml +++ b/metis-enrichment/metis-enrichment-common/pom.xml @@ -8,6 +8,10 @@ metis-enrichment-common + + 0.0.1-SNAPSHOT + + eu.europeana.metis @@ -34,7 +38,7 @@ javax.xml.bind - jaxb-api + jaxb-api org.glassfish.jaxb @@ -58,13 +62,43 @@ org.apache.commons commons-lang3 + + org.apache.commons + commons-collections4 + io.swagger swagger-annotations ${version.swagger.annotations} + + + eu.europeana.api + entity-api-client + ${version.entity-api-client} + + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + + + com.fasterxml.jackson.core + jackson-databind + + + org.slf4j + jcl-over-slf4j + + + org.apache.logging.log4j + * + + + + + + org.mockito + mockito-core + - - - diff --git a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/ClientEntityResolver.java b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/ClientEntityResolver.java new file mode 100644 index 000000000..bb41db2e1 --- /dev/null +++ b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/ClientEntityResolver.java @@ -0,0 +1,242 @@ +package eu.europeana.enrichment.api.external.impl; + +import static eu.europeana.metis.network.ExternalRequestUtil.retryableExternalRequestForNetworkExceptionsThrowing; +import static java.lang.String.format; +import static java.util.function.Predicate.not; +import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; +import static org.apache.commons.collections4.ListUtils.partition; + +import com.fasterxml.jackson.core.JsonProcessingException; +import eu.europeana.enrichment.api.exceptions.UnknownException; +import eu.europeana.enrichment.api.external.model.EnrichmentBase; +import eu.europeana.enrichment.api.internal.EntityResolver; +import eu.europeana.enrichment.api.internal.ReferenceTerm; +import eu.europeana.enrichment.api.internal.SearchTerm; +import eu.europeana.enrichment.utils.EnrichmentBaseConverter; +import eu.europeana.enrichment.utils.LanguageCodeConverter; +import eu.europeana.entity.client.web.EntityClientApi; +import eu.europeana.entitymanagement.definitions.model.Entity; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.commons.lang3.StringUtils; + +/** + * An entity resolver that works by accessing a service via Entity Client API and obtains entities from Entity Management API + * + * @author Srishti.singh@europeana.eu + */ +public class ClientEntityResolver implements EntityResolver { + + private final int batchSize; + private final LanguageCodeConverter languageCodeConverter; + private final EntityClientApi entityClientApi; + + + /** + * Constructor with required parameters. + * + * @param entityClientApi the entity client api + * @param batchSize the batch size + */ + public ClientEntityResolver(EntityClientApi entityClientApi, int batchSize) { + this.batchSize = batchSize; + this.languageCodeConverter = new LanguageCodeConverter(); + this.entityClientApi = entityClientApi; + } + + @Override + public Map> resolveByText(Set searchTerms) { + return performInBatches(searchTerms); + } + + @Override + public Map> resolveByUri(Set referenceTerms) { + return performInBatches(referenceTerms, true); + } + + @Override + public Map resolveById(Set referenceTerms) { + final Map> batches = performInBatches(referenceTerms); + return convertToMapWithSingleValues(batches); + } + + private Map> performInBatches(Set inputValues) { + return performInBatches(inputValues, false); + } + + private HashMap convertToMapWithSingleValues( + Map> batches) { + return batches.entrySet().stream().collect(HashMap::new, (map, entry) -> map.put(entry.getKey(), + entry.getValue().stream().findFirst().orElse(null)), HashMap::putAll); + } + + /** + * Perform search in batches. + * + * @param the input value type. Can be OR + * @param inputValues the set of values for which enrichment will be performed + * @param uriSearch boolean indicating if it is an uri search or not(then it is an id search) + * @return the results mapped per input value + */ + private Map> performInBatches(Set inputValues, boolean uriSearch) { + final Map> result = new HashMap<>(); + for (List batch : partition(new ArrayList<>(inputValues), batchSize)) { + result.putAll(performBatch(uriSearch, batch)); + } + return result; + } + + private Map> performBatch(boolean uriSearch, List batch) { + final Map> result = new HashMap<>(); + // TODO: 02/06/2022 This is actually bypassing the batching.. This is the selected way to perform this for now. + for (I batchItem : batch) { + List enrichmentBaseList = performItem(batchItem, uriSearch); + result.put(batchItem, enrichmentBaseList); + } + return result; + } + + private List performItem(I batchItem, boolean uriSearch) { + List entities = resolveEntities(batchItem, uriSearch); + List enrichmentBases = new ArrayList<>(); + if (isNotEmpty(entities)) { + entities = extendEntitiesWithParents(entities); + enrichmentBases = convertToEnrichmentBase(entities); + } + return enrichmentBases; + } + + private List resolveEntities(I batchItem, boolean uriSearch) { + if (batchItem instanceof ReferenceTerm) { + return resolveReference((ReferenceTerm) batchItem, uriSearch); + } else { + return resolveTextSearch((SearchTerm) batchItem); + } + } + + /** + * Get entities based on a reference. + *

    We always check first if the reference resembles a euroepeana entity identifier and if so then we search by id..

    + *

    For invocations that are uri searches({@code uriSearch} equals true) then we also invoke the remote uri search.

    + *

    For uri searches, this resembles the metis implementation where the about search is invoked and if no result return then + * a second invocation on the owlSameAs is performed.

    + * + * @param referenceTerm the reference term + * @param uriSearch indicates if the search is an uri or an id search + * @return the list of entities + */ + private List resolveReference(ReferenceTerm referenceTerm, boolean uriSearch) { + final String referenceValue = referenceTerm.getReference().toString(); + + List result = new ArrayList<>(); + if (europeanaLinkPattern.matcher(referenceValue).matches()) { + result = Optional.ofNullable(entityClientApi.getEntityById(referenceValue)).map(List::of).orElse(Collections.emptyList()); + } else if (uriSearch) { + result = entityClientApi.getEntityByUri(referenceValue); + } + return result; + } + + /** + * Get entities by text search. + *

    + * The result will always be a list of size 1. Internally the remote request might return more than one entities which in that + * case the return of this method will be an empty list. That is because the remote request would be ambiguous and therefore we + * do not know which of the entities is actually intended. + *

    + *

    + * ATTENTION: The described discarding of entities applies correctly in the case where the remote request does NOT + * contain parent entities and that the parent entities are fetched remotely i.e. {@link #extendEntitiesWithParents}. + *

    + * + * @param searchTerm the text search term + * @return the list of entities(at this point of size 0 or 1) + */ + private List resolveTextSearch(SearchTerm searchTerm) { + final String entityTypesConcatenated = searchTerm.getCandidateTypes().stream() + .map(entityType -> entityType.name().toLowerCase(Locale.US)) + .collect(Collectors.joining(",")); + final String language = languageCodeConverter.convertLanguageCode(searchTerm.getLanguage()); + final List entities; + try { + entities = retryableExternalRequestForNetworkExceptionsThrowing( + () -> entityClientApi.getEnrichment(searchTerm.getTextValue(), language, entityTypesConcatenated, null)); + return entities.size() == 1 ? entities : Collections.emptyList(); + } catch (JsonProcessingException e) { + throw new UnknownException( + format("SearchTerm request failed for textValue: %s, language: %s, entityTypes: %s.", searchTerm.getTextValue(), + searchTerm.getLanguage(), entityTypesConcatenated), e); + } + } + + /** + * Creates a copy list that is then extended with any parents found. + * + * @param entities the entities + * @return the extended entities + */ + private List extendEntitiesWithParents(List entities) { + //Copy list so that we can extend + final ArrayList copyEntities = new ArrayList<>(entities); + return findParentEntitiesRecursive(copyEntities, copyEntities); + } + + /** + * Converts the list of entities to a list of {@link EnrichmentBase}s. + * + * @param entities the entities + * @return the converted list + */ + private List convertToEnrichmentBase(List entities) { + return EnrichmentBaseConverter.convertEntitiesToEnrichmentBase(entities); + } + + /** + * Finds parent entities and extends recursively. + *

    For each recursion it will, iterate over {@link Entity#getIsPartOfArray} bypassing blank values and entities already + * encountered. Each recursion will extended list if more parents have been found. + *

    + * + * @param collectedEntities the collected entities + * @param children the children to check their parents for + * @return the extended list of entities + */ + private List findParentEntitiesRecursive(List collectedEntities, List children) { + List parentEntities = + Stream.ofNullable(children).flatMap(Collection::stream) + .map(Entity::getIsPartOfArray).filter(Objects::nonNull).flatMap(Collection::stream) + .filter(StringUtils::isNotBlank) + .filter(not(parentEntityId -> doesEntityExist(parentEntityId, collectedEntities))) + .map(entityClientApi::getEntityById) + .filter(Objects::nonNull) + .collect(Collectors.toCollection(ArrayList::new)); + + if (isNotEmpty(parentEntities)) { + collectedEntities.addAll(parentEntities); + //Now check again parents of parents + findParentEntitiesRecursive(collectedEntities, parentEntities); + } + return collectedEntities; + } + + /** + * Checks if an entity identifier matches an identifier of the entities provided. + * + * @param entityIdToCheck the entity identifier to check + * @param entities the entity list + * @return true if it matches otherwise false + */ + private static boolean doesEntityExist(String entityIdToCheck, List entities) { + return entities.stream().anyMatch(entity -> entity.getEntityId().equals(entityIdToCheck)); + } +} diff --git a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/RemoteEntityResolver.java b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/RemoteEntityResolver.java similarity index 80% rename from metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/RemoteEntityResolver.java rename to metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/RemoteEntityResolver.java index 10f8290a9..f8db213b8 100644 --- a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/RemoteEntityResolver.java +++ b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/RemoteEntityResolver.java @@ -1,9 +1,10 @@ -package eu.europeana.enrichment.rest.client.enrichment; +package eu.europeana.enrichment.api.external.impl; import static eu.europeana.metis.network.ExternalRequestUtil.retryableExternalRequestForNetworkExceptions; import static eu.europeana.metis.utils.RestEndpoints.ENRICH_ENTITY_EQUIVALENCE; import static eu.europeana.metis.utils.RestEndpoints.ENRICH_ENTITY_ID; import static eu.europeana.metis.utils.RestEndpoints.ENRICH_ENTITY_SEARCH; +import static org.apache.commons.collections4.ListUtils.partition; import eu.europeana.enrichment.api.exceptions.UnknownException; import eu.europeana.enrichment.api.external.EnrichmentReference; @@ -34,20 +35,24 @@ import org.springframework.http.MediaType; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; - /** - * An entity resolver that works by accessing a service through HTTP/REST and obtains entities from - * there. + * An entity resolver that works by accessing a service through HTTP/REST and obtains entities from there. */ public class RemoteEntityResolver implements EntityResolver { private final int batchSize; - private final RestTemplate template; + private final RestTemplate restTemplate; private final URL enrichmentServiceUrl; - public RemoteEntityResolver(URL enrichmentServiceUrl, int batchSize, RestTemplate template) { + /** + * Constructor with required parameters + * @param enrichmentServiceUrl the enrichment service url + * @param batchSize the batch size + * @param restTemplate the rest template + */ + public RemoteEntityResolver(URL enrichmentServiceUrl, int batchSize, RestTemplate restTemplate) { this.enrichmentServiceUrl = enrichmentServiceUrl; - this.template = template; + this.restTemplate = restTemplate; this.batchSize = batchSize; } @@ -55,7 +60,7 @@ public RemoteEntityResolver(URL enrichmentServiceUrl, int batchSize, RestTemplat public Map> resolveByText(Set searchTerms) { final Function, EnrichmentSearch> inputFunction = partition -> { final List searchValues = partition.stream() - .map(term -> new SearchValue(term.getTextValue(), term.getLanguage(), + .map(term -> new SearchValue(term.getTextValue(), term.getLanguage(), term.getCandidateTypes().toArray(EntityType[]::new))) .collect(Collectors.toList()); final EnrichmentSearch enrichmentSearch = new EnrichmentSearch(); @@ -97,33 +102,21 @@ private Map performInBatches(String endpointPath, Set inputVa try { final URI parentUri = enrichmentServiceUrl.toURI(); uri = new URI(parentUri.getScheme(), parentUri.getUserInfo(), parentUri.getHost(), - parentUri.getPort(), parentUri.getPath() + "/" + endpointPath, - parentUri.getQuery(), parentUri.getFragment()).normalize(); + parentUri.getPort(), parentUri.getPath() + "/" + endpointPath, + parentUri.getQuery(), parentUri.getFragment()).normalize(); } catch (URISyntaxException e) { throw new UnknownException( - "URL syntax issue with service url: " + enrichmentServiceUrl + ".", e); + "URL syntax issue with service url: " + enrichmentServiceUrl + ".", e); } - // Create partitions - final List> partitions = new ArrayList<>(); - partitions.add(new ArrayList<>()); - inputValues.forEach(item -> { - List currentPartition = partitions.get(partitions.size() - 1); - if (currentPartition.size() >= batchSize) { - currentPartition = new ArrayList<>(); - partitions.add(currentPartition); - } - currentPartition.add(item); - }); - - // Process partitions + final List> batches = partition(new ArrayList<>(inputValues), batchSize); final Map result = new HashMap<>(); - for (List partition : partitions) { - final EnrichmentResultList enrichmentResultList = executeRequest(uri, bodyCreator, partition); - for (int i = 0; i < partition.size(); i++) { - final I inputItem = partition.get(i); + for (List batch : batches) { + final EnrichmentResultList enrichmentResultList = executeRequest(uri, bodyCreator, batch); + for (int i = 0; i < batch.size(); i++) { + final I inputItem = batch.get(i); Optional.ofNullable(enrichmentResultList - .getEnrichmentBaseResultWrapperList().get(i)) + .getEnrichmentBaseResultWrapperList().get(i)) .map(EnrichmentResultBaseWrapper::getEnrichmentBaseList) .filter(list -> !list.isEmpty()) .map(resultParser).ifPresent(resultItem -> result.put(inputItem, resultItem)); @@ -135,10 +128,10 @@ private Map performInBatches(String endpointPath, Set inputVa } private EnrichmentResultList executeRequest(URI uri, Function, B> bodyCreator, - List partition) { + List batch) { // Create the request - final B body = bodyCreator.apply(partition); + final B body = bodyCreator.apply(batch); final HttpHeaders headers = new HttpHeaders(); if (body != null) { headers.setContentType(MediaType.APPLICATION_JSON); @@ -150,7 +143,7 @@ private EnrichmentResultList executeRequest(URI uri, Function, B> final EnrichmentResultList enrichmentResultList; try { enrichmentResultList = retryableExternalRequestForNetworkExceptions( - () -> template.postForObject(uri, httpEntity, EnrichmentResultList.class)); + () -> restTemplate.postForObject(uri, httpEntity, EnrichmentResultList.class)); } catch (RestClientException e) { throw new UnknownException("Enrichment client POST call failed: " + uri + ".", e); @@ -158,7 +151,7 @@ private EnrichmentResultList executeRequest(URI uri, Function, B> if (enrichmentResultList == null) { throw new UnknownException("Empty body from server (" + uri + ")."); } - if (enrichmentResultList.getEnrichmentBaseResultWrapperList().size() != partition.size()) { + if (enrichmentResultList.getEnrichmentBaseResultWrapperList().size() != batch.size()) { throw new UnknownException("Server returned unexpected number of results (" + uri + ")."); } diff --git a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/Agent.java b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/Agent.java index a28c4c5a6..83157d09a 100644 --- a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/Agent.java +++ b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/Agent.java @@ -11,4 +11,14 @@ @XmlAccessorType(XmlAccessType.FIELD) public class Agent extends AgentBase { + public Agent() {} + + public Agent(eu.europeana.entitymanagement.definitions.model.Agent entity) { + super(entity); + } + + public Agent(eu.europeana.entitymanagement.definitions.model.Organization entity) { + super(entity); + } + } diff --git a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/AgentBase.java b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/AgentBase.java index 158b86ef3..f43c432a2 100644 --- a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/AgentBase.java +++ b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/AgentBase.java @@ -1,5 +1,12 @@ package eu.europeana.enrichment.api.external.model; +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertListToLabel; +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertListToLabelResource; +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertListToPart; +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertListToResource; +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertMapToLabels; +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertResourceOrLiteral; + import java.util.ArrayList; import java.util.List; import javax.xml.bind.annotation.XmlAccessType; @@ -53,6 +60,19 @@ public abstract class AgentBase extends EnrichmentBase { @XmlElement(name = "sameAs", namespace = "http://www.w3.org/2002/07/owl#") private List sameAs = new ArrayList<>(); + protected AgentBase() { + } + + protected AgentBase(eu.europeana.entitymanagement.definitions.model.Organization organization) { + super(organization); + } + + // Used for creating XML entity from EM model class + protected AgentBase(eu.europeana.entitymanagement.definitions.model.Agent agent) { + super(agent); + init(agent); + } + public List     + + org.springframework + spring-webmvc + ${version.spring} + org.springframework spring-core @@ -74,6 +79,17 @@ springfox-swagger-ui ${version.swagger} + + + org.springframework.boot + spring-boot-autoconfigure + ${version.spring-boot-autoconfigure} + com.jayway.jsonpath json-path-assert @@ -93,11 +109,6 @@ javax.servlet-api ${version.servlet.api} - - org.springframework - spring-webmvc - ${version.spring} - org.springframework spring-test diff --git a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/EnrichmentController.java b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/EnrichmentController.java index e9490095f..87c771ea9 100644 --- a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/EnrichmentController.java +++ b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/EnrichmentController.java @@ -13,9 +13,8 @@ import io.swagger.annotations.ApiParam; import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; -import java.util.Collections; -import java.util.List; -import java.util.Optional; + +import java.util.*; import java.util.stream.Collectors; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; diff --git a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/Application.java b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/Application.java index 171b8f1e4..ded647fdd 100644 --- a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/Application.java +++ b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/Application.java @@ -1,15 +1,22 @@ package eu.europeana.enrichment.rest.config; +import static java.lang.String.format; + import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule; import com.mongodb.client.MongoClient; import eu.europeana.corelib.web.socks.SocksProxy; +import eu.europeana.enrichment.api.external.impl.ClientEntityResolver; +import eu.europeana.enrichment.api.internal.EntityResolver; import eu.europeana.enrichment.service.EnrichmentService; import eu.europeana.enrichment.service.PersistentEntityResolver; import eu.europeana.enrichment.service.dao.EnrichmentDao; +import eu.europeana.entity.client.config.EntityClientConfiguration; +import eu.europeana.entity.client.web.EntityClientApiImpl; import eu.europeana.metis.mongo.connection.MongoClientProvider; import eu.europeana.metis.mongo.connection.MongoProperties; -import java.util.Collections; +import java.util.Properties; import javax.annotation.PreDestroy; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; @@ -19,24 +26,16 @@ import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder; import org.springframework.web.servlet.config.annotation.EnableWebMvc; -import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; -import org.springframework.web.servlet.config.annotation.ViewControllerRegistry; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; -import springfox.documentation.builders.PathSelectors; -import springfox.documentation.builders.RequestHandlerSelectors; -import springfox.documentation.service.ApiInfo; -import springfox.documentation.service.Contact; -import springfox.documentation.spi.DocumentationType; -import springfox.documentation.spring.web.plugins.Docket; -import springfox.documentation.swagger2.annotations.EnableSwagger2; +/** + * Main Spring Configuration class + */ @Configuration @ComponentScan(basePackages = {"eu.europeana.enrichment.rest", "eu.europeana.enrichment.rest.exception"}) @PropertySource("classpath:enrichment.properties") @EnableWebMvc -@EnableSwagger2 -public class Application implements WebMvcConfigurer, InitializingBean { +public class Application implements InitializingBean { //Socks proxy @Value("${socks.proxy.enabled}") @@ -59,6 +58,22 @@ public class Application implements WebMvcConfigurer, InitializingBean { @Value("${enrichment.mongo.application.name}") private String enrichmentMongoApplicationName; + @Value("${enrichment.batch.size:20}") + private int enrichmentBatchSize; + + @Value("${enrichment.entity.resolver.type:PERSISTENT}") + private EntityResolverType entityResolverType; + + @Value("${entity.management.url}") + private String entityManagementUrl; + + @Value("${entity.api.url}") + private String entityApiUrl; + + @Value("${entity.api.key}") + private String entityApiKey; + + private MongoClient mongoClient; /** @@ -71,21 +86,30 @@ public void afterPropertiesSet() { } } - @Override - public void addViewControllers(ViewControllerRegistry registry) { - registry.addRedirectViewController("/", "/swagger-ui/index.html"); - } - - @Override - public void addResourceHandlers(ResourceHandlerRegistry registry) { - registry.addResourceHandler("/swagger-ui/**") - .addResourceLocations("classpath:/META-INF/resources/webjars/springfox-swagger-ui/") - .resourceChain(false); + @Bean + EnrichmentService getEnrichmentService(EntityResolver entityResolver) { + return new EnrichmentService(entityResolver); } @Bean - EnrichmentService getEnrichmentService(EnrichmentDao enrichmentDao) { - return new EnrichmentService(new PersistentEntityResolver(enrichmentDao)); + EntityResolver getEntityResolver() { + final EntityResolver entityResolver; + if (entityResolverType == EntityResolverType.ENTITY_CLIENT) { + //Sanity check + if (StringUtils.isAnyBlank(entityManagementUrl, entityApiUrl, entityApiKey)) { + throw new IllegalArgumentException( + format("Requested %s resolver but configuration is missing", EntityResolverType.ENTITY_CLIENT)); + } + final Properties properties = new Properties(); + properties.put("entity.management.url", entityManagementUrl); + properties.put("entity.api.url", entityApiUrl); + properties.put("entity.api.key", entityApiKey); + entityResolver = new ClientEntityResolver(new EntityClientApiImpl(new EntityClientConfiguration(properties)), + enrichmentBatchSize); + } else { + entityResolver = new PersistentEntityResolver(new EnrichmentDao(mongoClient, enrichmentMongoDatabase)); + } + return entityResolver; } @Bean @@ -99,11 +123,6 @@ MongoClient getMongoClient() { return mongoClient; } - @Bean - EnrichmentDao getEnrichmentDao(MongoClient mongoClient) { - return new EnrichmentDao(mongoClient, enrichmentMongoDatabase); - } - @Bean public Jackson2ObjectMapperBuilder objectMapperBuilder() { Jackson2ObjectMapperBuilder builder = new Jackson2ObjectMapperBuilder(); @@ -126,29 +145,4 @@ public void close() { mongoClient.close(); } } - - @Bean - public Docket api() { - return new Docket(DocumentationType.SWAGGER_2) - .useDefaultResponseMessages(false) - .select() - .apis(RequestHandlerSelectors.any()) - .paths(PathSelectors.regex("/.*")) - .build() - .apiInfo(apiInfo()); - } - - private ApiInfo apiInfo() { - Contact contact = new Contact("Europeana", "http:\\www.europeana.eu", - "development@europeana.eu"); - - return new ApiInfo( - "Enrichment REST API", - "Enrichment REST API for Europeana", - "v1", - "API TOS", - contact, - "EUPL Licence v1.2", - "https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12", Collections.emptyList()); - } } diff --git a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/EntityResolverType.java b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/EntityResolverType.java new file mode 100644 index 000000000..5f6580977 --- /dev/null +++ b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/EntityResolverType.java @@ -0,0 +1,8 @@ +package eu.europeana.enrichment.rest.config; + +/** + * Entity resolver type for choosing the requested implementation + */ +public enum EntityResolverType { + PERSISTENT, ENTITY_CLIENT +} diff --git a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/SwaggerConfig.java b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/SwaggerConfig.java new file mode 100644 index 000000000..50d8fe5da --- /dev/null +++ b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/SwaggerConfig.java @@ -0,0 +1,65 @@ +package eu.europeana.enrichment.rest.config; + +import java.util.Collections; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; +import org.springframework.web.servlet.config.annotation.ViewControllerRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; +import springfox.documentation.builders.PathSelectors; +import springfox.documentation.builders.RequestHandlerSelectors; +import springfox.documentation.service.ApiInfo; +import springfox.documentation.service.Contact; +import springfox.documentation.spi.DocumentationType; +import springfox.documentation.spring.web.plugins.Docket; +import springfox.documentation.swagger2.annotations.EnableSwagger2; + +/** + * Config for Swagger documentation. + */ +@Configuration +@EnableSwagger2 +public class SwaggerConfig implements WebMvcConfigurer { + + @Override + public void addViewControllers(ViewControllerRegistry registry) { + registry.addRedirectViewController("/", "/swagger-ui/index.html"); + } + + @Override + public void addResourceHandlers(ResourceHandlerRegistry registry) { + registry.addResourceHandler("/swagger-ui/**") + .addResourceLocations("classpath:/META-INF/resources/webjars/springfox-swagger-ui/") + .resourceChain(false); + } + + /** + * Initialize Swagger Documentation + * + * @return Swagger Docket for this API + */ + @Bean + public Docket api() { + return new Docket(DocumentationType.SWAGGER_2) + .useDefaultResponseMessages(false) + .select() + .apis(RequestHandlerSelectors.any()) + .paths(PathSelectors.regex("/.*")) + .build() + .apiInfo(apiInfo()); + } + + private ApiInfo apiInfo() { + Contact contact = new Contact("Europeana", "http:\\www.europeana.eu", + "development@europeana.eu"); + + return new ApiInfo( + "Enrichment REST API", + "Enrichment REST API for Europeana", + "v1", + "API TOS", + contact, + "EUPL Licence v1.2", + "https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12", Collections.emptyList()); + } +} diff --git a/metis-enrichment/metis-enrichment-rest/src/main/resources/enrichment.properties.example b/metis-enrichment/metis-enrichment-rest/src/main/resources/enrichment.properties.example index 68ad705a3..0741764f5 100644 --- a/metis-enrichment/metis-enrichment-rest/src/main/resources/enrichment.properties.example +++ b/metis-enrichment/metis-enrichment-rest/src/main/resources/enrichment.properties.example @@ -9,4 +9,10 @@ socks.proxy.password= enrichment.mongo.host= enrichment.mongo.database= enrichment.mongo.port= -enrichment.mongo.application.name= \ No newline at end of file +enrichment.mongo.application.name= + + +#If not provided defaults to 20 +enrichment.batch.size= +#Options PERSISTENT, ENTITY_CLIENT. Defaults to PERSISTENT +entity.resolver.type= \ No newline at end of file diff --git a/metis-enrichment/metis-enrichment-service/.gitignore b/metis-enrichment/metis-enrichment-service/.gitignore deleted file mode 100644 index 0b021c7ac..000000000 --- a/metis-enrichment/metis-enrichment-service/.gitignore +++ /dev/null @@ -1 +0,0 @@ -**.properties \ No newline at end of file diff --git a/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/EnrichmentService.java b/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/EnrichmentService.java index 7b6fb0805..7e5582a62 100644 --- a/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/EnrichmentService.java +++ b/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/EnrichmentService.java @@ -4,20 +4,18 @@ import eu.europeana.enrichment.api.external.SearchValue; import eu.europeana.enrichment.api.external.model.EnrichmentBase; import eu.europeana.enrichment.api.external.model.EnrichmentResultBaseWrapper; +import eu.europeana.enrichment.api.internal.EntityResolver; import eu.europeana.enrichment.api.internal.ReferenceTerm; import eu.europeana.enrichment.api.internal.ReferenceTermImpl; import eu.europeana.enrichment.api.internal.SearchTerm; import eu.europeana.enrichment.api.internal.SearchTermImpl; -import eu.europeana.enrichment.internal.model.OrganizationEnrichmentEntity; import eu.europeana.enrichment.service.dao.EnrichmentDao; import java.net.MalformedURLException; import java.net.URL; import java.util.Collections; -import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.slf4j.Logger; @@ -26,26 +24,26 @@ import org.springframework.stereotype.Service; /** - * Contains functionality for accessing entities from the enrichment database using {@link - * EnrichmentDao}. + * Contains functionality for accessing entities from the enrichment database using {@link EnrichmentDao}. * * @author Simon Tzanakis * @since 2020-07-16 */ @Service public class EnrichmentService { + private static final Logger LOGGER = LoggerFactory.getLogger(EnrichmentService.class); - private final PersistentEntityResolver persistentEntityResolver; + private final EntityResolver entityResolver; /** * Parameter constructor. * - * @param persistentEntityResolver the entity resolver + * @param entityResolver the entity resolver */ @Autowired - public EnrichmentService(PersistentEntityResolver persistentEntityResolver) { - this.persistentEntityResolver = persistentEntityResolver; + public EnrichmentService(EntityResolver entityResolver) { + this.entityResolver = entityResolver; } /** @@ -57,12 +55,12 @@ public EnrichmentService(PersistentEntityResolver persistentEntityResolver) { public List enrichByEnrichmentSearchValues( List searchValues) { final List orderedSearchTerms = searchValues.stream().map( - search -> new SearchTermImpl(search.getValue(), search.getLanguage(), - Set.copyOf(search.getEntityTypes()))).collect(Collectors.toList()); - final Map> result = persistentEntityResolver - .resolveByText(new HashSet<>(orderedSearchTerms)); + search -> new SearchTermImpl(search.getValue(), search.getLanguage(), + Set.copyOf(search.getEntityTypes()))).collect(Collectors.toList()); + final Map> result = entityResolver + .resolveByText(new HashSet<>(orderedSearchTerms)); return orderedSearchTerms.stream().map(result::get).map(EnrichmentResultBaseWrapper::new) - .collect(Collectors.toList()); + .collect(Collectors.toList()); } /** @@ -74,9 +72,9 @@ public List enrichByEnrichmentSearchValues( public List enrichByEquivalenceValues(ReferenceValue referenceValue) { try { final ReferenceTerm referenceTerm = new ReferenceTermImpl( - new URL(referenceValue.getReference()), Set.copyOf(referenceValue.getEntityTypes())); - return persistentEntityResolver.resolveByUri(Set.of(referenceTerm)) - .getOrDefault(referenceTerm, Collections.emptyList()); + new URL(referenceValue.getReference()), Set.copyOf(referenceValue.getEntityTypes())); + return entityResolver.resolveByUri(Set.of(referenceTerm)) + .getOrDefault(referenceTerm, Collections.emptyList()); } catch (MalformedURLException e) { LOGGER.debug("There was a problem converting the input to ReferenceTermType"); throw new IllegalArgumentException("The input values are invalid", e); @@ -92,73 +90,12 @@ public List enrichByEquivalenceValues(ReferenceValue referenceVa public EnrichmentBase enrichById(String entityAbout) { try { final ReferenceTerm referenceTerm = new ReferenceTermImpl(new URL(entityAbout), - new HashSet<>()); - return persistentEntityResolver.resolveById(Set.of(referenceTerm)).get(referenceTerm); + new HashSet<>()); + return entityResolver.resolveById(Set.of(referenceTerm)).get(referenceTerm); } catch (MalformedURLException e) { LOGGER.debug("There was a problem converting the input to ReferenceTermType"); throw new IllegalArgumentException("The input values are invalid", e); } } - /* --- Organization specific methods, used by the annotations api --- */ - - /** - * Save an organization to the database - * - * @param organizationEnrichmentEntity the organization to save - * @param created the created date to be used - * @param updated the updated date to be used - * @return the saved organization - */ - public OrganizationEnrichmentEntity saveOrganization( - OrganizationEnrichmentEntity organizationEnrichmentEntity, Date created, Date updated) { - return persistentEntityResolver.saveOrganization(organizationEnrichmentEntity, created, updated); - } - - /** - * Return the list of ids for existing organizations from database - * - * @param organizationIds The organization ids to check existence - * @return list of ids of existing organizations - */ - public List findExistingOrganizations(List organizationIds) { - return persistentEntityResolver.findExistingOrganizations(organizationIds); - } - - /** - * Get an organization by uri - * - * @param uri The EDM organization uri - * @return OrganizationImpl object - */ - public Optional getOrganizationByUri(String uri) { - return persistentEntityResolver.getOrganizationByUri(uri); - } - - /** - * Delete organizations from database by given organization ids - * - * @param organizationIds The organization ids - */ - public void deleteOrganizations(List organizationIds) { - persistentEntityResolver.deleteOrganizations(organizationIds); - } - - /** - * This method removes organization from database by given organization id. - * - * @param organizationId The organization id - */ - public void deleteOrganization(String organizationId) { - persistentEntityResolver.deleteOrganization(organizationId); - } - - /** - * Get the date of the latest updated organization. - * - * @return the date of the latest updated organization - */ - public Date getDateOfLastUpdatedOrganization() { - return persistentEntityResolver.getDateOfLastUpdatedOrganization(); - } } diff --git a/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/PersistentEntityResolver.java b/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/PersistentEntityResolver.java index 69d11039b..57d321668 100644 --- a/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/PersistentEntityResolver.java +++ b/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/PersistentEntityResolver.java @@ -5,19 +5,16 @@ import eu.europeana.enrichment.api.internal.ReferenceTerm; import eu.europeana.enrichment.api.internal.SearchTerm; import eu.europeana.enrichment.internal.model.EnrichmentTerm; -import eu.europeana.enrichment.internal.model.OrganizationEnrichmentEntity; import eu.europeana.enrichment.service.dao.EnrichmentDao; +import eu.europeana.enrichment.service.utils.EnrichmentTermsToEnrichmentBaseConverter; import eu.europeana.enrichment.utils.EntityType; +import eu.europeana.enrichment.utils.LanguageCodeConverter; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -26,7 +23,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; -import org.bson.types.ObjectId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,25 +32,11 @@ public class PersistentEntityResolver implements EntityResolver { private static final Logger LOGGER = LoggerFactory.getLogger(PersistentEntityResolver.class); - private static final Set ALL_2CODE_LANGUAGES; - private static final Map ALL_3CODE_TO_2CODE_LANGUAGES; private static final Pattern PATTERN_MATCHING_VERY_BROAD_TIMESPANS = Pattern .compile("http://semium.org/time/(ChronologicalPeriod$|Time$|(AD|BC)[1-9]x{3}$)"); - public static final int THREE_CHARACTER_LANGUAGE_LENGTH = 3; - public static final int TWO_CHARACTER_LANGUAGE_LENGTH = 2; - - static { - HashSet all2CodeLanguages = new HashSet<>(); - Map all3CodeLanguages = new HashMap<>(); - Arrays.stream(Locale.getISOLanguages()).map(Locale::new).forEach(locale -> { - all2CodeLanguages.add(locale.getLanguage()); - all3CodeLanguages.put(locale.getISO3Language(), locale.getLanguage()); - }); - ALL_2CODE_LANGUAGES = Collections.unmodifiableSet(all2CodeLanguages); - ALL_3CODE_TO_2CODE_LANGUAGES = Collections.unmodifiableMap(all3CodeLanguages); - } private final EnrichmentDao enrichmentDao; + private final LanguageCodeConverter languageCodeConverter; /** * Constructor with the persistence dao parameter. @@ -63,6 +45,7 @@ public class PersistentEntityResolver implements EntityResolver { */ public PersistentEntityResolver(EnrichmentDao enrichmentDao) { this.enrichmentDao = enrichmentDao; + languageCodeConverter = new LanguageCodeConverter(); } @Override @@ -141,17 +124,7 @@ private void findEnrichmentEntitiesBySearchTerm( if (!StringUtils.isBlank(value)) { final Set entityTypes = searchTerm.getCandidateTypes(); //Language has to be a valid 2 or 3 code, otherwise we do not use it - final String inputValueLanguage = searchTerm.getLanguage(); - final String language; - if (inputValueLanguage != null - && inputValueLanguage.length() == THREE_CHARACTER_LANGUAGE_LENGTH) { - language = ALL_3CODE_TO_2CODE_LANGUAGES.get(inputValueLanguage); - } else if (inputValueLanguage != null - && inputValueLanguage.length() == TWO_CHARACTER_LANGUAGE_LENGTH) { - language = ALL_2CODE_LANGUAGES.contains(inputValueLanguage) ? inputValueLanguage : null; - } else { - language = null; - } + final String language = languageCodeConverter.convertLanguageCode(searchTerm.getLanguage()); if (CollectionUtils.isEmpty(entityTypes)) { searchTermListMap.put(searchTerm, findEnrichmentTerms(null, value, language)); @@ -188,12 +161,13 @@ private List findEnrichmentTerms(EntityType entityType, String t final List enrichmentTerms = enrichmentDao .getAllEnrichmentTermsByFields(fieldNameMap); final List parentEnrichmentTerms = enrichmentTerms.stream() - .map(this::findParentEntities).flatMap(List::stream).collect(Collectors.toList()); + .map(this::findParentEntities).flatMap(List::stream) + .collect(Collectors.toList()); final List enrichmentBases = new ArrayList<>(); //Convert to EnrichmentBases - enrichmentBases.addAll(Converter.convert(enrichmentTerms)); - enrichmentBases.addAll(Converter.convert(parentEnrichmentTerms)); + enrichmentBases.addAll(EnrichmentTermsToEnrichmentBaseConverter.convert(enrichmentTerms)); + enrichmentBases.addAll(EnrichmentTermsToEnrichmentBaseConverter.convert(parentEnrichmentTerms)); return enrichmentBases; } @@ -250,7 +224,7 @@ private List searchBasesFirstAboutThenOwlSameAs(String reference private List getEnrichmentTermsAndConvert( List> fieldNamesAndValues) { final List enrichmentTerms = getEnrichmentTerms(fieldNamesAndValues); - return Converter.convert(enrichmentTerms); + return EnrichmentTermsToEnrichmentBaseConverter.convert(enrichmentTerms); } private List getEnrichmentTerms(List> fieldNamesAndValues) { @@ -259,87 +233,4 @@ private List getEnrichmentTerms(List> field return enrichmentDao.getAllEnrichmentTermsByFields(fieldNameMap); } - /* --- Organization specific methods, used by the annotations api --- */ - - /** - * Save an organization to the database - * - * @param organizationEnrichmentEntity the organization to save - * @param created the created date to be used - * @param updated the updated date to be used - * @return the saved organization - */ - public OrganizationEnrichmentEntity saveOrganization( - OrganizationEnrichmentEntity organizationEnrichmentEntity, Date created, Date updated) { - - final EnrichmentTerm enrichmentTerm = Converter - .organizationImplToEnrichmentTerm(organizationEnrichmentEntity, created, updated); - - final Optional objectId = enrichmentDao - .getEnrichmentTermObjectIdByField(EnrichmentDao.ENTITY_ABOUT_FIELD, - organizationEnrichmentEntity.getAbout()); - objectId.ifPresent(enrichmentTerm::setId); - - //Save term list - final String id = enrichmentDao.saveEnrichmentTerm(enrichmentTerm); - return enrichmentDao.getEnrichmentTermByField(EnrichmentDao.ID_FIELD, id) - .map(EnrichmentTerm::getEnrichmentEntity).map(OrganizationEnrichmentEntity.class::cast) - .orElse(null); - } - - /** - * Return the list of ids for existing organizations from database - * - * @param organizationIds The organization ids to check existence - * @return list of ids of existing organizations - */ - public List findExistingOrganizations(List organizationIds) { - List existingOrganizationIds = new ArrayList<>(); - for (String id : organizationIds) { - Optional organization = getOrganizationByUri(id); - organization.ifPresent(value -> existingOrganizationIds.add(value.getAbout())); - } - return existingOrganizationIds; - } - - /** - * Get an organization by uri - * - * @param uri The EDM organization uri - * @return OrganizationImpl object - */ - public Optional getOrganizationByUri(String uri) { - final List enrichmentTerm = getEnrichmentTerms( - Collections.singletonList(new ImmutablePair<>(EnrichmentDao.ENTITY_ABOUT_FIELD, uri))); - return enrichmentTerm.stream().findFirst().map(EnrichmentTerm::getEnrichmentEntity) - .map(OrganizationEnrichmentEntity.class::cast); - } - - /** - * Delete organizations from database by given organization ids - * - * @param organizationIds The organization ids - */ - public void deleteOrganizations(List organizationIds) { - enrichmentDao.deleteEnrichmentTerms(EntityType.ORGANIZATION, organizationIds); - } - - /** - * This method removes organization from database by given organization id. - * - * @param organizationId The organization id - */ - public void deleteOrganization(String organizationId) { - deleteOrganizations(Collections.singletonList(organizationId)); - } - - /** - * Get the date of the latest updated organization. - * - * @return the date of the latest updated organization - */ - public Date getDateOfLastUpdatedOrganization() { - return enrichmentDao.getDateOfLastUpdatedEnrichmentTerm(EntityType.ORGANIZATION); - } - } diff --git a/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/Converter.java b/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/utils/EnrichmentTermsToEnrichmentBaseConverter.java similarity index 66% rename from metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/Converter.java rename to metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/utils/EnrichmentTermsToEnrichmentBaseConverter.java index 54d429f11..37587b108 100644 --- a/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/Converter.java +++ b/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/utils/EnrichmentTermsToEnrichmentBaseConverter.java @@ -1,9 +1,14 @@ -package eu.europeana.enrichment.service; +package eu.europeana.enrichment.service.utils; + +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertListToPart; +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertMapToLabels; +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertMultilingualMapToLabel; +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertResourceOrLiteral; +import static eu.europeana.enrichment.utils.EntityValuesConverter.convertToResourceList; import eu.europeana.enrichment.api.external.model.Agent; import eu.europeana.enrichment.api.external.model.Concept; import eu.europeana.enrichment.api.external.model.EnrichmentBase; -import eu.europeana.enrichment.api.external.model.Label; import eu.europeana.enrichment.api.external.model.LabelInfo; import eu.europeana.enrichment.api.external.model.LabelResource; import eu.europeana.enrichment.api.external.model.Organization; @@ -22,16 +27,11 @@ import eu.europeana.enrichment.internal.model.PlaceEnrichmentEntity; import eu.europeana.enrichment.internal.model.TimespanEnrichmentEntity; import eu.europeana.enrichment.utils.EntityType; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.Stream; - import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -39,18 +39,19 @@ /** * Contains functionality for converting from an incoming Object to a different one. */ -public final class Converter { +public final class EnrichmentTermsToEnrichmentBaseConverter { - private Converter() { + private EnrichmentTermsToEnrichmentBaseConverter() { } /** * Converter from list of {@link EnrichmentTerm} to list of {@link EnrichmentBase}. + * * @param enrichmentTerms the enrichment terms to convert * @return the converted enrichment bases */ public static List convert(List enrichmentTerms) { - return enrichmentTerms.stream().map(Converter::convert).collect(Collectors.toList()); + return enrichmentTerms.stream().map(EnrichmentTermsToEnrichmentBaseConverter::convert).collect(Collectors.toList()); } /** @@ -93,14 +94,14 @@ private static TimeSpan convertTimespan(TimespanEnrichmentEntity timespanEnrichm TimeSpan output = new TimeSpan(); output.setAbout(timespanEnrichmentEntity.getAbout()); - output.setPrefLabelList(convert(timespanEnrichmentEntity.getPrefLabel())); - output.setAltLabelList(convert(timespanEnrichmentEntity.getAltLabel())); - output.setBegin(convert(timespanEnrichmentEntity.getBegin()).get(0)); - output.setEnd(convert(timespanEnrichmentEntity.getEnd()).get(0)); + output.setPrefLabelList(convertMultilingualMapToLabel(timespanEnrichmentEntity.getPrefLabel())); + output.setAltLabelList(convertMultilingualMapToLabel(timespanEnrichmentEntity.getAltLabel())); + output.setBegin(convertMultilingualMapToLabel(timespanEnrichmentEntity.getBegin()).get(0)); + output.setEnd(convertMultilingualMapToLabel(timespanEnrichmentEntity.getEnd()).get(0)); output.setHasPartsList(convertResourceOrLiteral(timespanEnrichmentEntity.getDctermsHasPart())); - output.setHiddenLabel(convert(timespanEnrichmentEntity.getHiddenLabel())); - output.setNotes(convert(timespanEnrichmentEntity.getNote())); - output.setSameAs(convertToPartsList(timespanEnrichmentEntity.getOwlSameAs())); + output.setHiddenLabel(convertMultilingualMapToLabel(timespanEnrichmentEntity.getHiddenLabel())); + output.setNotes(convertMultilingualMapToLabel(timespanEnrichmentEntity.getNote())); + output.setSameAs(convertListToPart(timespanEnrichmentEntity.getOwlSameAs())); if (StringUtils.isNotBlank(timespanEnrichmentEntity.getIsPartOf())) { output.setIsPartOf(List.of(new LabelResource(timespanEnrichmentEntity.getIsPartOf()))); @@ -117,11 +118,11 @@ private static Concept convertConcept(ConceptEnrichmentEntity conceptEnrichmentE Concept output = new Concept(); output.setAbout(conceptEnrichmentEntity.getAbout()); - output.setPrefLabelList(convert(conceptEnrichmentEntity.getPrefLabel())); - output.setAltLabelList(convert(conceptEnrichmentEntity.getAltLabel())); - output.setHiddenLabel(convert(conceptEnrichmentEntity.getHiddenLabel())); - output.setNotation(convert(conceptEnrichmentEntity.getNotation())); - output.setNotes(convert(conceptEnrichmentEntity.getNote())); + output.setPrefLabelList(convertMultilingualMapToLabel(conceptEnrichmentEntity.getPrefLabel())); + output.setAltLabelList(convertMultilingualMapToLabel(conceptEnrichmentEntity.getAltLabel())); + output.setHiddenLabel(convertMultilingualMapToLabel(conceptEnrichmentEntity.getHiddenLabel())); + output.setNotation(convertMultilingualMapToLabel(conceptEnrichmentEntity.getNotation())); + output.setNotes(convertMultilingualMapToLabel(conceptEnrichmentEntity.getNote())); output.setBroader(convertToResourceList(conceptEnrichmentEntity.getBroader())); output.setBroadMatch(convertToResourceList(conceptEnrichmentEntity.getBroadMatch())); output.setCloseMatch(convertToResourceList(conceptEnrichmentEntity.getCloseMatch())); @@ -141,12 +142,12 @@ private static Place convertPlace(PlaceEnrichmentEntity placeEnrichmentEntity) { Place output = new Place(); output.setAbout(placeEnrichmentEntity.getAbout()); - output.setPrefLabelList(convert(placeEnrichmentEntity.getPrefLabel())); - output.setAltLabelList(convert(placeEnrichmentEntity.getAltLabel())); + output.setPrefLabelList(convertMultilingualMapToLabel(placeEnrichmentEntity.getPrefLabel())); + output.setAltLabelList(convertMultilingualMapToLabel(placeEnrichmentEntity.getAltLabel())); output.setHasPartsList(convertResourceOrLiteral(placeEnrichmentEntity.getDcTermsHasPart())); - output.setNotes(convert(placeEnrichmentEntity.getNote())); - output.setSameAs(convertToPartsList(placeEnrichmentEntity.getOwlSameAs())); + output.setNotes(convertMultilingualMapToLabel(placeEnrichmentEntity.getNote())); + output.setSameAs(convertListToPart(placeEnrichmentEntity.getOwlSameAs())); if (StringUtils.isNotBlank(placeEnrichmentEntity.getIsPartOf())) { output.setIsPartOf(List.of(new LabelResource(placeEnrichmentEntity.getIsPartOf()))); @@ -169,33 +170,33 @@ private static Agent convertAgent(AgentEnrichmentEntity agentEntityEnrichment) { Agent output = new Agent(); output.setAbout(agentEntityEnrichment.getAbout()); - output.setPrefLabelList(convert(agentEntityEnrichment.getPrefLabel())); - output.setAltLabelList(convert(agentEntityEnrichment.getAltLabel())); - output.setHiddenLabel(convert(agentEntityEnrichment.getHiddenLabel())); - output.setFoafName(convert(agentEntityEnrichment.getFoafName())); - output.setNotes(convert(agentEntityEnrichment.getNote())); + output.setPrefLabelList(convertMultilingualMapToLabel(agentEntityEnrichment.getPrefLabel())); + output.setAltLabelList(convertMultilingualMapToLabel(agentEntityEnrichment.getAltLabel())); + output.setHiddenLabel(convertMultilingualMapToLabel(agentEntityEnrichment.getHiddenLabel())); + output.setFoafName(convertMultilingualMapToLabel(agentEntityEnrichment.getFoafName())); + output.setNotes(convertMultilingualMapToLabel(agentEntityEnrichment.getNote())); - output.setBeginList(convert(agentEntityEnrichment.getBegin())); - output.setEndList(convert(agentEntityEnrichment.getEnd())); + output.setBeginList(convertMultilingualMapToLabel(agentEntityEnrichment.getBegin())); + output.setEndList(convertMultilingualMapToLabel(agentEntityEnrichment.getEnd())); - output.setIdentifier(convert(agentEntityEnrichment.getDcIdentifier())); + output.setIdentifier(convertMultilingualMapToLabel(agentEntityEnrichment.getDcIdentifier())); output.setHasMet(convertToResourceList(agentEntityEnrichment.getEdmHasMet())); output.setBiographicalInformation( convertResourceOrLiteral(agentEntityEnrichment.getRdaGr2BiographicalInformation())); output.setPlaceOfBirth(convertResourceOrLiteral(agentEntityEnrichment.getRdaGr2PlaceOfBirth())); output.setPlaceOfDeath(convertResourceOrLiteral(agentEntityEnrichment.getRdaGr2PlaceOfDeath())); - output.setDateOfBirth(convert(agentEntityEnrichment.getRdaGr2DateOfBirth())); - output.setDateOfDeath(convert(agentEntityEnrichment.getRdaGr2DateOfDeath())); - output.setDateOfEstablishment(convert(agentEntityEnrichment.getRdaGr2DateOfEstablishment())); - output.setDateOfTermination(convert(agentEntityEnrichment.getRdaGr2DateOfTermination())); - output.setGender(convert(agentEntityEnrichment.getRdaGr2Gender())); + output.setDateOfBirth(convertMultilingualMapToLabel(agentEntityEnrichment.getRdaGr2DateOfBirth())); + output.setDateOfDeath(convertMultilingualMapToLabel(agentEntityEnrichment.getRdaGr2DateOfDeath())); + output.setDateOfEstablishment(convertMultilingualMapToLabel(agentEntityEnrichment.getRdaGr2DateOfEstablishment())); + output.setDateOfTermination(convertMultilingualMapToLabel(agentEntityEnrichment.getRdaGr2DateOfTermination())); + output.setGender(convertMultilingualMapToLabel(agentEntityEnrichment.getRdaGr2Gender())); output.setDate(convertResourceOrLiteral(agentEntityEnrichment.getDcDate())); output.setProfessionOrOccupation( convertResourceOrLiteral(agentEntityEnrichment.getRdaGr2ProfessionOrOccupation())); output.setWasPresentAt(convertToResourceList(agentEntityEnrichment.getEdmWasPresentAt())); - output.setSameAs(convertToPartsList(agentEntityEnrichment.getOwlSameAs())); + output.setSameAs(convertListToPart(agentEntityEnrichment.getOwlSameAs())); return output; } @@ -205,10 +206,10 @@ private static Organization convertOrganization( Organization output = new Organization(); output.setAbout(organizationEnrichmentEntity.getAbout()); - output.setPrefLabelList(convert(organizationEnrichmentEntity.getPrefLabel())); - output.setAltLabelList(convert(organizationEnrichmentEntity.getAltLabel())); - output.setNotes(convert(organizationEnrichmentEntity.getNote())); - output.setSameAs(convertToPartsList(organizationEnrichmentEntity.getOwlSameAs())); + output.setPrefLabelList(convertMultilingualMapToLabel(organizationEnrichmentEntity.getPrefLabel())); + output.setAltLabelList(convertMultilingualMapToLabel(organizationEnrichmentEntity.getAltLabel())); + output.setNotes(convertMultilingualMapToLabel(organizationEnrichmentEntity.getNote())); + output.setSameAs(convertListToPart(organizationEnrichmentEntity.getOwlSameAs())); if (MapUtils.isNotEmpty(organizationEnrichmentEntity.getEdmCountry())) { output.setCountry( organizationEnrichmentEntity.getEdmCountry().entrySet().iterator().next().getValue()); @@ -222,7 +223,7 @@ private static Organization convertOrganization( output.setHomepage(new Resource(organizationEnrichmentEntity.getFoafHomepage())); output.setLogo(new Resource(organizationEnrichmentEntity.getFoafLogo())); output.setDepiction(new Resource(organizationEnrichmentEntity.getFoafDepiction())); - output.setAcronyms(convert(organizationEnrichmentEntity.getEdmAcronym())); + output.setAcronyms(convertMultilingualMapToLabel(organizationEnrichmentEntity.getEdmAcronym())); output.setDescriptions(convertMapToLabels(organizationEnrichmentEntity.getDcDescription())); final Address address = organizationEnrichmentEntity.getAddress(); @@ -240,18 +241,6 @@ private static Organization convertOrganization( return output; } - static EnrichmentTerm organizationImplToEnrichmentTerm( - OrganizationEnrichmentEntity organizationEnrichmentEntity, Date created, Date updated) { - final EnrichmentTerm enrichmentTerm = new EnrichmentTerm(); - enrichmentTerm.setEnrichmentEntity(organizationEnrichmentEntity); - enrichmentTerm.setEntityType(EntityType.ORGANIZATION); - enrichmentTerm.setCreated(Objects.requireNonNullElseGet(created, Date::new)); - enrichmentTerm.setUpdated(updated); - enrichmentTerm.setLabelInfos(createLabelInfoList(organizationEnrichmentEntity)); - - return enrichmentTerm; - } - /** * Generates the list of {@link LabelInfo} values. * @param abstractEnrichmentEntity the entity to generate them for @@ -283,52 +272,4 @@ private static void copyToCombinedLabels(Map> combinedLabel }); } } - - private static List + + org.apache.commons + commons-collections4 + org.apache.commons commons-pool2 diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/solr/property/AggregationSolrCreator.java b/metis-indexing/src/main/java/eu/europeana/indexing/solr/property/AggregationSolrCreator.java index 64c23c9e4..0d7d75708 100644 --- a/metis-indexing/src/main/java/eu/europeana/indexing/solr/property/AggregationSolrCreator.java +++ b/metis-indexing/src/main/java/eu/europeana/indexing/solr/property/AggregationSolrCreator.java @@ -15,8 +15,8 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/utils/RdfWrapper.java b/metis-indexing/src/main/java/eu/europeana/indexing/utils/RdfWrapper.java index 7fefe12b2..1ab838a24 100644 --- a/metis-indexing/src/main/java/eu/europeana/indexing/utils/RdfWrapper.java +++ b/metis-indexing/src/main/java/eu/europeana/indexing/utils/RdfWrapper.java @@ -42,7 +42,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; /** diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/metadata/EnablingElementsBreakdownClassifierTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/metadata/EnablingElementsBreakdownClassifierTest.java index 9219362aa..afc4a2a33 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/metadata/EnablingElementsBreakdownClassifierTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/metadata/EnablingElementsBreakdownClassifierTest.java @@ -32,7 +32,7 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.junit.jupiter.api.Test; class EnablingElementsBreakdownClassifierTest { diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/ContentTierBreakdownTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/ContentTierBreakdownTest.java index 4378062b2..3fedd9a06 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/ContentTierBreakdownTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/ContentTierBreakdownTest.java @@ -11,7 +11,7 @@ import eu.europeana.metis.schema.model.MediaType; import java.util.Collections; import java.util.List; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.junit.jupiter.api.Test; class ContentTierBreakdownTest { diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/ContextualClassesBreakdownTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/ContextualClassesBreakdownTest.java index f7bdd44c0..ef02849a1 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/ContextualClassesBreakdownTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/ContextualClassesBreakdownTest.java @@ -7,7 +7,7 @@ import eu.europeana.metis.schema.jibx.PlaceType; import eu.europeana.metis.schema.jibx.TimeSpanType; import java.util.Set; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.junit.jupiter.api.Test; class ContextualClassesBreakdownTest { diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/EnablingElementsBreakdownTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/EnablingElementsBreakdownTest.java index 2731b4f3a..581f14415 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/EnablingElementsBreakdownTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/EnablingElementsBreakdownTest.java @@ -7,7 +7,7 @@ import eu.europeana.indexing.tiers.metadata.EnablingElement; import eu.europeana.indexing.tiers.model.MetadataTier; import java.util.Set; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.junit.jupiter.api.Test; class EnablingElementsBreakdownTest { diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/MediaResourceTechnicalMetadataBuilderTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/MediaResourceTechnicalMetadataBuilderTest.java index cf66fe101..650f28d92 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/MediaResourceTechnicalMetadataBuilderTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/view/MediaResourceTechnicalMetadataBuilderTest.java @@ -16,7 +16,7 @@ import eu.europeana.metis.schema.model.MediaType; import java.util.Collections; import java.util.Set; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.junit.jupiter.api.Test; class MediaResourceTechnicalMetadataBuilderTest { diff --git a/pom.xml b/pom.xml index c716efe26..1200a8957 100644 --- a/pom.xml +++ b/pom.xml @@ -126,14 +126,15 @@ 2.9.0 5.1 + 1.4 + 2.9.0 + 3.12.0 + 4.4 1.7.0 1.68 1.4 2.27.2 3.0.1 - 1.4 - 2.9.0 - 3.12.0 3.2.2 1.9 1.11 @@ -180,6 +181,7 @@ 1.7.30 8.8.2 5.3.7 + 2.7.0 ${version.spring} ${version.spring} 5.5.0 @@ -221,6 +223,11 @@ commons-lang3 ${version.commons.lang3} + + org.apache.commons + commons-collections4 + ${version.commons-collections4} + eu.europeana.corelib corelib-storage @@ -385,6 +392,10 @@ corelib-definitions ${version.corelib} + + commons-collections + commons-collections + dev.morphia.morphia core From 9a5cebde4d93227ff9ff21d876f47a171cd3a508 Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Wed, 8 Jun 2022 14:57:50 +0200 Subject: [PATCH 53/73] EA-2890 Update example properties with entity configuration --- .../src/main/resources/enrichment.properties.example | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/metis-enrichment/metis-enrichment-rest/src/main/resources/enrichment.properties.example b/metis-enrichment/metis-enrichment-rest/src/main/resources/enrichment.properties.example index 0741764f5..6f8489fbe 100644 --- a/metis-enrichment/metis-enrichment-rest/src/main/resources/enrichment.properties.example +++ b/metis-enrichment/metis-enrichment-rest/src/main/resources/enrichment.properties.example @@ -15,4 +15,13 @@ enrichment.mongo.application.name= #If not provided defaults to 20 enrichment.batch.size= #Options PERSISTENT, ENTITY_CLIENT. Defaults to PERSISTENT -entity.resolver.type= \ No newline at end of file +entity.resolver.type= + +#Entity Management Base Url for Entity Retrieval +entity.management.url= + +#Entity Api V2 url for search and suggest +entity.api.url= + +#Api key +entity.api.key= \ No newline at end of file From e362078f8634641876dc00305386b8f9b3dc0890 Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Wed, 8 Jun 2022 15:20:25 +0200 Subject: [PATCH 54/73] EA-2890 Fix build after merge --- .../enrichment/api/external/impl}/EntityResolverType.java | 2 +- .../eu/europeana/enrichment/rest/config/Application.java | 1 + .../europeana/indexing/solr/SolrDocumentPopulatorTest.java | 2 +- metis-pattern-analysis/pom.xml | 5 ++--- .../eu/europeana/patternanalysis/ProblemPatternAnalyzer.java | 2 +- .../view/DatasetProblemPatternAnalysisTest.java | 2 +- .../patternanalysis/view/ProblemOccurrenceTest.java | 2 +- .../europeana/patternanalysis/view/ProblemPatternTest.java | 2 +- .../europeana/patternanalysis/view/RecordAnalysisTest.java | 2 +- pom.xml | 1 - 10 files changed, 10 insertions(+), 11 deletions(-) rename metis-enrichment/{metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config => metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl}/EntityResolverType.java (73%) diff --git a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/EntityResolverType.java b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/EntityResolverType.java similarity index 73% rename from metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/EntityResolverType.java rename to metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/EntityResolverType.java index 5f6580977..48dea0d79 100644 --- a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/EntityResolverType.java +++ b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/EntityResolverType.java @@ -1,4 +1,4 @@ -package eu.europeana.enrichment.rest.config; +package eu.europeana.enrichment.api.external.impl; /** * Entity resolver type for choosing the requested implementation diff --git a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/Application.java b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/Application.java index ded647fdd..ca30e0818 100644 --- a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/Application.java +++ b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/Application.java @@ -6,6 +6,7 @@ import com.mongodb.client.MongoClient; import eu.europeana.corelib.web.socks.SocksProxy; import eu.europeana.enrichment.api.external.impl.ClientEntityResolver; +import eu.europeana.enrichment.api.external.impl.EntityResolverType; import eu.europeana.enrichment.api.internal.EntityResolver; import eu.europeana.enrichment.service.EnrichmentService; import eu.europeana.enrichment.service.PersistentEntityResolver; diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/SolrDocumentPopulatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/SolrDocumentPopulatorTest.java index 541d5a549..aaac7616a 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/solr/SolrDocumentPopulatorTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/SolrDocumentPopulatorTest.java @@ -16,7 +16,7 @@ import java.io.FileInputStream; import java.nio.charset.StandardCharsets; import java.util.List; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.IOUtils; import org.apache.solr.common.SolrInputDocument; import org.junit.jupiter.api.Test; diff --git a/metis-pattern-analysis/pom.xml b/metis-pattern-analysis/pom.xml index 00ffc9309..3b9ca23fb 100644 --- a/metis-pattern-analysis/pom.xml +++ b/metis-pattern-analysis/pom.xml @@ -19,9 +19,8 @@ ${version.jackson} - commons-collections - commons-collections - ${version.commons.collections} + org.apache.commons + commons-collections4 commons-io diff --git a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/ProblemPatternAnalyzer.java b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/ProblemPatternAnalyzer.java index 047b69a65..bcad10778 100644 --- a/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/ProblemPatternAnalyzer.java +++ b/metis-pattern-analysis/src/main/java/eu/europeana/patternanalysis/ProblemPatternAnalyzer.java @@ -37,7 +37,7 @@ import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.stream.Collectors; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.similarity.LongestCommonSubsequence; diff --git a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/DatasetProblemPatternAnalysisTest.java b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/DatasetProblemPatternAnalysisTest.java index d287b30c6..208c79256 100644 --- a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/DatasetProblemPatternAnalysisTest.java +++ b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/DatasetProblemPatternAnalysisTest.java @@ -6,7 +6,7 @@ import java.time.LocalDateTime; import java.util.List; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.junit.jupiter.api.Test; class DatasetProblemPatternAnalysisTest { diff --git a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemOccurrenceTest.java b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemOccurrenceTest.java index f92d04a3e..85de050bb 100644 --- a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemOccurrenceTest.java +++ b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemOccurrenceTest.java @@ -5,7 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.junit.jupiter.api.Test; class ProblemOccurrenceTest { diff --git a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemPatternTest.java b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemPatternTest.java index d4112cbff..a72838240 100644 --- a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemPatternTest.java +++ b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/ProblemPatternTest.java @@ -5,7 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.junit.jupiter.api.Test; class ProblemPatternTest { diff --git a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/RecordAnalysisTest.java b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/RecordAnalysisTest.java index 3bcb60b29..85539c8b6 100644 --- a/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/RecordAnalysisTest.java +++ b/metis-pattern-analysis/src/test/java/eu/europeana/patternanalysis/view/RecordAnalysisTest.java @@ -5,7 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.junit.jupiter.api.Test; class RecordAnalysisTest { diff --git a/pom.xml b/pom.xml index 1200a8957..57d5c4589 100644 --- a/pom.xml +++ b/pom.xml @@ -135,7 +135,6 @@ 1.4 2.27.2 3.0.1 - 3.2.2 1.9 1.11 2.15.3 From adbe0239a37282508876bfecd985b38e1951686e Mon Sep 17 00:00:00 2001 From: Jorge Ortiz Date: Wed, 8 Jun 2022 17:52:42 +0200 Subject: [PATCH 55/73] MET-4440_MET-4569 add ProxySolrCreator unit tests (#554) * MET-4440_MET-4569 add ProxySolrCreator unit tests * MET-4440_MET-4569 refactor unit test to remove code smells from sonar --- .../solr/property/ProxySolrCreatorTest.java | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ProxySolrCreatorTest.java diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ProxySolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ProxySolrCreatorTest.java new file mode 100644 index 000000000..8ba1b3fe7 --- /dev/null +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ProxySolrCreatorTest.java @@ -0,0 +1,132 @@ +package eu.europeana.indexing.solr.property; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import eu.europeana.corelib.solr.entity.ProxyImpl; +import eu.europeana.indexing.solr.EdmLabel; +import java.util.List; +import java.util.Map; +import org.apache.solr.common.SolrInputDocument; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit test for {@link ProxySolrCreator} class + */ +class ProxySolrCreatorTest { + + private ProxySolrCreator proxySolrCreator; + private SolrInputDocument solrInputDocument; + private ProxyImpl proxy; + + @BeforeEach + void setup() { + solrInputDocument = new SolrInputDocument(); + proxySolrCreator = new ProxySolrCreator(); + proxy = new ProxyImpl(); + } + + @Test + void addToDocument() { + proxy = getTestProxy(); + + proxySolrCreator.addToDocument(solrInputDocument, proxy); + + assertKeys(solrInputDocument); + assertDocumentContent(solrInputDocument); + } + + private void assertDocumentContent(SolrInputDocument solrInputDocument) { + assertEquals(List.of("locationA", "locationB"), solrInputDocument.getFieldValues(EdmLabel.PROXY_EDM_CURRENT_LOCATION + ".location")); + assertEquals(List.of("contributorA", "contributorB"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_CONTRIBUTOR + ".contributor")); + assertEquals(List.of("coverage1", "coverage2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_COVERAGE + ".coverage")); + assertEquals(List.of("creator1", "creator2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_CREATOR + ".creator")); + assertEquals(List.of("date1", "date2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_DATE + ".date")); + assertEquals(List.of("description1", "description2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_DESCRIPTION + ".description")); + assertEquals(List.of("format1", "format2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_FORMAT + ".format")); + assertEquals(List.of("identifier1", "identifier2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_IDENTIFIER + ".identifier")); + assertEquals(List.of("taal1", "taal2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_LANGUAGE + ".language")); + assertEquals(List.of("publisher1", "publisher2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_PUBLISHER + ".publisher")); + assertEquals(List.of("rights1", "rights2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_RIGHTS + ".rights")); + assertEquals(List.of("source1", "source2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_SOURCE + ".source")); + assertEquals(List.of("subject1", "subject2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_SUBJECT + ".subject")); + assertEquals(List.of("title1", "title2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_TITLE + ".title")); + assertEquals(List.of("type1", "type2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_TYPE + ".type")); + assertEquals(List.of("alt1", "alt2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DCTERMS_ALTERNATIVE + ".alternative")); + assertEquals(List.of("created1", "created2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DCTERMS_CREATED + ".created")); + assertEquals(List.of("hasPart1", "hasPart2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DCTERMS_HAS_PART + ".hasPart")); + assertEquals(List.of("isPartOf1", "isPartOf2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DCTERMS_IS_PART_OF + ".isPartOf")); + assertEquals(List.of("issued1", "issued2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DCTERMS_ISSUED + ".issued")); + assertEquals(List.of("medium1", "medium2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DCTERMS_MEDIUM + ".medium")); + assertEquals(List.of("provenance1", "provenance2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DCTERMS_PROVENANCE + ".provenance")); + assertEquals(List.of("spatial1", "spatial2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DCTERMS_SPATIAL + ".spatial")); + assertEquals(List.of("temp1", "temp2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DCTERMS_TEMPORAL + ".temporal")); + assertEquals(List.of("year1", "year2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_EDM_YEAR + ".year")); + assertEquals(List.of("hasMet1", "hasMet2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_EDM_HAS_MET + ".hasMet")); + assertEquals(List.of("isRelatedTo1", "isRelatedTo2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_EDM_ISRELATEDTO + ".isRelatedTo")); + } + + private void assertKeys(SolrInputDocument solrInputDocument) { + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_EDM_CURRENT_LOCATION + ".location")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_CONTRIBUTOR + ".contributor")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_COVERAGE + ".coverage")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_CREATOR + ".creator")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_DATE + ".date")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_DESCRIPTION + ".description")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_FORMAT + ".format")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_IDENTIFIER + ".identifier")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_LANGUAGE + ".language")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_PUBLISHER + ".publisher")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_RIGHTS + ".rights")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_SOURCE + ".source")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_SUBJECT + ".subject")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_TITLE + ".title")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_TYPE + ".type")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DCTERMS_ALTERNATIVE + ".alternative")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DCTERMS_CREATED + ".created")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DCTERMS_HAS_PART + ".hasPart")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DCTERMS_IS_PART_OF + ".isPartOf")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DCTERMS_ISSUED + ".issued")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DCTERMS_MEDIUM + ".medium")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DCTERMS_PROVENANCE + ".provenance")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DCTERMS_SPATIAL + ".spatial")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DCTERMS_TEMPORAL + ".temporal")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_EDM_YEAR + ".year")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_EDM_HAS_MET + ".hasMet")); + assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_EDM_ISRELATEDTO + ".isRelatedTo")); + } + + private ProxyImpl getTestProxy() { + ProxyImpl proxy = new ProxyImpl(); + proxy.setAbout("proxy"); + proxy.setEdmCurrentLocation(Map.of("location", List.of("locationA", "locationB"))); + proxy.setDcContributor(Map.of("contributor", List.of("contributorA", "contributorB"))); + proxy.setDcCoverage(Map.of("coverage", List.of("coverage1", "coverage2"))); + proxy.setDcCreator(Map.of("creator", List.of("creator1", "creator2"))); + proxy.setDcDate(Map.of("date", List.of("date1", "date2"))); + proxy.setDcDescription(Map.of("description", List.of("description1", "description2"))); + proxy.setDcFormat(Map.of("format", List.of("format1", "format2"))); + proxy.setDcIdentifier(Map.of("identifier", List.of("identifier1", "identifier2"))); + proxy.setDcLanguage(Map.of("language", List.of("taal1", "taal2"))); + proxy.setDcPublisher(Map.of("publisher", List.of("publisher1", "publisher2"))); + proxy.setDcRights(Map.of("rights", List.of("rights1", "rights2"))); + proxy.setDcSource(Map.of("source", List.of("source1", "source2"))); + proxy.setDcSubject(Map.of("subject", List.of("subject1", "subject2"))); + proxy.setDcTitle(Map.of("title", List.of("title1", "title2"))); + proxy.setDcType(Map.of("type", List.of("type1", "type2"))); + proxy.setDctermsAlternative(Map.of("alternative", List.of("alt1", "alt2"))); + proxy.setDctermsCreated(Map.of("created", List.of("created1", "created2"))); + proxy.setDctermsHasPart(Map.of("hasPart", List.of("hasPart1", "hasPart2"))); + proxy.setDctermsIsPartOf(Map.of("isPartOf", List.of("isPartOf1", "isPartOf2"))); + proxy.setDctermsIssued(Map.of("issued", List.of("issued1", "issued2"))); + proxy.setDctermsMedium(Map.of("medium", List.of("medium1", "medium2"))); + proxy.setDctermsProvenance(Map.of("provenance", List.of("provenance1", "provenance2"))); + proxy.setDctermsSpatial(Map.of("spatial", List.of("spatial1", "spatial2"))); + proxy.setDctermsTemporal(Map.of("temporal", List.of("temp1", "temp2"))); + proxy.setYear(Map.of("year", List.of("year1", "year2"))); + proxy.setEdmHasMet(Map.of("hasMet", List.of("hasMet1", "hasMet2"))); + proxy.setEdmIsRelatedTo(Map.of("isRelatedTo", List.of("isRelatedTo1", "isRelatedTo2"))); + return proxy; + } +} From b5b9b3d69a69834fe23f93ef77bd3f93baa77e99 Mon Sep 17 00:00:00 2001 From: Jorge Ortiz Date: Wed, 8 Jun 2022 18:02:03 +0200 Subject: [PATCH 56/73] MET-4440_MET-4576 FacetEncoder basic unit tests (#553) --- .../indexing/solr/facet/FacetEncoderTest.java | 301 ++++++++++++++++++ 1 file changed, 301 insertions(+) create mode 100644 metis-indexing/src/test/java/eu/europeana/indexing/solr/facet/FacetEncoderTest.java diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/facet/FacetEncoderTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/facet/FacetEncoderTest.java new file mode 100644 index 000000000..0cfbecc9f --- /dev/null +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/facet/FacetEncoderTest.java @@ -0,0 +1,301 @@ +package eu.europeana.indexing.solr.facet; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import eu.europeana.indexing.solr.facet.FacetEncoder.FacetWithValues; +import eu.europeana.indexing.solr.facet.value.AudioDuration; +import eu.europeana.indexing.solr.facet.value.AudioQuality; +import eu.europeana.indexing.solr.facet.value.ImageAspectRatio; +import eu.europeana.indexing.solr.facet.value.ImageColorEncoding; +import eu.europeana.indexing.solr.facet.value.ImageColorSpace; +import eu.europeana.indexing.solr.facet.value.ImageSize; +import eu.europeana.indexing.solr.facet.value.MimeTypeEncoding; +import eu.europeana.indexing.solr.facet.value.VideoDuration; +import eu.europeana.indexing.solr.facet.value.VideoQuality; +import eu.europeana.indexing.utils.RdfWrapper; +import eu.europeana.indexing.utils.WebResourceWrapper; +import eu.europeana.metis.schema.jibx.Aggregation; +import eu.europeana.metis.schema.jibx.CodecName; +import eu.europeana.metis.schema.jibx.HasMimeType; +import eu.europeana.metis.schema.jibx.HasView; +import eu.europeana.metis.schema.jibx.IsShownAt; +import eu.europeana.metis.schema.jibx.RDF; +import eu.europeana.metis.schema.jibx.Type1; +import eu.europeana.metis.schema.jibx.WebResourceType; +import java.util.List; +import java.util.Set; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit test for {@link FacetEncoder} class + */ +class FacetEncoderTest { + + private FacetEncoder encoder; + + Set audioMimeTypeEncodings = Set.of(MimeTypeEncoding.TYPE_589, MimeTypeEncoding.TYPE_590, + MimeTypeEncoding.TYPE_591, + MimeTypeEncoding.TYPE_592, MimeTypeEncoding.TYPE_593, MimeTypeEncoding.TYPE_594, + MimeTypeEncoding.TYPE_595, MimeTypeEncoding.TYPE_596, MimeTypeEncoding.TYPE_597, + MimeTypeEncoding.TYPE_598, MimeTypeEncoding.TYPE_599, MimeTypeEncoding.TYPE_600, + MimeTypeEncoding.TYPE_601, MimeTypeEncoding.TYPE_602, MimeTypeEncoding.TYPE_603, + MimeTypeEncoding.TYPE_604, MimeTypeEncoding.TYPE_605, MimeTypeEncoding.TYPE_606, + MimeTypeEncoding.TYPE_607, MimeTypeEncoding.TYPE_608, MimeTypeEncoding.TYPE_609, + MimeTypeEncoding.TYPE_610, MimeTypeEncoding.TYPE_611, MimeTypeEncoding.TYPE_612, + MimeTypeEncoding.TYPE_613, MimeTypeEncoding.TYPE_614, MimeTypeEncoding.TYPE_615, + MimeTypeEncoding.TYPE_616, MimeTypeEncoding.TYPE_617, MimeTypeEncoding.TYPE_618, + MimeTypeEncoding.TYPE_619, MimeTypeEncoding.TYPE_620); + Set audioQualities = Set.of(AudioQuality.HIGH); + Set audioDurations = Set.of(AudioDuration.TINY, AudioDuration.SHORT, AudioDuration.MEDIUM, AudioDuration.LONG); + + Set videoMimeTypeEncodings = Set.of( + MimeTypeEncoding.TYPE_726, MimeTypeEncoding.TYPE_727, MimeTypeEncoding.TYPE_728, + MimeTypeEncoding.TYPE_729, MimeTypeEncoding.TYPE_730, MimeTypeEncoding.TYPE_731, + MimeTypeEncoding.TYPE_732, MimeTypeEncoding.TYPE_733, MimeTypeEncoding.TYPE_734, + MimeTypeEncoding.TYPE_735, MimeTypeEncoding.TYPE_736, MimeTypeEncoding.TYPE_737, + MimeTypeEncoding.TYPE_738, MimeTypeEncoding.TYPE_739, MimeTypeEncoding.TYPE_740, + MimeTypeEncoding.TYPE_741, MimeTypeEncoding.TYPE_742, MimeTypeEncoding.TYPE_743, + MimeTypeEncoding.TYPE_744, MimeTypeEncoding.TYPE_745, MimeTypeEncoding.TYPE_746, + MimeTypeEncoding.TYPE_747, MimeTypeEncoding.TYPE_748, MimeTypeEncoding.TYPE_749, + MimeTypeEncoding.TYPE_750, MimeTypeEncoding.TYPE_751, MimeTypeEncoding.TYPE_752, + MimeTypeEncoding.TYPE_753, MimeTypeEncoding.TYPE_754, MimeTypeEncoding.TYPE_755, + MimeTypeEncoding.TYPE_756, MimeTypeEncoding.TYPE_757, MimeTypeEncoding.TYPE_758, + MimeTypeEncoding.TYPE_759, MimeTypeEncoding.TYPE_760, MimeTypeEncoding.TYPE_761, + MimeTypeEncoding.TYPE_762, MimeTypeEncoding.TYPE_763, MimeTypeEncoding.TYPE_764 + ); + Set videoQualities = Set.of(VideoQuality.HIGH); + Set videoDurations = Set.of(VideoDuration.SHORT, VideoDuration.MEDIUM, VideoDuration.LONG); + + Set imageMimeTypeEncodings = Set.of( + MimeTypeEncoding.TYPE_627, MimeTypeEncoding.TYPE_628, MimeTypeEncoding.TYPE_629, + MimeTypeEncoding.TYPE_630, MimeTypeEncoding.TYPE_631, MimeTypeEncoding.TYPE_632, + MimeTypeEncoding.TYPE_633, MimeTypeEncoding.TYPE_634, MimeTypeEncoding.TYPE_635, + MimeTypeEncoding.TYPE_636, MimeTypeEncoding.TYPE_637, MimeTypeEncoding.TYPE_638, + MimeTypeEncoding.TYPE_639, MimeTypeEncoding.TYPE_640, MimeTypeEncoding.TYPE_641, + MimeTypeEncoding.TYPE_642, MimeTypeEncoding.TYPE_643, MimeTypeEncoding.TYPE_644, + MimeTypeEncoding.TYPE_645, MimeTypeEncoding.TYPE_646, MimeTypeEncoding.TYPE_647, + MimeTypeEncoding.TYPE_648, MimeTypeEncoding.TYPE_649, MimeTypeEncoding.TYPE_650, + MimeTypeEncoding.TYPE_651, MimeTypeEncoding.TYPE_652, MimeTypeEncoding.TYPE_653, + MimeTypeEncoding.TYPE_654, MimeTypeEncoding.TYPE_655, MimeTypeEncoding.TYPE_656, + MimeTypeEncoding.TYPE_657, MimeTypeEncoding.TYPE_658, MimeTypeEncoding.TYPE_659, + MimeTypeEncoding.TYPE_660, MimeTypeEncoding.TYPE_661, MimeTypeEncoding.TYPE_662, + MimeTypeEncoding.TYPE_663, MimeTypeEncoding.TYPE_664, MimeTypeEncoding.TYPE_665, + MimeTypeEncoding.TYPE_666, MimeTypeEncoding.TYPE_667, MimeTypeEncoding.TYPE_668, + MimeTypeEncoding.TYPE_669, MimeTypeEncoding.TYPE_670, MimeTypeEncoding.TYPE_671, + MimeTypeEncoding.TYPE_672 + ); + Set imageColorSpaces = Set.of( + ImageColorSpace.GRAYSCALE, ImageColorSpace.COLOR, ImageColorSpace.OTHER + ); + Set imageSizes = Set.of(ImageSize.SMALL, ImageSize.MEDIUM, ImageSize.LARGE, ImageSize.HUGE); + Set imageAspectRatios = Set.of(ImageAspectRatio.LANDSCAPE, ImageAspectRatio.PORTRAIT); + Set imageColorEncodings = Set.of(ImageColorEncoding.COLOR_1, + ImageColorEncoding.COLOR_2, ImageColorEncoding.COLOR_3, ImageColorEncoding.COLOR_4, + ImageColorEncoding.COLOR_5, ImageColorEncoding.COLOR_6, ImageColorEncoding.COLOR_7, + ImageColorEncoding.COLOR_8, ImageColorEncoding.COLOR_9, ImageColorEncoding.COLOR_10, + ImageColorEncoding.COLOR_11, ImageColorEncoding.COLOR_12, ImageColorEncoding.COLOR_13, + ImageColorEncoding.COLOR_14, ImageColorEncoding.COLOR_15, ImageColorEncoding.COLOR_16, + ImageColorEncoding.COLOR_17, ImageColorEncoding.COLOR_18, ImageColorEncoding.COLOR_19, + ImageColorEncoding.COLOR_20, ImageColorEncoding.COLOR_21, ImageColorEncoding.COLOR_22, + ImageColorEncoding.COLOR_23, ImageColorEncoding.COLOR_24, ImageColorEncoding.COLOR_25, + ImageColorEncoding.COLOR_26, ImageColorEncoding.COLOR_27, ImageColorEncoding.COLOR_28, + ImageColorEncoding.COLOR_29, ImageColorEncoding.COLOR_30, ImageColorEncoding.COLOR_31, + ImageColorEncoding.COLOR_32, ImageColorEncoding.COLOR_33, ImageColorEncoding.COLOR_34, + ImageColorEncoding.COLOR_35, ImageColorEncoding.COLOR_36, ImageColorEncoding.COLOR_37, + ImageColorEncoding.COLOR_38, ImageColorEncoding.COLOR_39, ImageColorEncoding.COLOR_40, + ImageColorEncoding.COLOR_41, ImageColorEncoding.COLOR_42, ImageColorEncoding.COLOR_43, + ImageColorEncoding.COLOR_44, ImageColorEncoding.COLOR_45, ImageColorEncoding.COLOR_46, + ImageColorEncoding.COLOR_47, ImageColorEncoding.COLOR_48, ImageColorEncoding.COLOR_49, + ImageColorEncoding.COLOR_50, ImageColorEncoding.COLOR_51, ImageColorEncoding.COLOR_52, + ImageColorEncoding.COLOR_53, ImageColorEncoding.COLOR_54, ImageColorEncoding.COLOR_55, + ImageColorEncoding.COLOR_56, ImageColorEncoding.COLOR_57, ImageColorEncoding.COLOR_58, + ImageColorEncoding.COLOR_59, ImageColorEncoding.COLOR_60, ImageColorEncoding.COLOR_61, + ImageColorEncoding.COLOR_62, ImageColorEncoding.COLOR_63, ImageColorEncoding.COLOR_64, + ImageColorEncoding.COLOR_65, ImageColorEncoding.COLOR_66, ImageColorEncoding.COLOR_67, + ImageColorEncoding.COLOR_68, ImageColorEncoding.COLOR_69, ImageColorEncoding.COLOR_70, + ImageColorEncoding.COLOR_71, ImageColorEncoding.COLOR_72, ImageColorEncoding.COLOR_73, + ImageColorEncoding.COLOR_74, ImageColorEncoding.COLOR_75, ImageColorEncoding.COLOR_76, + ImageColorEncoding.COLOR_77, ImageColorEncoding.COLOR_78, ImageColorEncoding.COLOR_79, + ImageColorEncoding.COLOR_80, ImageColorEncoding.COLOR_81, ImageColorEncoding.COLOR_82, + ImageColorEncoding.COLOR_83, ImageColorEncoding.COLOR_84, ImageColorEncoding.COLOR_85, + ImageColorEncoding.COLOR_86, ImageColorEncoding.COLOR_87, ImageColorEncoding.COLOR_88, + ImageColorEncoding.COLOR_89, ImageColorEncoding.COLOR_90, ImageColorEncoding.COLOR_91, + ImageColorEncoding.COLOR_92, ImageColorEncoding.COLOR_93, ImageColorEncoding.COLOR_94, + ImageColorEncoding.COLOR_95, ImageColorEncoding.COLOR_96, ImageColorEncoding.COLOR_97, + ImageColorEncoding.COLOR_98, ImageColorEncoding.COLOR_99, ImageColorEncoding.COLOR_100, + ImageColorEncoding.COLOR_101, ImageColorEncoding.COLOR_102, ImageColorEncoding.COLOR_103, + ImageColorEncoding.COLOR_104, ImageColorEncoding.COLOR_105, ImageColorEncoding.COLOR_106, + ImageColorEncoding.COLOR_107, ImageColorEncoding.COLOR_108, ImageColorEncoding.COLOR_109, + ImageColorEncoding.COLOR_110, ImageColorEncoding.COLOR_111, ImageColorEncoding.COLOR_112, + ImageColorEncoding.COLOR_113, ImageColorEncoding.COLOR_114, ImageColorEncoding.COLOR_115, + ImageColorEncoding.COLOR_116, ImageColorEncoding.COLOR_117, ImageColorEncoding.COLOR_118, + ImageColorEncoding.COLOR_119, ImageColorEncoding.COLOR_120, ImageColorEncoding.COLOR_121, + ImageColorEncoding.COLOR_122, ImageColorEncoding.COLOR_123, ImageColorEncoding.COLOR_124, + ImageColorEncoding.COLOR_125, ImageColorEncoding.COLOR_126, ImageColorEncoding.COLOR_127, + ImageColorEncoding.COLOR_128, ImageColorEncoding.COLOR_129, ImageColorEncoding.COLOR_130, + ImageColorEncoding.COLOR_131, ImageColorEncoding.COLOR_132, ImageColorEncoding.COLOR_133, + ImageColorEncoding.COLOR_134, ImageColorEncoding.COLOR_135, ImageColorEncoding.COLOR_136, + ImageColorEncoding.COLOR_137, ImageColorEncoding.COLOR_138); + Set textMimeTypeEncodings = Set.of( + MimeTypeEncoding.TYPE_621, MimeTypeEncoding.TYPE_622, MimeTypeEncoding.TYPE_623, + MimeTypeEncoding.TYPE_624, MimeTypeEncoding.TYPE_625, MimeTypeEncoding.TYPE_626); + + List getBasicWebResourceWrappers() { + Type1 type1 = new Type1(); + type1.setResource("resource"); + + CodecName codecName = new CodecName(); + codecName.setCodecName("codec"); + + HasMimeType hasMimeType = new HasMimeType(); + hasMimeType.setHasMimeType("hasMimeType"); + + Aggregation aggregation = new Aggregation(); + aggregation.setAbout("about"); + + HasView hasView = new HasView(); + hasView.setResource("resourceHasView"); + aggregation.setHasViewList(List.of(hasView)); + + IsShownAt isShownAt = new IsShownAt(); + isShownAt.setResource("resourceIsShown"); + aggregation.setIsShownAt(isShownAt); + + WebResourceType webResourceType = new WebResourceType(); + webResourceType.setType(type1); + webResourceType.setCodecName(codecName); + webResourceType.setHasMimeType(hasMimeType); + webResourceType.setAbout("about"); + + RDF rdf = new RDF(); + rdf.setAggregationList(List.of(aggregation)); + rdf.setWebResourceList(List.of(webResourceType)); + + RdfWrapper rdfWrapper = new RdfWrapper(rdf); + + List webResourceWrappers = rdfWrapper.getWebResourceWrappers(); + return webResourceWrappers; + } + + @BeforeEach + void setup() { + encoder = new FacetEncoder(); + } + + @Test + void getFacetSearchCodes() { + Set resultAudio = FacetEncoder.getFacetSearchCodes(EncodedFacetCollection.AUDIO, + new FacetWithValues<>(EncodedFacet.AUDIO_QUALITY, audioQualities)); + + assertEquals(1, resultAudio.size()); + assertTrue(resultAudio.contains(67117056)); + } + + @Test + void getAudioFacetSearchCodes() { + Set result = encoder.getAudioFacetSearchCodes(audioMimeTypeEncodings, audioQualities, audioDurations); + + assertEquals(128, result.size()); + } + + @Test + void getVideoFacetSearchCodes() { + Set result = encoder.getVideoFacetSearchCodes(videoMimeTypeEncodings, videoQualities, videoDurations); + + assertEquals(117, result.size()); + } + + @Test + void getImageFacetSearchCodes() { + Set result = encoder.getImageFacetSearchCodes(imageMimeTypeEncodings, imageSizes, imageColorSpaces, + imageAspectRatios, + imageColorEncodings); + + assertEquals(152352, result.size()); + } + + @Test + void getTextFacetSearchCodes() { + Set result = encoder.getTextFacetSearchCodes(textMimeTypeEncodings); + + assertEquals(6, result.size()); + } + + @Test + void getFacetFilterCodes() { + List webResourceWrappers = getBasicWebResourceWrappers(); + + Set result = encoder.getFacetFilterCodes(webResourceWrappers.get(0)); + + assertEquals(0, result.size()); + } + + @Test + void getAudioFacetFilterCodes() { + Set result = encoder.getAudioFacetFilterCodes(audioMimeTypeEncodings, audioQualities, audioDurations); + + assertEquals(330, result.size()); + } + + @Test + void getVideoFacetFilterCodes() { + Set result = encoder.getVideoFacetFilterCodes(videoMimeTypeEncodings, videoQualities, videoDurations); + + assertEquals(320, result.size()); + } + + @Test + void getImageFacetFilterCodes() { + Set result = encoder.getImageFacetFilterCodes(imageMimeTypeEncodings, imageSizes, imageColorSpaces, + imageAspectRatios, imageColorEncodings); + + assertEquals(391980, result.size()); + } + + @Test + void getTextFacetFilterCodes() { + Set result = encoder.getTextFacetFilterCodes(textMimeTypeEncodings); + + assertEquals(7, result.size()); + } + + @Test + void getFacetValueCodes() { + List webResourceWrappers = getBasicWebResourceWrappers(); + + Set result = encoder.getFacetValueCodes(webResourceWrappers.get(0)); + + assertEquals(0, result.size()); + } + + @Test + void getAudioFacetValueCodes() { + Set result = encoder.getAudioFacetValueCodes(audioMimeTypeEncodings, audioQualities, audioDurations); + + assertEquals(37, result.size()); + } + + @Test + void getVideoFacetValueCodes() { + Set result = encoder.getVideoFacetValueCodes(videoMimeTypeEncodings, videoQualities, videoDurations); + + assertEquals(43, result.size()); + } + + @Test + void getImageFacetValueCodes() { + Set result = encoder.getImageFacetValueCodes(imageMimeTypeEncodings, imageSizes, imageColorSpaces, imageAspectRatios, + imageColorEncodings); + + assertEquals(193, result.size()); + } + + @Test + void getTextFacetValueCodes() { + Set result = encoder.getTextFacetValueCodes(textMimeTypeEncodings); + + assertEquals(6, result.size()); + } +} From e74c00f020f85a52f2641bbc1bbb95a9a6f6f44e Mon Sep 17 00:00:00 2001 From: Jorge Ortiz Date: Wed, 8 Jun 2022 18:19:12 +0200 Subject: [PATCH 57/73] MET-4440_MET-4570 add ConceptSolrCreator unit tests (#546) * MET-4440_MET-4570 add ConceptSolrCreator unit tests * MET-4440_MET-4570 use Map.of to initialize objects --- .../solr/property/ConceptSolrCreatorTest.java | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ConceptSolrCreatorTest.java diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ConceptSolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ConceptSolrCreatorTest.java new file mode 100644 index 000000000..ff0d060c8 --- /dev/null +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ConceptSolrCreatorTest.java @@ -0,0 +1,108 @@ +package eu.europeana.indexing.solr.property; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import eu.europeana.corelib.solr.entity.ConceptImpl; +import eu.europeana.indexing.solr.EdmLabel; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.solr.common.SolrInputDocument; +import org.bson.types.ObjectId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit test for {@link ConceptSolrCreator} class + */ +class ConceptSolrCreatorTest { + + private ConceptSolrCreator conceptSolrCreator; + private SolrInputDocument solrInputDocument; + private ConceptImpl concept; + + @BeforeEach + void setup() { + solrInputDocument = new SolrInputDocument(); + conceptSolrCreator = new ConceptSolrCreator(); + concept = new ConceptImpl(); + } + + @Test + void addToDocument_withConceptPrefValueAndAltValue() { + concept.setId(new ObjectId("6294c725de3fe70c48388a88")); + concept.setAbout("concept"); + concept.setPrefLabel(Map.of("keyPref", List.of("prefValue1", "prefValue2"))); + concept.setAltLabel(Map.of("keyAlt", List.of("altValue1", "altValue2"))); + + conceptSolrCreator.addToDocument(solrInputDocument, concept); + + assertTrue(solrInputDocument.containsKey(EdmLabel.SKOS_CONCEPT.toString()) && + solrInputDocument.containsKey(EdmLabel.CC_SKOS_PREF_LABEL + ".keyPref") && + solrInputDocument.containsKey(EdmLabel.CC_SKOS_ALT_LABEL + ".keyAlt")); + assertEquals("concept", solrInputDocument.getFieldValue(EdmLabel.SKOS_CONCEPT.toString())); + assertEquals(List.of("prefValue1", "prefValue2"), + solrInputDocument.getFieldValues(EdmLabel.CC_SKOS_PREF_LABEL + ".keyPref")); + assertEquals(List.of("altValue1", "altValue2"), + solrInputDocument.getFieldValues(EdmLabel.CC_SKOS_ALT_LABEL + ".keyAlt")); + assertEquals(3, solrInputDocument.size()); + } + + @Test + void addToDocument_withoutConcept() { + concept.setId(new ObjectId("6294c725de3fe70c48388a88")); + concept.setPrefLabel(Map.of("keyPref", List.of("prefValue1", "prefValue2"))); + concept.setAltLabel(Map.of("keyAlt", List.of("altValue1", "altValue2"))); + + conceptSolrCreator.addToDocument(solrInputDocument, concept); + + assertFalse(solrInputDocument.containsKey(EdmLabel.SKOS_CONCEPT.toString())); + assertTrue(solrInputDocument.containsKey(EdmLabel.CC_SKOS_PREF_LABEL + ".keyPref") && + solrInputDocument.containsKey(EdmLabel.CC_SKOS_ALT_LABEL + ".keyAlt")); + assertNull(solrInputDocument.getFieldValue(EdmLabel.SKOS_CONCEPT.toString())); + assertEquals(List.of("prefValue1", "prefValue2"), + solrInputDocument.getFieldValues(EdmLabel.CC_SKOS_PREF_LABEL + ".keyPref")); + assertEquals(List.of("altValue1", "altValue2"), + solrInputDocument.getFieldValues(EdmLabel.CC_SKOS_ALT_LABEL + ".keyAlt")); + assertEquals(2, solrInputDocument.size()); + } + + @Test + void addToDocument_withoutPrefValue() { + concept.setId(new ObjectId("6294c725de3fe70c48388a88")); + concept.setAbout("concept"); + concept.setAltLabel(Map.of("keyAlt", List.of("altValue1", "altValue2"))); + + conceptSolrCreator.addToDocument(solrInputDocument, concept); + + assertFalse(solrInputDocument.containsKey(EdmLabel.CC_SKOS_PREF_LABEL + ".keyPref")); + assertTrue(solrInputDocument.containsKey(EdmLabel.SKOS_CONCEPT.toString()) && + solrInputDocument.containsKey(EdmLabel.CC_SKOS_ALT_LABEL + ".keyAlt")); + assertEquals("concept", solrInputDocument.getFieldValue(EdmLabel.SKOS_CONCEPT.toString())); + assertNull(solrInputDocument.getFieldValues(EdmLabel.CC_SKOS_PREF_LABEL + ".keyPref")); + assertEquals(List.of("altValue1", "altValue2"), + solrInputDocument.getFieldValues(EdmLabel.CC_SKOS_ALT_LABEL + ".keyAlt")); + assertEquals(2, solrInputDocument.size()); + } + + @Test + void addToDocument_withoutAltValue() { + concept.setId(new ObjectId("6294c725de3fe70c48388a88")); + concept.setAbout("concept"); + concept.setPrefLabel(Map.of("keyPref", List.of("prefValue1", "prefValue2"))); + + conceptSolrCreator.addToDocument(solrInputDocument, concept); + + assertFalse(solrInputDocument.containsKey(EdmLabel.CC_SKOS_ALT_LABEL + ".keyAlt")); + assertTrue(solrInputDocument.containsKey(EdmLabel.SKOS_CONCEPT.toString()) && + solrInputDocument.containsKey(EdmLabel.CC_SKOS_PREF_LABEL + ".keyPref")); + assertEquals("concept", solrInputDocument.getFieldValue(EdmLabel.SKOS_CONCEPT.toString())); + assertEquals(List.of("prefValue1", "prefValue2"), + solrInputDocument.getFieldValues(EdmLabel.CC_SKOS_PREF_LABEL + ".keyPref")); + assertNull(solrInputDocument.getFieldValues(EdmLabel.CC_SKOS_ALT_LABEL + ".keyAlt")); + assertEquals(2, solrInputDocument.size()); + } +} From f563f8525085b3e0b8d1cdfaf5e89e400112f380 Mon Sep 17 00:00:00 2001 From: Jorge Ortiz Date: Wed, 8 Jun 2022 18:20:59 +0200 Subject: [PATCH 58/73] MET-4440_MET-4571 add TimespanSolrCreator unit tests (#547) --- .../property/TimespanSolrCreatorTest.java | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 metis-indexing/src/test/java/eu/europeana/indexing/solr/property/TimespanSolrCreatorTest.java diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/TimespanSolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/TimespanSolrCreatorTest.java new file mode 100644 index 000000000..62d07200c --- /dev/null +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/TimespanSolrCreatorTest.java @@ -0,0 +1,139 @@ +package eu.europeana.indexing.solr.property; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import eu.europeana.corelib.solr.entity.TimespanImpl; +import eu.europeana.indexing.solr.EdmLabel; +import java.util.List; +import java.util.Map; +import org.apache.solr.common.SolrInputDocument; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit test for {@link TimespanSolrCreator} class + */ +class TimespanSolrCreatorTest { + + private TimespanSolrCreator timespanSolrCreator; + private SolrInputDocument solrInputDocument; + private TimespanImpl timespan; + + @BeforeEach + void setup() { + solrInputDocument = new SolrInputDocument(); + timespanSolrCreator = new TimespanSolrCreator(); + timespan = new TimespanImpl(); + } + + @Test + void addToDocument_withTimeSpan_PrefValue_AltValue_And_OwlSameAs() { + timespan.setAbout("timespan"); + timespan.setPrefLabel(Map.of("keyPref", List.of("prefValue1", "prefValue2"))); + timespan.setAltLabel(Map.of("keyAlt", List.of("altValue1", "altValue2"))); + timespan.setOwlSameAs(List.of("value1", "value2").toArray(String[]::new)); + + timespanSolrCreator.addToDocument(solrInputDocument, timespan); + + assertTrue(solrInputDocument.containsKey(EdmLabel.EDM_TIMESPAN.toString()) && + solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref") && + solrInputDocument.containsKey(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt") && + solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL.toString())); + assertEquals("timespan", solrInputDocument.getFieldValue(EdmLabel.EDM_TIMESPAN.toString())); + assertEquals(List.of("prefValue1", "prefValue2"), + solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref")); + assertEquals(List.of("altValue1", "altValue2"), + solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt")); + assertEquals(List.of("value1", "value2"), + solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL.toString())); + assertEquals(4, solrInputDocument.size()); + } + + @Test + void addToDocument_withoutTimeSpan() { + timespan.setPrefLabel(Map.of("keyPref", List.of("prefValue1", "prefValue2"))); + timespan.setAltLabel(Map.of("keyAlt", List.of("altValue1", "altValue2"))); + timespan.setOwlSameAs(List.of("value1", "value2").toArray(String[]::new)); + + timespanSolrCreator.addToDocument(solrInputDocument, timespan); + + assertFalse(solrInputDocument.containsKey(EdmLabel.EDM_TIMESPAN.toString())); + assertTrue(solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref") && + solrInputDocument.containsKey(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt") && + solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL.toString())); + assertNull(solrInputDocument.getFieldValue(EdmLabel.EDM_TIMESPAN.toString())); + assertEquals(List.of("prefValue1", "prefValue2"), + solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref")); + assertEquals(List.of("altValue1", "altValue2"), + solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt")); + assertEquals(List.of("value1", "value2"), + solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL.toString())); + assertEquals(3, solrInputDocument.size()); + } + + @Test + void addToDocument_without_PrefValue() { + timespan.setAbout("timespan"); + timespan.setAltLabel(Map.of("keyAlt", List.of("altValue1", "altValue2"))); + timespan.setOwlSameAs(List.of("value1", "value2").toArray(String[]::new)); + + timespanSolrCreator.addToDocument(solrInputDocument, timespan); + + assertTrue(solrInputDocument.containsKey(EdmLabel.EDM_TIMESPAN.toString()) && + solrInputDocument.containsKey(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt") && + solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL.toString())); + assertFalse(solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref")); + assertEquals("timespan", solrInputDocument.getFieldValue(EdmLabel.EDM_TIMESPAN.toString())); + assertNull(solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref")); + assertEquals(List.of("altValue1", "altValue2"), + solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt")); + assertEquals(List.of("value1", "value2"), + solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL.toString())); + assertEquals(3, solrInputDocument.size()); + } + + @Test + void addToDocument_without_AltValue() { + timespan.setAbout("timespan"); + timespan.setPrefLabel(Map.of("keyPref", List.of("prefValue1", "prefValue2"))); + timespan.setOwlSameAs(List.of("value1", "value2").toArray(String[]::new)); + + timespanSolrCreator.addToDocument(solrInputDocument, timespan); + + assertTrue(solrInputDocument.containsKey(EdmLabel.EDM_TIMESPAN.toString()) && + solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref") && + solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL.toString())); + assertFalse(solrInputDocument.containsKey(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt")); + assertEquals("timespan", solrInputDocument.getFieldValue(EdmLabel.EDM_TIMESPAN.toString())); + assertEquals(List.of("prefValue1", "prefValue2"), + solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref")); + assertNull(solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt")); + assertEquals(List.of("value1", "value2"), + solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL.toString())); + assertEquals(3, solrInputDocument.size()); + } + + @Test + void addToDocument_without_OwlSameAs() { + timespan.setAbout("timespan"); + timespan.setPrefLabel(Map.of("keyPref", List.of("prefValue1", "prefValue2"))); + timespan.setAltLabel(Map.of("keyAlt", List.of("altValue1", "altValue2"))); + + timespanSolrCreator.addToDocument(solrInputDocument, timespan); + + assertTrue(solrInputDocument.containsKey(EdmLabel.EDM_TIMESPAN.toString()) && + solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref") && + solrInputDocument.containsKey(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt")); + assertFalse(solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL.toString())); + assertEquals("timespan", solrInputDocument.getFieldValue(EdmLabel.EDM_TIMESPAN.toString())); + assertEquals(List.of("prefValue1", "prefValue2"), + solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref")); + assertEquals(List.of("altValue1", "altValue2"), + solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt")); + assertNull(solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL.toString())); + assertEquals(3, solrInputDocument.size()); + } +} From 54da18d1c82e835bef1204a068b916d2c6b8e82d Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Thu, 9 Jun 2022 08:53:42 +0200 Subject: [PATCH 59/73] EA-2890 Add semium pattern enrichment cleanup --- .../rest/client/enrichment/EnricherImpl.java | 12 +++++++----- .../enrichment/api/internal/EntityResolver.java | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/EnricherImpl.java b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/EnricherImpl.java index b94b4e8ef..029ac61af 100644 --- a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/EnricherImpl.java +++ b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/EnricherImpl.java @@ -1,6 +1,7 @@ package eu.europeana.enrichment.rest.client.enrichment; import static eu.europeana.enrichment.api.internal.EntityResolver.europeanaLinkPattern; +import static eu.europeana.enrichment.api.internal.EntityResolver.semiumLinkPattern; import eu.europeana.enrichment.api.external.model.EnrichmentBase; import eu.europeana.enrichment.api.internal.EntityResolver; @@ -117,10 +118,11 @@ public Map> enrichReferences( public void cleanupPreviousEnrichmentEntities(RDF rdf) { final ProxyType europeanaProxy = RdfEntityUtils.getEuropeanaProxy(rdf); //Find the correct links - final Set europeanaLinks = Arrays.stream(ProxyFieldType.values()) - .map(proxyFieldType -> proxyFieldType.extractFieldLinksForEnrichment(europeanaProxy)) - .flatMap(Collection::stream).filter(europeanaLinkPattern.asPredicate()) - .collect(Collectors.toSet()); - RdfEntityUtils.removeMatchingEntities(rdf, europeanaLinks); + final Set matchingLinks = Arrays.stream(ProxyFieldType.values()) + .map(proxyFieldType -> proxyFieldType.extractFieldLinksForEnrichment(europeanaProxy)) + .flatMap(Collection::stream) + .filter(europeanaLinkPattern.asPredicate().or(semiumLinkPattern.asPredicate())) + .collect(Collectors.toSet()); + RdfEntityUtils.removeMatchingEntities(rdf, matchingLinks); } } diff --git a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/internal/EntityResolver.java b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/internal/EntityResolver.java index 6c985aaa5..79f868b5c 100644 --- a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/internal/EntityResolver.java +++ b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/internal/EntityResolver.java @@ -12,6 +12,7 @@ public interface EntityResolver { Pattern europeanaLinkPattern = Pattern.compile("^https?://data.europeana.eu.*$"); + Pattern semiumLinkPattern = Pattern.compile("^https?://semium.org.*$"); /** * Resolve entities by a textual reference. From 2ece5ffea7d4b2a3681871efbc0ed3a5e511a444 Mon Sep 17 00:00:00 2001 From: Jorge Ortiz Date: Fri, 10 Jun 2022 09:54:22 +0200 Subject: [PATCH 60/73] MET-4440_MET-4575 LicenseSolrCreator unit tests (#551) * MET-4440_MET-4575 LicenseSolrCreator unit tests * MET-4440_MET-4575 simplify assertion checks --- .../solr/property/LicenseSolrCreatorTest.java | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 metis-indexing/src/test/java/eu/europeana/indexing/solr/property/LicenseSolrCreatorTest.java diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/LicenseSolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/LicenseSolrCreatorTest.java new file mode 100644 index 000000000..769ecfe7d --- /dev/null +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/LicenseSolrCreatorTest.java @@ -0,0 +1,111 @@ +package eu.europeana.indexing.solr.property; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import eu.europeana.corelib.solr.entity.LicenseImpl; +import eu.europeana.indexing.solr.EdmLabel; +import java.time.Instant; +import java.util.Date; +import java.util.Set; +import org.apache.solr.common.SolrInputDocument; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit test for {@link LicenseSolrCreator} class + */ +class LicenseSolrCreatorTest { + + private LicenseSolrCreator licenseSolrCreator; + private SolrInputDocument solrInputDocument; + private LicenseImpl license; + + @BeforeEach + void setup() { + solrInputDocument = new SolrInputDocument(); + Set defRights = Set.of("license1", "license2", "license3", "license4", "license5"); + licenseSolrCreator = new LicenseSolrCreator(lic -> defRights.contains(license.getAbout())); + license = new LicenseImpl(); + } + + @Test + void addToDocument_WithAggregationAndDeprecatedOn() { + license.setAbout("license2"); + license.setCcDeprecatedOn(Date.from(Instant.parse("2022-06-01T12:22:55.888Z"))); + license.setOdrlInheritFrom("OdrlInheritFrom"); + + licenseSolrCreator.addToDocument(solrInputDocument, license); + + assertEquals("license2", solrInputDocument.getFieldValue(EdmLabel.PROVIDER_AGGREGATION_CC_LICENSE.toString())); + assertEquals(Date.from(Instant.parse("2022-06-01T12:22:55.888Z")), + solrInputDocument.getFieldValue(EdmLabel.PROVIDER_AGGREGATION_CC_DEPRECATED_ON.toString())); + assertEquals("OdrlInheritFrom", + solrInputDocument.getFieldValue(EdmLabel.PROVIDER_AGGREGATION_ODRL_INHERITED_FROM.toString())); + assertNull(solrInputDocument.getFieldValue(EdmLabel.WR_CC_DEPRECATED_ON.toString())); + assertEquals(3, solrInputDocument.size()); + } + + @Test + void addToDocument_WithAggregationAndWithoutDeprecatedOn() { + license.setAbout("license2"); + license.setCcDeprecatedOn(null); + license.setOdrlInheritFrom("OdrlInheritFrom"); + + licenseSolrCreator.addToDocument(solrInputDocument, license); + + assertEquals("license2", solrInputDocument.getFieldValue(EdmLabel.PROVIDER_AGGREGATION_CC_LICENSE.toString())); + assertNull(solrInputDocument.getFieldValue(EdmLabel.PROVIDER_AGGREGATION_CC_DEPRECATED_ON.toString())); + assertNull(solrInputDocument.getFieldValue(EdmLabel.WR_CC_DEPRECATED_ON.toString())); + assertEquals("OdrlInheritFrom", + solrInputDocument.getFieldValue(EdmLabel.PROVIDER_AGGREGATION_ODRL_INHERITED_FROM.toString())); + assertEquals(2, solrInputDocument.size()); + } + + @Test + void addToDocument_WithAggregationAndDeprecatedOnWithoutOdrlInherit() { + license.setAbout("license5"); + license.setCcDeprecatedOn(Date.from(Instant.parse("2022-06-01T12:22:55.888Z"))); + + licenseSolrCreator.addToDocument(solrInputDocument, license); + + assertEquals("license5", solrInputDocument.getFieldValue(EdmLabel.PROVIDER_AGGREGATION_CC_LICENSE.toString())); + assertEquals(Date.from(Instant.parse("2022-06-01T12:22:55.888Z")), + solrInputDocument.getFieldValue(EdmLabel.PROVIDER_AGGREGATION_CC_DEPRECATED_ON.toString())); + assertNull(solrInputDocument.getFieldValue(EdmLabel.PROVIDER_AGGREGATION_ODRL_INHERITED_FROM.toString())); + assertEquals(2, solrInputDocument.size()); + } + + @Test + void addToDocument_WithoutAggregation() { + license.setAbout("license0"); + license.setCcDeprecatedOn(Date.from(Instant.parse("2022-06-01T12:22:55.888Z"))); + license.setOdrlInheritFrom("OdrlInheritFrom"); + + licenseSolrCreator.addToDocument(solrInputDocument, license); + + assertEquals("license0", solrInputDocument.getFieldValue(EdmLabel.WR_CC_LICENSE.toString())); + assertEquals(Date.from(Instant.parse("2022-06-01T12:22:55.888Z")), + solrInputDocument.getFieldValue(EdmLabel.WR_CC_DEPRECATED_ON.toString())); + assertNull(solrInputDocument.getFieldValue(EdmLabel.PROVIDER_AGGREGATION_CC_LICENSE.toString())); + assertNull(solrInputDocument.getFieldValue(EdmLabel.PROVIDER_AGGREGATION_ODRL_INHERITED_FROM.toString())); + assertEquals(2, solrInputDocument.size()); + } + + @Test + void addToDocument_WithoutAggregationAndWithoutCcDeprecatedOn() { + license.setAbout("license0"); + license.setCcDeprecatedOn(null); + license.setOdrlInheritFrom("OdrlInheritFrom"); + + licenseSolrCreator.addToDocument(solrInputDocument, license); + + assertEquals("license0", solrInputDocument.getFieldValue(EdmLabel.WR_CC_LICENSE.toString())); + assertNull(solrInputDocument.getFieldValue(EdmLabel.WR_CC_DEPRECATED_ON.toString())); + assertNull(solrInputDocument.getFieldValue(EdmLabel.PROVIDER_AGGREGATION_CC_LICENSE.toString())); + assertNull(solrInputDocument.getFieldValue(EdmLabel.PROVIDER_AGGREGATION_ODRL_INHERITED_FROM.toString())); + assertEquals(1, solrInputDocument.size()); + } +} From d6e0d37ad9371e699a2d5f84eda1ff940b1f1e42 Mon Sep 17 00:00:00 2001 From: Adolfo Peixinho <91942157+adolfopeixinho@users.noreply.github.com> Date: Mon, 13 Jun 2022 15:46:01 +0200 Subject: [PATCH 61/73] Feat/met 2608 improve dereference cache (#556) * MET-2608 Improve dereference cache. * MET-2608 Improve dereference cache. First Joana Review * MET-2608 Improve dereference cache. First Joana Review, forgot the javadoc * MET-2608 Improve dereference cache. Sonar still complains about code smell, missing param doc. * MET-2608 Added filtering of ProcessedEntities by empty or null XML. Changed application properties to main App. Added controller and DAO endpoints * MET-2608 corrected wrong cron expression for springframework. Added clearer macros for cron expressions. Corrected wrong comparing of values in filter * MET-2608 Added javadoc in Application.java. Removed this keyword from MongoDereferencingManagementService.java * MET-2608 Added more tests for cache purge * MET-2608 add dereferencingcontroller coverage unit tests * MET-2608 Added more tests for cache purge empty or null XML * MET-2608 add DeferencingManagementController unit tests * MET-2608 fix name of variables * MET-2608 Added unit tests for the DAO * MET-2608 Code cleanup * MET-2608 MongoDereferencingManagementService add unit tests * MET-2608 add more unit tests * MET-2608 remove unnecessary bean annotation, from scheduled method. * MET-2608 Code review. Use of RestEndpoint Constants in tests. * MET-2608 change of http verb from POST to delete in DereferencingManagementController * MET-2608 change of http verb from POST to delete in DereferencingManagementController, test fix Co-authored-by: Jorge Ortiz --- .../dereference/rest/DereferencingManagementController.java | 4 ++-- .../rest/DereferencingManagementControllerTest.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/DereferencingManagementController.java b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/DereferencingManagementController.java index eeade969f..3a8e4f426 100644 --- a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/DereferencingManagementController.java +++ b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/DereferencingManagementController.java @@ -89,7 +89,7 @@ public void emptyCacheByEmptyXml() { * Empty the cache for a specific resource * @param resourceId The resourceId to empty the cache for * */ - @PostMapping(value = RestEndpoints.CACHE_EMPTY_RESOURCE) + @DeleteMapping(value = RestEndpoints.CACHE_EMPTY_RESOURCE) @ResponseBody @ApiOperation(value = "Empty the cache by resource Id") public void emptyCacheByResourceId( @@ -101,7 +101,7 @@ public void emptyCacheByResourceId( * Empty the cache for a specific vocabulary, with all associated entities * @param vocabularyId The vocabularyId to empty the cache for * */ - @PostMapping(value = RestEndpoints.CACHE_EMPTY_VOCABULARY) + @DeleteMapping(value = RestEndpoints.CACHE_EMPTY_VOCABULARY) @ResponseBody @ApiOperation(value = "Empty the cache by vocabulary Id") public void emptyCacheByVocabularyId( diff --git a/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingManagementControllerTest.java b/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingManagementControllerTest.java index 8136d7eea..c12ef839d 100644 --- a/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingManagementControllerTest.java +++ b/metis-dereference/metis-dereference-rest/src/test/java/eu/europeana/metis/dereference/rest/DereferencingManagementControllerTest.java @@ -141,7 +141,7 @@ void testEmptyCacheByResourceId() throws Exception { return null; }).when(deRefManagementServiceMock).purgeByResourceId(any(String.class)); - deRefManagementControllerMock.perform(post(RestEndpoints.CACHE_EMPTY_RESOURCE) + deRefManagementControllerMock.perform(delete(RestEndpoints.CACHE_EMPTY_RESOURCE) .param("resourceId", "resourceId") .param("resourceId", "12345")) .andExpect(status().is(200)); @@ -157,7 +157,7 @@ void testEmptyCacheByVocabularyId() throws Exception { return null; }).when(deRefManagementServiceMock).purgeByVocabularyId(any(String.class)); - deRefManagementControllerMock.perform(post(RestEndpoints.CACHE_EMPTY_VOCABULARY) + deRefManagementControllerMock.perform(delete(RestEndpoints.CACHE_EMPTY_VOCABULARY) .param("vocabularyId", "12345")) .andExpect(status().is(200)); From 11bdfad566b5f7eacd6f8ea2cf12a43a058a2a70 Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Mon, 13 Jun 2022 16:55:32 +0200 Subject: [PATCH 62/73] MET-2608 Fix swagger webflux coming from transitive entity-api-client --- metis-dereference/metis-dereference-rest/pom.xml | 11 +++++++++++ metis-enrichment/metis-enrichment-common/pom.xml | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/metis-dereference/metis-dereference-rest/pom.xml b/metis-dereference/metis-dereference-rest/pom.xml index 87bfb7fe1..4817793b4 100644 --- a/metis-dereference/metis-dereference-rest/pom.xml +++ b/metis-dereference/metis-dereference-rest/pom.xml @@ -69,6 +69,17 @@ springfox-swagger-ui ${version.swagger} + + + org.springframework.boot + spring-boot-autoconfigure + ${version.spring-boot-autoconfigure} + com.jayway.jsonpath json-path-assert diff --git a/metis-enrichment/metis-enrichment-common/pom.xml b/metis-enrichment/metis-enrichment-common/pom.xml index 7006a252c..7d7479178 100644 --- a/metis-enrichment/metis-enrichment-common/pom.xml +++ b/metis-enrichment/metis-enrichment-common/pom.xml @@ -9,7 +9,7 @@ metis-enrichment-common - 0.0.1-SNAPSHOT + 2.0 From 15a93654691721441241a3be9f8d838d02b2be44 Mon Sep 17 00:00:00 2001 From: Adolfo Peixinho <91942157+adolfopeixinho@users.noreply.github.com> Date: Wed, 15 Jun 2022 10:09:36 +0200 Subject: [PATCH 63/73] MET-4568 EuropeanaAggregationSolrCreator Unit Tests (#548) * MET-4568 added test for EuropeanaAggregationSolrCreator * MET-4558 changes from code review * MET-4568 cleanup some assertions. QualityAnnotationSolrCreator not fully tested * EA-4568 Add test forEuropeanaAggregationSolrCreator Co-authored-by: Simon Tzanakis --- .../EuropeanaAggregationSolrCreator.java | 13 +- .../EuropeanaAggregationSolrCreatorTest.java | 136 ++++++++++++++++++ 2 files changed, 141 insertions(+), 8 deletions(-) create mode 100644 metis-indexing/src/test/java/eu/europeana/indexing/solr/property/EuropeanaAggregationSolrCreatorTest.java diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/solr/property/EuropeanaAggregationSolrCreator.java b/metis-indexing/src/main/java/eu/europeana/indexing/solr/property/EuropeanaAggregationSolrCreator.java index ca8308192..29a858ced 100644 --- a/metis-indexing/src/main/java/eu/europeana/indexing/solr/property/EuropeanaAggregationSolrCreator.java +++ b/metis-indexing/src/main/java/eu/europeana/indexing/solr/property/EuropeanaAggregationSolrCreator.java @@ -11,7 +11,6 @@ import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.solr.common.SolrInputDocument; @@ -26,11 +25,9 @@ public class EuropeanaAggregationSolrCreator implements PropertySolrCreator licenses, Function qualityAnnotationGetter) { @@ -49,8 +46,8 @@ public void addToDocument(SolrInputDocument doc, EuropeanaAggregation europeanaA new WebResourceSolrCreator(licenses) .addAllToDocument(doc, europeanaAggregation.getWebResources()); final List annotationsToAdd = Optional - .ofNullable(europeanaAggregation.getDqvHasQualityAnnotation()).map(Arrays::stream) - .orElseGet(Stream::empty).filter(StringUtils::isNotBlank).distinct() + .ofNullable(europeanaAggregation.getDqvHasQualityAnnotation()).stream().flatMap(Arrays::stream) + .filter(StringUtils::isNotBlank).distinct() .map(qualityAnnotationGetter).filter(Objects::nonNull).collect(Collectors.toList()); new QualityAnnotationSolrCreator().addAllToDocument(doc, annotationsToAdd); } diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/EuropeanaAggregationSolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/EuropeanaAggregationSolrCreatorTest.java new file mode 100644 index 000000000..ebf73573a --- /dev/null +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/EuropeanaAggregationSolrCreatorTest.java @@ -0,0 +1,136 @@ +package eu.europeana.indexing.solr.property; + +import static org.apache.commons.collections4.CollectionUtils.union; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import eu.europeana.corelib.definitions.edm.entity.EuropeanaAggregation; +import eu.europeana.corelib.definitions.edm.entity.License; +import eu.europeana.corelib.definitions.edm.entity.QualityAnnotation; +import eu.europeana.corelib.definitions.edm.entity.WebResource; +import eu.europeana.corelib.solr.entity.EuropeanaAggregationImpl; +import eu.europeana.corelib.solr.entity.LicenseImpl; +import eu.europeana.corelib.solr.entity.QualityAnnotationImpl; +import eu.europeana.corelib.solr.entity.WebResourceImpl; +import eu.europeana.indexing.solr.EdmLabel; +import eu.europeana.indexing.utils.RdfTier; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.apache.solr.common.SolrInputDocument; +import org.bson.types.ObjectId; +import org.junit.jupiter.api.Test; + +/** + * Unit test for {@link EuropeanaAggregationSolrCreator} class + */ + +class EuropeanaAggregationSolrCreatorTest { + + @Test + void addToDocument() { + SolrInputDocument solrInputDocument = new SolrInputDocument(); + License license1 = new LicenseImpl(); + license1.setAbout("license1About"); + license1.setOdrlInheritFrom("license1OdrlInheritFrom"); + License license2 = new LicenseImpl(); + license2.setAbout("license2About"); + license2.setOdrlInheritFrom("license2OdrlInheritFrom"); + + QualityAnnotation qualityAnnotationContent1 = new QualityAnnotationImpl(); + qualityAnnotationContent1.setAbout("qualityAnnotationContent1About"); + qualityAnnotationContent1.setBody(RdfTier.CONTENT_TIER_0.getUri()); + QualityAnnotation qualityAnnotationContent2 = new QualityAnnotationImpl(); + qualityAnnotationContent2.setAbout("qualityAnnotationContent2About"); + qualityAnnotationContent2.setBody(RdfTier.CONTENT_TIER_1.getUri()); + QualityAnnotation qualityAnnotationMetadata1 = new QualityAnnotationImpl(); + qualityAnnotationMetadata1.setAbout("qualityAnnotationMetadata1About"); + qualityAnnotationMetadata1.setBody(RdfTier.METADATA_TIER_0.getUri()); + + final Map qualityAnnotationMap = Map.of(qualityAnnotationContent1.getAbout(), + qualityAnnotationContent1, qualityAnnotationContent2.getAbout(), qualityAnnotationContent2, + qualityAnnotationMetadata1.getAbout(), qualityAnnotationMetadata1); + + final EuropeanaAggregationSolrCreator europeanaAggregationSolrCreator = new EuropeanaAggregationSolrCreator( + List.of(license1, license2), qualityAnnotationMap::get); + + WebResource webResource1 = new WebResourceImpl(); + webResource1.setAbout("webResource1About"); + webResource1.setIsNextInSequence("webResource1IsNextInSequence"); + webResource1.setWebResourceEdmRights( + Map.of("webResource1EdmRightsKey", List.of("webResource1EdmRightsValue", license2.getAbout()))); + webResource1.setWebResourceDcRights( + Map.of("webResource1DcRightsKey", List.of("webResource1DcRightsValue", license1.getAbout()))); + final List webResource1SvcsHasService = List.of("webResource1SvcsHasService"); + webResource1.setSvcsHasService(webResource1SvcsHasService.toArray(new String[0])); + final List webResource1DctermsIsReferencedBy = List.of("webResource1DctermsIsReferencedBy"); + webResource1.setDctermsIsReferencedBy(webResource1DctermsIsReferencedBy.toArray(new String[0])); + + WebResource webResource2 = new WebResourceImpl(); + webResource2.setAbout("webResource2About"); + webResource2.setIsNextInSequence("webResource2IsNextInSequence"); + webResource2.setWebResourceEdmRights(Map.of("webResource2EdmRightsKey", List.of("webResource2EdmRightsValue"))); + webResource2.setWebResourceDcRights(Map.of("webResource2DcRightsKey", List.of("webResource2DcRightsValue"))); + final List webResource2SvcsHasService = List.of("webResource2SvcsHasService"); + webResource2.setSvcsHasService(webResource2SvcsHasService.toArray(new String[0])); + final List webResource2DctermsIsReferencedBy = List.of("webResource2DctermsIsReferencedBy"); + webResource2.setDctermsIsReferencedBy(webResource2DctermsIsReferencedBy.toArray(new String[0])); + + EuropeanaAggregation europeanaAggregation = new EuropeanaAggregationImpl(); + europeanaAggregation.setId(new ObjectId(String.valueOf(ObjectId.get()))); + europeanaAggregation.setAbout("europeanaAggregationAbout"); + europeanaAggregation.setWebResources(List.of(webResource1, webResource2)); + europeanaAggregation.setEdmCountry(Map.of("edmCountryKey", List.of("countryValue"))); + europeanaAggregation.setEdmLanguage(Map.of("edmLanguageKey", List.of("languageValue"))); + europeanaAggregation.setEdmPreview("previewValue"); + europeanaAggregation.setDqvHasQualityAnnotation( + new String[]{qualityAnnotationContent1.getAbout(), qualityAnnotationContent2.getAbout(), + qualityAnnotationMetadata1.getAbout()}); + + europeanaAggregationSolrCreator.addToDocument(solrInputDocument, europeanaAggregation); + + //Verify Europeana Aggregation fields + verifyMap(solrInputDocument, EdmLabel.EUROPEANA_AGGREGATION_EDM_COUNTRY, europeanaAggregation.getEdmCountry()); + verifyMap(solrInputDocument, EdmLabel.EUROPEANA_AGGREGATION_EDM_LANGUAGE, europeanaAggregation.getEdmLanguage()); + assertEquals(europeanaAggregation.getEdmPreview(), + solrInputDocument.getFieldValue(EdmLabel.EUROPEANA_AGGREGATION_EDM_PREVIEW.toString())); + + //Verify web resources fields + verifyCollection(solrInputDocument, EdmLabel.EDM_WEB_RESOURCE, List.of(webResource1.getAbout(), webResource2.getAbout())); + verifyCollection(solrInputDocument, EdmLabel.WR_EDM_IS_NEXT_IN_SEQUENCE, + List.of(webResource1.getIsNextInSequence(), webResource2.getIsNextInSequence())); + verifyCollection(solrInputDocument, EdmLabel.WR_SVCS_HAS_SERVICE, + union(webResource1SvcsHasService, webResource2SvcsHasService)); + verifyCollection(solrInputDocument, EdmLabel.WR_DCTERMS_ISREFERENCEDBY, + union(webResource1DctermsIsReferencedBy, webResource2DctermsIsReferencedBy)); + verifyCollection(solrInputDocument, EdmLabel.WR_CC_ODRL_INHERITED_FROM, + List.of(license1.getOdrlInheritFrom(), license2.getOdrlInheritFrom())); + + //Verify webResource1 + verifyMap(solrInputDocument, EdmLabel.WR_EDM_RIGHTS, webResource1.getWebResourceEdmRights()); + verifyMap(solrInputDocument, EdmLabel.WR_DC_RIGHTS, webResource1.getWebResourceDcRights()); + //Verify webResource2 + verifyMap(solrInputDocument, EdmLabel.WR_EDM_RIGHTS, webResource2.getWebResourceEdmRights()); + verifyMap(solrInputDocument, EdmLabel.WR_DC_RIGHTS, webResource2.getWebResourceDcRights()); + + //Verify QualityAnnotation Fields + verifyCollection(solrInputDocument, EdmLabel.CONTENT_TIER, + List.of(RdfTier.CONTENT_TIER_0.getTier().toString(), RdfTier.CONTENT_TIER_1.getTier().toString())); + verifyCollection(solrInputDocument, EdmLabel.METADATA_TIER, List.of(RdfTier.METADATA_TIER_0.getTier().toString())); + } + + void verifyCollection(SolrInputDocument solrInputDocument, EdmLabel edmLabel, Collection collection) { + final Collection fieldValues = solrInputDocument.getFieldValues(edmLabel.toString()); + assertTrue(fieldValues.containsAll(collection)); + assertEquals(fieldValues.size(), collection.size()); + } + + void verifyMap(SolrInputDocument solrInputDocument, EdmLabel edmLabel, Map> map) { + map.forEach((key, value) -> assertTrue(solrInputDocument.getFieldValues(computeSolrField(edmLabel, key)) + .containsAll(value))); + } + + private String computeSolrField(EdmLabel label, String value) { + return label.toString() + "." + value; + } +} \ No newline at end of file From 9d50fda56e3650195c7037e36b2ab1af2d5c636d Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Fri, 17 Jun 2022 14:14:16 +0200 Subject: [PATCH 64/73] MET-4612 Technical debt fixes (#557) * MET-4612 Reproduce * MET-4612 Reproduce * MET-4612 Suppress token name warning * MET-4612 Reproduce * MET-4612 Make temp file creation secure * MET-4612 Try sonar proposed way * MET-4612 Try creating once with attributes * MET-4612 Reproduce * MET-4612 Fix secure file for pdf image converter * MET-4612 Reproduce * MET-4612 Reproduce * MET-4612 Fix ThumbnailGenerator secure temp file * MET-4612 Reproduce * MET-4612 Fix ThumbnailWithSize secure temp file * MET-4612 Reproduce * MET-4612 Fix AbstractTemporaryFile secure temp file * MET-4612 Reproduce * MET-4612 Fix HttpHarvesterImpl secure temp file * MET-4612 Reproduce * MET-4612 Try fix zip extraction with stream * MET-4612 Write TempFileUtils tests * MET-4612 Small fixes * MET-4612 Add retryable wrapper on client calls * MET-4612 Cleanup tests --- .../service/AuthenticationService.java | 24 ++-- .../metis/network/AbstractHttpClient.java | 12 +- .../metis/utils/GeoUriWGS84Parser.java | 4 +- .../europeana/metis/utils/RestEndpoints.java | 15 ++- .../europeana/metis/utils/TempFileUtils.java | 119 ++++++++++++++++++ .../europeana/metis/utils/ZipFileReader.java | 32 +++-- .../metis/utils/TempFileUtilsTest.java | 86 +++++++++++++ .../core/rest/config/OrchestratorConfig.java | 4 +- .../external/impl/ClientEntityResolver.java | 9 +- .../RestResponseExceptionHandlerTest.java | 5 +- .../http/CompressedFileExtractor.java | 28 ++--- .../harvesting/http/HttpHarvesterImpl.java | 81 ++++++------ .../http/CompressedFileExtractorGzTest.java | 12 +- .../http/CompressedFileExtractorZipTest.java | 20 +-- .../oaipmh/CloseableHttpOaiClientTest.java | 16 +-- .../solr/property/SolrPropertyUtilsTest.java | 21 ++-- .../tiers/media/MediaClassifierTest.java | 2 +- .../indexing/tiers/model/MediaTierTest.java | 4 +- .../tiers/model/MetadataTierTest.java | 5 +- .../indexing/utils/SetUtilsTest.java | 11 +- .../extraction/PdfToImageConverter.java | 56 +++++---- .../extraction/TextProcessor.java | 2 +- .../extraction/ThumbnailGenerator.java | 63 +++++----- .../http/ResourceDownloadClient.java | 2 - .../model/AbstractTemporaryFile.java | 11 +- .../mediaprocessing/model/ResourceImpl.java | 2 +- .../extraction/PdfToImageConverterTest.java | 16 +-- .../extraction/TextProcessorTest.java | 2 +- .../extraction/ThumbnailGeneratorTest.java | 5 +- .../model/ConfidenceLevelTest.java | 2 +- .../CleanMarkupTagsNormalizerTest.java | 10 +- ...emoveDuplicateStatementNormalizerTest.java | 41 +++--- .../src/test/java/TestValidationClient.java | 16 +-- .../validation/service/SchemaProvider.java | 43 +++---- 34 files changed, 490 insertions(+), 291 deletions(-) create mode 100644 metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/TempFileUtils.java create mode 100644 metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/TempFileUtilsTest.java diff --git a/metis-authentication/metis-authentication-service/src/main/java/eu/europeana/metis/authentication/service/AuthenticationService.java b/metis-authentication/metis-authentication-service/src/main/java/eu/europeana/metis/authentication/service/AuthenticationService.java index bcd0cec80..d02017fbd 100644 --- a/metis-authentication/metis-authentication-service/src/main/java/eu/europeana/metis/authentication/service/AuthenticationService.java +++ b/metis-authentication/metis-authentication-service/src/main/java/eu/europeana/metis/authentication/service/AuthenticationService.java @@ -4,9 +4,9 @@ import eu.europeana.metis.authentication.dao.PsqlMetisUserDao; import eu.europeana.metis.authentication.user.AccountRole; import eu.europeana.metis.authentication.user.Credentials; -import eu.europeana.metis.authentication.user.MetisUserView; -import eu.europeana.metis.authentication.user.MetisUserAccessToken; import eu.europeana.metis.authentication.user.MetisUser; +import eu.europeana.metis.authentication.user.MetisUserAccessToken; +import eu.europeana.metis.authentication.user.MetisUserView; import eu.europeana.metis.authentication.utils.ZohoMetisUserUtils; import eu.europeana.metis.exception.BadContentException; import eu.europeana.metis.exception.GenericMetisException; @@ -36,8 +36,7 @@ import org.springframework.stereotype.Service; /** - * Service that handles all related operations to authentication including communication between a - * psql database and Zoho. + * Service that handles all related operations to authentication including communication between a psql database and Zoho. * * @author Simon Tzanakis (Simon.Tzanakis@europeana.eu) * @since 2018-12-05 @@ -47,10 +46,10 @@ public class AuthenticationService { private static final int LOG_ROUNDS = 13; private static final int CREDENTIAL_FIELDS_NUMBER = 2; + @SuppressWarnings("java:S6418") // It is not an actual token private static final String ACCESS_TOKEN_CHARACTER_BASKET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; private static final int ACCESS_TOKEN_LENGTH = 32; - private static final Pattern TOKEN_MATCHING_PATTERN = Pattern - .compile("^[" + ACCESS_TOKEN_CHARACTER_BASKET + "]*$"); + private static final Pattern TOKEN_MATCHING_PATTERN = Pattern.compile("^[" + ACCESS_TOKEN_CHARACTER_BASKET + "]*$"); public static final Supplier COULD_NOT_CONVERT_EXCEPTION_SUPPLIER = () -> new BadContentException( "Could not convert internal user"); private final PsqlMetisUserDao psqlMetisUserDao; @@ -243,7 +242,7 @@ public String validateAuthorizationHeaderWithAccessToken(String authorization) } //Check that the token is of valid structure if (accessToken.length() != ACCESS_TOKEN_LENGTH || !TOKEN_MATCHING_PATTERN.matcher(accessToken) - .matches()) { + .matches()) { throw new UserUnauthorizedException("Access token invalid"); } return accessToken; @@ -368,7 +367,8 @@ public boolean hasPermissionToRequestUserUpdate(String accessToken, String userE } MetisUser storedMetisUser = authenticateUserInternal(accessToken); return storedMetisUser.getAccountRole() == AccountRole.METIS_ADMIN || storedMetisUser.getEmail() - .equals(storedMetisUserToUpdate.getEmail()); + .equals( + storedMetisUserToUpdate.getEmail()); } String generateAccessToken() { @@ -480,14 +480,14 @@ public List getAllUsers() { return convert(psqlMetisUserDao.getAllMetisUsers()); } - private static MetisUserView convert(MetisUser record) throws BadContentException { - return Optional.ofNullable(record).map(MetisUserView::new) - .orElseThrow(COULD_NOT_CONVERT_EXCEPTION_SUPPLIER); + private static MetisUserView convert(MetisUser metisUser) throws BadContentException { + return Optional.ofNullable(metisUser).map(MetisUserView::new) + .orElseThrow(COULD_NOT_CONVERT_EXCEPTION_SUPPLIER); } private static List convert(List records) { return Optional.ofNullable(records).stream().flatMap(Collection::stream).map(MetisUserView::new) - .collect(Collectors.toList()); + .collect(Collectors.toList()); } } diff --git a/metis-common/metis-common-network/src/main/java/eu/europeana/metis/network/AbstractHttpClient.java b/metis-common/metis-common-network/src/main/java/eu/europeana/metis/network/AbstractHttpClient.java index 876548abf..e30d4144f 100644 --- a/metis-common/metis-common-network/src/main/java/eu/europeana/metis/network/AbstractHttpClient.java +++ b/metis-common/metis-common-network/src/main/java/eu/europeana/metis/network/AbstractHttpClient.java @@ -135,8 +135,8 @@ public R download(I link) throws IOException { public R download(I link, Map requestHeaders) throws IOException { // Set up the connection. - final String resourceUlr = getResourceUrl(link); - final HttpGet httpGet = new HttpGet(resourceUlr); + final String resourceUrl = getResourceUrl(link); + final HttpGet httpGet = new HttpGet(resourceUrl); requestHeaders.forEach(httpGet::setHeader); final HttpClientContext context = HttpClientContext.create(); @@ -146,7 +146,7 @@ public R download(I link, Map requestHeaders) throws IOException public void run() { synchronized (httpGet) { if (httpGet.cancel()) { - LOGGER.info("Aborting request due to time limit: {}.", resourceUlr); + LOGGER.info("Aborting request due to time limit: {}.", resourceUrl); } } } @@ -166,8 +166,8 @@ public void run() { final HttpEntity responseEntity = performThrowingFunction(responseObject, response -> { final int status = response.getCode(); if (!httpCallIsSuccessful(status)) { - throw new IOException("Download failed of resource " + resourceUlr + ". Status code " + - status + " (message: " + response.getReasonPhrase() + ")."); + throw new IOException("Download failed of resource " + resourceUrl + ". Status code " + + status + " (message: " + response.getReasonPhrase() + ")."); } return response.getEntity(); }); @@ -210,7 +210,7 @@ public void run() { // Cancel the request to stop downloading. synchronized (httpGet) { if (httpGet.cancel()) { - LOGGER.debug("Aborting request after all processing is completed: {}.", resourceUlr); + LOGGER.debug("Aborting request after all processing is completed: {}.", resourceUrl); } } diff --git a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/GeoUriWGS84Parser.java b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/GeoUriWGS84Parser.java index a2981ae50..9bed28c0c 100644 --- a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/GeoUriWGS84Parser.java +++ b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/GeoUriWGS84Parser.java @@ -15,10 +15,10 @@ public final class GeoUriWGS84Parser { private static final String DECIMAL_POINT_REGEX = "(?:\\.\\d+)?"; private static final String ZEROES_DECIMAL_POINT_REGEX = "(?:\\.0+)?"; private static final String LATITUDE_REGEX = - "^[+-]?(?:90" + ZEROES_DECIMAL_POINT_REGEX + "|(?:[0-9]|[1-8][0-9])" + DECIMAL_POINT_REGEX + ")$"; + "^[+-]?(?:90" + ZEROES_DECIMAL_POINT_REGEX + "|(?:\\d|[1-8]\\d)" + DECIMAL_POINT_REGEX + ")$"; private static final Pattern LATITUDE_PATTERN = Pattern.compile(LATITUDE_REGEX); private static final String LONGITUDE_REGEX = - "^[+-]?(?:180" + ZEROES_DECIMAL_POINT_REGEX + "|(?:[0-9]|[1-9][0-9]|1[0-7][0-9])" + DECIMAL_POINT_REGEX + ")$"; + "^[+-]?(?:180" + ZEROES_DECIMAL_POINT_REGEX + "|(?:\\d|[1-9]\\d|1[0-7]\\d)" + DECIMAL_POINT_REGEX + ")$"; private static final Pattern LONGITUDE_PATTERN = Pattern.compile(LONGITUDE_REGEX); private static final String ALTITUDE_REGEX = "^[+-]?\\d+" + DECIMAL_POINT_REGEX + "$"; private static final Pattern ALTITUDE_PATTERN = Pattern.compile(ALTITUDE_REGEX); diff --git a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/RestEndpoints.java b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/RestEndpoints.java index b3f1dfe16..103c05c3b 100644 --- a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/RestEndpoints.java +++ b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/RestEndpoints.java @@ -48,12 +48,16 @@ public final class RestEndpoints { public static final String ORCHESTRATOR_WORKFLOWS_SCHEDULE = "/orchestrator/workflows/schedule"; public static final String ORCHESTRATOR_WORKFLOWS_SCHEDULE_DATASETID = "/orchestrator/workflows/schedule/{datasetId}"; public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_EXECUTIONID = "/orchestrator/workflows/executions/{executionId}"; - public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_EXECUTIONID_PLUGINS_DATA_AVAILABILITY = "/orchestrator/workflows/executions/{executionId}/plugins/data-availability"; + public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_EXECUTIONID_PLUGINS_DATA_AVAILABILITY + = "/orchestrator/workflows/executions/{executionId}/plugins/data-availability"; public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID = "/orchestrator/workflows/executions/dataset/{datasetId}"; - public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID_ALLOWED_INCREMENTAL = "/orchestrator/workflows/executions/dataset/{datasetId}/allowed_incremental"; - public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID_ALLOWED_PLUGIN = "/orchestrator/workflows/executions/dataset/{datasetId}/allowed_plugin"; + public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID_ALLOWED_INCREMENTAL + = "/orchestrator/workflows/executions/dataset/{datasetId}/allowed_incremental"; + public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID_ALLOWED_PLUGIN + = "/orchestrator/workflows/executions/dataset/{datasetId}/allowed_plugin"; public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID_HISTORY = "/orchestrator/workflows/executions/dataset/{datasetId}/history"; - public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID_INFORMATION = "/orchestrator/workflows/executions/dataset/{datasetId}/information"; + public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID_INFORMATION + = "/orchestrator/workflows/executions/dataset/{datasetId}/information"; public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS = "/orchestrator/workflows/executions"; public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_OVERVIEW = "/orchestrator/workflows/executions/overview"; public static final String ORCHESTRATOR_WORKFLOWS_EVOLUTION = "/orchestrator/workflows/evolution/{workflowExecutionId}/{pluginType}"; @@ -94,8 +98,7 @@ private RestEndpoints() { } /** - * Resolves an endpoint with parameters wrapped around "{" and "}" by providing the endpoint and - * all the required parameters. + * Resolves an endpoint with parameters wrapped around "{" and "}" by providing the endpoint and all the required parameters. * * @param endpoint the endpoint to resolve * @param params all the parameters specified diff --git a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/TempFileUtils.java b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/TempFileUtils.java new file mode 100644 index 000000000..feb3373de --- /dev/null +++ b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/TempFileUtils.java @@ -0,0 +1,119 @@ +package eu.europeana.metis.utils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.EnumSet; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * File utilities class + */ +public final class TempFileUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(TempFileUtils.class); + private static final EnumSet OWNER_PERMISSIONS_ONLY_SET = EnumSet.of(PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE); + private static final FileAttribute> OWNER_PERMISSIONS_ONLY_FILE_ATTRIBUTE = PosixFilePermissions.asFileAttribute( + OWNER_PERMISSIONS_ONLY_SET); + public static final String PNG_FILE_EXTENSION = ".png"; + public static final String JPEG_FILE_EXTENSION = ".jpeg"; + + private TempFileUtils() { + //Private constructor + } + + /** + * Creates a secure temporary file(owner permissions only) for posix and other file systems. + *

    This method is not responsible of removing the temporary file. + * An implementation that uses this method should delete the temp files by itself.

    + * + * @param prefix the prefix, (e.g. the class simple name that generates the temp file) + * @param suffix the suffix + * @return the secure temporary file + * @throws IOException if the file failed to be created + */ + public static Path createSecureTempFile(String prefix, String suffix) throws IOException { + //Set permissions only to owner, posix style + final Path secureTempFile = Files.createTempFile(prefix, suffix, OWNER_PERMISSIONS_ONLY_FILE_ATTRIBUTE); + //Set again for non posix systems + setPosixIndependentOwnerOnlyFilePermissions(secureTempFile); + + return secureTempFile; + } + + /** + * Creates a secure temporary file(owner permissions only) for posix and other file systems. + *

    + * This is equivalent to calling {@link #createSecureTempFile(String, String)} and in addition it declares that it will remove + * the temporary file with {@link File#deleteOnExit()}. + *

    + * + *

    CAUTION: This method can have a memory impact if too many files are created, and that is because + * {@link File#deleteOnExit()} keeps an in memory cache of the file paths. If possible prefer the use of + * {@link #createSecureTempFile(String, String)} and make your implementation remove the temporary files created + * explicitly.

    + * + * @param prefix the prefix + * @param suffix the suffix + * @return the secure temporary file + * @throws IOException if the file failed to be created + */ + @SuppressWarnings("java:S2308") //Delete on exit is intended here and javadoc warns the user + public static Path createSecureTempFileDeleteOnExit(String prefix, String suffix) throws IOException { + final Path secureTempFile = createSecureTempFile(prefix, suffix); + secureTempFile.toFile().deleteOnExit(); + return secureTempFile; + } + + /** + * Creates a secure temporary directory(owner permissions only) with the {@code directoryPrefix} specified and then creates a + * secure temporary file(owner permissions only) inside that directory with the {@code prefix} and {@code suffix} specified. + * + * @param directoryPrefix the directory prefix + * @param prefix the file prefix + * @param suffix the file suffix + * @return the secure temporary file in the newly created secure temporary directory + * @throws IOException if the directory or file failed to be created + */ + public static Path createSecureTempDirectoryAndFile(String directoryPrefix, String prefix, String suffix) throws IOException { + Path tempSecureParentDir = createSecureTempDirectory(directoryPrefix); + //Set permissions only to owner, posix style + final Path secureTempFile = Files.createTempFile(tempSecureParentDir, prefix, suffix, OWNER_PERMISSIONS_ONLY_FILE_ATTRIBUTE); + //Set again for non posix systems + setPosixIndependentOwnerOnlyFilePermissions(secureTempFile); + + return secureTempFile; + } + + /** + * Creates a secure temporary directory(owner permissions only) with the {@code prefix} specified. + * + * @param prefix the prefix + * @return the secure temporary directory + * @throws IOException if the directory failed to be created + */ + public static Path createSecureTempDirectory(String prefix) throws IOException { + //Set permissions only to owner, posix style + final Path secureTempFile = Files.createTempDirectory(prefix, OWNER_PERMISSIONS_ONLY_FILE_ATTRIBUTE); + //Set again for non posix systems + setPosixIndependentOwnerOnlyFilePermissions(secureTempFile); + + return secureTempFile; + } + + private static void setPosixIndependentOwnerOnlyFilePermissions(Path path) { + File file = path.toFile(); + //Set again for non posix systems + if (!(file.setReadable(true, true) && file.setWritable(true, true) && file.setExecutable(true, true))) { + LOGGER.debug("Setting permissions failed on file {}", file.getAbsolutePath()); + } + } + +} diff --git a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/ZipFileReader.java b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/ZipFileReader.java index 0b610a0ad..f7c711fed 100644 --- a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/ZipFileReader.java +++ b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/ZipFileReader.java @@ -1,5 +1,7 @@ package eu.europeana.metis.utils; +import static eu.europeana.metis.utils.TempFileUtils.createSecureTempFile; + import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; @@ -8,7 +10,6 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.UUID; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import org.apache.commons.io.FileUtils; @@ -18,9 +19,8 @@ /** * This class provides the functionality of reading zip files. - * - * @author jochen * + * @author jochen */ public class ZipFileReader { @@ -37,33 +37,30 @@ public ZipFileReader() { } /** - * This method extracts all files from a ZIP file and returns them as strings. This method only - * considers files in the main directory. This method creates (and then removes) a temporary file. + * This method extracts all files from a ZIP file and returns them as strings. This method only considers files in the main + * directory. This method creates (and then removes) a temporary file. * - * @param providedZipFile Input stream containing the zip file. This method is not responsible for - * closing the stream. + * @param providedZipFile Input stream containing the zip file. This method is not responsible for closing the stream. * @return A list of records. * @throws IOException In case of problems with the temporary file or with reading the zip file. */ public List getRecordsFromZipFile(InputStream providedZipFile) throws IOException { - try (final ZipFile zipFile = createTempZipFile(providedZipFile)) { + try (final ZipFile zipFile = createInMemoryZipFileObject(providedZipFile)) { return getRecordsFromZipFile(zipFile); } } /** - * This method extracts all files from a ZIP file and returns them as byte arrays. This method - * only considers files in the main directory. This method creates (and then removes) a temporary - * file. + * This method extracts all files from a ZIP file and returns them as byte arrays. This method only considers files in the main + * directory. This method creates (and then removes) a temporary file. * - * @param providedZipFile Input stream containing the zip file. This method is not responsible for - * closing the stream. + * @param providedZipFile Input stream containing the zip file. This method is not responsible for closing the stream. * @return A list of records. * @throws IOException In case of problems with the temporary file or with reading the zip file. */ public List getContentFromZipFile(InputStream providedZipFile) - throws IOException { - try (final ZipFile zipFile = createTempZipFile(providedZipFile)) { + throws IOException { + try (final ZipFile zipFile = createInMemoryZipFileObject(providedZipFile)) { final List streams = getContentFromZipFile(zipFile); final List result = new ArrayList<>(streams.size()); for (InputStream stream : streams) { @@ -73,9 +70,8 @@ public List getContentFromZipFile(InputStream providedZipF } } - private ZipFile createTempZipFile(InputStream content) throws IOException { - final String prefix = UUID.randomUUID().toString(); - final File tempFile = File.createTempFile(prefix, ".zip"); + private ZipFile createInMemoryZipFileObject(InputStream content) throws IOException { + final File tempFile = createSecureTempFile(ZipFileReader.class.getSimpleName(), ".zip").toFile(); FileUtils.copyInputStreamToFile(content, tempFile); LOGGER.info("Temp file: {} created.", tempFile); return new ZipFile(tempFile, ZipFile.OPEN_READ | ZipFile.OPEN_DELETE); diff --git a/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/TempFileUtilsTest.java b/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/TempFileUtilsTest.java new file mode 100644 index 000000000..8cb2612dc --- /dev/null +++ b/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/TempFileUtilsTest.java @@ -0,0 +1,86 @@ +package eu.europeana.metis.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermission; +import java.util.EnumSet; +import java.util.LinkedHashSet; +import java.util.Set; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; + +class TempFileUtilsTest { + + public static final boolean IS_POSIX = FileSystems.getDefault().supportedFileAttributeViews().contains("posix"); + + @Test + void createSecureTempFile() throws IOException { + final Path secureTempFile = TempFileUtils.createSecureTempFile("prefix", "suffix"); + assertFilePermissions(secureTempFile); + assertTrue(Files.deleteIfExists(secureTempFile)); + } + + @Test + void createSecureTempFileDeleteOnExit() + throws ClassNotFoundException, IOException, NoSuchFieldException, IllegalAccessException { + final Path secureTempFile = TempFileUtils.createSecureTempFileDeleteOnExit("prefix", "suffix"); + assertFilePermissions(secureTempFile); + + final LinkedHashSet filesForDeletion = getFileListMarkedForDeletion(); + assertTrue(filesForDeletion.contains(secureTempFile.toString())); + } + + private LinkedHashSet getFileListMarkedForDeletion() + throws ClassNotFoundException, NoSuchFieldException, IllegalAccessException { + //Check that deletion on exit is in place + final Class deleteOnExitHook = Class.forName("java.io.DeleteOnExitHook"); + final Field filesField = deleteOnExitHook.getDeclaredField("files"); + filesField.setAccessible(true); + return castList(filesField.get(null)); + } + + private LinkedHashSet castList(Object objectList) { + @SuppressWarnings("rawtypes") final LinkedHashSet linkedHashSet = (LinkedHashSet) objectList; + LinkedHashSet result = new LinkedHashSet<>(); + for (Object object : linkedHashSet) { + result.add((String) object); + } + return result; + } + + @Test + void createSecureTempDirectoryAndFile() throws IOException { + final Path secureTempDirectoryAndFile = TempFileUtils.createSecureTempDirectoryAndFile("directoryPrefix", "prefix", "suffix"); + final Path parent = secureTempDirectoryAndFile.getParent(); + assertFilePermissions(parent); + assertFilePermissions(secureTempDirectoryAndFile); + FileUtils.deleteDirectory(parent.toFile()); + } + + @Test + void createSecureTempDirectory() throws IOException { + final Path secureTempDirectory = TempFileUtils.createSecureTempDirectory("directoryPrefix"); + assertFilePermissions(secureTempDirectory); + FileUtils.deleteDirectory(secureTempDirectory.toFile()); + } + + private void assertFilePermissions(Path secureTempFile) throws IOException { + if (IS_POSIX) { + final Set posixFilePermissions = Files.getPosixFilePermissions(secureTempFile); + assertTrue(posixFilePermissions.containsAll( + EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE))); + //Check that permissions to others are denied + assertEquals(3, posixFilePermissions.size()); + } + + assertTrue(Files.isReadable(secureTempFile)); + assertTrue(Files.isWritable(secureTempFile)); + assertTrue(Files.isExecutable(secureTempFile)); + } +} \ No newline at end of file diff --git a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/OrchestratorConfig.java b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/OrchestratorConfig.java index a3069107b..605ef384b 100644 --- a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/OrchestratorConfig.java +++ b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/OrchestratorConfig.java @@ -25,8 +25,8 @@ import eu.europeana.metis.core.service.ProxiesService; import eu.europeana.metis.core.service.ScheduleWorkflowService; import eu.europeana.metis.core.service.WorkflowExecutionFactory; -import java.io.File; import java.net.MalformedURLException; +import java.nio.file.Paths; import java.time.Duration; import java.util.concurrent.TimeUnit; import javax.annotation.PreDestroy; @@ -86,7 +86,7 @@ RedissonClient getRedissonClient() throws MalformedURLException { LOGGER.info("Redis enabled SSL"); if (propertiesHolder.isRedisEnableCustomTruststore()) { singleServerConfig - .setSslTruststore(new File(propertiesHolder.getTruststorePath()).toURI().toURL()); + .setSslTruststore(Paths.get(propertiesHolder.getTruststorePath()).toUri().toURL()); singleServerConfig.setSslTruststorePassword(propertiesHolder.getTruststorePassword()); LOGGER.info("Redis enabled SSL using custom Truststore"); } diff --git a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/ClientEntityResolver.java b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/ClientEntityResolver.java index bb41db2e1..5f505e9b2 100644 --- a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/ClientEntityResolver.java +++ b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/impl/ClientEntityResolver.java @@ -140,9 +140,11 @@ private List resolveReference(ReferenceTerm referenceTerm, boolean uriSe List result = new ArrayList<>(); if (europeanaLinkPattern.matcher(referenceValue).matches()) { - result = Optional.ofNullable(entityClientApi.getEntityById(referenceValue)).map(List::of).orElse(Collections.emptyList()); + result = Optional.ofNullable(retryableExternalRequestForNetworkExceptionsThrowing( + () -> entityClientApi.getEntityById(referenceValue))).map(List::of).orElse(Collections.emptyList()); } else if (uriSearch) { - result = entityClientApi.getEntityByUri(referenceValue); + result = retryableExternalRequestForNetworkExceptionsThrowing( + () -> entityClientApi.getEntityByUri(referenceValue)); } return result; } @@ -217,7 +219,8 @@ private List findParentEntitiesRecursive(List collectedEntities, .map(Entity::getIsPartOfArray).filter(Objects::nonNull).flatMap(Collection::stream) .filter(StringUtils::isNotBlank) .filter(not(parentEntityId -> doesEntityExist(parentEntityId, collectedEntities))) - .map(entityClientApi::getEntityById) + .map(parentEntityId -> retryableExternalRequestForNetworkExceptionsThrowing( + () -> entityClientApi.getEntityById(parentEntityId))) .filter(Objects::nonNull) .collect(Collectors.toCollection(ArrayList::new)); diff --git a/metis-enrichment/metis-enrichment-rest/src/test/java/eu/europeana/enrichment/rest/exception/RestResponseExceptionHandlerTest.java b/metis-enrichment/metis-enrichment-rest/src/test/java/eu/europeana/enrichment/rest/exception/RestResponseExceptionHandlerTest.java index 7f4ae493a..a2eee4679 100644 --- a/metis-enrichment/metis-enrichment-rest/src/test/java/eu/europeana/enrichment/rest/exception/RestResponseExceptionHandlerTest.java +++ b/metis-enrichment/metis-enrichment-rest/src/test/java/eu/europeana/enrichment/rest/exception/RestResponseExceptionHandlerTest.java @@ -7,19 +7,18 @@ import eu.europeana.metis.exception.StructuredExceptionWrapper; import javax.servlet.http.HttpServletResponse; - import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import org.springframework.http.HttpStatus; -public class RestResponseExceptionHandlerTest { +class RestResponseExceptionHandlerTest { private static final RestResponseExceptionHandler REST_RESPONSE_EXCEPTION_HANDLER = new RestResponseExceptionHandler(); private static final String ERROR_MESSAGE = "error message"; ArgumentCaptor valueCaptor = ArgumentCaptor.forClass(Integer.class); @Test - void testHandleResponse(){ + void testHandleResponse() { HttpServletResponse response = mock(HttpServletResponse.class); Exception exception = new Exception(ERROR_MESSAGE); diff --git a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/CompressedFileExtractor.java b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/CompressedFileExtractor.java index d89ea5a86..615fa0521 100644 --- a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/CompressedFileExtractor.java +++ b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/CompressedFileExtractor.java @@ -1,6 +1,7 @@ package eu.europeana.metis.harvesting.http; import static eu.europeana.metis.utils.SonarqubeNullcheckAvoidanceUtils.performFunction; + import java.io.IOException; import java.io.OutputStream; import java.nio.file.Files; @@ -35,9 +36,9 @@ private CompressedFileExtractor() { * @throws IOException If there was a problem with the extraction. */ public static void extractFile(final Path compressedFile, final Path destinationFolder) - throws IOException { + throws IOException { final CompressedFileExtension compressingExtension = CompressedFileExtension - .forPath(compressedFile); + .forPath(compressedFile); if (compressingExtension == null) { throw new IOException("Can't process archive of this type: " + compressedFile); } @@ -54,12 +55,11 @@ public static void extractFile(final Path compressedFile, final Path destination break; default: throw new IllegalStateException( - "Shouldn't be here. Extension found: " + compressingExtension.name()); + "Shouldn't be here. Extension found: " + compressingExtension.name()); } } - private static void extractZipFile(final Path compressedFile, - final Path destinationFolder) throws IOException { + private static void extractZipFile(final Path compressedFile, final Path destinationFolder) throws IOException { final List nestedCompressedFiles = new ArrayList<>(); ZipUtil.unpack(compressedFile.toFile(), destinationFolder.toFile(), name -> { if (CompressedFileExtension.hasCompressedFileExtension(name)) { @@ -73,17 +73,17 @@ private static void extractZipFile(final Path compressedFile, } private static void extractTarGzFile(final Path compressedFile, final Path destinationFolder) - throws IOException { + throws IOException { final Archiver archiver = ArchiverFactory - .createArchiver(ArchiveFormat.TAR, CompressionType.GZIP); + .createArchiver(ArchiveFormat.TAR, CompressionType.GZIP); archiver.extract(compressedFile.toFile(), destinationFolder.toFile()); final Path newDestination = CompressedFileExtension - .removeExtension(destinationFolder.resolve(compressedFile.getFileName())); + .removeExtension(destinationFolder.resolve(compressedFile.getFileName())); final Set nestedCompressedFiles; try (Stream nestedFilesStream = Files.walk(newDestination)) { nestedCompressedFiles = performFunction(nestedFilesStream, stream -> stream - .filter(CompressedFileExtension::hasCompressedFileExtension) - .collect(Collectors.toSet())); + .filter(CompressedFileExtension::hasCompressedFileExtension) + .collect(Collectors.toSet())); } for (Path file : nestedCompressedFiles) { extractFile(file, file.getParent()); @@ -91,13 +91,13 @@ private static void extractTarGzFile(final Path compressedFile, final Path desti } private static void extractGzFile(final Path compressedFile, final Path destinationFolder) - throws IOException { + throws IOException { // Note: .gz files just contain one file. final Path destination = CompressedFileExtension - .removeExtension(destinationFolder.resolve(compressedFile.getFileName())); + .removeExtension(destinationFolder.resolve(compressedFile.getFileName())); try (final GzipCompressorInputStream inputStream = new GzipCompressorInputStream( - Files.newInputStream(compressedFile)); - final OutputStream outputStream = Files.newOutputStream(destination)) { + Files.newInputStream(compressedFile)); + final OutputStream outputStream = Files.newOutputStream(destination)) { IOUtils.copy(inputStream, outputStream); } } diff --git a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java index adccc46fe..09d9da12a 100644 --- a/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java +++ b/metis-harvesting/src/main/java/eu/europeana/metis/harvesting/http/HttpHarvesterImpl.java @@ -1,6 +1,8 @@ package eu.europeana.metis.harvesting.http; import static eu.europeana.metis.utils.SonarqubeNullcheckAvoidanceUtils.performFunction; +import static eu.europeana.metis.utils.TempFileUtils.createSecureTempDirectoryAndFile; +import static org.apache.commons.io.FileUtils.copyInputStreamToFile; import eu.europeana.metis.harvesting.HarvesterException; import eu.europeana.metis.harvesting.ReportingIteration; @@ -22,7 +24,6 @@ import java.util.Iterator; import java.util.List; import java.util.Set; -import java.util.UUID; import java.util.function.Consumer; import java.util.stream.Stream; import org.apache.commons.io.FileUtils; @@ -46,26 +47,26 @@ public class HttpHarvesterImpl implements HttpHarvester { public void harvestRecords(InputStream inputStream, CompressedFileExtension compressedFileType, Consumer action) throws HarvesterException { - // Now perform the harvesting - go by each file. - final HttpRecordIterator iterator = createTemporaryHttpHarvestIterator(inputStream, compressedFileType); - List> exception = new ArrayList<>(1); - iterator.forEach(path -> { - try (InputStream content = Files.newInputStream(path)) { - action.accept(new ArchiveEntryImpl(path.getFileName().toString(), - new ByteArrayInputStream(IOUtils.toByteArray(content)))); - return IterationResult.CONTINUE; - } catch (IOException | RuntimeException e) { - exception.add(new ImmutablePair<>(path, e)); - return IterationResult.TERMINATE; - } - }); + // Now perform the harvesting - go by each file. + final HttpRecordIterator iterator = createTemporaryHttpHarvestIterator(inputStream, compressedFileType); + List> exception = new ArrayList<>(1); + iterator.forEach(path -> { + try (InputStream content = Files.newInputStream(path)) { + action.accept(new ArchiveEntryImpl(path.getFileName().toString(), + new ByteArrayInputStream(IOUtils.toByteArray(content)))); + return IterationResult.CONTINUE; + } catch (IOException | RuntimeException e) { + exception.add(new ImmutablePair<>(path, e)); + return IterationResult.TERMINATE; + } + }); - iterator.deleteIteratorContent(); + iterator.deleteIteratorContent(); - if (!exception.isEmpty()) { - throw new HarvesterException("Could not process path " + exception.get(0).getKey() + ".", - exception.get(0).getValue()); - } + if (!exception.isEmpty()) { + throw new HarvesterException("Could not process path " + exception.get(0).getKey() + ".", + exception.get(0).getValue()); + } } @Override @@ -88,23 +89,16 @@ public HttpRecordIterator harvestRecords(String archiveUrl, String downloadDirec } @Override - public HttpRecordIterator createTemporaryHttpHarvestIterator(InputStream input, CompressedFileExtension compressedFileType) throws HarvesterException { - // We chose where to store the temporary file. - @SuppressWarnings("findsecbugs:PATH_TRAVERSAL_IN") - Path tempDir; - final Path tempFile; + public HttpRecordIterator createTemporaryHttpHarvestIterator(InputStream input, CompressedFileExtension compressedFileType) + throws HarvesterException { try { - // Save the zip file in a temporary directory (and close the input stream). - final String prefix = UUID.randomUUID().toString(); - - tempDir = Files.createTempDirectory(prefix); - tempFile = Files.createTempFile(tempDir, prefix, compressedFileType.getExtension()); - FileUtils.copyInputStreamToFile(input, tempFile.toFile()); - + final Path tempFile = createSecureTempDirectoryAndFile(HttpHarvesterImpl.class.getSimpleName(), + HttpHarvesterImpl.class.getSimpleName(), compressedFileType.getExtension()); + copyInputStreamToFile(input, tempFile.toFile()); return harvestRecords(tempFile); - } catch (IOException e) { - throw new HarvesterException("Problem saving archive.", e); - } + } catch (IOException e) { + throw new HarvesterException("Problem saving archive.", e); + } } @@ -133,16 +127,16 @@ private HttpRecordIterator harvestRecords(Path archiveFile) throws HarvesterExce } private Path downloadFile(String archiveUrlString, Path downloadDirectory) throws IOException { - final Path directory = Files.createDirectories(downloadDirectory); - final Path file = directory.resolve(FilenameUtils.getName(archiveUrlString)); final URL archiveUrl = new URL(archiveUrlString); if (!SUPPORTED_PROTOCOLS.contains(archiveUrl.getProtocol())) { throw new IOException("This functionality does not support this protocol (" + archiveUrl.getProtocol() + ")."); } + final Path directory = Files.createDirectories(downloadDirectory); + final Path file = directory.resolve(FilenameUtils.getName(archiveUrlString)); // Note: we allow any download URL for http harvesting. This is the functionality we support. - @SuppressWarnings("findsecbugs:URLCONNECTION_SSRF_FD") final URLConnection conn = archiveUrl.openConnection(); - try (final InputStream inputStream = conn.getInputStream(); + @SuppressWarnings("findsecbugs:URLCONNECTION_SSRF_FD") final URLConnection urlConnection = archiveUrl.openConnection(); + try (final InputStream inputStream = urlConnection.getInputStream(); final OutputStream outputStream = Files.newOutputStream(file)) { IOUtils.copyLarge(inputStream, outputStream); } @@ -150,12 +144,11 @@ private Path downloadFile(String archiveUrlString, Path downloadDirectory) throw } /** - * Method corrects rights on Linux systems, where created new directory and extracted files have - * not right copied from parent folder, and they have not any right for others users. Also group - * is not preserved from parent. It is a problem cause apache server could not reach files cause - * it typically works as special apache_user. The purpose of this method is to copy rights from - * parent directory, that should have correctly configured right to passed as parameter directory - * and any directory or file inside. + * Method corrects rights on Linux systems, where created new directory and extracted files have not right copied from parent + * folder, and they have not any right for others users. Also group is not preserved from parent. It is a problem cause apache + * server could not reach files cause it typically works as special apache_user. The purpose of this method is to copy rights + * from parent directory, that should have correctly configured right to passed as parameter directory and any directory or file + * inside. * * @param directory directory for which rights will be updated * @throws IOException in case of rights update failure diff --git a/metis-harvesting/src/test/java/eu/europeana/metis/harvesting/http/CompressedFileExtractorGzTest.java b/metis-harvesting/src/test/java/eu/europeana/metis/harvesting/http/CompressedFileExtractorGzTest.java index 6d5b78f35..f4684c929 100644 --- a/metis-harvesting/src/test/java/eu/europeana/metis/harvesting/http/CompressedFileExtractorGzTest.java +++ b/metis-harvesting/src/test/java/eu/europeana/metis/harvesting/http/CompressedFileExtractorGzTest.java @@ -22,23 +22,23 @@ public class CompressedFileExtractorGzTest { public static final String FILE_EXTENSION = ".tar.gz"; @Test - public void shouldUnpackTheTarGzFilesRecursively() throws IOException { - CompressedFileExtractor.extractFile(Path.of(DESTINATION_DIR + FILE_NAME + FILE_EXTENSION),Path.of(DESTINATION_DIR)); + void shouldUnpackTheTarGzFilesRecursively() throws IOException { + CompressedFileExtractor.extractFile(Path.of(DESTINATION_DIR + FILE_NAME + FILE_EXTENSION), Path.of(DESTINATION_DIR)); Collection files = getXMLFiles(DESTINATION_DIR + FILE_NAME); assertNotNull(files); assertEquals(XML_FILES_COUNT, files.size()); } @Test - public void shouldUnpackTheTarGzFilesRecursivelyWithCompressedXMLFiles() throws IOException { - CompressedFileExtractor.extractFile(Path.of(DESTINATION_DIR + FILE_NAME2 + FILE_EXTENSION),Path.of(DESTINATION_DIR)); + void shouldUnpackTheTarGzFilesRecursivelyWithCompressedXMLFiles() throws IOException { + CompressedFileExtractor.extractFile(Path.of(DESTINATION_DIR + FILE_NAME2 + FILE_EXTENSION), Path.of(DESTINATION_DIR)); Collection files = getXMLFiles(DESTINATION_DIR + FILE_NAME2); assertNotNull(files); assertEquals(XML_FILES_COUNT, files.size()); } @Test - public void shouldUnpackTheTGZFilesRecursivelyWithCompressedXMLFiles() throws IOException { + void shouldUnpackTheTGZFilesRecursivelyWithCompressedXMLFiles() throws IOException { CompressedFileExtractor.extractFile(Path.of(DESTINATION_DIR + FILE_NAME2 + FILE_EXTENSION), Path.of(DESTINATION_DIR)); Collection files = getXMLFiles(DESTINATION_DIR + FILE_NAME2); assertNotNull(files); @@ -46,7 +46,7 @@ public void shouldUnpackTheTGZFilesRecursivelyWithCompressedXMLFiles() throws IO } @Test - public void shouldUnpackTheTarGzFilesRecursivelyWithMixedNestedCompressedFiles() throws IOException { + void shouldUnpackTheTarGzFilesRecursivelyWithMixedNestedCompressedFiles() throws IOException { CompressedFileExtractor.extractFile(Path.of(DESTINATION_DIR + FILE_NAME3 + FILE_EXTENSION), Path.of(DESTINATION_DIR)); Collection files = getXMLFiles(DESTINATION_DIR + FILE_NAME3); assertNotNull(files); diff --git a/metis-harvesting/src/test/java/eu/europeana/metis/harvesting/http/CompressedFileExtractorZipTest.java b/metis-harvesting/src/test/java/eu/europeana/metis/harvesting/http/CompressedFileExtractorZipTest.java index 491ee0479..9b07c205e 100644 --- a/metis-harvesting/src/test/java/eu/europeana/metis/harvesting/http/CompressedFileExtractorZipTest.java +++ b/metis-harvesting/src/test/java/eu/europeana/metis/harvesting/http/CompressedFileExtractorZipTest.java @@ -11,7 +11,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; -public class CompressedFileExtractorZipTest { +class CompressedFileExtractorZipTest { private final static String DESTINATION_DIR = String.format("src%1$stest%1$sresources%1$s__files%1$s", File.separator); private final static int XML_FILES_COUNT = 13; @@ -23,27 +23,27 @@ public class CompressedFileExtractorZipTest { public static final String FILE_EXTENSION = ".zip"; @Test - public void shouldUnpackTheZipFilesRecursively() throws IOException { - CompressedFileExtractor.extractFile(Path.of(DESTINATION_DIR + FILE_NAME + FILE_EXTENSION), - Path.of(DESTINATION_DIR)); + void shouldUnpackTheZipFilesRecursively() throws IOException { + CompressedFileExtractor.extractFile(Path.of(DESTINATION_DIR + FILE_NAME + FILE_EXTENSION), + Path.of(DESTINATION_DIR)); Collection files = getXMLFiles(DESTINATION_DIR + DEFAULT_DESTINATION_NAME); assertNotNull(files); assertEquals(XML_FILES_COUNT, files.size()); } @Test - public void shouldUnpackTheZipFilesWithNestedFoldersRecursively() throws IOException { - CompressedFileExtractor.extractFile(Path.of(DESTINATION_DIR + FILE_NAME2 + FILE_EXTENSION), - Path.of(DESTINATION_DIR)); + void shouldUnpackTheZipFilesWithNestedFoldersRecursively() throws IOException { + CompressedFileExtractor.extractFile(Path.of(DESTINATION_DIR + FILE_NAME2 + FILE_EXTENSION), + Path.of(DESTINATION_DIR)); Collection files = getXMLFiles(DESTINATION_DIR + DEFAULT_DESTINATION_NAME); assertNotNull(files); assertEquals(XML_FILES_COUNT, files.size()); } @Test - public void shouldUnpackTheZipFilesWithNestedMixedCompressedFiles() throws IOException { - CompressedFileExtractor.extractFile(Path.of(DESTINATION_DIR + FILE_NAME3 + FILE_EXTENSION), - Path.of(DESTINATION_DIR)); + void shouldUnpackTheZipFilesWithNestedMixedCompressedFiles() throws IOException { + CompressedFileExtractor.extractFile(Path.of(DESTINATION_DIR + FILE_NAME3 + FILE_EXTENSION), + Path.of(DESTINATION_DIR)); Collection files = getXMLFiles(DESTINATION_DIR + DEFAULT_DESTINATION_NAME); assertNotNull(files); assertEquals(XML_FILES_COUNT, files.size()); diff --git a/metis-harvesting/src/test/java/eu/europeana/metis/harvesting/oaipmh/CloseableHttpOaiClientTest.java b/metis-harvesting/src/test/java/eu/europeana/metis/harvesting/oaipmh/CloseableHttpOaiClientTest.java index 458dcc7e6..d12dec09d 100644 --- a/metis-harvesting/src/test/java/eu/europeana/metis/harvesting/oaipmh/CloseableHttpOaiClientTest.java +++ b/metis-harvesting/src/test/java/eu/europeana/metis/harvesting/oaipmh/CloseableHttpOaiClientTest.java @@ -19,7 +19,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -public class CloseableHttpOaiClientTest { +class CloseableHttpOaiClientTest { private static WireMockServer WIREMOCK_SERVER; private static final String ENDPOINT = "ENDPOINT"; @@ -38,7 +38,7 @@ static void prepare() throws IOException { } @Test - public void shouldReturnACorrectValue() throws HttpException, IOException { + void shouldReturnACorrectValue() throws HttpException, IOException { final String fileContent = "FILE CONTENT"; WIREMOCK_SERVER.stubFor( get(urlEqualTo(PATH)).willReturn(WiremockHelper.response200XmlContent(fileContent))); @@ -50,7 +50,7 @@ public void shouldReturnACorrectValue() throws HttpException, IOException { } @Test - public void shouldHandleTimeout() throws HttpException { + void shouldHandleTimeout() throws HttpException { WIREMOCK_SERVER.stubFor(get(urlEqualTo(PATH)).willReturn(WiremockHelper .responsTimeoutGreaterThanSocketTimeout("FILE CONTENT", TestHelper.TEST_SOCKET_TIMEOUT))); final Parameters parameters = mock(Parameters.class); @@ -61,7 +61,7 @@ public void shouldHandleTimeout() throws HttpException { } @Test - public void shouldRetryAndFail() throws HttpException { + void shouldRetryAndFail() throws HttpException { WIREMOCK_SERVER.stubFor(get(urlEqualTo(PATH)).willReturn(WiremockHelper.response404())); final Parameters parameters = mock(Parameters.class); when(parameters.toUrl(ENDPOINT)).thenReturn(URL); @@ -71,14 +71,14 @@ public void shouldRetryAndFail() throws HttpException { } @Test - public void shouldRetryAndReturnACorrectValue() throws Exception { + void shouldRetryAndReturnACorrectValue() throws Exception { final String fileContent = "FILE CONTENT"; WIREMOCK_SERVER.stubFor( get(urlEqualTo(PATH)).inScenario("Retry and success scenario").whenScenarioStateIs(STARTED) - .willSetStateTo("one time requested").willReturn(WiremockHelper.response404())); + .willSetStateTo("one time requested").willReturn(WiremockHelper.response404())); WIREMOCK_SERVER.stubFor(get(urlEqualTo(PATH)).inScenario("Retry and success scenario") - .whenScenarioStateIs("one time requested") - .willReturn(WiremockHelper.response200XmlContent(fileContent))); + .whenScenarioStateIs("one time requested") + .willReturn(WiremockHelper.response200XmlContent(fileContent))); final Parameters parameters = mock(Parameters.class); when(parameters.toUrl(ENDPOINT)).thenReturn(URL); try (final CloseableHttpOaiClient client = CONNECTION_CLIENT_FACTORY.get()) { diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/SolrPropertyUtilsTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/SolrPropertyUtilsTest.java index 639907fc3..9dd60a346 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/SolrPropertyUtilsTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/SolrPropertyUtilsTest.java @@ -4,6 +4,12 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import eu.europeana.indexing.solr.EdmLabel; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -15,14 +21,11 @@ import java.util.stream.Stream; import org.apache.solr.common.SolrInputDocument; import org.junit.jupiter.api.Test; -import eu.europeana.indexing.solr.EdmLabel; -import eu.europeana.indexing.solr.property.SolrPropertyUtils; -import static org.mockito.Mockito.*; -public class SolrPropertyUtilsTest { +class SolrPropertyUtilsTest { @Test - public void testAddValues() { + void testAddValues() { // Start with empty document: verify that it is empty. final SolrInputDocument document = new SolrInputDocument(); @@ -64,7 +67,7 @@ public void testAddValues() { } @Test - public void testAddNullValues() { + void testAddNullValues() { // Start with empty document: verify that it is empty. final SolrInputDocument document = new SolrInputDocument(); @@ -87,7 +90,7 @@ public void testAddNullValues() { } @Test - public void testRightsForMap() { + void testRightsForMap() { // Test null map final Stream fromNullMap = SolrPropertyUtils.getRightsFromMap(null); @@ -113,7 +116,7 @@ public void testRightsForMap() { } @Test - public void testHasLicenseForRights() { + void testHasLicenseForRights() { // no rights required assertTrue(SolrPropertyUtils.hasLicenseForRights(Collections.emptyMap(), license -> false)); @@ -127,7 +130,7 @@ public void testHasLicenseForRights() { assertFalse(SolrPropertyUtils.hasLicenseForRights(rights, contradiction)); verify(contradiction, times(3)).test(anyString()); for (String inputString : input) { - verify(contradiction, times(1)).test(eq(inputString)); + verify(contradiction, times(1)).test(inputString); } // rights required and licenses available diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/media/MediaClassifierTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/media/MediaClassifierTest.java index a96adefe6..665a82629 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/media/MediaClassifierTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/media/MediaClassifierTest.java @@ -15,7 +15,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -public class MediaClassifierTest { +class MediaClassifierTest { private static AudioClassifier audioClassifier; private static ImageClassifier imageClassifier; diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/model/MediaTierTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/model/MediaTierTest.java index 4fa3a6664..b524f6e04 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/model/MediaTierTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/model/MediaTierTest.java @@ -8,12 +8,12 @@ import java.util.Comparator; import org.junit.jupiter.api.Test; -public class MediaTierTest { +class MediaTierTest { @Test void checkValues() { final MediaTier[] tiersOrderedByLevel = Arrays.stream(MediaTier.values()) - .sorted(Comparator.comparingInt(MediaTier::getLevel)).toArray(MediaTier[]::new); + .sorted(Comparator.comparingInt(MediaTier::getLevel)).toArray(MediaTier[]::new); assertEquals(5, tiersOrderedByLevel.length); final MediaTier[] expectedOrder = diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/model/MetadataTierTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/model/MetadataTierTest.java index c9846ccde..eb1da706d 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/tiers/model/MetadataTierTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/tiers/model/MetadataTierTest.java @@ -8,12 +8,13 @@ import java.util.Comparator; import org.junit.jupiter.api.Test; -public class MetadataTierTest { +class MetadataTierTest { @Test void checkValues() { final MetadataTier[] tiersOrderedByLevel = Arrays.stream(MetadataTier.values()) - .sorted(Comparator.comparingInt(MetadataTier::getLevel)).toArray(MetadataTier[]::new); + .sorted(Comparator.comparingInt(MetadataTier::getLevel)) + .toArray(MetadataTier[]::new); assertEquals(4, tiersOrderedByLevel.length); final MetadataTier[] expectedOrder = new MetadataTier[]{MetadataTier.T0, MetadataTier.TA, MetadataTier.TB, MetadataTier.TC}; diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/utils/SetUtilsTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/utils/SetUtilsTest.java index 772dc4dc2..669ace5b3 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/utils/SetUtilsTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/utils/SetUtilsTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -11,7 +12,7 @@ import java.util.stream.Stream; import org.junit.jupiter.api.Test; -public class SetUtilsTest { +class SetUtilsTest { /* * We test this by generating all strings that satisfy the following conditions: @@ -25,14 +26,14 @@ public class SetUtilsTest { * 4. After that it may contain one upper case letter (only X) */ @Test - public void testGenerateOptionalCombinations() { + void testGenerateOptionalCombinations() { // Create sets of options final Set allowedDigits = Stream.of('1', '2', '3').collect(Collectors.toSet()); final Set allowedlowerCase = Stream.of('a', 'b').collect(Collectors.toSet()); final Set allowedUpperCase = Collections.singleton('X'); final List> options = - Arrays.asList(allowedDigits, allowedlowerCase, allowedUpperCase); + Arrays.asList(allowedDigits, allowedlowerCase, allowedUpperCase); // Generate the combinations. final Set result = SetUtils @@ -71,14 +72,14 @@ public void testGenerateOptionalCombinations() { * 4. After that it must contain one upper case letter (only X) */ @Test - public void testGenerateForcedCombinations() { + void testGenerateForcedCombinations() { // Create sets of options final Set allowedDigits = Stream.of('1', '2', '3').collect(Collectors.toSet()); final Set allowedlowerCase = Stream.of('a', 'b').collect(Collectors.toSet()); final Set allowedUpperCase = Collections.singleton('X'); final List> options = - Arrays.asList(allowedDigits, allowedlowerCase, allowedUpperCase); + Arrays.asList(allowedDigits, allowedlowerCase, allowedUpperCase); // Generate the combinations. final Set result = SetUtils diff --git a/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/extraction/PdfToImageConverter.java b/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/extraction/PdfToImageConverter.java index f2a4c389b..520654cff 100644 --- a/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/extraction/PdfToImageConverter.java +++ b/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/extraction/PdfToImageConverter.java @@ -1,7 +1,10 @@ package eu.europeana.metis.mediaprocessing.extraction; +import static eu.europeana.metis.utils.TempFileUtils.createSecureTempFile; + import eu.europeana.metis.mediaprocessing.exception.MediaExtractionException; import eu.europeana.metis.mediaprocessing.exception.MediaProcessorException; +import eu.europeana.metis.utils.TempFileUtils; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -24,9 +27,8 @@ class PdfToImageConverter { private final CommandExecutor commandExecutor; /** - * Constructor. This is a wrapper for {@link PdfToImageConverter#PdfToImageConverter(CommandExecutor, - * String)} where the property is detected. It is advisable to use this constructor for - * non-testing purposes. + * Constructor. This is a wrapper for {@link PdfToImageConverter#PdfToImageConverter(CommandExecutor, String)} where the + * property is detected. It is advisable to use this constructor for non-testing purposes. * * @param commandExecutor A command executor. * @throws MediaProcessorException In case the properties could not be initialized. @@ -47,7 +49,7 @@ class PdfToImageConverter { } private static String getGlobalGhostScriptCommand(CommandExecutor commandExecutor) - throws MediaProcessorException { + throws MediaProcessorException { synchronized (AudioVideoProcessor.class) { if (globalGhostScriptCommand == null) { globalGhostScriptCommand = discoverGhostScriptCommand(commandExecutor); @@ -57,13 +59,13 @@ private static String getGlobalGhostScriptCommand(CommandExecutor commandExecuto } static String discoverGhostScriptCommand(CommandExecutor commandExecutor) - throws MediaProcessorException { + throws MediaProcessorException { // Check whether ghostscript is installed. final String command = "gs"; final String output; output = commandExecutor.execute(Arrays.asList(command, "--version"), true, message -> - new MediaProcessorException("Error while looking for ghostscript tools: " + message)); + new MediaProcessorException("Error while looking for ghostscript tools: " + message)); if (!output.startsWith("9.")) { throw new MediaProcessorException("Ghostscript 9.x not found."); } @@ -73,14 +75,14 @@ static String discoverGhostScriptCommand(CommandExecutor commandExecutor) } /** - * The main method of this class. It takes a PDF input file and converts the frist page of it to a - * PNG image. It saves it as a file and returns the reference to it. + * The main method of this class. It takes a PDF input file and converts the first page of it to a PNG image. It saves it as a + * file and returns the reference to it. * * @param content the PDF input. * @return The PNG output. * @throws MediaExtractionException In case something went wrong during the conversion. */ - Path convertToPdf(Path content) throws MediaExtractionException { + Path convertPdfFirstPageToImage(Path content) throws MediaExtractionException { // Sanity checking if (content == null) { @@ -113,7 +115,7 @@ void removePdfImageFileSilently(Path file) { Path createPdfImageFile() throws MediaExtractionException { try { - return Files.createTempFile("metis_pdf_image_", null); + return createSecureTempFile("metis_pdf_image_", TempFileUtils.PNG_FILE_EXTENSION); } catch (IOException e) { throw new MediaExtractionException("Could not create temporary file.", e); } @@ -121,23 +123,23 @@ Path createPdfImageFile() throws MediaExtractionException { List createPdfConversionCommand(Path inputFile, Path outputFile) { return Arrays.asList(ghostScriptCmd, - "-q", - "-dQUIET", - "-dSAFER", - "-dBATCH", - "-dNOPAUSE", - "-dNOPROMPT", - "-dMaxBitmap=500000000", - "-dAlignToPixels=0", - "-dGridFitTT=2", - "-sDEVICE=pngalpha", - "-dTextAlphaBits=4", - "-dGraphicsAlphaBits=4", - "-r72x72", - "-dFirstPage=1", - "-dLastPage=1", - "-sOutputFile=" + outputFile.toAbsolutePath().toString(), - "-f" + inputFile.toAbsolutePath().toString() + "-q", + "-dQUIET", + "-dSAFER", + "-dBATCH", + "-dNOPAUSE", + "-dNOPROMPT", + "-dMaxBitmap=500000000", + "-dAlignToPixels=0", + "-dGridFitTT=2", + "-sDEVICE=pngalpha", + "-dTextAlphaBits=4", + "-dGraphicsAlphaBits=4", + "-r72x72", + "-dFirstPage=1", + "-dLastPage=1", + "-sOutputFile=" + outputFile.toAbsolutePath(), + "-f" + inputFile.toAbsolutePath() ); } } diff --git a/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/extraction/TextProcessor.java b/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/extraction/TextProcessor.java index ee02a9fd5..16f451cd4 100644 --- a/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/extraction/TextProcessor.java +++ b/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/extraction/TextProcessor.java @@ -86,7 +86,7 @@ public ResourceExtractionResultImpl extractMetadata(Resource resource, String de final List thumbnails; if (PDF_MIME_TYPE.equals(detectedMimeType) && generateThumbnailForPdf(resource, mainThumbnailAvailable)) { - final Path pdfImage = pdfToImageConverter.convertToPdf(resource.getContentPath()); + final Path pdfImage = pdfToImageConverter.convertPdfFirstPageToImage(resource.getContentPath()); try { thumbnails = thumbnailGenerator.generateThumbnails(resource.getResourceUrl(), PNG_MIME_TYPE, pdfImage.toFile(), true).getRight(); diff --git a/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/extraction/ThumbnailGenerator.java b/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/extraction/ThumbnailGenerator.java index 391e949de..d7a4d79de 100644 --- a/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/extraction/ThumbnailGenerator.java +++ b/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/extraction/ThumbnailGenerator.java @@ -6,6 +6,7 @@ import eu.europeana.metis.mediaprocessing.model.ThumbnailImpl; import eu.europeana.metis.mediaprocessing.model.ThumbnailKind; import eu.europeana.metis.schema.model.MediaType; +import eu.europeana.metis.utils.TempFileUtils; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -51,6 +52,7 @@ class ThumbnailGenerator { private static final int COMMAND_RESULT_COLORSPACE_LINE = 2; private static final int COMMAND_RESULT_MAX_COLORS = 6; public static final String COLORMAP_PNG = "colormap.png"; + public static final int EXPECTED_CONTENT_MARKERS_COUNT = 5; private static String globalMagickCommand; private static Path globalColormapFile; @@ -61,12 +63,10 @@ class ThumbnailGenerator { private final CommandExecutor commandExecutor; /** - * Constructor. This is a wrapper for {@link ThumbnailGenerator#ThumbnailGenerator(CommandExecutor, - * String, String)} where the properties are detected. It is advisable to use this constructor for - * non-testing purposes. + * Constructor. This is a wrapper for {@link ThumbnailGenerator#ThumbnailGenerator(CommandExecutor, String, String)} where the + * properties are detected. It is advisable to use this constructor for non-testing purposes. * - * @param commandExecutor A command executor. The calling class is responsible for closing this - * object. + * @param commandExecutor A command executor. The calling class is responsible for closing this object. * @throws MediaProcessorException In case the properties could not be initialized. */ ThumbnailGenerator(CommandExecutor commandExecutor) throws MediaProcessorException { @@ -76,8 +76,7 @@ class ThumbnailGenerator { /** * Constructor. * - * @param commandExecutor A command executor.The calling class is responsible for closing this - * object + * @param commandExecutor A command executor.The calling class is responsible for closing this object * @param magickCommand The magick command (how to trigger imageMagick). * @param colorMapFile The location of the color map file. */ @@ -102,16 +101,13 @@ private static Path initColorMap() throws MediaProcessorException { if (colorMapInputStream == null) { throw new MediaProcessorException("Could not load color map file: could not find file."); } - colormapTempFile = Files.createTempFile("colormap", ".png"); + colormapTempFile = TempFileUtils.createSecureTempFileDeleteOnExit("colormap", ".png"); Files.copy(colorMapInputStream, colormapTempFile, StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { throw new MediaProcessorException( String.format("Could not load color map file: %s", COLORMAP_PNG), e); } - // Ensure that the temporary file is removed (should not be needed). - colormapTempFile.toFile().deleteOnExit(); - // So everything went well. We set this as the new color map file. globalColormapFile = colormapTempFile; return globalColormapFile; @@ -180,7 +176,7 @@ static String discoverImageMagickCommand(CommandExecutor commandExecutor) private static List splitByNewLine(String input) { return Stream.of(input.split("\\R")).filter(StringUtils::isNotBlank) - .collect(Collectors.toList()); + .collect(Collectors.toList()); } /** @@ -190,8 +186,8 @@ private static List splitByNewLine(String input) { * @param detectedMimeType The detected mime type of the content. * @param content The resource content for which to generate thumbnails. * @param removeAlpha Whether any alpha should be removed and replaced with a white background. - * @return The metadata of the image as gathered during processing, together with the thumbnails. - * The list can be null or empty, but does not contain null values or thumbnails without content. + * @return The metadata of the image as gathered during processing, together with the thumbnails. The list can be null or empty, + * but does not contain null values or thumbnails without content. * @throws MediaExtractionException In case a problem occurred. */ Pair> generateThumbnails(String url, String detectedMimeType, @@ -232,7 +228,7 @@ Pair> generateThumbnails(String url, String detec // Done. final List resultThumbnails = thumbnails.stream() - .map(ThumbnailWithSize::getThumbnail).collect(Collectors.toList()); + .map(ThumbnailWithSize::getThumbnail).collect(Collectors.toList()); return new ImmutablePair<>(image, resultThumbnails); } @@ -282,11 +278,9 @@ private ImageMetadata generateThumbnailsInternal(List thumbna // Generate the thumbnails and read image properties. final String contentMarker = UUID.randomUUID().toString(); - final List command = - createThumbnailGenerationCommand(thumbnails, removeAlpha, content, contentMarker); + final List command = createThumbnailGenerationCommand(thumbnails, removeAlpha, content, contentMarker); final String response = commandExecutor.execute(command, false, message -> - new MediaExtractionException( - "Could not analyze content and generate thumbnails: " + message)); + new MediaExtractionException("Could not analyze content and generate thumbnails: " + message)); final ImageMetadata result = parseCommandResponse(response, contentMarker); // Check the thumbnails. @@ -336,12 +330,15 @@ List prepareThumbnailFiles(String url, String detectedMimeTyp // Decide on the thumbnail file type final String imageMagickThumbnailTypePrefix; final String thumbnailMimeType; + final String thumbnailFileSuffix; if (PNG_MIME_TYPE.equals(detectedMimeType)) { imageMagickThumbnailTypePrefix = "png:"; thumbnailMimeType = PNG_MIME_TYPE; + thumbnailFileSuffix = TempFileUtils.PNG_FILE_EXTENSION; } else { imageMagickThumbnailTypePrefix = "jpeg:"; thumbnailMimeType = JPEG_MIME_TYPE; + thumbnailFileSuffix = TempFileUtils.JPEG_FILE_EXTENSION; } // Create the thumbnails: one for each kind @@ -351,10 +348,9 @@ List prepareThumbnailFiles(String url, String detectedMimeTyp for (ThumbnailKind thumbnailKind : ThumbnailKind.values()) { final String targetName = md5 + thumbnailKind.getNameSuffix(); // False positive - we don't want to close the thumbnail here. - @SuppressWarnings("squid:S2095") - final ThumbnailImpl thumbnail = new ThumbnailImpl(url, thumbnailMimeType, targetName); - result.add(new ThumbnailWithSize(thumbnail, thumbnailKind.getImageSize(), - imageMagickThumbnailTypePrefix)); + @SuppressWarnings("squid:S2095") final ThumbnailImpl thumbnail = new ThumbnailImpl(url, thumbnailMimeType, targetName); + result.add( + new ThumbnailWithSize(thumbnail, thumbnailKind.getImageSize(), imageMagickThumbnailTypePrefix, thumbnailFileSuffix)); } } catch (RuntimeException | IOException e) { closeAllThumbnailsSilently(result); @@ -384,12 +380,12 @@ ImageMetadata parseCommandResponse(String response, String contentMarker) // Divide in segments and check their number. final String[] segments = response.split(Pattern.quote(contentMarker), 6); - if (segments.length < 5) { + if (segments.length < EXPECTED_CONTENT_MARKERS_COUNT) { throw new MediaExtractionException(String.format( "Could not parse ImageMagick response(there are not enough content markers):%s%s", System.lineSeparator(), response)); } - if (segments.length > 5) { + if (segments.length > EXPECTED_CONTENT_MARKERS_COUNT) { throw new MediaExtractionException(String .format("Could not parse ImageMagick response(there are too many content markers):%s%s", System.lineSeparator(), response)); @@ -399,8 +395,8 @@ ImageMetadata parseCommandResponse(String response, String contentMarker) // segments are empty. If there is any unexpected content, this could be an error message. final String unexpectedContent = IntStream.range(0, segments.length).filter(index -> index % 2 == 0) - .mapToObj(index -> segments[index]) - .collect(Collectors.joining(System.lineSeparator())); + .mapToObj(index -> segments[index]) + .collect(Collectors.joining(System.lineSeparator())); if (StringUtils.isNotBlank(unexpectedContent)) { throw new MediaExtractionException(String .format("Unexpected content found in ImageMagick response: %s%s", @@ -411,14 +407,14 @@ ImageMetadata parseCommandResponse(String response, String contentMarker) final Pattern pattern = Pattern.compile("#([0-9A-F]{6})"); final List colorStrings = splitByNewLine(segments[3]).stream().sorted(Collections.reverseOrder()) - .limit(COMMAND_RESULT_MAX_COLORS).collect(Collectors.toList()); + .limit(COMMAND_RESULT_MAX_COLORS).collect(Collectors.toList()); final Supplier> streamMatcherSupplier = () -> colorStrings.stream() - .map(pattern::matcher); + .map(pattern::matcher); if (!streamMatcherSupplier.get().allMatch(Matcher::find)) { throw new IllegalStateException("Invalid color line found."); } final List dominantColors = streamMatcherSupplier.get().filter(Matcher::find) - .map(matcher -> matcher.group(1)).collect(Collectors.toList()); + .map(matcher -> matcher.group(1)).collect(Collectors.toList()); // Get width, height and color space final List metadata = splitByNewLine(segments[1]); @@ -450,9 +446,10 @@ static class ThumbnailWithSize { this.imageMagickTypePrefix = imageMagickTypePrefix; } - ThumbnailWithSize(ThumbnailImpl thumbnail, int imageSize, String imageMagickTypePrefix) + ThumbnailWithSize(ThumbnailImpl thumbnail, int imageSize, String imageMagickTypePrefix, String thumbnailFileSuffix) throws IOException { - this(thumbnail, imageSize, Files.createTempFile("thumbnail_", null), imageMagickTypePrefix); + this(thumbnail, imageSize, TempFileUtils.createSecureTempFile("thumbnail_", thumbnailFileSuffix), + imageMagickTypePrefix); } ThumbnailImpl getThumbnail() { @@ -473,7 +470,7 @@ String getImageMagickTypePrefix() { void deleteTempFileSilently() { try { - Files.delete(getTempFileForThumbnail()); + Files.deleteIfExists(getTempFileForThumbnail()); } catch (IOException e) { LOGGER.warn("Could not close thumbnail: {}", getTempFileForThumbnail(), e); } diff --git a/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/http/ResourceDownloadClient.java b/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/http/ResourceDownloadClient.java index 5606e917e..99a3996ab 100644 --- a/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/http/ResourceDownloadClient.java +++ b/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/http/ResourceDownloadClient.java @@ -127,12 +127,10 @@ protected Resource createResult(Pair providedLin resource.markAsNoContent(); } } catch (IOException | RuntimeException e) { - // Close the resource if a problem occurs. resource.close(); throw e; } - // Done: return the resource. return resource; } diff --git a/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/model/AbstractTemporaryFile.java b/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/model/AbstractTemporaryFile.java index e1995533a..69105a13b 100644 --- a/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/model/AbstractTemporaryFile.java +++ b/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/model/AbstractTemporaryFile.java @@ -1,5 +1,6 @@ package eu.europeana.metis.mediaprocessing.model; +import eu.europeana.metis.utils.TempFileUtils; import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; @@ -36,7 +37,7 @@ abstract class AbstractTemporaryFile implements ResourceRelatedFile { */ AbstractTemporaryFile(String resourceUrl, String prefix, String suffix) { this.resourceUrl = resourceUrl; - this.contentFileCreator = () -> Files.createTempFile(prefix, suffix); + this.contentFileCreator = () -> TempFileUtils.createSecureTempFile(prefix, suffix); } @Override @@ -59,8 +60,7 @@ public void markAsWithContent(InputStream newContent) throws IOException { private Long computeContentSizeInternal() throws IOException { // If the content path does not exist, remove the reference. - // Note: should use Files.exists instead when migrating away from Java 8. - if (this.contentPath != null && !this.contentPath.toFile().exists()) { + if (this.contentPath != null && Files.notExists(this.contentPath)) { this.contentPath = null; } @@ -94,9 +94,8 @@ public Long getContentSize() throws IOException { @Override public void markAsNoContent() throws IOException { try { - // Note: should use Files.exists instead when migrating away from Java 8. - if (this.contentPath != null && this.contentPath.toFile().exists()) { - Files.delete(this.contentPath); + if (this.contentPath != null) { + Files.deleteIfExists(this.contentPath); } } finally { this.contentPath = null; diff --git a/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/model/ResourceImpl.java b/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/model/ResourceImpl.java index 826061d87..ab6dee639 100644 --- a/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/model/ResourceImpl.java +++ b/metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/model/ResourceImpl.java @@ -14,7 +14,7 @@ public class ResourceImpl extends AbstractTemporaryFile implements Resource { private static final String DEFAULT_MIME_TYPE = "application/octet-stream"; - private static final Long DEFAULT_FILE_SIZE = Long.valueOf(0L); + private static final Long DEFAULT_FILE_SIZE = 0L; private final String providedMimeType; private final Long providedFileSize; private final Set urlTypes; diff --git a/metis-media-service/src/test/java/eu/europeana/metis/mediaprocessing/extraction/PdfToImageConverterTest.java b/metis-media-service/src/test/java/eu/europeana/metis/mediaprocessing/extraction/PdfToImageConverterTest.java index fa59f788b..f0a4cab75 100644 --- a/metis-media-service/src/test/java/eu/europeana/metis/mediaprocessing/extraction/PdfToImageConverterTest.java +++ b/metis-media-service/src/test/java/eu/europeana/metis/mediaprocessing/extraction/PdfToImageConverterTest.java @@ -23,7 +23,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -public class PdfToImageConverterTest { +class PdfToImageConverterTest { private static final String GHOST_SCRIPT_COMMAND = "gs command"; @@ -121,23 +121,23 @@ void testConvertToPdf() throws MediaExtractionException { doReturn("").when(commandExecutor).execute(eq(command), eq(false), any()); // Execute happy flow - assertEquals(outputFile, pdfToImageConverter.convertToPdf(inputFile)); + assertEquals(outputFile, pdfToImageConverter.convertPdfFirstPageToImage(inputFile)); // Test for empty input - assertThrows(MediaExtractionException.class, () -> pdfToImageConverter.convertToPdf(null)); + assertThrows(MediaExtractionException.class, () -> pdfToImageConverter.convertPdfFirstPageToImage(null)); // Test for problem creating the file - revert and check all is well doThrow(MediaExtractionException.class).when(pdfToImageConverter).createPdfImageFile(); - assertThrows(MediaExtractionException.class, () -> pdfToImageConverter.convertToPdf(inputFile)); + assertThrows(MediaExtractionException.class, () -> pdfToImageConverter.convertPdfFirstPageToImage(inputFile)); doReturn(outputFile).when(pdfToImageConverter).createPdfImageFile(); - assertEquals(outputFile, pdfToImageConverter.convertToPdf(inputFile)); + assertEquals(outputFile, pdfToImageConverter.convertPdfFirstPageToImage(inputFile)); // Test for exceptions in executing the execution doNothing().when(pdfToImageConverter).removePdfImageFileSilently(any()); doThrow(MediaExtractionException.class).when(commandExecutor) - .execute(eq(command), eq(false), any()); - assertThrows(MediaExtractionException.class, () -> pdfToImageConverter.convertToPdf(inputFile)); - verify(pdfToImageConverter, times(1)).removePdfImageFileSilently(eq(outputFile)); + .execute(eq(command), eq(false), any()); + assertThrows(MediaExtractionException.class, () -> pdfToImageConverter.convertPdfFirstPageToImage(inputFile)); + verify(pdfToImageConverter, times(1)).removePdfImageFileSilently(outputFile); verify(pdfToImageConverter, times(1)).removePdfImageFileSilently(any()); } } diff --git a/metis-media-service/src/test/java/eu/europeana/metis/mediaprocessing/extraction/TextProcessorTest.java b/metis-media-service/src/test/java/eu/europeana/metis/mediaprocessing/extraction/TextProcessorTest.java index 5be65d589..be3881c28 100644 --- a/metis-media-service/src/test/java/eu/europeana/metis/mediaprocessing/extraction/TextProcessorTest.java +++ b/metis-media-service/src/test/java/eu/europeana/metis/mediaprocessing/extraction/TextProcessorTest.java @@ -178,7 +178,7 @@ void testExtractForPdf() throws IOException, MediaExtractionException { final Path pdfImagePath = mock(Path.class); final File pdfImageFile = new File("PDF image"); doReturn(pdfImageFile).when(pdfImagePath).toFile(); - doReturn(pdfImagePath).when(pdfToImageConverter).convertToPdf(contentPath); + doReturn(pdfImagePath).when(pdfToImageConverter).convertPdfFirstPageToImage(contentPath); doNothing().when(pdfToImageConverter).removePdfImageFileSilently(pdfImagePath); // Define output and mock thumbnail generator - resource type for which metadata is generated. diff --git a/metis-media-service/src/test/java/eu/europeana/metis/mediaprocessing/extraction/ThumbnailGeneratorTest.java b/metis-media-service/src/test/java/eu/europeana/metis/mediaprocessing/extraction/ThumbnailGeneratorTest.java index 5b1fe14b8..bdcd1d264 100644 --- a/metis-media-service/src/test/java/eu/europeana/metis/mediaprocessing/extraction/ThumbnailGeneratorTest.java +++ b/metis-media-service/src/test/java/eu/europeana/metis/mediaprocessing/extraction/ThumbnailGeneratorTest.java @@ -10,7 +10,6 @@ import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.notNull; import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; @@ -432,7 +431,9 @@ private void testPrepareThumbnailFiles(String detectedMimeType, String expectedM assertEquals(contentSize, thumbnail1.getContentSize()); assertEquals(url, thumbnail1.getResourceUrl()); assertEquals(expectedMimeType, thumbnail1.getMimeType()); - } + //Delete the temp files now + thumbnailWithSize1.deleteTempFileSilently(); + thumbnailWithSize2.deleteTempFileSilently(); } } diff --git a/metis-normalization/src/test/java/eu/europeana/normalization/model/ConfidenceLevelTest.java b/metis-normalization/src/test/java/eu/europeana/normalization/model/ConfidenceLevelTest.java index e1f3c8759..bce6652a9 100644 --- a/metis-normalization/src/test/java/eu/europeana/normalization/model/ConfidenceLevelTest.java +++ b/metis-normalization/src/test/java/eu/europeana/normalization/model/ConfidenceLevelTest.java @@ -5,7 +5,7 @@ import org.junit.jupiter.api.Test; -public class ConfidenceLevelTest { +class ConfidenceLevelTest { @Test void testGetForConfidence() { diff --git a/metis-normalization/src/test/java/eu/europeana/normalization/normalizers/CleanMarkupTagsNormalizerTest.java b/metis-normalization/src/test/java/eu/europeana/normalization/normalizers/CleanMarkupTagsNormalizerTest.java index 8726963c7..e7bdb51c1 100644 --- a/metis-normalization/src/test/java/eu/europeana/normalization/normalizers/CleanMarkupTagsNormalizerTest.java +++ b/metis-normalization/src/test/java/eu/europeana/normalization/normalizers/CleanMarkupTagsNormalizerTest.java @@ -9,7 +9,7 @@ import java.util.stream.Collectors; import org.junit.jupiter.api.Test; -public class CleanMarkupTagsNormalizerTest { +class CleanMarkupTagsNormalizerTest { private String html = ""; @Test - public void testHtmlMarkup() { + void testHtmlMarkup() { CleanMarkupTagsNormalizer cleaner = new CleanMarkupTagsNormalizer(CleanMarkupTagsMode.HTML_ONLY); List cleaned = cleaner.normalizeValue(html).stream() - .map(NormalizedValueWithConfidence::getNormalizedValue).collect(Collectors.toList()); + .map(NormalizedValueWithConfidence::getNormalizedValue).collect(Collectors.toList()); System.out.println(html); System.out.println(cleaned); assertEquals(1, cleaned.size()); @@ -34,10 +34,10 @@ public void testHtmlMarkup() { } @Test - public void testAllMarkup() { + void testAllMarkup() { CleanMarkupTagsNormalizer cleaner = new CleanMarkupTagsNormalizer(CleanMarkupTagsMode.ALL_MARKUP); List cleaned = cleaner.normalizeValue(html).stream() - .map(NormalizedValueWithConfidence::getNormalizedValue).collect(Collectors.toList()); + .map(NormalizedValueWithConfidence::getNormalizedValue).collect(Collectors.toList()); System.out.println(cleaned); assertEquals(1, cleaned.size()); assertTrue(cleaned.get(0).contains("ire this guy")); diff --git a/metis-normalization/src/test/java/eu/europeana/normalization/normalizers/RemoveDuplicateStatementNormalizerTest.java b/metis-normalization/src/test/java/eu/europeana/normalization/normalizers/RemoveDuplicateStatementNormalizerTest.java index f9c9b5bf5..451552276 100644 --- a/metis-normalization/src/test/java/eu/europeana/normalization/normalizers/RemoveDuplicateStatementNormalizerTest.java +++ b/metis-normalization/src/test/java/eu/europeana/normalization/normalizers/RemoveDuplicateStatementNormalizerTest.java @@ -1,7 +1,8 @@ package eu.europeana.normalization.normalizers; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.doAnswer; @@ -17,10 +18,10 @@ import org.w3c.dom.NamedNodeMap; -public class RemoveDuplicateStatementNormalizerTest { +class RemoveDuplicateStatementNormalizerTest { @Test - public void testConvertToMap() { + void testConvertToMap() { // Create attributes with proper values, two of which are very similar final String name1 = "a1"; @@ -65,36 +66,34 @@ private NamedNodeMap createNamedNodeMap(Attr... attributes) { } @Test - public void testPairEquals() { + void testPairEquals() { // Test with incompatible objects final TextAttributesPair pair = new TextAttributesPair(null, null); - assertFalse(pair.equals(null)); - assertFalse(pair.equals(new Object())); + assertNotNull(pair); + assertNotEquals(pair, new Object()); // Test with itself assertEquals(pair, pair); // Test with actual and empty names - assertTrue(new TextAttributesPair("A", null).equals(new TextAttributesPair("A", null))); - assertFalse(new TextAttributesPair("A", null).equals(new TextAttributesPair("B", null))); - assertTrue(new TextAttributesPair("", null).equals(new TextAttributesPair(null, null))); + assertEquals(new TextAttributesPair("A", null), new TextAttributesPair("A", null)); + assertNotEquals(new TextAttributesPair("A", null), new TextAttributesPair("B", null)); + assertEquals(new TextAttributesPair("", null), new TextAttributesPair(null, null)); // Test with attribute values - assertTrue(new TextAttributesPair("", Collections.singletonMap("A", "X")) - .equals(new TextAttributesPair("", Collections.singletonMap("A", "X")))); - assertFalse(new TextAttributesPair("", Collections.singletonMap("A", "X")) - .equals(new TextAttributesPair("", Collections.singletonMap("A", "x")))); - assertFalse(new TextAttributesPair("", Collections.singletonMap("A", "X")) - .equals(new TextAttributesPair("", Collections.singletonMap("a", "X")))); - assertFalse(new TextAttributesPair("", Collections.singletonMap("A", "X")) - .equals(new TextAttributesPair("", null))); + assertEquals(new TextAttributesPair("", Collections.singletonMap("A", "X")), + new TextAttributesPair("", Collections.singletonMap("A", "X"))); + assertNotEquals(new TextAttributesPair("", Collections.singletonMap("A", "X")), + new TextAttributesPair("", Collections.singletonMap("A", "x"))); + assertNotEquals(new TextAttributesPair("", Collections.singletonMap("A", "X")), + new TextAttributesPair("", Collections.singletonMap("a", "X"))); + assertNotEquals(new TextAttributesPair("", Collections.singletonMap("A", "X")), new TextAttributesPair("", null)); Map multipleAttributes = new HashMap<>(); multipleAttributes.put("A", "X"); multipleAttributes.put("B", "Y"); - assertFalse(new TextAttributesPair("", Collections.singletonMap("A", "X")) - .equals(new TextAttributesPair("", multipleAttributes))); - assertTrue(new TextAttributesPair("", multipleAttributes) - .equals(new TextAttributesPair("", multipleAttributes))); + assertNotEquals(new TextAttributesPair("", Collections.singletonMap("A", "X")), + new TextAttributesPair("", multipleAttributes)); + assertEquals(new TextAttributesPair("", multipleAttributes), new TextAttributesPair("", multipleAttributes)); } } diff --git a/metis-validation/metis-validation-client/src/test/java/TestValidationClient.java b/metis-validation/metis-validation-client/src/test/java/TestValidationClient.java index 0a9ceb84b..efc79a07b 100644 --- a/metis-validation/metis-validation-client/src/test/java/TestValidationClient.java +++ b/metis-validation/metis-validation-client/src/test/java/TestValidationClient.java @@ -13,8 +13,9 @@ import eu.europeana.validation.client.ValidationClient; import eu.europeana.validation.model.ValidationResult; import eu.europeana.validation.model.ValidationResultList; -import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -110,11 +111,12 @@ void shouldCreateProperValidationResultForCorrectZipFile() throws IOException { "}"))); ValidationClient client = new ValidationClient("http://127.0.0.1:" + portForWireMock); - File tempFile = File.createTempFile("temp_file", ".tmp"); - tempFile.deleteOnExit(); - ValidationResultList result = client.validateRecordsInFile("EDM-INTERNAL", tempFile); + Path tempFile = Files.createTempFile("temp_file", ".tmp"); + ValidationResultList result = client.validateRecordsInFile("EDM-INTERNAL", tempFile.toFile()); + Files.delete(tempFile); assertTrue(result.isSuccess()); assertEquals(0, result.getResultList().size()); + } @Test @@ -136,9 +138,9 @@ void shouldCreateProperValidationResultForWrongZipFile() throws IOException { "}"))); ValidationClient client = new ValidationClient("http://127.0.0.1:" + portForWireMock); - File tempFile = File.createTempFile("temp_file", ".tmp"); - tempFile.deleteOnExit(); - ValidationResultList result = client.validateRecordsInFile("EDM-INTERNAL", tempFile); + Path tempFile = Files.createTempFile("temp_file", ".tmp"); + ValidationResultList result = client.validateRecordsInFile("EDM-INTERNAL", tempFile.toFile()); + Files.delete(tempFile); assertTrue(result.isSuccess()); assertEquals(1, result.getResultList().size()); assertFalse(result.getResultList().get(0).isSuccess()); diff --git a/metis-validation/metis-validation-service/src/main/java/eu/europeana/validation/service/SchemaProvider.java b/metis-validation/metis-validation-service/src/main/java/eu/europeana/validation/service/SchemaProvider.java index 50f89084a..baa723e59 100644 --- a/metis-validation/metis-validation-service/src/main/java/eu/europeana/validation/service/SchemaProvider.java +++ b/metis-validation/metis-validation-service/src/main/java/eu/europeana/validation/service/SchemaProvider.java @@ -1,5 +1,7 @@ package eu.europeana.validation.service; +import static java.lang.String.format; + import eu.europeana.validation.model.Schema; import java.io.File; import java.io.IOException; @@ -13,7 +15,7 @@ import java.net.http.HttpResponse.BodyHandlers; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Enumeration; +import java.util.Iterator; import java.util.UUID; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -23,13 +25,12 @@ import org.slf4j.LoggerFactory; /** - * Provides schemas based on url of predefined name (EDM-INTERNAL on EDM-EXTERNAL). Per instance of - * this class there is a UUID generated that is used to create the directory path of where the - * schemas will be stored. This is done so, to avoid file system collisions of processes that run - * the exact same code and are independent from each other. + * Provides schemas based on url of predefined name (EDM-INTERNAL on EDM-EXTERNAL). Per instance of this class there is a UUID + * generated that is used to create the directory path of where the schemas will be stored. This is done so, to avoid file system + * collisions of processes that run the exact same code and are independent from each other. *

    - * The {@link #TMP_DIR} field is set through a system variable named "java.io.tmpdir". This value - * should be sanitized and controlled otherwise this class can become unsecure. + * The {@link #TMP_DIR} field is set through a system variable named "java.io.tmpdir". This value should be sanitized and + * controlled otherwise this class can become unsecure. *

    *

    * Created by pwozniak on 12/20/17 @@ -48,9 +49,8 @@ public class SchemaProvider { private final PredefinedSchemas predefinedSchemasLocations; /** - * Creates {@link SchemaProvider} for given {@link PredefinedSchemas} object. The {@link - * #schemasRootDirectory} is also calculated which includes the UUID of this {@link - * SchemaProvider}. + * Creates {@link SchemaProvider} for given {@link PredefinedSchemas} object. The {@link #schemasRootDirectory} is also + * calculated which includes the UUID of this {@link SchemaProvider}. * * @param predefinedSchemasLocations the wrapper class with all the schema locations */ @@ -70,8 +70,8 @@ public SchemaProvider(PredefinedSchemas predefinedSchemasLocations) { * Retrieves schema object from given (remote) location. * * @param zipUrl place where (remote) zip file is located. Accepts url to file. - * @param rootFileLocation indicates where root xsd file is located inside zip. The caller is - * responsible to provide a valid path to that file + * @param rootFileLocation indicates where root xsd file is located inside zip. The caller is responsible to provide a valid + * path to that file * @param schematronLocation place where schematron file is located * @return schema object * @throws SchemaProviderException any exception that can occur during retrieving schema files @@ -143,10 +143,7 @@ public boolean isPredefined(String name) { private File downloadZipIfNeeded(String zipLocation, String destinationDir) throws SchemaProviderException { - HttpRequest httpRequest = HttpRequest.newBuilder() - .GET() - .uri(URI.create(zipLocation)) - .build(); + HttpRequest httpRequest = HttpRequest.newBuilder().GET().uri(URI.create(zipLocation)).build(); File schemasLocation = new File(schemasRootDirectory, destinationDir); @@ -167,8 +164,8 @@ private File downloadZipIfNeeded(String zipLocation, String destinationDir) httpResponse = httpClient.send(httpRequest, BodyHandlers.ofFile(Paths.get(destinationFile.toURI()))); destinationFile = httpResponse.body().toFile(); } catch (IOException e) { - LOGGER.info("There was some trouble sending a request to {}", schemasLocation); - } catch (InterruptedException e){ + LOGGER.info(format("There was some trouble sending a request to %s", schemasLocation), e); + } catch (InterruptedException e) { Thread.currentThread().interrupt(); LOGGER.info("The thread was interrupted"); } @@ -187,11 +184,11 @@ private void unzipArchiveIfNeeded(File downloadedFile, String rootFileLocation) } private void unzipArchive(File downloadedFile) throws SchemaProviderException { - try (ZipFile zip = new ZipFile(downloadedFile)) { - Enumeration entries = zip.entries(); - while (entries.hasMoreElements()) { - ZipEntry entry = entries.nextElement(); - handleZipEntry(downloadedFile, zip, entry); + try (ZipFile zipFile = new ZipFile(downloadedFile)) { + final Iterator entries = zipFile.stream().iterator(); + while (entries.hasNext()) { + final ZipEntry zipEntry = entries.next(); + handleZipEntry(downloadedFile, zipFile, zipEntry); } } catch (IOException e) { throw new SchemaProviderException("Exception while unzipping file", e); From c1be489512831c22ecb8dfad3e5c04d154927c89 Mon Sep 17 00:00:00 2001 From: Adolfo Peixinho <91942157+adolfopeixinho@users.noreply.github.com> Date: Mon, 20 Jun 2022 15:23:34 +0200 Subject: [PATCH 65/73] MET-4566 Unit tests for FullBeanSolrProperties (#555) * MET-4566 Unit tests for FullBeanSolrPropertiesTest * MET-4566 add test with null proxy. Fixed possible NPE in FullBeanSolrProperties * MET-4566 Code cleanup. Use of correct EdmLabels. * MET-4566 FullBeanSolrPropertiesTest fixes * MET-4566 FullBeanSolrPropertiesTest review inverted logic --- .../solr/property/FullBeanSolrProperties.java | 6 +- .../property/FullBeanSolrPropertiesTest.java | 164 ++++++++++++++++++ 2 files changed, 168 insertions(+), 2 deletions(-) create mode 100644 metis-indexing/src/test/java/eu/europeana/indexing/solr/property/FullBeanSolrPropertiesTest.java diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/solr/property/FullBeanSolrProperties.java b/metis-indexing/src/main/java/eu/europeana/indexing/solr/property/FullBeanSolrProperties.java index 8653e5177..497c4c2df 100644 --- a/metis-indexing/src/main/java/eu/europeana/indexing/solr/property/FullBeanSolrProperties.java +++ b/metis-indexing/src/main/java/eu/europeana/indexing/solr/property/FullBeanSolrProperties.java @@ -65,10 +65,12 @@ private void setGeospatialFields(SolrInputDocument document, FullBeanImpl fullBe (place1, place2) -> place1)); final Set currentLocationStrings = new HashSet<>(); final Set coverageLocationStrings = new HashSet<>(); - for (ProxyImpl proxy : proxies) { + + proxies.stream().filter(Objects::nonNull).forEach(proxy -> { currentLocationStrings.addAll(getCurrentLocationStrings(proxy)); coverageLocationStrings.addAll(getCoverageLocationStrings(proxy)); - } + }); + final Set currentLocationPoints = new HashSet<>( getReferencedPlacesLocationPoints(placesAboutMap, currentLocationStrings)); currentLocationPoints.addAll(getWGS84LocationPoints(currentLocationStrings)); diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/FullBeanSolrPropertiesTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/FullBeanSolrPropertiesTest.java new file mode 100644 index 000000000..8ec4f9f14 --- /dev/null +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/FullBeanSolrPropertiesTest.java @@ -0,0 +1,164 @@ +package eu.europeana.indexing.solr.property; + +import static java.time.temporal.ChronoUnit.DAYS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import eu.europeana.corelib.definitions.edm.entity.Place; +import eu.europeana.corelib.solr.bean.impl.FullBeanImpl; +import eu.europeana.corelib.solr.entity.AgentImpl; +import eu.europeana.corelib.solr.entity.PlaceImpl; +import eu.europeana.corelib.solr.entity.ProxyImpl; +import eu.europeana.indexing.solr.EdmLabel; +import eu.europeana.metis.schema.jibx.EdmType; +import java.sql.Date; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.apache.solr.common.SolrInputDocument; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class FullBeanSolrPropertiesTest { + + private FullBeanSolrProperties fullBeanSolrProperties; + private SolrInputDocument solrInputDocument; + + + @BeforeEach + void setup() { + solrInputDocument = new SolrInputDocument(); + fullBeanSolrProperties = new FullBeanSolrProperties(); + } + + + @Test + void fullBeanSolrSetPropertiesTest() { + + FullBeanImpl fullBean = new FullBeanImpl(); + // agent setup + AgentImpl agent = new AgentImpl(); + agent.setAbout("About Agent"); + fullBean.setAgents(List.of(agent)); + + // place setup + Place place = new PlaceImpl(); + place.setAbout("Nowhere"); + place.setLatitude(10.0f); + place.setLongitude(33.0f); + place.setAltitude(7.0f); + fullBean.setPlaces(List.of(place)); + + // proxy setup + ProxyImpl proxy = new ProxyImpl(); + proxy.setAbout("About Proxy"); + proxy.setEuropeanaProxy(true); + proxy.setEdmType(EdmType.TEXT.name()); + proxy.setEdmCurrentLocation(Map.of(EdmLabel.PROXY_EDM_CURRENT_LOCATION.name(), List.of(place.getAbout()))); + proxy.setDctermsSpatial(Map.of(EdmLabel.PROXY_DCTERMS_SPATIAL.name(), List.of(place.getAbout()))); + proxy.setDcCoverage(Map.of(EdmLabel.PROXY_DC_COVERAGE.name(), List.of(place.getAbout()))); + fullBean.setProxies(List.of(proxy)); + + fullBean.setEuropeanaCompleteness(0); + fullBean.setEuropeanaCollectionName(new String[]{"Europeana Collection Name"}); + fullBean.setTimestampCreated(Date.from(Instant.now().truncatedTo(DAYS))); + fullBean.setTimestampUpdated(Date.from(Instant.now().truncatedTo(DAYS).plus(1, DAYS))); + + // method to test + fullBeanSolrProperties.setProperties(solrInputDocument, fullBean); + + // assertions + verifyCollection(solrInputDocument, EdmLabel.PROVIDER_EDM_TYPE, List.of(EdmType.TEXT.toString())); + verifyCollection(solrInputDocument, EdmLabel.CURRENT_LOCATION_WGS, List.of("10,33")); + verifyCollection(solrInputDocument, EdmLabel.COVERAGE_LOCATION_WGS, List.of("10,33")); + verifyCollection(solrInputDocument, EdmLabel.LOCATION_WGS, List.of("10,33")); + assertEquals(0, solrInputDocument.getFieldValue(EdmLabel.EUROPEANA_COMPLETENESS.toString())); + assertEquals("Europeana Collection Name", solrInputDocument.getFieldValue(EdmLabel.EUROPEANA_COLLECTIONNAME.toString())); + assertEquals(solrInputDocument.getFieldValue(EdmLabel.TIMESTAMP_CREATED.toString()), + Date.from(Instant.now().truncatedTo(DAYS))); + assertEquals(solrInputDocument.getFieldValue(EdmLabel.TIMESTAMP_UPDATED.toString()), + Date.from(Instant.now().truncatedTo(DAYS).plus(1, DAYS))); + assertEquals(8, solrInputDocument.size()); + + } + + @Test + void fullBeanSolrSetPropertiesMultipleProxiesTest() { + + FullBeanImpl fullBean = new FullBeanImpl(); + + // agent setup + AgentImpl agent = new AgentImpl(); + agent.setAbout("About Agent"); + fullBean.setAgents(List.of(agent)); + + // place setup + Place place1 = new PlaceImpl(); + place1.setAbout("Somewhere1"); + place1.setLatitude(10.0f); + place1.setLongitude(33.0f); + place1.setAltitude(7.0f); + + Place place2 = new PlaceImpl(); + place2.setAbout("Somewhere2"); + place2.setLatitude(40.0f); + place2.setLongitude(53.0f); + place2.setAltitude(36.0f); + + fullBean.setPlaces(List.of(place1, place2)); + + // proxy setup + ProxyImpl proxy1 = new ProxyImpl(); + proxy1.setAbout("About Proxy1"); + proxy1.setEuropeanaProxy(true); + proxy1.setEdmType(EdmType.IMAGE.name()); + proxy1.setEdmCurrentLocation(Map.of(EdmLabel.PROXY_EDM_CURRENT_LOCATION.name(), List.of(place1.getAbout()))); + proxy1.setDctermsSpatial(Map.of(EdmLabel.PROXY_DCTERMS_SPATIAL.name(), List.of(place1.getAbout()))); + proxy1.setDcCoverage(Map.of(EdmLabel.PROXY_DC_COVERAGE.name(), List.of(place1.getAbout()))); + proxy1.setEdmHasType(Map.of(EdmType.IMAGE.name(), List.of(EdmType.IMAGE.name()))); + + ProxyImpl proxy2 = new ProxyImpl(); + proxy2.setAbout("About Proxy2"); + proxy2.setEuropeanaProxy(true); + proxy2.setEdmType(EdmType.SOUND.name()); + proxy2.setEdmCurrentLocation(Map.of(EdmLabel.PROXY_EDM_CURRENT_LOCATION.name(), List.of(place2.getAbout()))); + proxy2.setDctermsSpatial(Map.of(EdmLabel.PROXY_DCTERMS_SPATIAL.name(), List.of(place2.getAbout()))); + proxy2.setDcCoverage(Map.of(EdmLabel.PROXY_DC_COVERAGE.name(), List.of(place2.getAbout()))); + proxy2.setEdmHasType(Map.of(EdmType.SOUND.name(), List.of(EdmType.SOUND.name()))); + + List proxies = new ArrayList<>(); + proxies.add(proxy1); + proxies.add(proxy2); + proxies.add(null); + fullBean.setProxies(proxies); + + fullBean.setEuropeanaCompleteness(0); + fullBean.setEuropeanaCollectionName(new String[]{"Europeana Collection Name"}); + fullBean.setTimestampCreated(Date.from(Instant.now().truncatedTo(DAYS))); + fullBean.setTimestampUpdated(Date.from(Instant.now().truncatedTo(DAYS).plus(1, DAYS))); + + // method to test + fullBeanSolrProperties.setProperties(solrInputDocument, fullBean); + + // assertions + verifyCollection(solrInputDocument, EdmLabel.PROVIDER_EDM_TYPE, List.of(EdmType.IMAGE.name(), EdmType.SOUND.name())); + verifyCollection(solrInputDocument, EdmLabel.CURRENT_LOCATION_WGS, List.of("10,33", "40,53")); + verifyCollection(solrInputDocument, EdmLabel.COVERAGE_LOCATION_WGS, List.of("10,33", "40,53")); + verifyCollection(solrInputDocument, EdmLabel.LOCATION_WGS, List.of("10,33", "40,53")); + assertEquals(0, solrInputDocument.getFieldValue(EdmLabel.EUROPEANA_COMPLETENESS.toString())); + assertEquals("Europeana Collection Name", solrInputDocument.getFieldValue(EdmLabel.EUROPEANA_COLLECTIONNAME.toString())); + assertEquals(solrInputDocument.getFieldValue(EdmLabel.TIMESTAMP_CREATED.toString()), + Date.from(Instant.now().truncatedTo(DAYS))); + assertEquals(solrInputDocument.getFieldValue(EdmLabel.TIMESTAMP_UPDATED.toString()), + Date.from(Instant.now().truncatedTo(DAYS).plus(1, DAYS))); + assertEquals(8, solrInputDocument.size()); + } + void verifyCollection(SolrInputDocument solrInputDocument, EdmLabel edmLabel, Collection collection) { + final Collection fieldValues = solrInputDocument.getFieldValues(edmLabel.toString()); + assertTrue(fieldValues.containsAll(collection)); + assertEquals(collection.size(), fieldValues.size()); + } +} From 9485b83d7fcafb3c5f62ead478f19596b266c339 Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Mon, 20 Jun 2022 15:39:56 +0200 Subject: [PATCH 66/73] MET-4437 Update metis-schema library (#559) --- .../test/resources/__files/test_schema.zip | Bin 45798 -> 45808 bytes pom.xml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/metis-validation/metis-validation-rest/src/test/resources/__files/test_schema.zip b/metis-validation/metis-validation-rest/src/test/resources/__files/test_schema.zip index 2d7761fd86be22e0caf3a8dbc314084b7fe4b3e8..9afe21a544854b3ea9df6a9b3d25a310ee988adb 100644 GIT binary patch delta 11340 zcmZ{K1yt1C^ZqW~-QC^Y-Q5k6(%rcN0@AU7vEs^S>xIuN@~r zgMj*Hxj&uw*YbY4JOq?ON{1HMPlExjR66yCb(uyQ4E>{Z)U>~E0}ue9f_w~11dvh{ zS7Y&Ux75~x2SEN8e2(Y?knS-G7aL=o4lDowO#uMFKA5X&NdIU3UvO(nXV@Q9tRcyN zQAvP&4NLoo`Q778X!{|J@&{a)#~|YW#rO~LlNk{BBgTUHuXJiGM1QWz=~MrT?5SV( zbJ+uh?ALM@%Ki_)jT2qN`UkMhha?{XB_;m`OaYGljFIE~TUooE-$Eh$s(B*HFEZ7f zg1~=jSbvWIFJQY3qkxv3wnw&$ z+RgG@#u3FWmn@bV_2U*JwjG5o*{b={`>MdY83ttG&Ask4wjn|nF`g`y#Bc66%cfXPWdhIfhyN?0x^CdbKFA^UU4-CQ!T0@|5Kn-v| z=LTYxWrF1h3&p>fM6#$*+$|gCs@XQ-R#qH=AnXJ|BbQANA6A}%sXbg@sZnKIahkpa zY{w)H>;soEag%*jNN(>+Eg{BPr@;lmXfv_geHR{pw{Pb_KUx|(FZ=)oGKYY4(YLFI zOWC(7d<(Q26kw73QH&JF`PU;eXjL6O2n6G%+=e*4inoNo@n`*?_`Z9ecDnk??jJ-T zkJdZk?zwlcWKQuq*on-T#}Pw=zr7qSH%6TKdNTEHk)^fw6kxECkfhN?U1D|TD_k%4M?`+>2V zmS_jkw+<@02lxYy*5Ohv7U@G!q442I7pUo=q_V1NumlSuXfz(}DjlR5I%-PT2fm5#cwIF#cQp>Rjb*4_EVWS&u1IS z#^}_k>h4fa5j8IsWb0XnxtgeJXG3oMp!dgx<}9^~R~Vl$sz=oLO(nBZyf2ZI)JlB3 z72`h}J2v}Z%@*!0tW1aQD9o&Z8CZ!xmE8*UF;!ipO)dSfil)yLLTP|@Mhr3$qu!CX zVHrz{6787_Px`hlUFt48A~Mj6snD^~kE!1+77wtufZUAwt9NM0CKJmAzx%(*2t7g0lauRocO}08HKCQO~e$Pq^ zUEXQ*#UFgaN{TOFh@gz#!fv$=_09O4CKQX#L2ue~Zdx%8OwYzjbgmYL`o@1Lyc;A| z!Nme+q0OY6bBz6ca}mA$3hcr53d66-Vz$;(Acs**0$YeZ#iL3bOwCSBz(eZHfw;C( zU*}-?t>+dfiwMm!lA8v9-24q}$<70G24}L90F}Wl?5Z$N?IL*FOTefcnn>IhI2_Kz zvzsN+&T3#M4ql?&$pL=<>yE~8{{Z9`AuY?=h5FPZ&ZCMPoepYn7Y8X#{q0ECQ@jwa zQET_zwUTDtudC591U=Q_7A_k&7xqKV{Zgn(4=&|rLrOU% zG|TZ?C>#0AyAR&tH=v@|&XLl`qPN%9FZGAOh<}sjrAfMjaB1V4KtWqxd(mzw_o@MG zC2$0c1#1g30v*A@g3s`s&5uSniK5d-jOxENq(l!CrTe~oX8mfy8%!Wn0lK<(KAQI< z>@Pp|2ZgP^w0st`G2$ssHMwbS4PjYpo3!Ymjs-bi=om{q%sAx5u`PK&aMyg7bebip zku2wlG2`?^APv};CzUAMmNo1xLP4TY{w6GG(21A{Tj1heW}pwxAKM#0px>vRNgQ|7 zBDRXStth}^wwa5ESTw6(3Ir)JRBCT)GY+a)r-%IZ79C^oH z89qAS8iQcW&-65pejNs$X4ptuQ7|=#k4DIwM>WT8lYKtnCL%Jyiy3P8z`SqSG0C2?&%wM1oL)p!o3X%a0P&El#Tsy|56hJjPHvyT+FEsqn zy7mzVc*UR+QxBO7c5l2Yks}+HKxOYmZtc!vNtLoUmmEi4jvL7j$Ro4V-%ltNL$8+3 z#os-vcpF6k-7PjmLX;*|&q2^tq3{WiUzEl&R#Zi^C$8^mOAQ;0(#~UkXdcp`0u+xt z?<$*o1;LHJtV55Ci~$PXR!U|@X@EiBm$r@B&z+c28m&pZ z?@7?SukrT0+c>^#nCV~c$TGV9tl81Jc&J=?vw0hF*m_XWmx{C8<-)^ZCOkvN+I71k znQ233@t!iTaVLY{Z8A;EVt%Sy=4xokv>Pd+_+CFEE8F!=SU4z(N-{?cM7yz1grNjM z9fGc2C>nqP)0W2BVLImbRUV^6*>lqgJt&bbtq>nMR?BPXjYeYt$F1wgb%Gl~k!x11 z2;67ZbQiDm2$J-eS!mo&^kxw6NR-6lm)7dSkU*SJqq5tiEL%5St6f$1H<<(apRXC( z-`9$9E6P~DdQAfA4>1loNi5RG161`U*XzYS2oO{NM>n~{VYWTrb5}yG{Ut9DlhX{PGXF^y`@sn9aLuFXza%zy|EXe>}qx9oxXUxVu_rl z%t+YG)-g5+lG-Bg_h)>t%G@q{Nu)675<1wG?G9#m1I;rs1fF%?^(lnx@U7rlW>5*7 z9gA^L_+LcR*rBhXVZtW|ZXm&1NiL^sm6Qjdr3u%@kUuGVvfM}y8Mme8Ihht1j-*^^ z+IoX)i)5$`iW`WPXy%0p(Vx){_@EyI5E+LM#(^t5fkk+4##~fOw~SRZeqPB>{_Nv4 zj@lEnF4L@*LIb(TYTFRF(${LK3xZ&$Ldm2wSXhHRXM)gZX2R02fl(Uj`xJSUb0bL; zW*KWv5`x{DFFE=Qts(IOdpgub>rMPq7^%Wg!&f~JVLe9jvQ z`&psds*^tS8RaO9*=ZngJM6I~hPzVLR{_siU67vxa$-=MraZS7QTr@;=Q}mB-qldJ zsAc8)hLdyyZEd$`HfE^2X2~TqhddfgF;>=GUmukyN%3Ct+fxEYG|&78DDhnHHy_ST^15Ew`{?eZUtjJTisLdxrLzQW*yln%{yuR)<|01b+ibNwgpC8N)B{OA--0a!j@I z!gLg``|?05DcW34zbusHTY>UETT(9}S~RG+R8@vn-{~mSSfKmOR4YdjF;7ktHH978 zn`&jcSHtEQhsfnBf!NH6LID3>JPIZclnC*0yjlmjI**-=+(6xT`}>^yR7){P^{7li zJ|k@MTtsbZv3W5VjaHD+vJnG_6l9Fa=5GMg6b4N6Ett7m_Aozg;gN;Xz;R)j)LL3%TaBR ze`kpn3H_>x&t4^j!=eEGc@t&Wu6(-Z!rpphL!>Ejl%{(3oNUJMGmNP+Sb@m?IO`B3 z{N*?u+}cQ9P2_j3fj(mK25bZ<%N@StzDWH z7o)Iy?9xoPF8)hkMZpW-Vje5RTVxO1srJ{y-J97uCrJq}dfpW#Q-U3nSW?pxav*KB z5!5hxM1$!C2h37SY0vHzOVB)gieTX5(nSFnYPyjSm;^Z;eGyViP{%?wi#N$EOXVvb z=3o7J*QWYKmgm)C1SuFkXsXDKtv8F45X4&Qvk*4X#4#kuT7gmsOGHpbkL^pUM$tw?YkH zAjvwenxU#x2o;^9jnA{|s(9+yGO)SJ+dfEa@#_5-dVwh(lW%6JuR(M00`NnlYys$+ zkP2~H_#u4_8e~3rT6%D%zJjf-{)kUWtEl6Zv=yYJ#WKZBGCmVmG(o7mr_{yNd8eTt zu}&*Bx+Cc0dm}S3Q3G7;dPF|9#Z?b5nR!MRW9V>p`C6{2;pn>6vwmF@OHuEdcP~5q{s;h z{h_g3DsRV@dudji=DzC_(fboYJ{#$cu*4PuUu-%lvEzzDQN51;^8JhL(cw(= z#7oJ!`<>m@Gw~oH&`nj_^rh7@g4h>RS2^ z^tOXqChJaR@oPN%w|IoS;d7bdhD`F$T!krNLgCt;bT6ho89}SNH6L|CxGGqv@Oe93 zlM*L#(;JlrRx1}ndpdSv*uPMWTgDc$s8#Y)<+px?+A4oFoD|K@!Te zr5*!B9LOKy)x7<0TGE|;2N3kAqQ&5|E;K$wk^d|C3;q^(wGKKtj3{qUw3maXw6=m-nx8N0 zzj|70>nu{QF59-S1NGfXK!5+X4TsUlU%)8a9rf_=(C6)NtMMzLfLcu*hd?V45=wF< zRSNQ%v|zQ6yAz4H)%D(Z^4Db)HAb{I*%E`~!C1-MLN>76&R-!}-<;0bx`XzN^1?HU zu%j$Q{51{DhqBYXQ9KwupuA`_*wPb@D{iW!fq}2F49s?5!q4Qx(X0-sa2xtL+zZJk zO^2Yv884*hbZGRTzE0XS0xOSJkX$ofHz;vSnRv>U@!i1>3A?Qb$G@64#Pkcel)c=r zcTgkk*939BQrY4JT_1a1kb=DMMy-eYUv8`soNYwenfB+~*PF03!+4xHn(TOd4h0}=sz7&AjqR4JqIXwU6YPRfj z^IkslQ(U~459>Fw z9>rNK#}}B!vN`W(lh^Lp!O3tsTIRhVEH^ZdRqDZC&bW+AL;fVIr9yB`CRY09a(9#o zSCj!q^weyfs0}Q)^wbL|yBFLg!pqefADG(?fLV+Y1 z^IDW~`JML(6ffpoqm_%Jkzlte@uYLJk<9QW(5XJ9?ZS~o-A1KAfv||zHP(TZ5$x4w zaGpe8#;A}B=w!tPBF5<(PN4LCAm%U&ha=6N)aXE*>P+S3Y#p1K%)B;Z`K;P~MX~?` z)76uMR!&k|xHc!VG0-SCJ9_p)B4PlzZQ-;g_-V>dPJKh3A59)vQg5ZZ+Cph zoVl2YJ>UqY^p;p}F}W`HYG!2L;_AZDl!!;;-aUEOalL){9V*-8)jiLphbv2w*GEC& zb`iqd(mhV~tq-`g?@)H{FJzCto~-Py8SQ)<^XouPp4_YD+&BgK9Lw6EYzyEKqGYIL zlOY>!$5n|<>UYMzkQQSP$(m>1Ozz3AFnGa#voB*G_)pReE(#qmPA>j9rc~09cEyML)&JKLQ9$wT7 zk^1a;sNSx}CZ;Bc-;TxCBYtW?=-B@h4u^M&TwHP%%G83~6k%VlT|l!8T@-R}(XJlD zGi8H`P@`0Kzc~m3O367hr3o?$mKS9CifdQj2^pY4iKK-M<994-6NssHbbW7|Ga92_ zI@9s=3cp7!jP%*GsLY{gCQ?kq6xU9VITxcWZ%8=*8pk}S^UA*40NJPiLW?pK0jH1+ zIEyM{?!Bby?5xL@u0W^5eZbSabLEO3dI?qX1hME1(NvN^QKx9g_UDMOz-fkyX^69C zIO%PgWsR5#h&aV7dK)DUnpobX-uou4&6gRaq`1Qq+TCYgzFAt#0X{oW!7UhiTxXp4C|^uPl;P|6(eEpXeJW5CU2g~4 zvff)Pn!&0gG8=u!&=b~({>aNt|A7ZKDGIXISiv;B1~dcXQIfWvlSU`hCT6jJiMZ15 zC*QTORJanWYf%~8Y-x&+c;$c8=Itr8P5(+Id9pP6Ly%2g$TRhPwnzz0b`L9?N%#)J z0m6axh@D)QGlt<~2>x}EM$->!r95d$3ZAIkpyY6IGOs{X?G!sKBAZujH6X{Ro>CFMWg677^=~AW;voFHPDpm%ukBQO)IVs&P^i zX7-v2-nZtZ%-tJp5qvnM@$QwmgUB~b_bIejQU+sV>!tFyM zZrDNeP=)Ls_fP`6n8#P%TAgS!POA~zMb}6pZ2jBDzl~k zSS`w7$E1$0f`w?HqpM<~80QN$y>%YI_4W=#^j!*RMuYAx(x7z4a!n4yj)b>Tb|6z& zT>%FRK_qtGum%-fvHR0*C;ccY>C4`Cs89LvYh}(&>Zs3HrI)*FKz)H$>3VL4V(VL? zjL{a?IpQN>_2pwyUfkp-lNp@S7c`O>$>Rw$D5Zo6;g0eEp0X(=0#3#0)v%~RjFd}B zmuC#)Dqpc0wLdEvlR2-3GcCmjINhDqOpE%=?UJ_JoEzS-8s2UW>xkAi1?ulMK54j4 zx0)E%xW66Nx$p5J6%jfClLcafOtAFoI89G#rs5l4kk^mM2bhemAcLkpJp4mz_)Ax= zH?>JO$^zPQAB!a}_Ty{Y(FKlcjPt+l-9F!e40#6Bh4)Q{o~FZ@4pGPkG9sKzq(@5 z!yjedw6fWl(dv+Ib2Ka-_IrxC&yLfi|_f=1JuMiRQ`WCF6b z^@8dWV?Ig`i{;!KNd-mWIXKjFWj!~5FLAp8CaZI;#a)PC&Jyc+3ziXzBJ@kH1?<-} zC|zWe_Ep0Ncq$+z(lU#gSEPgat4(p2V%X!kCf3W!tNz~wz(}{AsUVUTUoL+bVs8N5 z*8=moEBOy#`cP1N-zCZgs=dY6>M)J2SehKnip>{Y5KxgU1rj}#wTmGOJs7JOF2qg3 zOr%cKEsg*u!_}p7Dqql?QX~Nw?J|HWnEJS#I&mDSwS>F43nO?!M@>9 zRS}fJz8g&zP)le%j2bacwz7zr8F5E|~p zVs@#hau|y|K)G^eSgHlQSYz6EUl0s57s^ME=w4zUW3TW)xCEbgBQ63g>t~H#c%*^ zV8I!J04ubI&xa$xaT_DL&r&(0(9sYqNu-lK0M*@vwI#P@-0AQZ6A-5rmBhk-*&2qt z=t8&lPLqsWE&*g@2X_Wv`NBN&_#3{Lz35rHRDn+iiU!k}d9sFwAte2}Zl(cg!;u}P z%VF9qfw1Jqw|Yl@WBt2rCguu(n-n{^l8}R+?HQaT>2m@nSY&_ z9xTR6pSL2@TsnmN)DvGzn0n~2g8}O4cdn52mg(Je@_YeOj?n5D3u`}lj)rlQo1F_l9;ZK`?7e8?8ZhDP%D_33m9! zX#9Q8EjpnMiVHWA8FoPxuo#K#wra>=c9<@JYlWCo%Gzzh5nA;MF*?`g=EZk0tcwhD zF;~}Q6igP7KzEM12RBEzLN9%WGzRlm9oY0aDjo~*L%6m#z8;iB>;cMe)utE&_tAoI zL$ie~``9qY$oIVUL!Pchn|B*^yeW#v$(9w`)secv7UW533kXJfW|^Y3gm_ng=KR{XU z=`)(jk8#+mlxe9-6of`v?8PG0z*XH;?nSiVP(9pPn6o6=A+z;K%6dJ2TQS-D?&g5_ z!cnQme1+86KCfJ7aGZq&VhFX}$#eXMqo@!daLyHaxo`76RC<4tjn`^gD&I)@bRD(S zqU~%4G~%zETA#F`+CyAhCMF@VPd>K3kLA>T_dQ+l0RJ?UQ-!@D%N%sZa=Gie(!8Zd zm*UMbZE`rm%ug{W2n)R4$6HTViAA7d;KPp^ZT_4A-teqpse}b>(b{~aRca@nSN$40 z*lp~9{PO60X7tlN@+MJ!6Vz}e$p==Ly{u^m(B554tI2Skk1?NP}z0;<9~ zW3by3jO3@SLD`u=W~K|4>VqXHmD_8jqvL@Ts?fFt-5V3UZ#mzG;UHl&R&OG^XU4se zw3c&mdqy%N*f@~4AtoK+%;5)I(Q$fKAopo=A;q#^^N!a`a=a~P`UbaD-H2JQ7$B(b zpZ)M*7$lge)VSh)Dmx66yo3F9SN~^fOf!__*VI^KXgcN(U2C1}w>c{$@@+^IAOs2I zuQ@3&sFdLGH-MD7&W}qu^p9~euxTkF=i~jIM2$F;?cwhJ3Hnh}HW$besC zYY!Ix2H|~VrJu0J`{H4~&5RZRK>C9fEH(fjsj8wSt*7;WFnja`yH*fjP7^&u|2fb0 zJN!Qe?+Wfn92!=tw;d?X(?r9`3E!&74g4Kc^;!py!S(ye4u(p z0RSNWK~;hLp$L)>zRG_yWq)-2a~|&Z+MClqWchPC?l<}zk6ugTWIM@F06^_SC1C#m z)MW$!G^Bn?n7_IGNXej>0Go*o01%h@KluDKsV2ZqV1L;5|0d^ti}DZUidc%n?T1zY z;69G<{Z5&k>oMEE%1QpmOkFkrfcb})z5oD>N{s{ifb*!o{MVL$3cSDLu$esK{44kV z#_ylvEHQewfS4!jWo&Qnk59I`e@-`3srJ^A(DW@#0@>9P4Eg!MR&Az4E_^$UL zPqR=10N8(2?wjq8?tC~1|8ys_vbem;U*+d|4DzQ&_j`6Z?hh<~DR#f1|5MF1P@S&L z1N!ra-u|&igGywoKz^%A`$b&%nyDIpQ=)dho_Lm~~2jiR&aAg%e z)W6n0o_#k}1Q1q<4}ZV~vsA!ErZOQ-Ua{sMtR}CqQR`KIg_WuEF C>*@~x delta 11099 zcmZvibyyTi`}cS0m6Go6loAA~rKP*OyFq#hNs(H*K^o~0kd$talx~m?5kcYwJ3iAJ~{VDeUu@>U`ZEj;O zAOgdK%ZMp&&w3yLKn?p48!(TM+@yNcs6%rB1; z`M-Iz&)t&M%JT62m30!I9RdWo>ng|H7x={-KS$cw zjZ7-_qzLM7pADB2-iVfaOTI5+Ik`?ga|!-JgZ^oQ3CEgPLweqLNxZ_hc7_pEU}f)P zGTQ)=qX6FHjcW4nMd5w)jkNaFrQnGW%7nD^P=c;u*Evo zPUE1pj$T=9j2ii6;FBIkr0MS%*z70`jNAy1!l-q?x<1vwt;`>vR#=@_kK$3hi%CU` z3c$`82%ZgAF*lOnM})yAkCe07gb5H|P%>ACXvw}88Ize}D1=Ud!N77Tp2$=31L^qI z4)YR{#5Gzx2*MO2>+Q8bAIPSa1LJ6^_ndYFYAX^*sf4jpH7LWrUGABuRxgc+@&Q$s zbBuQ*Je5wt&WZ4O)TAx+u_&IUU7bDNo=Y3+`~;7!wcw0#R0|yRB5Alx4|(R`WU^(E zHP46_H@8lY-99pcI)9>Jd=E=qKC-tq-W~i#Pbqd525w-bIWHKckqSZlfZBVDV7{|n zH$Gj9-Y$#XieRX(O2DRXq`CQ z-0l{IkJK&jDlK*6(q8t_GInZ2h*iUMtdPcn%G>lg)5?=EX0H-l8+bJ4V@CWwQAJ|uI*>a^X!+pPMw2}&mZ3|_8Xcby2TD^5nh zrSnJ*qlw*Y1MWi4jV1`2-d1zU?eo;$8)&@v6XGMqVDx#aqM)-&j`cot^Udz0B3H@-|JVG$gD_5|0;r_dM)8p3)K2M*-5mAYD6ql?ZPpd>Op z>ri$w=(v#wIx4{p)PN#Lk^)JfG?EGk^46g|t$9!{NfneQCb%39B(qy3uN~x|pCx&S zHzxXcz4_YSk9qq5oA^~tYv$_`9XO84Gu7M3pv+Qaw81w+9p?CfT*DSFyK5zl8fU9a zk%XO9U=znp+;0RE`{g6Y4Br@_PEyZ+D$o+CComTQ(0M5t42QO_w2c6A51@BKy-43s zTYe4C3v3|J04giZfT$KMxNhmI1@)131~Nd8q*;J(p(HYKz;F}XnH8F0vk&7DUng8 z`~171g_%95vVt~{7Fwo20Yg>?9aA_0hC~PZ@}?8s$jd0Ly4(MZ)6QKozvZy^|$4gt4?E$FczM#Evqv16#p!b;JZWi zS!l*UInzZd0<&0tZSFt@k+WShp8hl}$>nmSSES$cJzC-;j2{pxQ%Bxcd{Ra@EjbqUj=$mErGAwUYVX9#v}C21WN!Dw|h%d{LAvg;dc&J4>a&#AAj zLcZ1&yy_n31fg08uz8MSI|so*&B{yiNo0hGu`-{h(J4GV!?%?$5bAhXT?)zaTL`>&MXxcc-ukNY`y+GdOhbD}XeC?1Zf9-u*2m0&%_yX0^6s7J$VkH{5mAVE9EE668`Rky z4MZD&*%3xLK_&=-l^9AfV6$ksqy|kD@VUH4YLmj2#fl7(V0^x*T&yGb=JuJnuM{up zrJ+h4Rg8w1bg*`Omzrco0caZca~)amqdZGXLF4xn=N#eUDKSXI9{MDU zAgvKu#HKb{E*50^iJ-d|h zZ}#p`_}-0N!>$Q@Yt2k{xuKm4Is*=4&g`<9anYQF8I39`#clL#9^Olay{iI0&v ziAKe!Y%Sq%w_ zh7@R2F5bN63MpfRNNz~vED*tVyxU`J-DVa7kXZ(hBEwC5hYen<#GTZNcX*__b2XFx zSon)Ia?z78LpFIm6_&iNSuXIaDMk9hgNzqE6BI9of?!$3dERt%C%%~y*)WI3-3?HO zdX-aD;J&RPKu2}^>@>=ppw}Up;l}^A4}r7+sSWp-3G>X781iO$ZXfqnTaVA}nKl zMpLJt)Hw1+N60^VJj`L==SV#4hEzBK?Y zNb0Y3{lQUpi;zt6$&U?ltdE!XK8HIn#v?4opcfh%B~RIE^n5o?;zFH4g9bmZ?qelo zfG1wj5JVQC3v++Back2TZorh+IOq&@EEQn8ly!!R*#st=_9HpDS!1!WjN!9l;f3V~ zA6=z&!P@7~a;(II+hfgx*e+YqmSH_O!g*)&--#DIu=? zOzQ`pF(V}24nfxIY`P9V_KSsDmN-^ydky1DT`jDeOdPS5a~*PRT%U<3+{IJdDUwE^ zFGB+XJ$N%MSu`qex!aLs6EE4|l!TEEV`*Op1g2M|N|Yw&w!OLuz(8~F=$RxW1a>ld4HmH$A7Vg8rBV2n4){j$^8DJgOy}ec~_5M5{*b< z0TbhC*r!?A_)#-<&_#%PVQWO96sf;}F$xAQUuY+}Wuz82wRY22=dspOefMuE#F!2e zA#;ehLi~ow)P!jIMG{I9;q%o2-ET!V;4xxwq^p$utua^7VXyQugDX&B4k#G|Qa1Fq0M24IeLD)%Xo z1SG96qW~#}My%F6_zXidqn(kgpxk*STq8t;tFPgkhi5FKHyj6WoBiKP-za-aw#wS- z%9C-+nNVCu`bNR*Tero*k7eWisJa%HyBlYL2rh;$tltQAOx6-eX2|E=qJ{1=l8Rh) zmulQ?ksNOmCYNN7;HpW}x#Z=@y^B}-TpuDYSug{P6@SB_Vxy`zM#OgAq5P=ui&Uf8 zS}JGZE+#j?ruUor;-=|Nk0XpEsMXZlHL)c>@A%DhOs3;VLKJ1`#NoPX;A{msAn`~9 zGS#kj1Gib6G=MqNBoN!(h%2&0XzH2rna}q4)r_+mcmTn;0=+?mo_mvf+9lz&Rg`qK z_LDPPfsa}kpENOegKXkVT*e|ot6Vc+b{}CdBlNAqTGbypKo0A|Ljf8a-Y zFMVBb8hK-w%@o3jD|y{sQ&qwmdnp{~KnP7bx%JxM?cKPpkG#K59>eB8=F|C5D=rR8 z2pOfD&(r4!>k3rRO@B%UPGSE9C$bPLa-KKTbl`R>HZN9*)?G}$x=%H1CJeHkhRx_Q z>E#56L^?>8zW&^mRvfa_AjqFB$%2m>dbHeJw*{xWn8RQ+g<_T{q@8y_K8We&1fc>) zVLg>)OijpalsK|kEsY!mXef6ztEN>4-y$?2TQ8598J)?Odc2V8TGdw>i0$WofhQv? zH5yRM2u~lrn$;dpvcWbug>`jDqlU!CuNnl`J46Nm6_ttwfKW#b)&VhFlY}(=Y+qcf+-lxT6KX@H8rsVe@Z#JOfCDNqs@N zI7=I(?b)EJ*vwjT{~(=m%6cT>sKPqPsaEoJQjV^ySy(e{m0FqMfU1x>1rv-YYCW2n zT#eXihc53iYZIrD-;uQsl~k%T)TrUsGj%y9THc7N8eKcVCb1@}_`xFa!@I;}V z*td~jFEqm+Dg#e@HY}AK%g7xcsh0zF4a=~Joso_A0GUOJLp|MuxCBwJCZ$RWxMFls zMSY0VU!W}u=cu-7+=hno=S$n@G#erv#52P%42)0@(uVI8iO)$tB@_>(yXkehJ*j

    ^WaD~|AKze!ei0Ddn=h$_TH(P1Y;_~RD#04O6dls z_kajn^aCINToe|WA{M#qD@7#JH(|Fv-%oCyA4O(vI~aM?`MoP)@z~Wm$UJ zFtyY+v5ZM24=FsnM11u_5`KKs>#gei@MC+zi2)JP&JsTCuHB{th@+d96%&|H11~O; zXysXdpXJz)4(-PA%wfmsvTIMW6NY8AgbH5K3~}7}qG#aQ^r%^XH-1XG;tzTFr1k)b zYe%ZZ_9h_ygpvU5aBu&|Q-d(h8e{s2=C4G#85nBCmYwu6>$ymcIJZJHFQ3xX9pa=y zt@zts5sQEXS$#D%sNOGeH9>{EUXt2dl;_rH-_uJ&++DYbOxg z14Dr}z%l-ZPIlpv5q^;u7c!cEc}z_(a&#n3B&et!jJur_`4?s(zs42ldh!(MEqa~}>5F|!qxtKPf;#f%lSkn?9=Zb25>mhwRSZ>y9`1Mq5} zlV(FO`QjK!J(jm$-r^m3&F<<{b;ZLnFg-s`95EBfSUO1yG&D6CXRz+M`+hn;Q0HwuNw{fv-$gG_*_AL(;N1O%jH$CKGJbZ68naHk zp5w(T111WVtP<~Fp!?WZz0lW2Rl)75&7%~^7y=QPg==|RC2UTlkQuLLB;U_Cmii$Lvgf*gj3qTvM zK6sAdTBq!&z~&-J#Y9N1cmS~n`uc}&JrblutstT09;)2~kKO2xU$~kt3V95U?t(%g zN*+ROzUiZ^;H<17s(lYvt9RRyhe&)saHpNN7W7cylau+nBYSrO=FPv#lI#g&PcczB z%41ff2W5;ih}_QCW=e&I&?OPj@l8XH__)|LPBmhS-s=G8pv)a|X!JC6-!{6+hmw%E zc<8u;nJll0fq<0pQ4S%G$4n_Js2`;332hBu9Y7M;={mha08@WR5P;VlGbe$Mg2E3#LSlr$GhoF>i0-5Wn_Q{id zt6R{)LG;pR3{R?R7!}eA84O=p_vSR$#geHdgAf&q>_qjKWuACCH;dcWvsb8p?#^r1 zulzgr6u0c2m?GTMQU&l|Tcieo1o+P`L?-=>&$5vU0pT9M$WVs*p%NoG=tLMg-+$HQ zqB{J;XMK(SL#s78Ms5F0GVVwA-){ZxR}G|pI1iS_$iKwH0RS~P0082BAj)F@ZPtGY zYdBLO+@nYL;Q|0Z9a{vF|8)G|92y-+fblR2nCHh9LIVIe`~d(;sDCUz6uufAau0w1 z7vlff^9OpKF7w>DcS2b(YR{P%o{GyzE9>YqIuUIqyV*w5I82_t|*Z|ygokM*Ay zk#9kadb_yYq~4zjYi_=Oi`TZq`JuBJF|CrFf)s`kZBH}OoylZ0&E3^^_k(*7Vf}da zp%hPw)!q<36!cAZS5-gT5^3y3$im{{)5AIN-PY=A&TT#<7hQM1lERJdV@wr~68fXe zf{XyVI=NyV^N?}JutHJvU|G*eogY8sZU(m@P(L=x13N;3jK-!Ad0P&OsNS#UZ;K0C z!g*uEu$J>x0!dKX#6Pl;@PR}_u+gFHV3p2ntPGyL9~>t{%h{~41Pk9$P2MSlh05(4 zW$=;Ked(iYr`;#LyEBpQdwPQSK4HvhSjT!LEo2j=c%`FJqTGA z@{z!^D}zmsOK&XBrp?yCR^EAWjV(hc?iE=HaYIEA`f4Mt_LZa(_C7=3nWTtDcplDViQzC z1d+z9tpdEzqe73KN)mhW7$a6-$GvaVgGjvE1sCoy;2>7tDJKXRIAvZpW`fA6 z%m%UfCub;zimT(GS~oS~!cUw@ddWm3^Nv#*eIBn6vYadI=R(o)_yj(FZVtygN!3SW zg06(Ws!e=6HkprLBpY-`A^zA6ikqa2u7$Egg)g3+fntQujzK-0oQgyqaTigC4Z$Wj zr9TarSxjzVd&ObCIGY#1b%2Bf|Cq0vHIp1sf!sUS)zhMJyE(zX8K;z@HV_uiRo1PN(8eADXq1!uTXP>KFy z1~k+x^X(n!LyGONMfcA;vo9u2?O-#f$#C%kgm>85)q<(p| z{on9K)s2BN!XLrRZhU9qUm!3~qZhm-y=+Op3;4plCK`~Mef$(S`wS@==6Pm?Z3HT& zsbPywnfh9JN&2R!^?7p3m_nfc7q^-?SK5doIyc1eHnn!f;FZWLO4kGjh7n6(uh}&=cL4#kn@D`EO^@+RW_arT>&+cTqpNR9>)0#PVw{95 zBb1!3C$^gXCuel2(;N^LR0Bfoi?QU9CngY>53_-vyMcjqd{-e`l4;YJ>!?LGa761} zuCoE>j6-IG*Fp6ZS+;qVebHkdFS98S(&A68;~h4@&QW+3xX}@wSTbmXSEIw%D@r}` zD3q&(v(16sSc^u8O0Su9vz}J@<>BQ!SS!ccp;6-?ntzCIk2Hl;pzf&F^s`hcxq1Zj zB*P=G{P23wF~3x>BByRL8qjEJC>?v5^K$OPGt#fM3P-gUcotbfV`A{Yx4lM~p#|nC zb7T2fPCPL@F@xw6;MEsF zWXJNRuJTp0ZZ8eXI8Is0&8S)V2{FW3R2Q3>ttlbg!FzfVLYv?yG(f_kts-S%{eLJDvc3C@>rL`c) zaz3x1bCY5=KB0VfKB0a$>`o@ge|p)UzIa-0dnL2%Vn2iC5L5^pJUqVwerXXi_ue+^=6pHNsXahGgp6ykHV6a zoVqt}TKr=d!Oi1_l5y_USK)W-er+$=!{A;I0=wUXiZ#5E8RncoZ(D6*RwVRY(J5TW zXpb@ti{R>Q$6z=m7SyCkuPrc~%mX=zdOFT+_31FYM?ksFGTWRigRw0(-tEW)nHW#|_78QgY)zvV6;saiwciLtzy|Wid(5 ztHRna)X2))JNBfF=7PU<_(N@M`X#d_!7$6#@Yp~_vEOT%u8%9~@~whR_R&E+h&oX` zda7mi@Sk1JN3W@y8{QR1&PO9ZxamA_+EY~Gm}oL zBSnN$^T`S*W;Xmtx>g|!U~@JEemQ>9(wT}0*FWb|*v`OOln9gum8rEe$Zbo`*kP?q zE;u`;`XL&o3I=_W09!Rd{U@0*Vp8`Y0OcAVVdb5Akg&BKFL~p%<$l$E6qD_+?R)OF zx4P=>7XI@{?ZG|EEFLxqNDgIgD`jee%?z$g=u56v1UX$z?jTJ(_^?MOU-v1kNxmbm z=EZ_B{B~CrUk2LkLBx=k_LOWUAuAQnKJx=uS=P{m4I^;6AHy;7g~6S}jqg2&0Jsk@ zI|^FRW@`r#iGuBEg5J0$=fb6H5W%GH$A046;l~-n5NLyKKPwSi@vK8TZe}F&z-}r7 z@-K9>eD~WG`Or8M4R|KaAlhz0t(P;wwpB^vbFEk@aKeVDJ| zq`+rEaWApKET#F`fUkGN#`@Eja+Mq{5JeG{54x7p1m);TKEqHSPI_|f=qG7?!;8C* z=Oc%jLE2v4@ttxXCwUYS zrPRDA3I<4fcLpjd^+=dVaKm3vrYj^2q_tWc3sz}^zsfY%q&U*p-Ly%}^`O@j0>*i$ zzALqBbg*5D1=5`{#!#JT{T2z7FQO=ShTqjk7z3~{J#dr8Jvf!YWn9#Mr?0W%bVk?M7&ZbkITh?|Kmmc8eein|5XA8=XryXS zFIfKODr4z5hN&M#+QGoe)hN>E>xW%fG|TaMo7|-Mm+#uMEjm*!J{g+ojKLRdAss6V zV^cB*3_aJh7De^*qFAHaRM{O8S17`i<}5m%Bs41pC$PuL|0*A=hO8 z-L-#RlwD$3|GOv`$EINYVqTf-zg??4**#!>K4cbWb^DzQ>3`SlPcF*;4AZ;)b~%@I^8}#*VMt+rgK89Ig!fl* z001l|_m5lnU+<@&vE@YU4}fj+L_??Uo*Qi4(=!AB{(~{#{!c({fa(1YQPE$){-z>- zP-(^N@q{4)07i5G0LmXKEm$Zz>BK79?va@>`+8;^G>fX!bm0yTT$%)JV-SeMo=Q{qAf&H$_Hv;@i#C}u#^wtXR>`Z$f>y!J{`RB?UHojMhDa-zqBgX@t z;$mZpQomoB@b@e8&(b_Kz1RKL-=9?L-{w@76Jih9+*@e;J@M0r;$0p0ho;;Ilzvaq z9{o|g8au!}PXkLTbNn6B-|X%80>n5xDE}dG_Z9f3`#8TtxPU4`d^1-7z{SMU%+AQw z*}?van(Dpn|H3E#3(EZwT!}06u!;fhf7ZVaO1^4B81bO{5AdLN)u8`Ae8>~(aDOu4 zCjkH|G75j6OutrgRWTt>W6HhBzh~xe?fYdSQ$dKslnwxBh$;U)3)h1q2jRwe-ur^O z+; UTF-8 7-SNAPSHOT - 7-SNAPSHOT + 7 2.9.0 5.1 From 00348f2005ef88e581229ada9af8fe810c9f8751 Mon Sep 17 00:00:00 2001 From: Adolfo Peixinho <91942157+adolfopeixinho@users.noreply.github.com> Date: Mon, 20 Jun 2022 17:34:43 +0200 Subject: [PATCH 67/73] MET-4606 WebResourceSolrCreatorTest (#558) * MET-4606 WebResourceSolrCreatorTest * MET-4606 WebResourceSolrCreatorTest review --- .../property/WebResourceSolrCreatorTest.java | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 metis-indexing/src/test/java/eu/europeana/indexing/solr/property/WebResourceSolrCreatorTest.java diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/WebResourceSolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/WebResourceSolrCreatorTest.java new file mode 100644 index 000000000..4dca62ead --- /dev/null +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/WebResourceSolrCreatorTest.java @@ -0,0 +1,76 @@ +package eu.europeana.indexing.solr.property; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import eu.europeana.corelib.definitions.edm.entity.License; +import eu.europeana.corelib.definitions.edm.entity.WebResource; +import eu.europeana.corelib.solr.entity.LicenseImpl; +import eu.europeana.corelib.solr.entity.WebResourceImpl; +import eu.europeana.indexing.solr.EdmLabel; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.apache.solr.common.SolrInputDocument; +import org.junit.jupiter.api.Test; + +class WebResourceSolrCreatorTest { + + @Test + void addToDocument() { + + // given + License license1 = new LicenseImpl(); + license1.setAbout("license1About"); + license1.setOdrlInheritFrom("license1OdrlInheritFrom"); + License license2 = new LicenseImpl(); + license2.setAbout("license2About"); + license2.setOdrlInheritFrom("license2OdrlInheritFrom"); + + WebResource webResource = new WebResourceImpl(); + webResource.setAbout("webResourceAbout"); + webResource.setIsNextInSequence("webResourceIsNextInSequence"); + webResource.setWebResourceEdmRights( + Map.of("webResourceEdmRightsKey", List.of("webResourceEdmRightsValue", license2.getAbout()))); + webResource.setWebResourceDcRights( + Map.of("webResourceDcRightsKey", List.of("webResourceDcRightsValue", license1.getAbout()))); + final List webResourceSvcsHasService = List.of("webResourceSvcsHasService"); + webResource.setSvcsHasService(webResourceSvcsHasService.toArray(new String[0])); + final List webResourceDctermsIsReferencedBy = List.of("webResourceDctermsIsReferencedBy"); + webResource.setDctermsIsReferencedBy(webResourceDctermsIsReferencedBy.toArray(new String[0])); + + final WebResourceSolrCreator webResourceSolrCreator = new WebResourceSolrCreator(List.of(license1, license2)); + final SolrInputDocument solrInputDocument = new SolrInputDocument(); + + // When + webResourceSolrCreator.addToDocument(solrInputDocument, webResource); + + // Then + assertEquals(webResource.getAbout(), solrInputDocument.getFieldValue(EdmLabel.EDM_WEB_RESOURCE.toString())); + assertEquals(webResource.getIsNextInSequence(), + solrInputDocument.getFieldValue(EdmLabel.WR_EDM_IS_NEXT_IN_SEQUENCE.toString())); + verifyMap(solrInputDocument, EdmLabel.WR_EDM_RIGHTS, webResource.getWebResourceEdmRights()); + verifyMap(solrInputDocument, EdmLabel.WR_DC_RIGHTS, webResource.getWebResourceDcRights()); + verifyCollection(solrInputDocument, EdmLabel.WR_SVCS_HAS_SERVICE, webResourceSvcsHasService); + verifyCollection(solrInputDocument, EdmLabel.WR_DCTERMS_ISREFERENCEDBY, webResourceDctermsIsReferencedBy); + verifyCollection(solrInputDocument, EdmLabel.WR_CC_ODRL_INHERITED_FROM, + List.of(license1.getOdrlInheritFrom(), license2.getOdrlInheritFrom())); + } + + void verifyMap(SolrInputDocument solrInputDocument, EdmLabel edmLabel, Map> map) { + map.forEach((key, value) -> assertTrue(solrInputDocument.getFieldValues(computeSolrField(edmLabel, key)) + .containsAll(value))); + } + + private String computeSolrField(EdmLabel label, String value) { + return label.toString() + "." + value; + } + + void verifyCollection(SolrInputDocument solrInputDocument, EdmLabel edmLabel, Collection collection) { + final Collection fieldValues = solrInputDocument.getFieldValues(edmLabel.toString()); + assertTrue(fieldValues.containsAll(collection)); + assertEquals(collection.size(), fieldValues.size()); + } + + +} From 726a9d08f5eecd5e9c4a31205175fa8d48c2a392 Mon Sep 17 00:00:00 2001 From: Adolfo Peixinho <91942157+adolfopeixinho@users.noreply.github.com> Date: Mon, 20 Jun 2022 17:43:25 +0200 Subject: [PATCH 68/73] Feat/met 4440 met 4568 europeana aggregation solr creator unit tests (#560) * MET-4568 added test for EuropeanaAggregationSolrCreator * MET-4558 changes from code review * MET-4568 cleanup some assertions. QualityAnnotationSolrCreator not fully tested * MET-4568 added test for EuropeanaAggregationSolrCreator * MET-4558 changes from code review * MET-4568 cleanup some assertions. QualityAnnotationSolrCreator not fully tested * EA-4568 Add test forEuropeanaAggregationSolrCreator * MET-4568 EuropeanaAggregationSolrCreatorTest wrong assertion order Co-authored-by: Simon Tzanakis Co-authored-by: Jorge Ortiz --- .../solr/property/EuropeanaAggregationSolrCreatorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/EuropeanaAggregationSolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/EuropeanaAggregationSolrCreatorTest.java index ebf73573a..dd7f53d60 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/EuropeanaAggregationSolrCreatorTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/EuropeanaAggregationSolrCreatorTest.java @@ -122,7 +122,7 @@ void addToDocument() { void verifyCollection(SolrInputDocument solrInputDocument, EdmLabel edmLabel, Collection collection) { final Collection fieldValues = solrInputDocument.getFieldValues(edmLabel.toString()); assertTrue(fieldValues.containsAll(collection)); - assertEquals(fieldValues.size(), collection.size()); + assertEquals(collection.size(), fieldValues.size()); } void verifyMap(SolrInputDocument solrInputDocument, EdmLabel edmLabel, Map> map) { From 0b0316ef78a214d5b77976a3202a8e7402efe573 Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Tue, 21 Jun 2022 10:40:10 +0200 Subject: [PATCH 69/73] Update enrichment.properties.example --- .../src/main/resources/enrichment.properties.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metis-enrichment/metis-enrichment-rest/src/main/resources/enrichment.properties.example b/metis-enrichment/metis-enrichment-rest/src/main/resources/enrichment.properties.example index 6f8489fbe..69b503833 100644 --- a/metis-enrichment/metis-enrichment-rest/src/main/resources/enrichment.properties.example +++ b/metis-enrichment/metis-enrichment-rest/src/main/resources/enrichment.properties.example @@ -15,7 +15,7 @@ enrichment.mongo.application.name= #If not provided defaults to 20 enrichment.batch.size= #Options PERSISTENT, ENTITY_CLIENT. Defaults to PERSISTENT -entity.resolver.type= +enrichment.entity.resolver.type= #Entity Management Base Url for Entity Retrieval entity.management.url= From 7d4fecdd009906b4c83f9495f39acb86a238ae24 Mon Sep 17 00:00:00 2001 From: Jorge Ortiz Date: Wed, 22 Jun 2022 11:06:20 +0200 Subject: [PATCH 70/73] Feat/met 4440 met 4618 unify common test code (#561) * MET-4440_MET-4618 remove duplicate code and add java doc title * MET-4440_MET-4618 code cleanup remove duplicated code remove unused imports use verifyMap function --- .../solr/SolrDocumentPopulatorTest.java | 5 +- .../solr/property/AgentSolrCreatorTest.java | 39 +++----- .../solr/property/ConceptSolrCreatorTest.java | 20 ++-- .../EuropeanaAggregationSolrCreatorTest.java | 22 +---- .../property/FullBeanSolrPropertiesTest.java | 17 +--- .../solr/property/LicenseSolrCreatorTest.java | 2 - .../solr/property/PlaceSolrCreatorTest.java | 3 + .../property/PropertySolrCreatorTest.java | 6 +- .../solr/property/ProxySolrCreatorTest.java | 92 ++++++------------- .../QualityAnnotationSolrCreatorTest.java | 5 +- .../solr/property/SolrPropertyUtilsTest.java | 7 +- .../property/TimespanSolrCreatorTest.java | 63 +++++-------- .../property/WebResourceSolrCreatorTest.java | 23 +---- .../europeana/indexing/utils/TestUtils.java | 25 ++++- 14 files changed, 129 insertions(+), 200 deletions(-) diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/SolrDocumentPopulatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/SolrDocumentPopulatorTest.java index aaac7616a..285eb645d 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/solr/SolrDocumentPopulatorTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/SolrDocumentPopulatorTest.java @@ -21,6 +21,9 @@ import org.apache.solr.common.SolrInputDocument; import org.junit.jupiter.api.Test; +/** + * Unit test for {@link SolrInputDocument} class + */ class SolrDocumentPopulatorTest { @Test @@ -80,4 +83,4 @@ void populateWithProperties_WGS84Coordinates() throws Exception { assertTrue(CollectionUtils.isEqualCollection(document.get(LOCATION_WGS.toString()).getValues(), List.of("50,50", "40,40", "40.123456,40.1234567", "50.75,4.5"))); } -} \ No newline at end of file +} diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/AgentSolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/AgentSolrCreatorTest.java index eca466b90..cc37cde0e 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/AgentSolrCreatorTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/AgentSolrCreatorTest.java @@ -1,18 +1,21 @@ package eu.europeana.indexing.solr.property; +import static eu.europeana.indexing.utils.TestUtils.verifyMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import eu.europeana.corelib.definitions.edm.entity.Agent; import eu.europeana.corelib.solr.entity.AgentImpl; import eu.europeana.indexing.solr.EdmLabel; -import java.util.Arrays; import java.util.List; import java.util.Map; import org.apache.solr.common.SolrInputDocument; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +/** + * Unit test for {@link AgentSolrCreator} class + */ class AgentSolrCreatorTest { private SolrInputDocument solrInputDocument; @@ -28,7 +31,6 @@ void setup() { @Test void agentSolrCreatorAddToSolrDocument() { - Agent agent = new AgentImpl(); agent.setAbout("About Agent"); agent.setPrefLabel(Map.of("pref_label", List.of("val1", "val2"))); @@ -45,32 +47,17 @@ void agentSolrCreatorAddToSolrDocument() { // assertions assertTrue(solrInputDocument.containsKey(EdmLabel.EDM_AGENT.toString())); - assertTrue(solrInputDocument.containsKey(EdmLabel.AG_SKOS_PREF_LABEL + ".pref_label")); - assertTrue(solrInputDocument.containsKey(EdmLabel.AG_SKOS_ALT_LABEL + ".alt_label")); - assertTrue(solrInputDocument.containsKey(EdmLabel.AG_FOAF_NAME + ".foaf_name")); - assertTrue(solrInputDocument.containsKey(EdmLabel.AG_RDAGR2_DATEOFBIRTH + ".date_of_birth_label")); - assertTrue(solrInputDocument.containsKey(EdmLabel.AG_RDAGR2_DATEOFDEATH + ".date_of_death_label")); - assertTrue(solrInputDocument.containsKey(EdmLabel.AG_RDAGR2_PLACEOFBIRTH + ".place_of_birth_label")); - assertTrue(solrInputDocument.containsKey(EdmLabel.AG_RDAGR2_PLACEOFDEATH + ".place_of_death_label")); - assertTrue(solrInputDocument.containsKey(EdmLabel.AG_RDAGR2_PROFESSIONOROCCUPATION + ".profession_label")); - assertEquals("About Agent", solrInputDocument.getFieldValue(EdmLabel.EDM_AGENT.toString())); - assertEquals(Arrays.asList("val1", "val2"), solrInputDocument.getFieldValues(EdmLabel.AG_SKOS_PREF_LABEL + ".pref_label")); - assertEquals(Arrays.asList("val1", "val2"), solrInputDocument.getFieldValues(EdmLabel.AG_SKOS_ALT_LABEL + ".alt_label")); - assertEquals(Arrays.asList("val1", "val2"), solrInputDocument.getFieldValues(EdmLabel.AG_FOAF_NAME + ".foaf_name")); - assertEquals(Arrays.asList("some_date"), - solrInputDocument.getFieldValues(EdmLabel.AG_RDAGR2_DATEOFBIRTH + ".date_of_birth_label")); - assertEquals(Arrays.asList("other_date"), - solrInputDocument.getFieldValues(EdmLabel.AG_RDAGR2_DATEOFDEATH + ".date_of_death_label")); - assertEquals(Arrays.asList("some_place"), - solrInputDocument.getFieldValues(EdmLabel.AG_RDAGR2_PLACEOFBIRTH + ".place_of_birth_label")); - assertEquals(Arrays.asList("other_place"), - solrInputDocument.getFieldValues(EdmLabel.AG_RDAGR2_PLACEOFDEATH + ".place_of_death_label")); - assertEquals(Arrays.asList("lawyer", "chef"), - solrInputDocument.getFieldValues(EdmLabel.AG_RDAGR2_PROFESSIONOROCCUPATION + ".profession_label")); - assertEquals(9, solrInputDocument.size()); + verifyMap(solrInputDocument, EdmLabel.AG_SKOS_PREF_LABEL, agent.getPrefLabel()); + verifyMap(solrInputDocument, EdmLabel.AG_SKOS_ALT_LABEL, agent.getAltLabel()); + verifyMap(solrInputDocument, EdmLabel.AG_FOAF_NAME, agent.getFoafName()); + verifyMap(solrInputDocument, EdmLabel.AG_RDAGR2_DATEOFBIRTH, agent.getRdaGr2DateOfBirth()); + verifyMap(solrInputDocument, EdmLabel.AG_RDAGR2_DATEOFDEATH, agent.getRdaGr2DateOfDeath()); + verifyMap(solrInputDocument, EdmLabel.AG_RDAGR2_PLACEOFBIRTH, agent.getRdaGr2PlaceOfBirth()); + verifyMap(solrInputDocument, EdmLabel.AG_RDAGR2_PLACEOFDEATH, agent.getRdaGr2PlaceOfDeath()); + verifyMap(solrInputDocument, EdmLabel.AG_RDAGR2_PROFESSIONOROCCUPATION, agent.getRdaGr2ProfessionOrOccupation()); + assertEquals(9, solrInputDocument.size()); } - } diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ConceptSolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ConceptSolrCreatorTest.java index ff0d060c8..6299f0010 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ConceptSolrCreatorTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ConceptSolrCreatorTest.java @@ -1,5 +1,6 @@ package eu.europeana.indexing.solr.property; +import static eu.europeana.indexing.utils.TestUtils.verifyMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; @@ -7,7 +8,6 @@ import eu.europeana.corelib.solr.entity.ConceptImpl; import eu.europeana.indexing.solr.EdmLabel; -import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.solr.common.SolrInputDocument; @@ -44,10 +44,8 @@ void addToDocument_withConceptPrefValueAndAltValue() { solrInputDocument.containsKey(EdmLabel.CC_SKOS_PREF_LABEL + ".keyPref") && solrInputDocument.containsKey(EdmLabel.CC_SKOS_ALT_LABEL + ".keyAlt")); assertEquals("concept", solrInputDocument.getFieldValue(EdmLabel.SKOS_CONCEPT.toString())); - assertEquals(List.of("prefValue1", "prefValue2"), - solrInputDocument.getFieldValues(EdmLabel.CC_SKOS_PREF_LABEL + ".keyPref")); - assertEquals(List.of("altValue1", "altValue2"), - solrInputDocument.getFieldValues(EdmLabel.CC_SKOS_ALT_LABEL + ".keyAlt")); + verifyMap(solrInputDocument, EdmLabel.CC_SKOS_PREF_LABEL, concept.getPrefLabel()); + verifyMap(solrInputDocument, EdmLabel.CC_SKOS_ALT_LABEL, concept.getAltLabel()); assertEquals(3, solrInputDocument.size()); } @@ -63,10 +61,8 @@ void addToDocument_withoutConcept() { assertTrue(solrInputDocument.containsKey(EdmLabel.CC_SKOS_PREF_LABEL + ".keyPref") && solrInputDocument.containsKey(EdmLabel.CC_SKOS_ALT_LABEL + ".keyAlt")); assertNull(solrInputDocument.getFieldValue(EdmLabel.SKOS_CONCEPT.toString())); - assertEquals(List.of("prefValue1", "prefValue2"), - solrInputDocument.getFieldValues(EdmLabel.CC_SKOS_PREF_LABEL + ".keyPref")); - assertEquals(List.of("altValue1", "altValue2"), - solrInputDocument.getFieldValues(EdmLabel.CC_SKOS_ALT_LABEL + ".keyAlt")); + verifyMap(solrInputDocument, EdmLabel.CC_SKOS_PREF_LABEL, concept.getPrefLabel()); + verifyMap(solrInputDocument, EdmLabel.CC_SKOS_ALT_LABEL, concept.getAltLabel()); assertEquals(2, solrInputDocument.size()); } @@ -83,8 +79,7 @@ void addToDocument_withoutPrefValue() { solrInputDocument.containsKey(EdmLabel.CC_SKOS_ALT_LABEL + ".keyAlt")); assertEquals("concept", solrInputDocument.getFieldValue(EdmLabel.SKOS_CONCEPT.toString())); assertNull(solrInputDocument.getFieldValues(EdmLabel.CC_SKOS_PREF_LABEL + ".keyPref")); - assertEquals(List.of("altValue1", "altValue2"), - solrInputDocument.getFieldValues(EdmLabel.CC_SKOS_ALT_LABEL + ".keyAlt")); + verifyMap(solrInputDocument, EdmLabel.CC_SKOS_ALT_LABEL, concept.getAltLabel()); assertEquals(2, solrInputDocument.size()); } @@ -100,8 +95,7 @@ void addToDocument_withoutAltValue() { assertTrue(solrInputDocument.containsKey(EdmLabel.SKOS_CONCEPT.toString()) && solrInputDocument.containsKey(EdmLabel.CC_SKOS_PREF_LABEL + ".keyPref")); assertEquals("concept", solrInputDocument.getFieldValue(EdmLabel.SKOS_CONCEPT.toString())); - assertEquals(List.of("prefValue1", "prefValue2"), - solrInputDocument.getFieldValues(EdmLabel.CC_SKOS_PREF_LABEL + ".keyPref")); + verifyMap(solrInputDocument, EdmLabel.CC_SKOS_PREF_LABEL, concept.getPrefLabel()); assertNull(solrInputDocument.getFieldValues(EdmLabel.CC_SKOS_ALT_LABEL + ".keyAlt")); assertEquals(2, solrInputDocument.size()); } diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/EuropeanaAggregationSolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/EuropeanaAggregationSolrCreatorTest.java index dd7f53d60..ae8118a55 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/EuropeanaAggregationSolrCreatorTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/EuropeanaAggregationSolrCreatorTest.java @@ -1,8 +1,9 @@ package eu.europeana.indexing.solr.property; +import static eu.europeana.indexing.utils.TestUtils.verifyCollection; +import static eu.europeana.indexing.utils.TestUtils.verifyMap; import static org.apache.commons.collections4.CollectionUtils.union; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; import eu.europeana.corelib.definitions.edm.entity.EuropeanaAggregation; import eu.europeana.corelib.definitions.edm.entity.License; @@ -14,7 +15,6 @@ import eu.europeana.corelib.solr.entity.WebResourceImpl; import eu.europeana.indexing.solr.EdmLabel; import eu.europeana.indexing.utils.RdfTier; -import java.util.Collection; import java.util.List; import java.util.Map; import org.apache.solr.common.SolrInputDocument; @@ -24,7 +24,6 @@ /** * Unit test for {@link EuropeanaAggregationSolrCreator} class */ - class EuropeanaAggregationSolrCreatorTest { @Test @@ -118,19 +117,4 @@ void addToDocument() { List.of(RdfTier.CONTENT_TIER_0.getTier().toString(), RdfTier.CONTENT_TIER_1.getTier().toString())); verifyCollection(solrInputDocument, EdmLabel.METADATA_TIER, List.of(RdfTier.METADATA_TIER_0.getTier().toString())); } - - void verifyCollection(SolrInputDocument solrInputDocument, EdmLabel edmLabel, Collection collection) { - final Collection fieldValues = solrInputDocument.getFieldValues(edmLabel.toString()); - assertTrue(fieldValues.containsAll(collection)); - assertEquals(collection.size(), fieldValues.size()); - } - - void verifyMap(SolrInputDocument solrInputDocument, EdmLabel edmLabel, Map> map) { - map.forEach((key, value) -> assertTrue(solrInputDocument.getFieldValues(computeSolrField(edmLabel, key)) - .containsAll(value))); - } - - private String computeSolrField(EdmLabel label, String value) { - return label.toString() + "." + value; - } -} \ No newline at end of file +} diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/FullBeanSolrPropertiesTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/FullBeanSolrPropertiesTest.java index 8ec4f9f14..6fc571a05 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/FullBeanSolrPropertiesTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/FullBeanSolrPropertiesTest.java @@ -1,8 +1,8 @@ package eu.europeana.indexing.solr.property; +import static eu.europeana.indexing.utils.TestUtils.verifyCollection; import static java.time.temporal.ChronoUnit.DAYS; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; import eu.europeana.corelib.definitions.edm.entity.Place; import eu.europeana.corelib.solr.bean.impl.FullBeanImpl; @@ -14,30 +14,28 @@ import java.sql.Date; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; import java.util.List; import java.util.Map; import org.apache.solr.common.SolrInputDocument; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +/** + * Unit test for {@link FullBeanSolrProperties} class + */ class FullBeanSolrPropertiesTest { private FullBeanSolrProperties fullBeanSolrProperties; private SolrInputDocument solrInputDocument; - @BeforeEach void setup() { solrInputDocument = new SolrInputDocument(); fullBeanSolrProperties = new FullBeanSolrProperties(); } - @Test void fullBeanSolrSetPropertiesTest() { - FullBeanImpl fullBean = new FullBeanImpl(); // agent setup AgentImpl agent = new AgentImpl(); @@ -82,12 +80,10 @@ void fullBeanSolrSetPropertiesTest() { assertEquals(solrInputDocument.getFieldValue(EdmLabel.TIMESTAMP_UPDATED.toString()), Date.from(Instant.now().truncatedTo(DAYS).plus(1, DAYS))); assertEquals(8, solrInputDocument.size()); - } @Test void fullBeanSolrSetPropertiesMultipleProxiesTest() { - FullBeanImpl fullBean = new FullBeanImpl(); // agent setup @@ -156,9 +152,4 @@ void fullBeanSolrSetPropertiesMultipleProxiesTest() { Date.from(Instant.now().truncatedTo(DAYS).plus(1, DAYS))); assertEquals(8, solrInputDocument.size()); } - void verifyCollection(SolrInputDocument solrInputDocument, EdmLabel edmLabel, Collection collection) { - final Collection fieldValues = solrInputDocument.getFieldValues(edmLabel.toString()); - assertTrue(fieldValues.containsAll(collection)); - assertEquals(collection.size(), fieldValues.size()); - } } diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/LicenseSolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/LicenseSolrCreatorTest.java index 769ecfe7d..0bdb2874b 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/LicenseSolrCreatorTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/LicenseSolrCreatorTest.java @@ -1,9 +1,7 @@ package eu.europeana.indexing.solr.property; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; import eu.europeana.corelib.solr.entity.LicenseImpl; import eu.europeana.indexing.solr.EdmLabel; diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/PlaceSolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/PlaceSolrCreatorTest.java index 56d685f6b..82e6f893d 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/PlaceSolrCreatorTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/PlaceSolrCreatorTest.java @@ -15,6 +15,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +/** + * Unit test for {@link PlaceSolrCreator} class + */ class PlaceSolrCreatorTest { private SolrInputDocument solrInputDocument; diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/PropertySolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/PropertySolrCreatorTest.java index 5e4bea476..27ca6cd81 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/PropertySolrCreatorTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/PropertySolrCreatorTest.java @@ -12,6 +12,9 @@ import org.apache.solr.common.SolrInputDocument; import org.junit.jupiter.api.Test; +/** + * Unit test for {@link ProxySolrCreator} class + */ public class PropertySolrCreatorTest { @Test @@ -21,7 +24,8 @@ public void testAddAllToDocument() { final PropertySolrCreator creator = spy(new PropertySolrCreator() { @Override - public void addToDocument(SolrInputDocument document, PlaceImpl property) {} + public void addToDocument(SolrInputDocument document, PlaceImpl property) { + } }); diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ProxySolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ProxySolrCreatorTest.java index 8ba1b3fe7..be005419b 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ProxySolrCreatorTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/ProxySolrCreatorTest.java @@ -1,7 +1,6 @@ package eu.europeana.indexing.solr.property; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static eu.europeana.indexing.utils.TestUtils.verifyMap; import eu.europeana.corelib.solr.entity.ProxyImpl; import eu.europeana.indexing.solr.EdmLabel; @@ -33,68 +32,37 @@ void addToDocument() { proxySolrCreator.addToDocument(solrInputDocument, proxy); - assertKeys(solrInputDocument); - assertDocumentContent(solrInputDocument); + assertDocumentContent(proxy, solrInputDocument); } - private void assertDocumentContent(SolrInputDocument solrInputDocument) { - assertEquals(List.of("locationA", "locationB"), solrInputDocument.getFieldValues(EdmLabel.PROXY_EDM_CURRENT_LOCATION + ".location")); - assertEquals(List.of("contributorA", "contributorB"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_CONTRIBUTOR + ".contributor")); - assertEquals(List.of("coverage1", "coverage2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_COVERAGE + ".coverage")); - assertEquals(List.of("creator1", "creator2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_CREATOR + ".creator")); - assertEquals(List.of("date1", "date2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_DATE + ".date")); - assertEquals(List.of("description1", "description2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_DESCRIPTION + ".description")); - assertEquals(List.of("format1", "format2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_FORMAT + ".format")); - assertEquals(List.of("identifier1", "identifier2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_IDENTIFIER + ".identifier")); - assertEquals(List.of("taal1", "taal2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_LANGUAGE + ".language")); - assertEquals(List.of("publisher1", "publisher2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_PUBLISHER + ".publisher")); - assertEquals(List.of("rights1", "rights2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_RIGHTS + ".rights")); - assertEquals(List.of("source1", "source2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_SOURCE + ".source")); - assertEquals(List.of("subject1", "subject2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_SUBJECT + ".subject")); - assertEquals(List.of("title1", "title2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_TITLE + ".title")); - assertEquals(List.of("type1", "type2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DC_TYPE + ".type")); - assertEquals(List.of("alt1", "alt2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DCTERMS_ALTERNATIVE + ".alternative")); - assertEquals(List.of("created1", "created2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DCTERMS_CREATED + ".created")); - assertEquals(List.of("hasPart1", "hasPart2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DCTERMS_HAS_PART + ".hasPart")); - assertEquals(List.of("isPartOf1", "isPartOf2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DCTERMS_IS_PART_OF + ".isPartOf")); - assertEquals(List.of("issued1", "issued2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DCTERMS_ISSUED + ".issued")); - assertEquals(List.of("medium1", "medium2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DCTERMS_MEDIUM + ".medium")); - assertEquals(List.of("provenance1", "provenance2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DCTERMS_PROVENANCE + ".provenance")); - assertEquals(List.of("spatial1", "spatial2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DCTERMS_SPATIAL + ".spatial")); - assertEquals(List.of("temp1", "temp2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_DCTERMS_TEMPORAL + ".temporal")); - assertEquals(List.of("year1", "year2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_EDM_YEAR + ".year")); - assertEquals(List.of("hasMet1", "hasMet2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_EDM_HAS_MET + ".hasMet")); - assertEquals(List.of("isRelatedTo1", "isRelatedTo2"), solrInputDocument.getFieldValues(EdmLabel.PROXY_EDM_ISRELATEDTO + ".isRelatedTo")); - } - - private void assertKeys(SolrInputDocument solrInputDocument) { - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_EDM_CURRENT_LOCATION + ".location")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_CONTRIBUTOR + ".contributor")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_COVERAGE + ".coverage")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_CREATOR + ".creator")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_DATE + ".date")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_DESCRIPTION + ".description")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_FORMAT + ".format")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_IDENTIFIER + ".identifier")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_LANGUAGE + ".language")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_PUBLISHER + ".publisher")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_RIGHTS + ".rights")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_SOURCE + ".source")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_SUBJECT + ".subject")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_TITLE + ".title")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DC_TYPE + ".type")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DCTERMS_ALTERNATIVE + ".alternative")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DCTERMS_CREATED + ".created")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DCTERMS_HAS_PART + ".hasPart")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DCTERMS_IS_PART_OF + ".isPartOf")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DCTERMS_ISSUED + ".issued")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DCTERMS_MEDIUM + ".medium")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DCTERMS_PROVENANCE + ".provenance")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DCTERMS_SPATIAL + ".spatial")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_DCTERMS_TEMPORAL + ".temporal")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_EDM_YEAR + ".year")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_EDM_HAS_MET + ".hasMet")); - assertTrue(solrInputDocument.containsKey(EdmLabel.PROXY_EDM_ISRELATEDTO + ".isRelatedTo")); + private void assertDocumentContent(ProxyImpl expectedProxy, SolrInputDocument actualSolrInputDocument) { + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_EDM_CURRENT_LOCATION, expectedProxy.getEdmCurrentLocation()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DC_CONTRIBUTOR, expectedProxy.getDcContributor()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DC_COVERAGE, expectedProxy.getDcCoverage()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DC_CREATOR, expectedProxy.getDcCreator()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DC_DATE, expectedProxy.getDcDate()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DC_DESCRIPTION, expectedProxy.getDcDescription()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DC_FORMAT, expectedProxy.getDcFormat()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DC_IDENTIFIER, expectedProxy.getDcIdentifier()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DC_LANGUAGE, expectedProxy.getDcLanguage()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DC_PUBLISHER, expectedProxy.getDcPublisher()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DC_RIGHTS, expectedProxy.getDcRights()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DC_SOURCE, expectedProxy.getDcSource()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DC_SUBJECT, expectedProxy.getDcSubject()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DC_TITLE, expectedProxy.getDcTitle()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DC_TYPE, expectedProxy.getDcType()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DCTERMS_ALTERNATIVE, expectedProxy.getDctermsAlternative()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DCTERMS_CREATED, expectedProxy.getDctermsCreated()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DCTERMS_HAS_PART, expectedProxy.getDctermsHasPart()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DCTERMS_IS_PART_OF, expectedProxy.getDctermsIsPartOf()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DCTERMS_ISSUED, expectedProxy.getDctermsIssued()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DCTERMS_MEDIUM, expectedProxy.getDctermsMedium()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DCTERMS_PROVENANCE, expectedProxy.getDctermsProvenance()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DCTERMS_SPATIAL, expectedProxy.getDctermsSpatial()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_DCTERMS_TEMPORAL, expectedProxy.getDctermsTemporal()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_EDM_YEAR, expectedProxy.getYear()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_EDM_HAS_MET, expectedProxy.getEdmHasMet()); + verifyMap(actualSolrInputDocument, EdmLabel.PROXY_EDM_ISRELATEDTO, expectedProxy.getEdmIsRelatedTo()); } private ProxyImpl getTestProxy() { diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/QualityAnnotationSolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/QualityAnnotationSolrCreatorTest.java index 78ea2b9eb..acabbfa10 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/QualityAnnotationSolrCreatorTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/QualityAnnotationSolrCreatorTest.java @@ -11,6 +11,9 @@ import org.apache.solr.common.SolrInputField; import org.junit.jupiter.api.Test; +/** + * Unit test for {@link QualityAnnotationSolrCreator} class + */ class QualityAnnotationSolrCreatorTest { @Test @@ -24,6 +27,6 @@ void addToDocumentTest() { final SolrInputField solrInputField = document.get(metadataTierC.getEdmLabel().toString()); assertNotNull(solrInputField); assertEquals(metadataTierC.getEdmLabel().toString(), solrInputField.getName()); - assertEquals(metadataTierC.getTier().toString(), ((Collection)solrInputField.getValue()).iterator().next()); + assertEquals(metadataTierC.getTier().toString(), ((Collection) solrInputField.getValue()).iterator().next()); } } diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/SolrPropertyUtilsTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/SolrPropertyUtilsTest.java index 9dd60a346..1382fd1ca 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/SolrPropertyUtilsTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/SolrPropertyUtilsTest.java @@ -22,6 +22,9 @@ import org.apache.solr.common.SolrInputDocument; import org.junit.jupiter.api.Test; +/** + * Unit test {@link SolrPropertyUtils} class + */ class SolrPropertyUtilsTest { @Test @@ -43,10 +46,10 @@ void testAddValues() { assertEquals(2, document.getFieldValues(EdmLabel.PL_WGS84_POS_LAT.toString()).size()); assertTrue(document.getFieldValues(EdmLabel.PL_WGS84_POS_LAT.toString()).contains("A")); assertTrue(document.getFieldValues(EdmLabel.PL_WGS84_POS_LAT.toString()) - .contains(Float.valueOf("0.0"))); + .contains(Float.valueOf("0.0"))); // Add multiple values to second field - SolrPropertyUtils.addValues(document, EdmLabel.PL_WGS84_POS_LONG, new String[] {"C", "D"}); + SolrPropertyUtils.addValues(document, EdmLabel.PL_WGS84_POS_LONG, new String[]{"C", "D"}); assertEquals(2, document.getFieldNames().size()); assertEquals(2, document.getFieldValues(EdmLabel.PL_WGS84_POS_LONG.toString()).size()); assertTrue(document.getFieldValues(EdmLabel.PL_WGS84_POS_LONG.toString()).contains("C")); diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/TimespanSolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/TimespanSolrCreatorTest.java index 62d07200c..1556a1a06 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/TimespanSolrCreatorTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/TimespanSolrCreatorTest.java @@ -1,5 +1,6 @@ package eu.europeana.indexing.solr.property; +import static eu.europeana.indexing.utils.TestUtils.verifyMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; @@ -38,17 +39,12 @@ void addToDocument_withTimeSpan_PrefValue_AltValue_And_OwlSameAs() { timespanSolrCreator.addToDocument(solrInputDocument, timespan); - assertTrue(solrInputDocument.containsKey(EdmLabel.EDM_TIMESPAN.toString()) && - solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref") && - solrInputDocument.containsKey(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt") && - solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL.toString())); + assertTrue(solrInputDocument.containsKey(EdmLabel.EDM_TIMESPAN.toString()) && solrInputDocument.containsKey( + EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref") && solrInputDocument.containsKey(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt") + && solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL.toString())); assertEquals("timespan", solrInputDocument.getFieldValue(EdmLabel.EDM_TIMESPAN.toString())); - assertEquals(List.of("prefValue1", "prefValue2"), - solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref")); - assertEquals(List.of("altValue1", "altValue2"), - solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt")); - assertEquals(List.of("value1", "value2"), - solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL.toString())); + verifyMap(solrInputDocument, EdmLabel.TS_SKOS_PREF_LABEL, timespan.getPrefLabel()); + verifyMap(solrInputDocument, EdmLabel.TS_SKOS_ALT_LABEL, timespan.getAltLabel()); assertEquals(4, solrInputDocument.size()); } @@ -61,16 +57,12 @@ void addToDocument_withoutTimeSpan() { timespanSolrCreator.addToDocument(solrInputDocument, timespan); assertFalse(solrInputDocument.containsKey(EdmLabel.EDM_TIMESPAN.toString())); - assertTrue(solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref") && - solrInputDocument.containsKey(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt") && - solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL.toString())); + assertTrue(solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref") && solrInputDocument.containsKey( + EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt") && solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL.toString())); assertNull(solrInputDocument.getFieldValue(EdmLabel.EDM_TIMESPAN.toString())); - assertEquals(List.of("prefValue1", "prefValue2"), - solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref")); - assertEquals(List.of("altValue1", "altValue2"), - solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt")); - assertEquals(List.of("value1", "value2"), - solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL.toString())); + verifyMap(solrInputDocument, EdmLabel.TS_SKOS_PREF_LABEL, timespan.getPrefLabel()); + verifyMap(solrInputDocument, EdmLabel.TS_SKOS_ALT_LABEL, timespan.getAltLabel()); + assertEquals(List.of("value1", "value2"), solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL.toString())); assertEquals(3, solrInputDocument.size()); } @@ -82,16 +74,13 @@ void addToDocument_without_PrefValue() { timespanSolrCreator.addToDocument(solrInputDocument, timespan); - assertTrue(solrInputDocument.containsKey(EdmLabel.EDM_TIMESPAN.toString()) && - solrInputDocument.containsKey(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt") && - solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL.toString())); + assertTrue(solrInputDocument.containsKey(EdmLabel.EDM_TIMESPAN.toString()) && solrInputDocument.containsKey( + EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt") && solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL.toString())); assertFalse(solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref")); assertEquals("timespan", solrInputDocument.getFieldValue(EdmLabel.EDM_TIMESPAN.toString())); assertNull(solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref")); - assertEquals(List.of("altValue1", "altValue2"), - solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt")); - assertEquals(List.of("value1", "value2"), - solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL.toString())); + verifyMap(solrInputDocument, EdmLabel.TS_SKOS_ALT_LABEL, timespan.getAltLabel()); + assertEquals(List.of("value1", "value2"), solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL.toString())); assertEquals(3, solrInputDocument.size()); } @@ -103,16 +92,13 @@ void addToDocument_without_AltValue() { timespanSolrCreator.addToDocument(solrInputDocument, timespan); - assertTrue(solrInputDocument.containsKey(EdmLabel.EDM_TIMESPAN.toString()) && - solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref") && - solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL.toString())); + assertTrue(solrInputDocument.containsKey(EdmLabel.EDM_TIMESPAN.toString()) && solrInputDocument.containsKey( + EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref") && solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL.toString())); assertFalse(solrInputDocument.containsKey(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt")); assertEquals("timespan", solrInputDocument.getFieldValue(EdmLabel.EDM_TIMESPAN.toString())); - assertEquals(List.of("prefValue1", "prefValue2"), - solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref")); + verifyMap(solrInputDocument, EdmLabel.TS_SKOS_PREF_LABEL, timespan.getPrefLabel()); assertNull(solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt")); - assertEquals(List.of("value1", "value2"), - solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL.toString())); + assertEquals(List.of("value1", "value2"), solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL.toString())); assertEquals(3, solrInputDocument.size()); } @@ -124,15 +110,12 @@ void addToDocument_without_OwlSameAs() { timespanSolrCreator.addToDocument(solrInputDocument, timespan); - assertTrue(solrInputDocument.containsKey(EdmLabel.EDM_TIMESPAN.toString()) && - solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref") && - solrInputDocument.containsKey(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt")); + assertTrue(solrInputDocument.containsKey(EdmLabel.EDM_TIMESPAN.toString()) && solrInputDocument.containsKey( + EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref") && solrInputDocument.containsKey(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt")); assertFalse(solrInputDocument.containsKey(EdmLabel.TS_SKOS_PREF_LABEL.toString())); assertEquals("timespan", solrInputDocument.getFieldValue(EdmLabel.EDM_TIMESPAN.toString())); - assertEquals(List.of("prefValue1", "prefValue2"), - solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL + ".keyPref")); - assertEquals(List.of("altValue1", "altValue2"), - solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_ALT_LABEL + ".keyAlt")); + verifyMap(solrInputDocument, EdmLabel.TS_SKOS_PREF_LABEL, timespan.getPrefLabel()); + verifyMap(solrInputDocument, EdmLabel.TS_SKOS_ALT_LABEL, timespan.getAltLabel()); assertNull(solrInputDocument.getFieldValues(EdmLabel.TS_SKOS_PREF_LABEL.toString())); assertEquals(3, solrInputDocument.size()); } diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/WebResourceSolrCreatorTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/WebResourceSolrCreatorTest.java index 4dca62ead..3f6473ceb 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/WebResourceSolrCreatorTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/WebResourceSolrCreatorTest.java @@ -1,5 +1,7 @@ package eu.europeana.indexing.solr.property; +import static eu.europeana.indexing.utils.TestUtils.verifyCollection; +import static eu.europeana.indexing.utils.TestUtils.verifyMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -14,11 +16,13 @@ import org.apache.solr.common.SolrInputDocument; import org.junit.jupiter.api.Test; +/** + * Unit test for {@link WebResourceSolrCreator} class + */ class WebResourceSolrCreatorTest { @Test void addToDocument() { - // given License license1 = new LicenseImpl(); license1.setAbout("license1About"); @@ -56,21 +60,4 @@ void addToDocument() { verifyCollection(solrInputDocument, EdmLabel.WR_CC_ODRL_INHERITED_FROM, List.of(license1.getOdrlInheritFrom(), license2.getOdrlInheritFrom())); } - - void verifyMap(SolrInputDocument solrInputDocument, EdmLabel edmLabel, Map> map) { - map.forEach((key, value) -> assertTrue(solrInputDocument.getFieldValues(computeSolrField(edmLabel, key)) - .containsAll(value))); - } - - private String computeSolrField(EdmLabel label, String value) { - return label.toString() + "." + value; - } - - void verifyCollection(SolrInputDocument solrInputDocument, EdmLabel edmLabel, Collection collection) { - final Collection fieldValues = solrInputDocument.getFieldValues(edmLabel.toString()); - assertTrue(fieldValues.containsAll(collection)); - assertEquals(collection.size(), fieldValues.size()); - } - - } diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/utils/TestUtils.java b/metis-indexing/src/test/java/eu/europeana/indexing/utils/TestUtils.java index aafa19c31..7fbab99f8 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/utils/TestUtils.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/utils/TestUtils.java @@ -1,10 +1,18 @@ package eu.europeana.indexing.utils; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import eu.europeana.indexing.solr.EdmLabel; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.util.Collection; +import java.util.List; +import java.util.Map; import java.util.stream.Collectors; +import org.apache.solr.common.SolrInputDocument; public class TestUtils { @@ -14,7 +22,20 @@ public static String readFileToString(String file) throws IOException { if (inputStream == null) { throw new IOException("Failed reading file " + file); } - return new BufferedReader(new InputStreamReader(inputStream)).lines() - .collect(Collectors.joining("\n")); + return new BufferedReader(new InputStreamReader(inputStream)).lines().collect(Collectors.joining("\n")); + } + + public static void verifyMap(SolrInputDocument solrInputDocument, EdmLabel edmLabel, Map> map) { + map.forEach((key, value) -> assertTrue(solrInputDocument.getFieldValues(computeSolrField(edmLabel, key)).containsAll(value))); + } + + private static String computeSolrField(EdmLabel label, String value) { + return label.toString() + "." + value; + } + + public static void verifyCollection(SolrInputDocument solrInputDocument, EdmLabel edmLabel, Collection collection) { + final Collection fieldValues = solrInputDocument.getFieldValues(edmLabel.toString()); + assertTrue(fieldValues.containsAll(collection)); + assertEquals(collection.size(), fieldValues.size()); } } From a9ba8d2694be00d2bc3d0209a82c94bcc1503650 Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Wed, 22 Jun 2022 12:08:19 +0200 Subject: [PATCH 71/73] MET-4622 Support for tier recalculation based on types (#562) It is a recalculation --- .../eu/europeana/indexing/IndexerImpl.java | 31 ++++++----- .../eu/europeana/indexing/IndexerPool.java | 20 +++---- .../indexing/IndexingProperties.java | 55 ++++++++++++++----- 3 files changed, 66 insertions(+), 40 deletions(-) diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/IndexerImpl.java b/metis-indexing/src/main/java/eu/europeana/indexing/IndexerImpl.java index 4c2d29b11..d526bd62c 100644 --- a/metis-indexing/src/main/java/eu/europeana/indexing/IndexerImpl.java +++ b/metis-indexing/src/main/java/eu/europeana/indexing/IndexerImpl.java @@ -73,22 +73,22 @@ public void index(List records, IndexingProperties indexingProperties) LOGGER.info("Parsing {} records...", records.size()); final StringToFullBeanConverter stringToRdfConverter = stringToRdfConverterSupplier.get(); final List wrappedRecords = new ArrayList<>(records.size()); - for (String record : records) { - wrappedRecords.add(stringToRdfConverter.convertStringToRdf(record)); + for (String stringRdfRecord : records) { + wrappedRecords.add(stringToRdfConverter.convertStringToRdf(stringRdfRecord)); } indexRecords(wrappedRecords, indexingProperties); } @Override - public void index(InputStream record, IndexingProperties indexingProperties) + public void index(InputStream recordInputStream, IndexingProperties indexingProperties) throws IndexingException { final StringToFullBeanConverter stringToRdfConverter = stringToRdfConverterSupplier.get(); - indexRdf(stringToRdfConverter.convertToRdf(record), indexingProperties); + indexRdf(stringToRdfConverter.convertToRdf(recordInputStream), indexingProperties); } @Override - public void indexRdf(RDF record, IndexingProperties indexingProperties) throws IndexingException { - indexRdfs(List.of(record), indexingProperties); + public void indexRdf(RDF rdfRecord, IndexingProperties indexingProperties) throws IndexingException { + indexRdfs(List.of(rdfRecord), indexingProperties); } private void indexRecords(List records, IndexingProperties properties) @@ -101,13 +101,13 @@ private void indexRecords(List records, IndexingProperties properties) final FullBeanPublisher publisher = connectionProvider.getFullBeanPublisher(properties.isPreserveUpdateAndCreateTimesFromRdf()); - for (RDF record : records) { - preprocessRecord(record, properties.isPerformTierCalculation()); + for (RDF rdfRecord : records) { + preprocessRecord(rdfRecord, properties); if (properties.isPerformRedirects()) { - publisher.publishWithRedirects(new RdfWrapper(record), properties.getRecordDate(), + publisher.publishWithRedirects(new RdfWrapper(rdfRecord), properties.getRecordDate(), properties.getDatasetIdsForRedirection()); } else { - publisher.publish(new RdfWrapper(record), properties.getRecordDate(), + publisher.publish(new RdfWrapper(rdfRecord), properties.getRecordDate(), properties.getDatasetIdsForRedirection()); } } @@ -116,16 +116,17 @@ private void indexRecords(List records, IndexingProperties properties) } @Override - public void index(String record, IndexingProperties indexingProperties) throws IndexingException { - index(List.of(record), indexingProperties); + public void index(String stringRdfRecord, IndexingProperties indexingProperties) throws IndexingException { + index(List.of(stringRdfRecord), indexingProperties); } - private void preprocessRecord(RDF rdf, boolean performTierCalculation) + private void preprocessRecord(RDF rdf, IndexingProperties properties) throws IndexingException { // Perform the tier classification - if (performTierCalculation) { - final RdfWrapper rdfWrapper = new RdfWrapper(rdf); + final RdfWrapper rdfWrapper = new RdfWrapper(rdf); + if (properties.isPerformTierCalculation() && properties.getTypesEnabledForTierCalculation() + .contains(rdfWrapper.getEdmType())) { RdfTierUtils.setTier(rdf, mediaClassifier.classify(rdfWrapper).getTier()); RdfTierUtils.setTier(rdf, metadataClassifier.classify(rdfWrapper).getTier()); } diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/IndexerPool.java b/metis-indexing/src/main/java/eu/europeana/indexing/IndexerPool.java index 8c4058ccc..d56f1b19d 100644 --- a/metis-indexing/src/main/java/eu/europeana/indexing/IndexerPool.java +++ b/metis-indexing/src/main/java/eu/europeana/indexing/IndexerPool.java @@ -1,9 +1,9 @@ package eu.europeana.indexing; -import eu.europeana.metis.schema.jibx.RDF; import eu.europeana.indexing.exception.IndexerRelatedIndexingException; import eu.europeana.indexing.exception.IndexingException; import eu.europeana.indexing.exception.SetupRelatedIndexingException; +import eu.europeana.metis.schema.jibx.RDF; import java.io.Closeable; import java.time.Duration; import org.apache.commons.pool2.BasePooledObjectFactory; @@ -84,12 +84,12 @@ private static long convertSecsToMillis(long seconds) { * This method indexes a single record, using a free indexer in the pool. *

    * - * @param record The record to index (can be parsed to RDF). + * @param stringRdfRecord The record to index (can be parsed to RDF). * @param indexingProperties The properties of this indexing operation. * @throws IndexingException In case a problem occurred during indexing. indexer. */ - public void index(String record, IndexingProperties indexingProperties) throws IndexingException { - indexRecord(indexer -> indexer.index(record, indexingProperties)); + public void index(String stringRdfRecord, IndexingProperties indexingProperties) throws IndexingException { + indexRecord(indexer -> indexer.index(stringRdfRecord, indexingProperties)); } /** @@ -97,22 +97,22 @@ public void index(String record, IndexingProperties indexingProperties) throws I * This method indexes a single record, using a free indexer in the pool. *

    * - * @param record The record to index. + * @param stringRdfRecord The record to index. * @param indexingProperties The properties of this indexing operation. * @throws IndexingException In case a problem occurred during indexing. indexer. */ - public void indexRdf(RDF record, IndexingProperties indexingProperties) throws IndexingException { - indexRecord(indexer -> indexer.indexRdf(record, indexingProperties)); + public void indexRdf(RDF stringRdfRecord, IndexingProperties indexingProperties) throws IndexingException { + indexRecord(indexer -> indexer.indexRdf(stringRdfRecord, indexingProperties)); } /** * This method removes a single record, using a free indexer in the pool * - * @param record The record to be removed + * @param stringRdfRecord The record to be removed * @throws IndexingException In case something went wrong. */ - public void remove(String record) throws IndexingException { - indexRecord(indexer -> indexer.remove(record)); + public void remove(String stringRdfRecord) throws IndexingException { + indexRecord(indexer -> indexer.remove(stringRdfRecord)); } private void indexRecord(IndexTask indexTask) throws IndexingException { diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/IndexingProperties.java b/metis-indexing/src/main/java/eu/europeana/indexing/IndexingProperties.java index 0d26af2bf..d915748d2 100644 --- a/metis-indexing/src/main/java/eu/europeana/indexing/IndexingProperties.java +++ b/metis-indexing/src/main/java/eu/europeana/indexing/IndexingProperties.java @@ -1,14 +1,17 @@ package eu.europeana.indexing; +import eu.europeana.metis.schema.jibx.EdmType; import java.util.ArrayList; import java.util.Collections; import java.util.Date; +import java.util.EnumSet; import java.util.List; import java.util.Optional; +import java.util.Set; /** - * This class contains all properties that affect the behavior of the indexing functionality, it is - * the input class for all indexing methods in {@link Indexer} (and {@link IndexerPool}). + * This class contains all properties that affect the behavior of the indexing functionality, it is the input class for all + * indexing methods in {@link Indexer} (and {@link IndexerPool}). */ public class IndexingProperties { @@ -17,29 +20,45 @@ public class IndexingProperties { private final List datasetIdsForRedirection; private final boolean performRedirects; private final boolean performTierCalculation; + private final EnumSet typesEnabledForTierCalculation; /** * Constructor. * - * @param recordDate The date that would represent the created/updated date of a record. Can be - * null. - * @param preserveUpdateAndCreateTimesFromRdf This determines whether this indexer should use the - * updated and created times from the incoming RDFs, or whether it computes its own. - * @param datasetIdsForRedirection The dataset ids that their records need to be redirected. Can - * be null. + * @param recordDate The date that would represent the created/updated date of a record. Can be null. + * @param preserveUpdateAndCreateTimesFromRdf This determines whether this indexer should use the updated and created times from + * the incoming RDFs, or whether it computes its own. + * @param datasetIdsForRedirection The dataset ids that their records need to be redirected. Can be null. * @param performRedirects flag that indicates whether redirect should be performed. - * @param performTierCalculation flag that indicates whether tier calculation should be - * performed. + * @param performTierCalculation flag that indicates whether tier calculation should be performed. + */ + public IndexingProperties(Date recordDate, boolean preserveUpdateAndCreateTimesFromRdf, List datasetIdsForRedirection, + boolean performRedirects, boolean performTierCalculation) { + this(recordDate, preserveUpdateAndCreateTimesFromRdf, datasetIdsForRedirection, performRedirects, performTierCalculation, + EnumSet.allOf(EdmType.class)); + } + + /** + * Constructor allowing specific types for tier re-calculation. + * + * @param recordDate The date that would represent the created/updated date of a record. Can be null. + * @param preserveUpdateAndCreateTimesFromRdf This determines whether this indexer should use the updated and created times from + * the incoming RDFs, or whether it computes its own. + * @param datasetIdsForRedirection The dataset ids that their records need to be redirected. Can be null. + * @param performRedirects flag that indicates whether redirect should be performed. + * @param performTierCalculation flag that indicates whether tier calculation should be performed. + * @param typesEnabledForTierCalculation the types enabled for tier calculation if enabled. */ public IndexingProperties(Date recordDate, boolean preserveUpdateAndCreateTimesFromRdf, - List datasetIdsForRedirection, boolean performRedirects, - boolean performTierCalculation) { + List datasetIdsForRedirection, boolean performRedirects, + boolean performTierCalculation, Set typesEnabledForTierCalculation) { this.recordDate = recordDate == null ? null : new Date(recordDate.getTime()); this.preserveUpdateAndCreateTimesFromRdf = preserveUpdateAndCreateTimesFromRdf; this.datasetIdsForRedirection = Optional.ofNullable(datasetIdsForRedirection) - .>map(ArrayList::new).orElseGet(Collections::emptyList); + .>map(ArrayList::new).orElseGet(Collections::emptyList); this.performRedirects = performRedirects; this.performTierCalculation = performTierCalculation; + this.typesEnabledForTierCalculation = EnumSet.copyOf(typesEnabledForTierCalculation); } /** @@ -50,8 +69,7 @@ public Date getRecordDate() { } /** - * @return Whether this indexer should use the updated and created times from the incoming RDFs, - * or whether it computes its own. + * @return Whether this indexer should use the updated and created times from the incoming RDFs, or whether it computes its own. */ public boolean isPreserveUpdateAndCreateTimesFromRdf() { return preserveUpdateAndCreateTimesFromRdf; @@ -77,4 +95,11 @@ public boolean isPerformRedirects() { public boolean isPerformTierCalculation() { return performTierCalculation; } + + /** + * @return the edm types for tier re-calculation if enabled + */ + public Set getTypesEnabledForTierCalculation() { + return Collections.unmodifiableSet(typesEnabledForTierCalculation); + } } From 138eac16b68cdcb04da39dce80d2830155ca49dc Mon Sep 17 00:00:00 2001 From: Simon Tzanakis Date: Fri, 1 Jul 2022 10:04:44 +0200 Subject: [PATCH 72/73] bug/MET-4632 Fix place references coordinates not being validated (#563) MET-4632 Fix place references coordinates not validated --- .../metis/utils/GeoUriWGS84ParserTest.java | 4 +++ .../solr/property/FullBeanSolrProperties.java | 33 ++++++++++--------- .../property/FullBeanSolrPropertiesTest.java | 13 +++++--- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/GeoUriWGS84ParserTest.java b/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/GeoUriWGS84ParserTest.java index c060c26bc..048735b52 100644 --- a/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/GeoUriWGS84ParserTest.java +++ b/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/GeoUriWGS84ParserTest.java @@ -44,6 +44,10 @@ void parse_invalid() { assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:test,-122.399677;crs=wgs84")); //Invalid range coordinates assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:-100,200;crs=wgs84")); + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:578991.875,578991.875")); + assertThrows(BadContentException.class, () -> GeoUriWGS84Parser.parse("geo:-90.123456,100")); + + } @Test diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/solr/property/FullBeanSolrProperties.java b/metis-indexing/src/main/java/eu/europeana/indexing/solr/property/FullBeanSolrProperties.java index 497c4c2df..8c5e82fcc 100644 --- a/metis-indexing/src/main/java/eu/europeana/indexing/solr/property/FullBeanSolrProperties.java +++ b/metis-indexing/src/main/java/eu/europeana/indexing/solr/property/FullBeanSolrProperties.java @@ -71,13 +71,11 @@ private void setGeospatialFields(SolrInputDocument document, FullBeanImpl fullBe coverageLocationStrings.addAll(getCoverageLocationStrings(proxy)); }); - final Set currentLocationPoints = new HashSet<>( - getReferencedPlacesLocationPoints(placesAboutMap, currentLocationStrings)); - currentLocationPoints.addAll(getWGS84LocationPoints(currentLocationStrings)); + currentLocationStrings.addAll(getReferencedPlacesLocationStrings(placesAboutMap, currentLocationStrings)); + final Set currentLocationPoints = new HashSet<>(getWGS84LocationPoints(currentLocationStrings)); - final Set coverageLocationPoints = new HashSet<>( - getReferencedPlacesLocationPoints(placesAboutMap, coverageLocationStrings)); - coverageLocationPoints.addAll(getWGS84LocationPoints(coverageLocationStrings)); + coverageLocationStrings.addAll(getReferencedPlacesLocationStrings(placesAboutMap, coverageLocationStrings)); + final Set coverageLocationPoints = new HashSet<>(getWGS84LocationPoints(coverageLocationStrings)); SolrPropertyUtils.addValues(document, CURRENT_LOCATION_WGS, currentLocationPoints.stream().map(Object::toString).toArray(String[]::new)); @@ -93,10 +91,20 @@ private void setGeospatialFields(SolrInputDocument document, FullBeanImpl fullBe } - private Set getReferencedPlacesLocationPoints(Map placesAboutMap, - Set locationStrings) { + private Set getReferencedPlacesLocationStrings(Map placesAboutMap, Set locationStrings) { return locationStrings.stream().map(placesAboutMap::get).filter(Objects::nonNull) - .map(this::getPlaceLocationPoint).filter(Objects::nonNull).collect(Collectors.toSet()); + .filter(place -> place.getLatitude() != null) + .filter(place -> place.getLongitude() != null) + .map(place -> { + //Create string value -> geo:lat,lon,alt + final StringBuilder stringBuilder = new StringBuilder("geo:"); + stringBuilder.append(place.getLatitude()).append(",").append(place.getLongitude()); + if (place.getAltitude() != null) { + stringBuilder.append(",").append(place.getAltitude()); + } + return stringBuilder.toString(); + }) + .collect(Collectors.toSet()); } private Set getWGS84LocationPoints(Set locationStrings) { @@ -126,13 +134,6 @@ private Set getCoverageLocationStrings(ProxyImpl proxy) { return coverageLocations; } - private LocationPoint getPlaceLocationPoint(PlaceImpl place) { - if (place.getLatitude() != null && place.getLongitude() != null) { - return new LocationPoint(place.getLatitude().doubleValue(), place.getLongitude().doubleValue()); - } - return null; - } - private GeoCoordinates getValidGeoCoordinates(String s) { try { return GeoUriWGS84Parser.parse(s); diff --git a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/FullBeanSolrPropertiesTest.java b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/FullBeanSolrPropertiesTest.java index 6fc571a05..6ab36ad8b 100644 --- a/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/FullBeanSolrPropertiesTest.java +++ b/metis-indexing/src/test/java/eu/europeana/indexing/solr/property/FullBeanSolrPropertiesTest.java @@ -48,16 +48,21 @@ void fullBeanSolrSetPropertiesTest() { place.setLatitude(10.0f); place.setLongitude(33.0f); place.setAltitude(7.0f); - fullBean.setPlaces(List.of(place)); + + Place place2 = new PlaceImpl(); + place2.setAbout("Invalid range"); + place2.setLatitude(578991.875f); + place2.setLongitude(578991.875f); + fullBean.setPlaces(List.of(place, place2)); // proxy setup ProxyImpl proxy = new ProxyImpl(); proxy.setAbout("About Proxy"); proxy.setEuropeanaProxy(true); proxy.setEdmType(EdmType.TEXT.name()); - proxy.setEdmCurrentLocation(Map.of(EdmLabel.PROXY_EDM_CURRENT_LOCATION.name(), List.of(place.getAbout()))); - proxy.setDctermsSpatial(Map.of(EdmLabel.PROXY_DCTERMS_SPATIAL.name(), List.of(place.getAbout()))); - proxy.setDcCoverage(Map.of(EdmLabel.PROXY_DC_COVERAGE.name(), List.of(place.getAbout()))); + proxy.setEdmCurrentLocation(Map.of("def", List.of(place.getAbout(), place2.getAbout()))); + proxy.setDctermsSpatial(Map.of("def", List.of(place.getAbout()))); + proxy.setDcCoverage(Map.of("def", List.of(place.getAbout()))); fullBean.setProxies(List.of(proxy)); fullBean.setEuropeanaCompleteness(0); From c789501a56a85cbb9be9b961c1674d46cc3a9d47 Mon Sep 17 00:00:00 2001 From: Jorge Ortiz Date: Fri, 8 Jul 2022 10:57:46 +0200 Subject: [PATCH 73/73] [maven-release-plugin] prepare release v7 --- metis-authentication/metis-authentication-common/pom.xml | 2 +- metis-authentication/metis-authentication-rest-client/pom.xml | 2 +- metis-authentication/metis-authentication-rest/pom.xml | 2 +- metis-authentication/metis-authentication-service/pom.xml | 2 +- metis-authentication/pom.xml | 2 +- metis-common/metis-common-mongo/pom.xml | 2 +- metis-common/metis-common-network/pom.xml | 2 +- metis-common/metis-common-solr/pom.xml | 2 +- metis-common/metis-common-utils/pom.xml | 2 +- metis-common/metis-common-zoho/pom.xml | 2 +- metis-common/pom.xml | 2 +- metis-core/metis-core-common/pom.xml | 2 +- metis-core/metis-core-rest/pom.xml | 2 +- metis-core/metis-core-service/pom.xml | 2 +- metis-core/pom.xml | 2 +- metis-dereference/metis-dereference-common/pom.xml | 2 +- metis-dereference/metis-dereference-import/pom.xml | 2 +- metis-dereference/metis-dereference-rest/pom.xml | 2 +- metis-dereference/metis-dereference-service/pom.xml | 2 +- metis-dereference/pom.xml | 2 +- metis-enrichment/metis-enrichment-client/pom.xml | 2 +- metis-enrichment/metis-enrichment-common/pom.xml | 2 +- metis-enrichment/metis-enrichment-rest/pom.xml | 2 +- metis-enrichment/metis-enrichment-service/pom.xml | 2 +- metis-enrichment/pom.xml | 2 +- metis-harvesting/pom.xml | 2 +- metis-indexing/pom.xml | 2 +- metis-media-service/pom.xml | 2 +- metis-normalization/pom.xml | 2 +- metis-pattern-analysis/pom.xml | 2 +- metis-repository/pom.xml | 2 +- metis-transformation/metis-transformation-service/pom.xml | 2 +- metis-transformation/pom.xml | 2 +- metis-validation/metis-validation-client/pom.xml | 2 +- metis-validation/metis-validation-common/pom.xml | 2 +- metis-validation/metis-validation-rest/pom.xml | 2 +- metis-validation/metis-validation-service/pom.xml | 2 +- metis-validation/pom.xml | 2 +- pom.xml | 4 ++-- 39 files changed, 40 insertions(+), 40 deletions(-) diff --git a/metis-authentication/metis-authentication-common/pom.xml b/metis-authentication/metis-authentication-common/pom.xml index 478002639..30795e225 100644 --- a/metis-authentication/metis-authentication-common/pom.xml +++ b/metis-authentication/metis-authentication-common/pom.xml @@ -4,7 +4,7 @@ metis-authentication eu.europeana.metis - 7-SNAPSHOT + 7 metis-authentication-common diff --git a/metis-authentication/metis-authentication-rest-client/pom.xml b/metis-authentication/metis-authentication-rest-client/pom.xml index bc73c59bc..472ffd320 100644 --- a/metis-authentication/metis-authentication-rest-client/pom.xml +++ b/metis-authentication/metis-authentication-rest-client/pom.xml @@ -4,7 +4,7 @@ metis-authentication eu.europeana.metis - 7-SNAPSHOT + 7 metis-authentication-rest-client diff --git a/metis-authentication/metis-authentication-rest/pom.xml b/metis-authentication/metis-authentication-rest/pom.xml index eda05fa57..9146f6ce4 100644 --- a/metis-authentication/metis-authentication-rest/pom.xml +++ b/metis-authentication/metis-authentication-rest/pom.xml @@ -4,7 +4,7 @@ metis-authentication eu.europeana.metis - 7-SNAPSHOT + 7 metis-authentication-rest war diff --git a/metis-authentication/metis-authentication-service/pom.xml b/metis-authentication/metis-authentication-service/pom.xml index 59c1fdcba..57030a4f6 100644 --- a/metis-authentication/metis-authentication-service/pom.xml +++ b/metis-authentication/metis-authentication-service/pom.xml @@ -4,7 +4,7 @@ metis-authentication eu.europeana.metis - 7-SNAPSHOT + 7 metis-authentication-service diff --git a/metis-authentication/pom.xml b/metis-authentication/pom.xml index 3c276c648..096a31b2e 100644 --- a/metis-authentication/pom.xml +++ b/metis-authentication/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 7-SNAPSHOT + 7 metis-authentication pom diff --git a/metis-common/metis-common-mongo/pom.xml b/metis-common/metis-common-mongo/pom.xml index 234a79441..b395f5b97 100644 --- a/metis-common/metis-common-mongo/pom.xml +++ b/metis-common/metis-common-mongo/pom.xml @@ -4,7 +4,7 @@ metis-common eu.europeana.metis - 7-SNAPSHOT + 7 metis-common-mongo diff --git a/metis-common/metis-common-network/pom.xml b/metis-common/metis-common-network/pom.xml index 362a47011..276bc1c2e 100644 --- a/metis-common/metis-common-network/pom.xml +++ b/metis-common/metis-common-network/pom.xml @@ -4,7 +4,7 @@ metis-common eu.europeana.metis - 7-SNAPSHOT + 7 metis-common-network diff --git a/metis-common/metis-common-solr/pom.xml b/metis-common/metis-common-solr/pom.xml index 1ac8f2188..54283c869 100644 --- a/metis-common/metis-common-solr/pom.xml +++ b/metis-common/metis-common-solr/pom.xml @@ -4,7 +4,7 @@ metis-common eu.europeana.metis - 7-SNAPSHOT + 7 metis-common-solr diff --git a/metis-common/metis-common-utils/pom.xml b/metis-common/metis-common-utils/pom.xml index 706b1b911..cab5d2ff8 100644 --- a/metis-common/metis-common-utils/pom.xml +++ b/metis-common/metis-common-utils/pom.xml @@ -4,7 +4,7 @@ metis-common eu.europeana.metis - 7-SNAPSHOT + 7 metis-common-utils diff --git a/metis-common/metis-common-zoho/pom.xml b/metis-common/metis-common-zoho/pom.xml index 728ea62c4..ddbf39195 100644 --- a/metis-common/metis-common-zoho/pom.xml +++ b/metis-common/metis-common-zoho/pom.xml @@ -4,7 +4,7 @@ metis-common eu.europeana.metis - 7-SNAPSHOT + 7 metis-common-zoho diff --git a/metis-common/pom.xml b/metis-common/pom.xml index e1059bf25..818e476d0 100644 --- a/metis-common/pom.xml +++ b/metis-common/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 7-SNAPSHOT + 7 metis-common pom diff --git a/metis-core/metis-core-common/pom.xml b/metis-core/metis-core-common/pom.xml index 50760dd35..7cbf13c58 100644 --- a/metis-core/metis-core-common/pom.xml +++ b/metis-core/metis-core-common/pom.xml @@ -4,7 +4,7 @@ metis-core eu.europeana.metis - 7-SNAPSHOT + 7 metis-core-common diff --git a/metis-core/metis-core-rest/pom.xml b/metis-core/metis-core-rest/pom.xml index 0ae469d7e..f27213bf1 100644 --- a/metis-core/metis-core-rest/pom.xml +++ b/metis-core/metis-core-rest/pom.xml @@ -4,7 +4,7 @@ metis-core eu.europeana.metis - 7-SNAPSHOT + 7 metis-core-rest war diff --git a/metis-core/metis-core-service/pom.xml b/metis-core/metis-core-service/pom.xml index 366ae33de..85e4dccf2 100644 --- a/metis-core/metis-core-service/pom.xml +++ b/metis-core/metis-core-service/pom.xml @@ -4,7 +4,7 @@ metis-core eu.europeana.metis - 7-SNAPSHOT + 7 metis-core-service diff --git a/metis-core/pom.xml b/metis-core/pom.xml index 4cc3c771d..44736dfb9 100644 --- a/metis-core/pom.xml +++ b/metis-core/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 7-SNAPSHOT + 7 metis-core pom diff --git a/metis-dereference/metis-dereference-common/pom.xml b/metis-dereference/metis-dereference-common/pom.xml index 1b6c4a6d6..b99c31323 100644 --- a/metis-dereference/metis-dereference-common/pom.xml +++ b/metis-dereference/metis-dereference-common/pom.xml @@ -4,7 +4,7 @@ metis-dereference eu.europeana.metis - 7-SNAPSHOT + 7 metis-dereference-common diff --git a/metis-dereference/metis-dereference-import/pom.xml b/metis-dereference/metis-dereference-import/pom.xml index 5ef621cc5..ecd559383 100644 --- a/metis-dereference/metis-dereference-import/pom.xml +++ b/metis-dereference/metis-dereference-import/pom.xml @@ -3,7 +3,7 @@ metis-dereference eu.europeana.metis - 7-SNAPSHOT + 7 4.0.0 metis-dereference-import diff --git a/metis-dereference/metis-dereference-rest/pom.xml b/metis-dereference/metis-dereference-rest/pom.xml index 4817793b4..0d67b48d9 100644 --- a/metis-dereference/metis-dereference-rest/pom.xml +++ b/metis-dereference/metis-dereference-rest/pom.xml @@ -4,7 +4,7 @@ metis-dereference eu.europeana.metis - 7-SNAPSHOT + 7 metis-dereference-rest war diff --git a/metis-dereference/metis-dereference-service/pom.xml b/metis-dereference/metis-dereference-service/pom.xml index 62bc2d4f6..343232172 100644 --- a/metis-dereference/metis-dereference-service/pom.xml +++ b/metis-dereference/metis-dereference-service/pom.xml @@ -4,7 +4,7 @@ metis-dereference eu.europeana.metis - 7-SNAPSHOT + 7 metis-dereference-service diff --git a/metis-dereference/pom.xml b/metis-dereference/pom.xml index 3e1cc3f38..e339a7ee9 100644 --- a/metis-dereference/pom.xml +++ b/metis-dereference/pom.xml @@ -3,7 +3,7 @@ 4.0.0 eu.europeana.metis - 7-SNAPSHOT + 7 metis-framework diff --git a/metis-enrichment/metis-enrichment-client/pom.xml b/metis-enrichment/metis-enrichment-client/pom.xml index 3edb4850d..fab5664bf 100644 --- a/metis-enrichment/metis-enrichment-client/pom.xml +++ b/metis-enrichment/metis-enrichment-client/pom.xml @@ -4,7 +4,7 @@ metis-enrichment eu.europeana.metis - 7-SNAPSHOT + 7 metis-enrichment-client jar diff --git a/metis-enrichment/metis-enrichment-common/pom.xml b/metis-enrichment/metis-enrichment-common/pom.xml index 7d7479178..fa3be2982 100644 --- a/metis-enrichment/metis-enrichment-common/pom.xml +++ b/metis-enrichment/metis-enrichment-common/pom.xml @@ -4,7 +4,7 @@ metis-enrichment eu.europeana.metis - 7-SNAPSHOT + 7 metis-enrichment-common diff --git a/metis-enrichment/metis-enrichment-rest/pom.xml b/metis-enrichment/metis-enrichment-rest/pom.xml index 10e5b7f1c..cb339d5c0 100644 --- a/metis-enrichment/metis-enrichment-rest/pom.xml +++ b/metis-enrichment/metis-enrichment-rest/pom.xml @@ -4,7 +4,7 @@ metis-enrichment eu.europeana.metis - 7-SNAPSHOT + 7 metis-enrichment-rest war diff --git a/metis-enrichment/metis-enrichment-service/pom.xml b/metis-enrichment/metis-enrichment-service/pom.xml index 93508ac5c..60c468b0c 100644 --- a/metis-enrichment/metis-enrichment-service/pom.xml +++ b/metis-enrichment/metis-enrichment-service/pom.xml @@ -4,7 +4,7 @@ metis-enrichment eu.europeana.metis - 7-SNAPSHOT + 7 metis-enrichment-service jar diff --git a/metis-enrichment/pom.xml b/metis-enrichment/pom.xml index 7dedc1224..5ad937024 100644 --- a/metis-enrichment/pom.xml +++ b/metis-enrichment/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 7-SNAPSHOT + 7 metis-enrichment pom diff --git a/metis-harvesting/pom.xml b/metis-harvesting/pom.xml index 643242ff5..01fcb104d 100644 --- a/metis-harvesting/pom.xml +++ b/metis-harvesting/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 7-SNAPSHOT + 7 metis-harvesting diff --git a/metis-indexing/pom.xml b/metis-indexing/pom.xml index 0e45e7341..976eed1f2 100644 --- a/metis-indexing/pom.xml +++ b/metis-indexing/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 7-SNAPSHOT + 7 metis-indexing diff --git a/metis-media-service/pom.xml b/metis-media-service/pom.xml index 50cb6bc63..19c9774c7 100644 --- a/metis-media-service/pom.xml +++ b/metis-media-service/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 7-SNAPSHOT + 7 metis-media-service diff --git a/metis-normalization/pom.xml b/metis-normalization/pom.xml index d0fe155b5..080a3753f 100644 --- a/metis-normalization/pom.xml +++ b/metis-normalization/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 7-SNAPSHOT + 7 metis-normalization diff --git a/metis-pattern-analysis/pom.xml b/metis-pattern-analysis/pom.xml index 3b9ca23fb..1e45b87b0 100644 --- a/metis-pattern-analysis/pom.xml +++ b/metis-pattern-analysis/pom.xml @@ -3,7 +3,7 @@ metis-framework eu.europeana.metis - 7-SNAPSHOT + 7 4.0.0 metis-pattern-analysis diff --git a/metis-repository/pom.xml b/metis-repository/pom.xml index 0bcb0f6d4..6374802fd 100644 --- a/metis-repository/pom.xml +++ b/metis-repository/pom.xml @@ -3,7 +3,7 @@ metis-framework eu.europeana.metis - 7-SNAPSHOT + 7 4.0.0 metis-repository diff --git a/metis-transformation/metis-transformation-service/pom.xml b/metis-transformation/metis-transformation-service/pom.xml index 6435ce338..8575e4552 100644 --- a/metis-transformation/metis-transformation-service/pom.xml +++ b/metis-transformation/metis-transformation-service/pom.xml @@ -4,7 +4,7 @@ metis-transformation eu.europeana.metis - 7-SNAPSHOT + 7 metis-transformation-service diff --git a/metis-transformation/pom.xml b/metis-transformation/pom.xml index d7faa76dd..06f381a5c 100644 --- a/metis-transformation/pom.xml +++ b/metis-transformation/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 7-SNAPSHOT + 7 metis-transformation pom diff --git a/metis-validation/metis-validation-client/pom.xml b/metis-validation/metis-validation-client/pom.xml index a08e74249..2e09e4337 100644 --- a/metis-validation/metis-validation-client/pom.xml +++ b/metis-validation/metis-validation-client/pom.xml @@ -4,7 +4,7 @@ metis-validation eu.europeana.metis - 7-SNAPSHOT + 7 metis-validation-client diff --git a/metis-validation/metis-validation-common/pom.xml b/metis-validation/metis-validation-common/pom.xml index 0e0e7c860..981054da5 100644 --- a/metis-validation/metis-validation-common/pom.xml +++ b/metis-validation/metis-validation-common/pom.xml @@ -4,7 +4,7 @@ metis-validation eu.europeana.metis - 7-SNAPSHOT + 7 metis-validation-common diff --git a/metis-validation/metis-validation-rest/pom.xml b/metis-validation/metis-validation-rest/pom.xml index f808edc03..d704df98f 100644 --- a/metis-validation/metis-validation-rest/pom.xml +++ b/metis-validation/metis-validation-rest/pom.xml @@ -4,7 +4,7 @@ metis-validation eu.europeana.metis - 7-SNAPSHOT + 7 metis-validation-rest war diff --git a/metis-validation/metis-validation-service/pom.xml b/metis-validation/metis-validation-service/pom.xml index 00b46299d..99c345b45 100644 --- a/metis-validation/metis-validation-service/pom.xml +++ b/metis-validation/metis-validation-service/pom.xml @@ -4,7 +4,7 @@ metis-validation eu.europeana.metis - 7-SNAPSHOT + 7 metis-validation-service diff --git a/metis-validation/pom.xml b/metis-validation/pom.xml index 5c02f40ac..93bf2b3c0 100644 --- a/metis-validation/pom.xml +++ b/metis-validation/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 7-SNAPSHOT + 7 metis-validation pom diff --git a/pom.xml b/pom.xml index 8231a2de8..432929547 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ eu.europeana.metis metis-framework - 7-SNAPSHOT + 7 pom @@ -27,7 +27,7 @@ scm:git:https://github.com/europeana/metis-framework https://github.com/europeana/metis-framework - HEAD + v7 scm:git:https://github.com/europeana/metis-framework