Feat/MET-5763 Add support for stl files in media processing

metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/extraction/MediaExtractorImpl.java

@@ -11,6 +11,7 @@
 import eu.europeana.metis.mediaprocessing.model.Resource;
 import eu.europeana.metis.mediaprocessing.model.ResourceExtractionResult;
 import eu.europeana.metis.mediaprocessing.model.UrlType;
+import eu.europeana.metis.mediaprocessing.wrappers.TikaWrapper;
 import eu.europeana.metis.schema.model.MediaType;
 import eu.europeana.metis.utils.SonarqubeNullcheckAvoidanceUtils.ThrowingConsumer;
 import java.io.IOException;
@@ -44,7 +45,7 @@ enum ProcessingMode {FULL, REDUCED, NONE}
 
   private final ResourceDownloadClient resourceDownloadClient;
   private final MimeTypeDetectHttpClient mimeTypeDetectHttpClient;
-  private final Tika tika;
+  private final TikaWrapper tika;
 
   private final ImageProcessor imageProcessor;
   private final AudioVideoProcessor audioVideoProcessor;
@@ -62,7 +63,7 @@ enum ProcessingMode {FULL, REDUCED, NONE}
    * @param textProcessor A text processor.
    */
   MediaExtractorImpl(ResourceDownloadClient resourceDownloadClient,
-      MimeTypeDetectHttpClient mimeTypeDetectHttpClient, Tika tika, ImageProcessor imageProcessor,
+      MimeTypeDetectHttpClient mimeTypeDetectHttpClient, TikaWrapper tika, ImageProcessor imageProcessor,
       AudioVideoProcessor audioVideoProcessor, TextProcessor textProcessor, Media3dProcessor media3dProcessor) {
     this.resourceDownloadClient = resourceDownloadClient;
     this.mimeTypeDetectHttpClient = mimeTypeDetectHttpClient;
@@ -95,7 +96,7 @@ public MediaExtractorImpl(int redirectCount, int thumbnailGenerateTimeout,
             this::shouldDownloadForFullProcessing, connectTimeout, responseTimeout, downloadTimeout);
     this.mimeTypeDetectHttpClient = new MimeTypeDetectHttpClient(connectTimeout, responseTimeout,
         downloadTimeout);
-    this.tika = new Tika();
+    this.tika = new TikaWrapper();
     this.imageProcessor = new ImageProcessor(thumbnailGenerator);
     this.audioVideoProcessor = new AudioVideoProcessor(new CommandExecutor(audioVideoProbeTimeout));
     this.textProcessor = new TextProcessor(thumbnailGenerator,

metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/http/MimeTypeDetectHttpClient.java

@@ -1,5 +1,6 @@
 package eu.europeana.metis.mediaprocessing.http;
 
+import eu.europeana.metis.mediaprocessing.wrappers.TikaWrapper;
 import eu.europeana.metis.network.AbstractHttpClient;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.tika.Tika;
@@ -24,7 +25,7 @@
  */
 public class MimeTypeDetectHttpClient extends AbstractHttpClient<URL, String> {
 
-    private final Tika tika = new Tika();
+    private final TikaWrapper tika = new TikaWrapper();
 
     /**
      * Constructor.

metis-media-service/src/main/java/eu/europeana/metis/mediaprocessing/wrappers/TikaWrapper.java

@@ -0,0 +1,39 @@
+package eu.europeana.metis.mediaprocessing.wrappers;
+
+import java.io.IOException;
+import java.io.InputStream;
+import org.apache.tika.Tika;
+import org.apache.tika.metadata.Metadata;
+
+/**
+ * Wrapper class of Tika
+ */
+public class TikaWrapper {
+
+  private final Tika tika;
+
+  /**
+   * It creates a new instance of Tika
+   */
+  public TikaWrapper() {
+    this.tika = new Tika();
+  }
+
+  /**
+   * It uses tika's own detect method
+   * @param inputStream The input stream to detect from
+   * @param metadata The metadata associated with the input stream
+   * @return The mime type detected from the input stream
+   * @throws IOException
+   */
+  public String detect(InputStream inputStream, Metadata metadata) throws IOException {
+
+    String detectedMimeType = tika.detect(inputStream, metadata);
+
+    if(detectedMimeType.equals("application/vnd.ms-pki.stl")){
+      return "model/x.stl-binary";
+    }
+
+    return detectedMimeType;
+  }
+}

metis-media-service/src/main/resources/org/apache/tika/mime/custom-mimetypes.xml

@@ -11,5 +11,19 @@
   <mime-type type="model/gltf-binary">
     <glob pattern="*.glb"/>
   </mime-type>
+  <mime-type type="model/x.stl-ascii">
+    <sub-class-of type="text/plain"/>
+    <magic>
+      <!-- It must start with the word 'solid', following it may contain 0 or many whitespaces,
+       following with 1 or many letters/numbers and new line, which then must follow with the word 'facet'-->
+      <match value="^solid *[A-Za-z0-9,;'.?! \n]+facet" type="regex" offset="0"/>
+    </magic>
+<!--    <glob pattern="*.stl"/> Conflicts with already existing *.stl in tika-->
+  </mime-type>
+  <mime-type type="model/x.stl-binary">
+    <sub-class-of type="application/vnd.ms-pki.stl"/>
+  </mime-type>
+  <mime-type type="model/stl">
+    <sub-class-of type="application/vnd.ms-pki.stl"/>
+  </mime-type>
 </mime-info>
-

metis-media-service/src/test/java/eu/europeana/metis/mediaprocessing/extraction/MediaExtractorImplTest.java

@@ -27,6 +27,7 @@
 import eu.europeana.metis.mediaprocessing.model.Resource;
 import eu.europeana.metis.mediaprocessing.model.ResourceExtractionResultImpl;
 import eu.europeana.metis.mediaprocessing.model.UrlType;
+import eu.europeana.metis.mediaprocessing.wrappers.TikaWrapper;
 import eu.europeana.metis.schema.model.MediaType;
 import java.io.IOException;
 import java.io.InputStream;
@@ -70,7 +71,7 @@ static void prepare() {
     textProcessor = mock(TextProcessor.class);
     media3dProcessor = mock(Media3dProcessor.class);
     mediaExtractor = spy(new MediaExtractorImpl(resourceDownloadClient, mimeTypeDetectHttpClient,
-        tika, imageProcessor, audioVideoProcessor, textProcessor, media3dProcessor));
+        new TikaWrapper(), imageProcessor, audioVideoProcessor, textProcessor, media3dProcessor));
   }
 
   @BeforeEach

metis-media-service/src/test/java/eu/europeana/metis/mediaprocessing/http/MimeTypeDetectHttpClientTest.java

@@ -6,21 +6,23 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import com.github.tomakehurst.wiremock.junit5.WireMockExtension;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URISyntaxException;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
 
-class MimeTypeDetectHttpClientTest {
 
-  private static final String EXPECTED_MIME_TYPE = "audio/mpeg";
-  @RegisterExtension
-  static WireMockExtension wireMockExtension = WireMockExtension.newInstance()
-                                                                .options(wireMockConfig().dynamicPort().dynamicHttpsPort())
-                                                                .build();
+
+class MimeTypeDetectHttpClientTest {
+    private static final String EXPECTED_AUDIO_MIME_TYPE = "audio/mpeg";
+    @RegisterExtension
+    static WireMockExtension wireMockExtension = WireMockExtension.newInstance()
+            .options(wireMockConfig().dynamicPort().dynamicHttpsPort())
+            .build();
 
   private final MimeTypeDetectHttpClient mimeTypeDetectHttpClient
       = new MimeTypeDetectHttpClient(5000, 5000, 5000);
@@ -49,9 +51,64 @@ void download_withContentDisposition_expectSuccess() throws IOException, URISynt
     // when
     String detectedMimeType = mimeTypeDetectHttpClient.download(new URI(url).toURL());
 
-    // then
-    assertEquals(EXPECTED_MIME_TYPE, detectedMimeType);
-  }
+        // then
+        assertEquals(EXPECTED_AUDIO_MIME_TYPE, detectedMimeType);
+    }
+
+    @Test
+    void download_detectMimeTypeStlAscii_expectSuccess() throws IOException, URISyntaxException {
+        // given
+        try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("__files/block100.stl")) {
+            byte[] fileBytes = inputStream.readAllBytes();
+            wireMockExtension.stubFor(get("/imagen_id.do?idImagen=10610909").willReturn(aResponse()
+                .withStatus(200)
+                .withBody(fileBytes)
+                .withHeader("Content-Disposition", "inline; filename=\"block100.stl\"")));
+        }
+        final String url = String.format("http://localhost:%d/imagen_id.do?idImagen=10610909", wireMockExtension.getPort());
+        // when
+        String detectedMimeType = mimeTypeDetectHttpClient.download(new URI(url).toURL());
+
+        // then
+        assertEquals("model/x.stl-ascii", detectedMimeType);
+    }
+
+    @Test
+    void download_detectMimeTypeStlBinary_expectSuccess() throws IOException, URISyntaxException {
+        // given
+        try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("__files/Cube_3d_printing_sample.stl")) {
+            byte[] fileBytes = inputStream.readAllBytes();
+            wireMockExtension.stubFor(get("/imagen_id.do?idImagen=10610909").willReturn(aResponse()
+                .withStatus(200)
+                .withBody(fileBytes)
+                .withHeader("Content-Disposition", "inline; filename=\"Cube_3d_printing_sample.stl\"")));
+        }
+        final String url = String.format("http://localhost:%d/imagen_id.do?idImagen=10610909", wireMockExtension.getPort());
+        // when
+        String detectedMimeType = mimeTypeDetectHttpClient.download(new URI(url).toURL());
+
+        // then
+        assertEquals("model/x.stl-binary", detectedMimeType);
+    }
+
+    @Test
+    void download_returnProvidedStlMimeType_expectSuccess() throws IOException, URISyntaxException {
+        // given
+        try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("__files/Cube_3d_printing_sample.stl")) {
+            byte[] fileBytes = inputStream.readAllBytes();
+            wireMockExtension.stubFor(get("/imagen_id.do?idImagen=10610909").willReturn(aResponse()
+                .withStatus(200)
+                .withBody(fileBytes)
+                .withHeader("Content-Disposition", "inline; filename=\"Cube_3d_printing_sample.stl\"")
+                .withHeader("Content-Type", "model/stl")));
+        }
+        final String url = String.format("http://localhost:%d/imagen_id.do?idImagen=10610909", wireMockExtension.getPort());
+        // when
+        String detectedMimeType = mimeTypeDetectHttpClient.download(new URI(url).toURL());
+
+        // then
+        assertEquals("model/stl", detectedMimeType);
+    }
 
   @Test
   void download_detectMimeTypeGltf_expectSuccess() throws IOException, URISyntaxException {

metis-media-service/src/test/resources/__files/block100.stl

@@ -0,0 +1,86 @@
+solid block100
+   facet normal -1.000000e+000 0.000000e+000 0.000000e+000
+      outer loop
+         vertex 0.000000e+000 1.000000e+002 1.000000e+002
+         vertex 0.000000e+000 1.000000e+002 0.000000e+000
+         vertex 0.000000e+000 0.000000e+000 1.000000e+002
+      endloop
+   endfacet
+   facet normal -1.000000e+000 0.000000e+000 0.000000e+000
+      outer loop
+         vertex 0.000000e+000 0.000000e+000 1.000000e+002
+         vertex 0.000000e+000 1.000000e+002 0.000000e+000
+         vertex 0.000000e+000 0.000000e+000 0.000000e+000
+      endloop
+   endfacet
+   facet normal 0.000000e+000 0.000000e+000 1.000000e+000
+      outer loop
+         vertex 1.000000e+002 1.000000e+002 1.000000e+002
+         vertex 0.000000e+000 1.000000e+002 1.000000e+002
+         vertex 1.000000e+002 0.000000e+000 1.000000e+002
+      endloop
+   endfacet
+   facet normal 0.000000e+000 0.000000e+000 1.000000e+000
+      outer loop
+         vertex 1.000000e+002 0.000000e+000 1.000000e+002
+         vertex 0.000000e+000 1.000000e+002 1.000000e+002
+         vertex 0.000000e+000 0.000000e+000 1.000000e+002
+      endloop
+   endfacet
+   facet normal 1.000000e+000 0.000000e+000 0.000000e+000
+      outer loop
+         vertex 1.000000e+002 1.000000e+002 0.000000e+000
+         vertex 1.000000e+002 1.000000e+002 1.000000e+002
+         vertex 1.000000e+002 0.000000e+000 0.000000e+000
+      endloop
+   endfacet
+   facet normal 1.000000e+000 0.000000e+000 0.000000e+000
+      outer loop
+         vertex 1.000000e+002 0.000000e+000 0.000000e+000
+         vertex 1.000000e+002 1.000000e+002 1.000000e+002
+         vertex 1.000000e+002 0.000000e+000 1.000000e+002
+      endloop
+   endfacet
+   facet normal 0.000000e+000 0.000000e+000 -1.000000e+000
+      outer loop
+         vertex 0.000000e+000 1.000000e+002 0.000000e+000
+         vertex 1.000000e+002 1.000000e+002 0.000000e+000
+         vertex 0.000000e+000 0.000000e+000 0.000000e+000
+      endloop
+   endfacet
+   facet normal 0.000000e+000 0.000000e+000 -1.000000e+000
+      outer loop
+         vertex 0.000000e+000 0.000000e+000 0.000000e+000
+         vertex 1.000000e+002 1.000000e+002 0.000000e+000
+         vertex 1.000000e+002 0.000000e+000 0.000000e+000
+      endloop
+   endfacet
+   facet normal 0.000000e+000 1.000000e+000 0.000000e+000
+      outer loop
+         vertex 1.000000e+002 1.000000e+002 1.000000e+002
+         vertex 1.000000e+002 1.000000e+002 0.000000e+000
+         vertex 0.000000e+000 1.000000e+002 1.000000e+002
+      endloop
+   endfacet
+   facet normal 0.000000e+000 1.000000e+000 0.000000e+000
+      outer loop
+         vertex 0.000000e+000 1.000000e+002 1.000000e+002
+         vertex 1.000000e+002 1.000000e+002 0.000000e+000
+         vertex 0.000000e+000 1.000000e+002 0.000000e+000
+      endloop
+   endfacet
+   facet normal 0.000000e+000 -1.000000e+000 0.000000e+000
+      outer loop
+         vertex 1.000000e+002 0.000000e+000 0.000000e+000
+         vertex 1.000000e+002 0.000000e+000 1.000000e+002
+         vertex 0.000000e+000 0.000000e+000 0.000000e+000
+      endloop
+   endfacet
+   facet normal 0.000000e+000 -1.000000e+000 0.000000e+000
+      outer loop
+         vertex 0.000000e+000 0.000000e+000 0.000000e+000
+         vertex 1.000000e+002 0.000000e+000 1.000000e+002
+         vertex 0.000000e+000 0.000000e+000 1.000000e+002
+      endloop
+   endfacet
+endsolid