Datastore / Metadata files / FME API Support

core/pom.xml

@@ -316,6 +316,10 @@
       <groupId>org.apache.httpcomponents</groupId>
       <artifactId>httpclient</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpmime</artifactId>
+    </dependency>
     <dependency>
       <groupId>commons-net</groupId>
       <artifactId>commons-net</artifactId>
@@ -606,6 +610,10 @@
       <groupId>org.jasypt</groupId>
       <artifactId>jasypt</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpmime</artifactId>
+    </dependency>
   </dependencies>
 
   <build>

core/src/main/java/org/fao/geonet/api/records/attachments/CMISStore.java

@@ -186,6 +186,11 @@ public ResourceHolder getResource(final ServiceContext context, final String met
         }
     }
 
+    @Override
+    public void streamResource(ServiceContext context, String metadataUuid, String resourceId, Boolean approved, OutputStream out) throws Exception {
+        throw new UnsupportedOperationException(String.format("%s does not support streamResource.", this.getClass().getSimpleName()));
+    }
+
     @Override
     public ResourceHolder getResourceInternal(String metadataUuid, MetadataResourceVisibility visibility, String resourceId, Boolean approved) throws Exception {
         throw new UnsupportedOperationException("CMISStore does not support getResourceInternal.");

core/src/main/java/org/fao/geonet/api/records/attachments/FilesystemStore.java

@@ -42,6 +42,7 @@
 import javax.annotation.Nullable;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -142,6 +143,11 @@ public ResourceHolder getResourceInternal(
         }
     }
 
+    @Override
+    public void streamResource(ServiceContext context, String metadataUuid, String resourceId, Boolean approved, OutputStream out) throws Exception {
+        throw new UnsupportedOperationException(String.format("%s does not support streamResource.", this.getClass().getSimpleName()));
+    }
+
     public MetadataResource getResourceDescription(final ServiceContext context, String metadataUuid, MetadataResourceVisibility visibility,
                                                    String filename, Boolean approved) throws Exception {
         Path path = getPath(context, metadataUuid, visibility, filename, approved);

core/src/main/java/org/fao/geonet/api/records/attachments/JCloudStore.java

@@ -52,6 +52,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -166,6 +167,11 @@ public ResourceHolder getResourceInternal(String metadataUuid, MetadataResourceV
         throw new UnsupportedOperationException("JCloud does not support getResourceInternal.");
     }
 
+    @Override
+    public void streamResource(ServiceContext context, String metadataUuid, String resourceId, Boolean approved, OutputStream out) throws Exception {
+        throw new UnsupportedOperationException(String.format("%s does not support streamResource.", this.getClass().getSimpleName()));
+    }
+
     private String getKey(final ServiceContext context, String metadataUuid, int metadataId, MetadataResourceVisibility visibility, String resourceId) {
         checkResourceId(resourceId);
         final String metadataDir = getMetadataDir(context, metadataId);

core/src/main/java/org/fao/geonet/api/records/attachments/ResourceLoggerStore.java

@@ -43,6 +43,7 @@
 import org.springframework.context.ConfigurableApplicationContext;
 
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.util.Date;
 import java.util.List;
 import javax.annotation.Nullable;
@@ -52,6 +53,10 @@
  */
 public class ResourceLoggerStore extends AbstractStore {
 
+    public Store getDecoratedStore() {
+        return decoratedStore;
+    }
+
     private Store decoratedStore;
 
 
@@ -94,6 +99,14 @@ public ResourceHolder getResourceInternal(String metadataUuid, MetadataResourceV
         throw new UnsupportedOperationException("ResourceLoggerStore does not support getResourceInternal.");
     }
 
+    @Override
+    public void streamResource(ServiceContext context, String metadataUuid, String resourceId, Boolean approved, OutputStream out) throws Exception {
+        if (decoratedStore != null) {
+            decoratedStore.streamResource(context, metadataUuid, resourceId, approved, out);
+        }
+    }
+
+
     @Override
     public MetadataResource putResource(final ServiceContext context, final String metadataUuid, final String filename,
                                         final InputStream is, @Nullable final Date changeDate, final MetadataResourceVisibility visibility,

core/src/main/java/org/fao/geonet/api/records/attachments/S3Store.java

@@ -44,6 +44,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.nio.file.*;
 import java.util.ArrayList;
 import java.util.Date;
@@ -124,6 +125,11 @@ public ResourceHolder getResourceInternal(String metadataUuid, MetadataResourceV
         throw new UnsupportedOperationException("S3Store does not support getResourceInternal.");
     }
 
+    @Override
+    public void streamResource(ServiceContext context, String metadataUuid, String resourceId, Boolean approved, OutputStream out) throws Exception {
+        throw new UnsupportedOperationException(String.format("%s does not support streamResource.", this.getClass().getSimpleName()));
+    }
+
     private String getKey(String metadataUuid, int metadataId, MetadataResourceVisibility visibility, String resourceId) throws Exception {
         checkResourceId(resourceId);
         final String metadataDir = getMetadataDir(metadataId);

core/src/main/java/org/fao/geonet/api/records/attachments/Store.java

@@ -33,11 +33,13 @@
 
 import java.io.Closeable;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.net.URL;
 import java.nio.file.Path;
 import java.util.Date;
 import java.util.List;
 import javax.annotation.Nullable;
+import javax.servlet.ServletOutputStream;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -140,6 +142,9 @@ ResourceHolder getResource(ServiceContext context, String metadataUuid, Metadata
      */
     ResourceHolder getResource(ServiceContext context, String metadataUuid, String resourceId, Boolean approved) throws Exception;
 
+    void streamResource(ServiceContext context, String metadataUuid,
+                        String resourceId, Boolean approved, OutputStream out) throws Exception;
+
     /**
      * Retrieve a metadata resource path (for internal use eg. indexing)
      */
@@ -348,6 +353,7 @@ MetadataResource getResourceDescription(final ServiceContext context, String met
      */
     void copyResources(ServiceContext context, String sourceUuid, String targetUuid, MetadataResourceVisibility metadataResourceVisibility, boolean sourceApproved, boolean targetApproved) throws Exception;
 
+
     interface ResourceHolder extends Closeable {
         Path getPath();
         MetadataResource getMetadata();

core/src/main/resources/config-spring-geonetwork.xml

@@ -248,6 +248,7 @@
      default - Default file system store
      s3      - AWS S3 storage (see config-store/config-s3.xml for more details)
      cmis    - CMIS storage (see config-store/config-cmis.xml for more details)
+     fme     - FME storage (see config-store/config-fme.xml for more details)
   -->
   <import resource="config-store/config-${geonetwork.store.type:default}.xml"/>
 </beans>

core/src/main/resources/config-store/config-fme-overrides.properties

@@ -0,0 +1,2 @@
+fme.api.url=${FME_API_URL:#{null}}
+fme.token=${FME_TOKEN:#{null}}

core/src/main/resources/config-store/config-fme.xml

@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (C) 2001-2016 Food and Agriculture Organization of the
+  ~ United Nations (FAO-UN), United Nations World Food Programme (WFP)
+  ~ and United Nations Environment Programme (UNEP)
+  ~
+  ~ This program is free software; you can redistribute it and/or modify
+  ~ it under the terms of the GNU General Public License as published by
+  ~ the Free Software Foundation; either version 2 of the License, or (at
+  ~ your option) any later version.
+  ~
+  ~ This program is distributed in the hope that it will be useful, but
+  ~ WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  ~ General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU General Public License
+  ~ along with this program; if not, write to the Free Software
+  ~ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
+  ~
+  ~ Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2,
+  ~ Rome - Italy. email: geonetwork@osgeo.org
+  -->
+<beans
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xmlns="http://www.springframework.org/schema/beans"
+  xmlns:context="http://www.springframework.org/schema/context"
+  xsi:schemaLocation="
+          http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+          http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
+    ">
+
+  <context:property-placeholder location="classpath:config-store/config-fme-overrides.properties"
+                                file-encoding="UTF-8"
+                                ignore-unresolvable="true"
+  />
+
+  <bean id="filesystemStore"
+        class="org.fao.geonet.api.records.attachments.FMEStore">
+    <property name="fmeApiUrl" value="${fme.api.url}"/>
+    <property name="fmeToken" value="${fme.token}"/>
+  </bean>
+
+  <bean id="resourceStore"
+        class="org.fao.geonet.api.records.attachments.ResourceLoggerStore">
+    <constructor-arg index="0" ref="filesystemStore"/>
+  </bean>
+
+  <bean id="resources" class="org.fao.geonet.resources.FileResources"/>
+</beans>

services/src/main/java/org/fao/geonet/api/records/attachments/AttachmentsApi.java

@@ -31,6 +31,7 @@
 import io.swagger.v3.oas.annotations.tags.Tag;
 import jeeves.server.UserSession;
 import jeeves.server.context.ServiceContext;
+import org.apache.commons.io.IOUtils;
 import org.fao.geonet.ApplicationContextHolder;
 import org.fao.geonet.api.ApiParams;
 import org.fao.geonet.api.ApiUtils;
@@ -61,13 +62,17 @@
 
 import javax.annotation.PostConstruct;
 import javax.imageio.ImageIO;
+import javax.servlet.ServletOutputStream;
 import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
 import java.awt.image.BufferedImage;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.List;
 
 /**
@@ -160,6 +165,9 @@ public List<MetadataResource> getAllResources(
         @RequestParam(required = false, defaultValue = FilesystemStore.DEFAULT_FILTER) String filter,
         @Parameter(hidden = true) HttpServletRequest request) throws Exception {
         ServiceContext context = ApiUtils.createServiceContext(request);
+
+        ApiUtils.canViewRecord(metadataUuid, request);
+
         List<MetadataResource> list = store.getResources(context, metadataUuid, sort, filter, approved);
         return list;
     }
@@ -241,47 +249,62 @@ public MetadataResource putResourceFromURL(
     }
 
     @io.swagger.v3.oas.annotations.Operation(summary = "Get a metadata resource")
-    // @PreAuthorize("permitAll")
     @RequestMapping(value = "/{resourceId:.+}", method = RequestMethod.GET)
     @ResponseStatus(value = HttpStatus.OK)
     @ApiResponses(value = {@ApiResponse(responseCode = "201", description = "Record attachment."),
         @ApiResponse(responseCode = "403", description = "Operation not allowed. "
             + "User needs to be able to download the resource.")})
     @ResponseBody
-    public HttpEntity<byte[]> getResource(
+    public void getResource(
         @Parameter(description = "The metadata UUID", required = true, example = "43d7c186-2187-4bcd-8843-41e575a5ef56") @PathVariable String metadataUuid,
         @Parameter(description = "The resource identifier (ie. filename)", required = true) @PathVariable String resourceId,
         @Parameter(description = "Use approved version or not", example = "true") @RequestParam(required = false, defaultValue = "true") Boolean approved,
         @Parameter(description = "Size (only applies to images). From 1px to 2048px.", example = "200") @RequestParam(required = false) Integer size,
-        @Parameter(hidden = true) HttpServletRequest request) throws Exception {
+        @Parameter(hidden = true) HttpServletRequest request,
+        @Parameter(hidden = true) HttpServletResponse response) throws Exception {
         ServiceContext context = ApiUtils.createServiceContext(request);
-        try (Store.ResourceHolder file = store.getResource(context, metadataUuid, resourceId, approved)) {
-
-            ApiUtils.canViewRecord(metadataUuid, request);
-
-            MultiValueMap<String, String> headers = new HttpHeaders();
-            headers.add("Content-Disposition", "inline; filename=\"" + file.getMetadata().getFilename() + "\"");
-            headers.add("Cache-Control", "no-cache");
-            String contentType = getFileContentType(file.getPath());
-            headers.add("Content-Type", contentType);
-
-            if (contentType.startsWith("image/") && size != null) {
-                if (size >= MIN_IMAGE_SIZE && size <= MAX_IMAGE_SIZE) {
-                    BufferedImage image = ImageIO.read(file.getPath().toFile());
-                    BufferedImage resized = ImageUtil.resize(image, size);
-                    ByteArrayOutputStream output = new ByteArrayOutputStream();
-                    ImageIO.write(resized, "png", output);
-                    output.flush();
-                    byte[] imagesB = output.toByteArray();
-                    output.close();
-                    return new HttpEntity<>(imagesB, headers);
+
+        ApiUtils.canViewRecord(metadataUuid, request);
+
+        ServletOutputStream out = response.getOutputStream();
+
+        if (store instanceof FMEStore
+        || (store instanceof ResourceLoggerStore && ((ResourceLoggerStore) store).getDecoratedStore() instanceof FMEStore)) {
+            response.setHeader("Content-Disposition", "inline; filename=\"" + resourceId + "\"");
+            response.setHeader("Cache-Control", "no-cache");
+
+            store.streamResource(context, metadataUuid, resourceId, approved, out);
+        } else {
+            try (Store.ResourceHolder file = store.getResource(context, metadataUuid, resourceId, approved)) {
+                response.setHeader("Content-Disposition", "inline; filename=\"" + file.getMetadata().getFilename() + "\"");
+                response.setHeader("Cache-Control", "no-cache");
+
+                Path path = file.getPath();
+                String contentType = getFileContentType(path);
+                response.setHeader("Content-Type", contentType);
+
+                if (contentType.startsWith("image/") && size != null) {
+                    if (size >= MIN_IMAGE_SIZE && size <= MAX_IMAGE_SIZE) {
+                        BufferedImage image = ImageIO.read(path.toFile());
+                        BufferedImage resized = ImageUtil.resize(image, size);
+                        ByteArrayOutputStream output = new ByteArrayOutputStream();
+                        ImageIO.write(resized, "png", output);
+                        output.flush();
+                        byte[] imagesB = output.toByteArray();
+                        output.close();
+                        out.write(imagesB);
+                        out.flush();
+                        out.close();
+                    } else {
+                        throw new IllegalArgumentException(String.format(
+                            "Image can only be resized from %d to %d. You requested %d.",
+                            MIN_IMAGE_SIZE, MAX_IMAGE_SIZE, size));
+                    }
                 } else {
-                    throw new IllegalArgumentException(String.format(
-                        "Image can only be resized from %d to %d. You requested %d.",
-                        MIN_IMAGE_SIZE, MAX_IMAGE_SIZE, size));
+                    IOUtils.copy(Files.newInputStream(path, StandardOpenOption.READ), out);
+                    out.flush();
+                    out.close();
                 }
-            } else {
-                return new HttpEntity<>(Files.readAllBytes(file.getPath()), headers);
             }
         }
     }