Reuse response data in SharePoint client

codelibs · Mar 15, 2024 · a4cfc13 · a4cfc13
1 parent 55095b0
commit a4cfc13
Show file tree

Hide file tree

Showing 2 changed files with 53 additions and 9 deletions.
diff --git a/src/main/java/org/codelibs/fess/ds/sharepoint/client/api/file/getfile/GetFileResponse.java b/src/main/java/org/codelibs/fess/ds/sharepoint/client/api/file/getfile/GetFileResponse.java
@@ -15,25 +15,65 @@
  */
 package org.codelibs.fess.ds.sharepoint.client.api.file.getfile;
 
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 
+import org.apache.commons.io.output.DeferredFileOutputStream;
+import org.apache.http.HttpEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.util.EntityUtils;
+import org.codelibs.core.io.CopyUtil;
 import org.codelibs.fess.ds.sharepoint.client.api.SharePointApiResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class GetFileResponse implements SharePointApiResponse {
+    private static final Logger logger = LoggerFactory.getLogger(GetFileResponse.class);
+
     private final CloseableHttpResponse httpResponse;
 
+    private final int cacheFileSize = 1_000_000;
+
+    private byte[] responseData;
+
+    private File responseFile;
+
     public GetFileResponse(final CloseableHttpResponse httpResponse) {
         this.httpResponse = httpResponse;
     }
 
     public InputStream getFileContent() throws IOException {
-        return httpResponse.getEntity().getContent();
+        if (responseData == null && responseFile == null) {
+            HttpEntity entity = null;
+            try (DeferredFileOutputStream out =
+                    DeferredFileOutputStream.builder().setThreshold(cacheFileSize).setPrefix("fess-extractor-").setSuffix(".out").get()) {
+                entity = httpResponse.getEntity();
+                CopyUtil.copy(entity.getContent(), out);
+                out.flush();
+
+                if (out.isInMemory()) {
+                    responseData = out.getData();
+                } else {
+                    responseFile = out.getFile();
+                }
+            } finally {
+                EntityUtils.consumeQuietly(entity);
+            }
+        }
+        if (responseData != null) {
+            return new ByteArrayInputStream(responseData);
+        }
+        return new FileInputStream(responseFile);
     }
 
     @Override
     public void close() throws IOException {
+        if (responseFile != null && !responseFile.delete()) {
+            logger.warn("Failed to delete {}.", responseFile.getAbsolutePath());
+        }
         httpResponse.close();
     }
 }
diff --git a/src/main/java/org/codelibs/fess/ds/sharepoint/crawl/file/FileCrawl.java b/src/main/java/org/codelibs/fess/ds/sharepoint/crawl/file/FileCrawl.java
@@ -24,6 +24,7 @@
 import java.util.Queue;
 
 import org.apache.commons.lang3.StringUtils;
+import org.codelibs.core.exception.IORuntimeException;
 import org.codelibs.core.lang.StringUtil;
 import org.codelibs.fess.crawler.helper.MimeTypeHelper;
 import org.codelibs.fess.ds.sharepoint.client.SharePointClient;
@@ -86,10 +87,9 @@ public Map<String, Object> doCrawl(final DataConfig dataConfig, final Queue<Shar
     }
 
     private Map<String, Object> buildDataMap(final DataConfig dataConfig, final GetFileResponse response) throws IOException {
-        final InputStream is = response.getFileContent();
-        final String mimeType = getMimeType(fileName, is);
+        final String mimeType = getMimeType(fileName, response);
         final String fileType = getFileType(mimeType);
-        final String content = getContent(is, mimeType);
+        final String content = getContent(response, mimeType);
 
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final Map<String, Object> dataMap = new HashMap<>();
@@ -124,10 +124,10 @@ private Map<String, Object> buildDataMap(final DataConfig dataConfig, final GetF
         return dataMap;
     }
 
-    private String getContent(final InputStream is, final String mimeType) {
+    private String getContent(final GetFileResponse response, final String mimeType) {
         final StringBuilder content = new StringBuilder(1000);
 
-        try {
+        try (final InputStream is = response.getFileContent()) {
             final String fileText = ComponentUtil.getExtractorFactory().builder(is, null).extractorName(DEFAULT_EXTRACTOR_NAME)
                     .mimeType(mimeType).extract().getContent();
             if (StringUtils.isNotBlank(fileText)) {
@@ -152,9 +152,13 @@ private String getContent(final InputStream is, final String mimeType) {
         return content.toString();
     }
 
-    protected String getMimeType(final String filename, final InputStream is) {
-        final MimeTypeHelper mimeTypeHelper = ComponentUtil.getComponent(MimeTypeHelper.class);
-        return mimeTypeHelper.getContentType(is, filename);
+    protected String getMimeType(final String filename, final GetFileResponse response) {
+        try (final InputStream is = response.getFileContent()) {
+            final MimeTypeHelper mimeTypeHelper = ComponentUtil.getComponent(MimeTypeHelper.class);
+            return mimeTypeHelper.getContentType(is, filename);
+        } catch (final IOException e) {
+            throw new IORuntimeException(e);
+        }
     }
 
     protected String getFileType(final String mimeType) {