Skip to content
Permalink
Browse files

Merge pull request #4248 from pvalsecc/s3_master

Add a Store implementation for Amazon's S3
  • Loading branch information
jahow committed Dec 16, 2019
2 parents 97bcf53 + caa3607 commit 6c80a91606f203461f0512f4533d8dc7924ef43f
Showing with 3,026 additions and 2,671 deletions.
  1. +1 −1 common/src/main/java/org/fao/geonet/utils/nio/PathHttpEntity.java
  2. +5 −0 core/pom.xml
  3. +7 −0 core/src/main/java/jeeves/server/context/BasicContext.java
  4. +2 −2 core/src/main/java/jeeves/server/dispatchers/ServiceManager.java
  5. +223 −0 core/src/main/java/org/fao/geonet/api/records/attachments/AbstractStore.java
  6. +129 −308 core/src/main/java/org/fao/geonet/api/records/attachments/FilesystemStore.java
  7. +34 −12 core/src/main/java/org/fao/geonet/api/records/attachments/FilesystemStoreResource.java
  8. +57 −123 core/src/main/java/org/fao/geonet/api/records/attachments/ResourceLoggerStore.java
  9. +277 −0 core/src/main/java/org/fao/geonet/api/records/attachments/S3Store.java
  10. +73 −10 core/src/main/java/org/fao/geonet/api/records/attachments/Store.java
  11. +97 −0 core/src/main/java/org/fao/geonet/api/records/attachments/StoreUtils.java
  12. +0 −7 core/src/main/java/org/fao/geonet/constants/Geonet.java
  13. +1 −1 core/src/main/java/org/fao/geonet/kernel/AccessManager.java
  14. +26 −30 core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java
  15. +1 −2 core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java
  16. +3 −18 core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java
  17. +2 −2 core/src/main/java/org/fao/geonet/kernel/mef/IMEFVisitor.java
  18. +14 −27 core/src/main/java/org/fao/geonet/kernel/mef/Importer.java
  19. +19 −14 core/src/main/java/org/fao/geonet/kernel/mef/MEF2Exporter.java
  20. +36 −33 core/src/main/java/org/fao/geonet/kernel/mef/MEFExporter.java
  21. +10 −9 core/src/main/java/org/fao/geonet/kernel/mef/MEFLib.java
  22. +0 −17 core/src/main/java/org/fao/geonet/lib/ResourceLib.java
  23. +228 −0 core/src/main/java/org/fao/geonet/resources/FileResources.java
  24. +18 −17 core/src/main/java/org/fao/geonet/resources/ResourceFilter.java
  25. +89 −242 core/src/main/java/org/fao/geonet/resources/Resources.java
  26. +88 −0 core/src/main/java/org/fao/geonet/resources/S3Credentials.java
  27. +325 −0 core/src/main/java/org/fao/geonet/resources/S3Resources.java
  28. +1 −1 core/src/main/java/org/fao/geonet/services/main/Info.java
  29. +35 −54 core/src/main/java/org/fao/geonet/services/thumbnail/Set.java
  30. +28 −0 core/src/main/resources/config-spring-geonetwork.xml
  31. +1 −3 core/src/test/java/org/fao/geonet/GeonetTestFixture.java
  32. +2 −0 core/src/test/java/org/fao/geonet/kernel/ThesaurusTest.java
  33. +5 −8 core/src/test/java/org/fao/geonet/kernel/mef/MEFExporterIntegrationTest.java
  34. +4 −2 core/src/test/java/org/fao/geonet/kernel/search/facet/DimensionTest.java
  35. +8 −3 domain/src/main/java/org/fao/geonet/domain/MetadataResource.java
  36. +1 −1 harvesters/src/main/java/org/fao/geonet/kernel/harvest/HarvestManagerImpl.java
  37. +80 −36 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractHarvester.java
  38. +4 −1 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractParams.java
  39. +1 −1 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/HarversterJobListener.java
  40. +2 −2 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/HarvesterJob.java
  41. +14 −89 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java
  42. +5 −0 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEParams.java
  43. +0 −1 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java
  44. +6 −71 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/CswHarvester.java
  45. +6 −0 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/CswParams.java
  46. +5 −81 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/GeoPRESTHarvester.java
  47. +5 −0 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/GeoPRESTParams.java
  48. +41 −72 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java
  49. +16 −59 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/GeonetHarvester.java
  50. +5 −0 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/GeonetParams.java
  51. +8 −9 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Harvester.java
  52. +6 −65 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Geonet20Harvester.java
  53. +5 −0 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/GeonetParams.java
  54. +13 −74 ...c/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java
  55. +5 −0 .../src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemParams.java
  56. +5 −83 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/OaiPmhHarvester.java
  57. +5 −0 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/OaiPmhParams.java
  58. +4 −2 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/ogcwxs/Harvester.java
  59. +6 −81 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/ogcwxs/OgcWxSHarvester.java
  60. +6 −0 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/ogcwxs/OgcWxSParams.java
  61. +6 −83 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/ThreddsHarvester.java
  62. +6 −0 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/ThreddsParams.java
  63. +5 −65 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WebDavHarvester.java
  64. +5 −0 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WebDavParams.java
  65. +6 −86 ...sters/src/main/java/org/fao/geonet/kernel/harvest/harvester/wfsfeatures/WfsFeaturesHarvester.java
  66. +6 −1 harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/wfsfeatures/WfsFeaturesParams.java
  67. +2 −2 harvesters/src/main/java/org/fao/geonet/services/harvesting/Info.java
  68. +9 −7 pom.xml
  69. +34 −29 services/src/main/java/org/fao/geonet/api/groups/GroupsApi.java
  70. +9 −5 services/src/main/java/org/fao/geonet/api/mapservers/MapServersApi.java
  71. +19 −27 services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java
  72. +10 −9 services/src/main/java/org/fao/geonet/api/records/attachments/AttachmentsApi.java
  73. +1 −1 services/src/main/java/org/fao/geonet/api/records/backup/BackupArchiveApi.java
  74. +2 −2 services/src/main/java/org/fao/geonet/api/records/formatters/FormatterApi.java
  75. +15 −0 services/src/main/java/org/fao/geonet/api/records/formatters/groovy/Environment.java
  76. +9 −0 services/src/main/java/org/fao/geonet/api/records/formatters/groovy/EnvironmentImpl.java
  77. +9 −0 services/src/main/java/org/fao/geonet/api/records/formatters/groovy/EnvironmentProxy.java
  78. +9 −7 services/src/main/java/org/fao/geonet/api/records/formatters/groovy/util/Summary.java
  79. +42 −51 services/src/main/java/org/fao/geonet/api/site/LogosApi.java
  80. +40 −29 services/src/main/java/org/fao/geonet/api/site/SiteApi.java
  81. +8 −6 services/src/main/java/org/fao/geonet/api/sources/SourcesApi.java
  82. +15 −18 services/src/main/java/org/fao/geonet/services/feedback/AddLimitations.java
  83. +6 −7 services/src/main/java/org/fao/geonet/services/group/List.java
  84. +22 −14 services/src/main/java/org/fao/geonet/services/group/Update.java
  85. +13 −22 services/src/main/java/org/fao/geonet/services/logo/Add.java
  86. +4 −15 services/src/main/java/org/fao/geonet/services/logo/Delete.java
  87. +25 −22 services/src/main/java/org/fao/geonet/services/logo/Set.java
  88. +10 −15 services/src/main/java/org/fao/geonet/services/mef/ImportWebMap.java
  89. +7 −6 services/src/main/java/org/fao/geonet/services/metadata/BatchDelete.java
  90. +4 −13 services/src/main/java/org/fao/geonet/services/metadata/Create.java
  91. +3 −1 services/src/main/java/org/fao/geonet/services/metadata/Delete.java
  92. +15 −8 services/src/main/java/org/fao/geonet/services/metadata/PrepareFileDownload.java
  93. +2 −2 services/src/main/java/org/fao/geonet/services/metadata/schema/SchematronCriteriaTypeService.java
  94. +5 −3 services/src/main/java/org/fao/geonet/services/publisher/Do.java
  95. +61 −64 services/src/main/java/org/fao/geonet/services/resources/Download.java
  96. +42 −40 services/src/main/java/org/fao/geonet/services/resources/DownloadArchive.java
  97. +7 −4 services/src/main/java/org/fao/geonet/services/resources/Get.java
  98. +2 −1 services/src/main/java/org/fao/geonet/services/resources/Remove.java
  99. +2 −1 services/src/main/java/org/fao/geonet/services/resources/RemoveAndProcess.java
  100. +2 −1 services/src/main/java/org/fao/geonet/services/resources/Upload.java
  101. +1 −1 services/src/main/java/org/fao/geonet/services/resources/UploadAndProcess.java
  102. +11 −12 services/src/main/java/org/fao/geonet/services/resources/handlers/DefaultResourceRemoveHandler.java
  103. +34 −63 services/src/main/java/org/fao/geonet/services/resources/handlers/DefaultResourceUploadHandler.java
  104. +7 −16 services/src/main/java/org/fao/geonet/services/thumbnail/Unset.java
  105. +1 −7 services/src/main/resources/config-spring-geonetwork.xml
  106. +229 −0 services/src/test/java/org/fao/geonet/api/records/attachments/AbstractStoreTest.java
  107. +26 −204 services/src/test/java/org/fao/geonet/api/records/attachments/FilesystemStoreTest.java
  108. +90 −0 services/src/test/java/org/fao/geonet/api/records/attachments/S3StoreTest.java
  109. +10 −10 services/src/test/java/org/fao/geonet/services/metadata/BatchOpsMetadatReindexerTest.java
  110. +5 −4 services/src/test/java/org/fao/geonet/services/metadata/CreateTest.java
  111. +0 −4 web/pom.xml
  112. +10 −9 web/src/main/java/org/fao/geonet/Geonetwork.java
@@ -27,7 +27,6 @@

package org.fao.geonet.utils.nio;

import org.apache.http.annotation.NotThreadSafe;
import org.apache.http.entity.AbstractHttpEntity;
import org.apache.http.entity.ContentType;
import org.apache.http.util.Args;
@@ -38,6 +37,7 @@
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import javax.annotation.concurrent.NotThreadSafe;

/**
* A self contained, repeatable entity that obtains its content from a file.
@@ -513,6 +513,11 @@
<artifactId>spring-test</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.11.618</version>
</dependency>
</dependencies>

<build>
@@ -118,6 +118,13 @@ public ConfigurableApplicationContext getApplicationContext() {
return jeevesApplicationContext.getBean(beanType);
}

public
@Nonnull
<T> T getBean(String name, Class<T> beanType) {
return jeevesApplicationContext.getBean(name, beanType);
}


//--------------------------------------------------------------------------

public EntityManager getEntityManager() {
@@ -654,7 +654,7 @@ private String dispatchOutput(ServiceRequest req, ServiceContext context,
//--- FILE output

else {
final NodeInfo nodeInfo = context.getApplicationContext().getBean(NodeInfo.class);
final NodeInfo nodeInfo = context.getBean(NodeInfo.class);
if (outPage.isFile()) {
// PDF Output
if (outPage.getContentType().equals("application/pdf") && !outPage.getStyleSheet().equals("")) {
@@ -896,7 +896,7 @@ private void dispatchError(ServiceRequest req, ServiceContext context,
// Dispatch HTTP status code
req.setStatusCode(outPage.getStatusCode());

addPrefixes(guiElem, context.getLanguage(), req.getService(), context.getApplicationContext().getBean(NodeInfo.class).getId());
addPrefixes(guiElem, context.getLanguage(), req.getService(), context.getBean(NodeInfo.class).getId());

Element rootElem = new Element(Jeeves.Elem.ROOT)
.addContent(guiElem)
@@ -0,0 +1,223 @@
/*
* =============================================================================
* === Copyright (C) 2019 Food and Agriculture Organization of the
* === United Nations (FAO-UN), United Nations World Food Programme (WFP)
* === and United Nations Environment Programme (UNEP)
* ===
* === This program is free software; you can redistribute it and/or modify
* === it under the terms of the GNU General Public License as published by
* === the Free Software Foundation; either version 2 of the License, or (at
* === your option) any later version.
* ===
* === This program is distributed in the hope that it will be useful, but
* === WITHOUT ANY WARRANTY; without even the implied warranty of
* === MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* === General Public License for more details.
* ===
* === You should have received a copy of the GNU General Public License
* === along with this program; if not, write to the Free Software
* === Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
* ===
* === Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2,
* === Rome - Italy. email: geonetwork@osgeo.org
* ==============================================================================
*/
package org.fao.geonet.api.records.attachments;

import jeeves.server.context.ServiceContext;
import org.apache.commons.io.FilenameUtils;
import org.fao.geonet.ApplicationContextHolder;
import org.fao.geonet.api.exception.ResourceNotFoundException;
import org.fao.geonet.domain.AbstractMetadata;
import org.fao.geonet.domain.MetadataResource;
import org.fao.geonet.domain.MetadataResourceVisibility;
import org.fao.geonet.kernel.AccessManager;
import org.fao.geonet.kernel.datamanager.IMetadataUtils;
import org.fao.geonet.repository.MetadataRepository;
import org.springframework.context.ApplicationContext;
import org.springframework.web.multipart.MultipartFile;

import java.io.BufferedInputStream;
import java.io.InputStream;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;

public abstract class AbstractStore implements Store {
@Override
public final List<MetadataResource> getResources(final ServiceContext context, final String metadataUuid, final Sort sort,
final String filter) throws Exception {
return getResources(context, metadataUuid, sort, filter, true);
}

@Override
public List<MetadataResource> getResources(final ServiceContext context, final String metadataUuid,
final MetadataResourceVisibility metadataResourceVisibility, final String filter) throws Exception {
return getResources(context, metadataUuid, metadataResourceVisibility, filter, true);
}

@Override
public List<MetadataResource> getResources(ServiceContext context, String metadataUuid, Sort sort, String filter, Boolean approved)
throws Exception {
int metadataId = getAndCheckMetadataId(metadataUuid, approved);
boolean canEdit = getAccessManager(context).canEdit(context, String.valueOf(metadataId));

List<MetadataResource> resourceList = new ArrayList<>(
getResources(context, metadataUuid, MetadataResourceVisibility.PUBLIC, filter, approved));
if (canEdit) {
resourceList.addAll(getResources(context, metadataUuid, MetadataResourceVisibility.PRIVATE, filter, approved));
}

if (sort == Sort.name) {
resourceList.sort(MetadataResourceVisibility.sortByFileName);
}

return resourceList;
}

@Override
public final ResourceHolder getResource(ServiceContext context, String metadataUuid, String resourceId) throws Exception {
return getResource(context, metadataUuid, resourceId, true);
}

@Override
public final ResourceHolder getResource(ServiceContext context, String metadataUuid, String resourceId, Boolean approved)
throws Exception {
try {
return getResource(context, metadataUuid, MetadataResourceVisibility.PUBLIC, resourceId, approved);
} catch (ResourceNotFoundException ignored) {
}
return getResource(context, metadataUuid, MetadataResourceVisibility.PRIVATE, resourceId, approved);
}

protected static AccessManager getAccessManager(final ServiceContext context) {
return context.getBean(AccessManager.class);
}

public static int getAndCheckMetadataId(String metadataUuid, Boolean approved) throws Exception {
final ApplicationContext _appContext = ApplicationContextHolder.get();
final AbstractMetadata metadata;
if (approved) {
metadata = _appContext.getBean(MetadataRepository.class).findOneByUuid(metadataUuid);
} else {
metadata = _appContext.getBean(IMetadataUtils.class).findOneByUuid(metadataUuid);
}
if (metadata == null) {
throw new ResourceNotFoundException(String.format("Metadata with UUID '%s' not found.", metadataUuid));
}
return metadata.getId();
}

protected int canEdit(ServiceContext context, String metadataUuid, Boolean approved) throws Exception {
return canEdit(context, metadataUuid, null, approved);
}

protected int canEdit(ServiceContext context, String metadataUuid, MetadataResourceVisibility visibility, Boolean approved)
throws Exception {
int metadataId = getAndCheckMetadataId(metadataUuid, approved);
boolean canEdit = getAccessManager(context).canEdit(context, String.valueOf(metadataId));
if ((visibility == null && !canEdit) || (visibility == MetadataResourceVisibility.PRIVATE && !canEdit)) {
throw new SecurityException(String.format("User '%s' does not have privileges to access '%s' resources for metadata '%s'.",
context.getUserSession() != null ?
context.getUserSession().getUsername() + "/" + context.getUserSession()
.getProfile() :
"anonymous", visibility == null ? "any" : visibility, metadataUuid));
}
return metadataId;
}

protected int canDownload(ServiceContext context, String metadataUuid, MetadataResourceVisibility visibility, Boolean approved)
throws Exception {
int metadataId = getAndCheckMetadataId(metadataUuid, approved);
if (visibility == MetadataResourceVisibility.PRIVATE) {
boolean canDownload = getAccessManager(context).canDownload(context, String.valueOf(metadataId));
if (!canDownload) {
throw new SecurityException(String.format(
"Current user can't download resources for metadata '%s' and as such can't access the requested resource.",
metadataUuid));
}
}
return metadataId;
}

protected String getFilenameFromUrl(final URL fileUrl) {
String fileName = FilenameUtils.getName(fileUrl.getPath());
if (fileName.contains("?")) {
fileName = fileName.substring(0, fileName.indexOf("?"));
}
return fileName;
}

@Override
public final MetadataResource putResource(final ServiceContext context, final String metadataUuid, final MultipartFile file,
final MetadataResourceVisibility visibility) throws Exception {
return putResource(context, metadataUuid, file.getOriginalFilename(), file.getInputStream(), null, visibility, true);
}

@Override
public final MetadataResource putResource(final ServiceContext context, final String metadataUuid, final MultipartFile file,
final MetadataResourceVisibility visibility, Boolean approved) throws Exception {
return putResource(context, metadataUuid, file.getOriginalFilename(), file.getInputStream(), null, visibility, approved);
}

@Override
public final MetadataResource putResource(final ServiceContext context, final String metadataUuid, final Path file,
final MetadataResourceVisibility visibility) throws Exception {
return putResource(context, metadataUuid, file, visibility, true);
}

@Override
public final MetadataResource putResource(final ServiceContext context, final String metadataUuid, final Path file,
final MetadataResourceVisibility visibility, Boolean approved) throws Exception {
final InputStream is = new BufferedInputStream(Files.newInputStream(file));
return putResource(context, metadataUuid, file.getFileName().toString(), is, null, visibility, approved);
}

@Override
public final MetadataResource putResource(ServiceContext context, String metadataUuid, URL fileUrl,
MetadataResourceVisibility visibility) throws Exception {
return putResource(context, metadataUuid, fileUrl, visibility, true);
}

@Override
public final MetadataResource putResource(ServiceContext context, String metadataUuid, URL fileUrl,
MetadataResourceVisibility visibility, Boolean approved) throws Exception {
return putResource(context, metadataUuid, getFilenameFromUrl(fileUrl), fileUrl.openStream(), null, visibility, approved);
}

@Override
public String delResources(final ServiceContext context, final String metadataUuid) throws Exception {
return delResources(context, metadataUuid, true);
}

@Override
public String delResource(final ServiceContext context, final String metadataUuid, final String resourceId) throws Exception {
return delResource(context, metadataUuid, resourceId, true);
}

@Override
public MetadataResource patchResourceStatus(final ServiceContext context, final String metadataUuid, final String resourceId,
final MetadataResourceVisibility metadataResourceVisibility) throws Exception {
return patchResourceStatus(context, metadataUuid, resourceId, metadataResourceVisibility, true);
}

protected String getFilename(final String metadataUuid, final String resourceId) {
// It's not always clear when we get a resourceId or a filename
String prefix = metadataUuid + "/attachements/";
if (resourceId.startsWith(prefix)) {
// It was a resourceId
return resourceId.substring(prefix.length());
} else {
// It was a filename
return resourceId;
}
}

protected void checkResourceId(final String resourceId) {
if (resourceId.contains("..") || resourceId.startsWith("/") || resourceId.startsWith("file:/")) {
throw new SecurityException(String.format("Invalid resource identifier '%s'.", resourceId));
}
}
}

0 comments on commit 6c80a91

Please sign in to comment.
You can’t perform that action at this time.