PDS prepare upload remote data to shared storage

sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/LogSanitizer.java → sechub-pds-commons-core/src/main/java/com/mercedesbenz/sechub/pds/commons/core/PDSLogSanitizer.java

@@ -1,20 +1,17 @@
-// SPDX-License-Identifier: MIT
-package com.mercedesbenz.sechub.pds;
+package com.mercedesbenz.sechub.pds.commons.core;
 
 import java.util.regex.Pattern;
 
-import org.springframework.stereotype.Component;
+public class PDSLogSanitizer {
 
-/**
- * This log sanitizer component handles -
- * https://cwe.mitre.org/data/definitions/93.html -
- * https://cwe.mitre.org/data/definitions/117.html
- *
- * @author Albert Tregnaghi
- *
- */
-@Component
-public class LogSanitizer {
+    /**
+     * This log sanitizer component handles -
+     * https://cwe.mitre.org/data/definitions/93.html -
+     * https://cwe.mitre.org/data/definitions/117.html
+     *
+     * @author Albert Tregnaghi
+     *
+     */
 
     private static final Pattern FORGERY_PATTERN = Pattern.compile("[\t\n\r]");
 
@@ -64,5 +61,4 @@ public String sanitize(String maybeContaminated, int maxLength, boolean handleLo
         }
         return result;
     }
-
 }

sechub-pds-commons-core/src/main/java/com/mercedesbenz/sechub/pds/commons/core/config/PDSStorageConstants.java

@@ -0,0 +1,35 @@
+package com.mercedesbenz.sechub.pds.commons.core.config;
+
+public class PDSStorageConstants {
+
+    /* shared volume */
+    public static final String PDS_STORAGE_SHAREDVOLUME_UPLOAD_DIR = "pds.storage.sharedvolume.upload.dir";
+
+    /* s3 */
+    public static final String PDS_STORAGE_S3_ACCESSKEY = "pds.storage.s3.accesskey";
+
+    public static final String PDS_STORAGE_S3_SECRETKEY = "pds.storage.s3.secretkey";
+
+    public static final String PDS_STORAGE_S3_BUCKETNAME = "pds.storage.s3.bucketname";
+
+    public static final String PDS_STORAGE_S3_ENDPOINT = "pds.storage.s3.endpoint";
+
+    public static final String PDS_STORAGE_S3_TIMEOUT_CONNECTION_MILLISECONDS = "pds.storage.s3.timeout.connection.milliseconds";
+
+    public static final String PDS_STORAGE_S3_TIMEOUT_SOCKET_MILLISECONDS = "pds.storage.s3.timeout.socket.milliseconds";
+
+    public static final String PDS_STORAGE_S3_TIMEOUT_REQUEST_MILLISECONDS = "pds.storage.s3.timeout.request.milliseconds";
+
+    public static final String PDS_STORAGE_S3_TIMEOUT_EXECUTION_MILLISECONDS = "pds.storage.s3.timeout.execution.milliseconds";
+
+    public static final String PDS_STORAGE_S3_CONNECTION_MAX_POOLSIZE = "pds.storage.s3.connection.max.poolsize";
+
+    public static final String PDS_STORAGE_S3_CONNECTION_TTL_MILLISECONDS = "pds.storage.s3.connection.ttl.milliseconds";
+
+    public static final String PDS_STORAGE_S3_CONNECTION_IDLE_MAX_MILLISECONDS = "pds.storage.s3.connection.idle.max.milliseconds";
+
+    public static final String PDS_STORAGE_S3_CONNECTION_IDLE_VALIDATE_MILLISECONDS = "pds.storage.s3.connection.idle.validate.milliseconds";
+
+    public static final String PDS_STORAGE_S3_SIGNER_OVERRIDE = "pds.storage.s3.signer.override";
+
+}

sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/PDSPojoFactory.java

@@ -13,6 +13,7 @@
 import com.mercedesbenz.sechub.commons.model.CodeScanPathCollector;
 import com.mercedesbenz.sechub.commons.model.SecHubConfigurationModelSupport;
 import com.mercedesbenz.sechub.commons.model.SecHubDataConfigurationTypeListParser;
+import com.mercedesbenz.sechub.pds.commons.core.PDSLogSanitizer;
 
 /**
  * This factory creates some "plain old java" objects and inject them into
@@ -64,4 +65,9 @@ TextFileReader createTextFileReader() {
     SecHubConfigurationModelSupport createSecHubConfigurationModelSupport() {
         return new SecHubConfigurationModelSupport();
     }
+
+    @Bean
+    PDSLogSanitizer createLogSanitizer() {
+        return new PDSLogSanitizer();
+    }
 }

sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/config/PDSConfigService.java

@@ -11,11 +11,11 @@
 import org.springframework.stereotype.Service;
 import org.springframework.util.Assert;
 
-import com.mercedesbenz.sechub.pds.LogSanitizer;
 import com.mercedesbenz.sechub.pds.PDSNotAcceptableException;
 import com.mercedesbenz.sechub.pds.autocleanup.PDSAutoCleanupConfig;
 import com.mercedesbenz.sechub.pds.autocleanup.PDSAutoCleanupConfig.CleanupTime;
 import com.mercedesbenz.sechub.pds.autocleanup.PDSAutoCleanupDaysCalculator;
+import com.mercedesbenz.sechub.pds.commons.core.PDSLogSanitizer;
 import com.mercedesbenz.sechub.pds.usecase.PDSStep;
 import com.mercedesbenz.sechub.pds.usecase.UseCaseAdminFetchesAutoCleanupConfiguration;
 import com.mercedesbenz.sechub.pds.usecase.UseCaseAdminUpdatesAutoCleanupConfiguration;
@@ -26,7 +26,7 @@ public class PDSConfigService {
     private static final Logger LOG = LoggerFactory.getLogger(PDSConfigService.class);
 
     @Autowired
-    LogSanitizer logSanitizer;
+    PDSLogSanitizer PDSLogSanitizer;
 
     @Autowired
     PDSConfigRepository repository;
@@ -49,7 +49,7 @@ public void updateAutoCleanupConfiguration(PDSAutoCleanupConfig configuration) {
 
         String configurationAsJson = configuration.toJSON();
 
-        LOG.info("Admin updates auto cleanup configuration to: {}", logSanitizer.sanitize(configurationAsJson, 8192));
+        LOG.info("Admin updates auto cleanup configuration to: {}", PDSLogSanitizer.sanitize(configurationAsJson, 8192));
 
         PDSConfig config = getOrCreateConfig();
         config.autoCleanupConfiguration = configurationAsJson;

sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/job/PDSFileUploadJobService.java

@@ -31,9 +31,9 @@
 import com.mercedesbenz.sechub.commons.core.security.CheckSumSupport.CheckSumValidationResult;
 import com.mercedesbenz.sechub.commons.model.SecHubRuntimeException;
 import com.mercedesbenz.sechub.commons.pds.data.PDSJobStatusState;
-import com.mercedesbenz.sechub.pds.LogSanitizer;
 import com.mercedesbenz.sechub.pds.PDSBadRequestException;
 import com.mercedesbenz.sechub.pds.UploadSizeConfiguration;
+import com.mercedesbenz.sechub.pds.commons.core.PDSLogSanitizer;
 import com.mercedesbenz.sechub.pds.security.PDSRoleConstants;
 import com.mercedesbenz.sechub.pds.storage.PDSMultiStorageService;
 import com.mercedesbenz.sechub.pds.usecase.PDSStep;
@@ -71,7 +71,7 @@ public class PDSFileUploadJobService {
     UploadSizeConfiguration configuration;
 
     @Autowired
-    LogSanitizer logSanitizer;
+    PDSLogSanitizer PDSLogSanitizer;
 
     @Autowired
     PDSServletFileUploadFactory servletFileUploadFactory;
@@ -222,7 +222,7 @@ private void startUpload(UUID jobUUID, HttpServletRequest request, String fileNa
                 fileDefinedByUser = true;
                 break;
             default:
-                LOG.warn("Given field '{}' is not supported while uploading job data to project {}, {}", logSanitizer.sanitize(fieldName, 30), jobUUID);
+                LOG.warn("Given field '{}' is not supported while uploading job data to project {}, {}", PDSLogSanitizer.sanitize(fieldName, 30), jobUUID);
             }
         }
 

sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/storage/PDSS3PropertiesSetup.java

@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.pds.storage;
 
+import static com.mercedesbenz.sechub.pds.commons.core.config.PDSStorageConstants.*;
+
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
@@ -11,62 +13,63 @@
 public class PDSS3PropertiesSetup implements S3Setup {
 
     private static final String UNDEFINED = "undefined";
+
     @PDSMustBeDocumented(value = "Defines the access key for used s3 bucket", scope = "storage", secret = true)
-    @Value("${pds.storage.s3.accesskey:" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
+    @Value("${" + PDS_STORAGE_S3_ACCESSKEY + ":" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
     private String accessKey;
 
     @PDSMustBeDocumented(value = "Defines the secret key for used s3 bucket", scope = "storage", secret = true)
-    @Value("${pds.storage.s3.secretkey:" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
+    @Value("${" + PDS_STORAGE_S3_SECRETKEY + ":" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
     private String secretKey;
 
     @PDSMustBeDocumented(value = "Defines the s3 bucket name", scope = "storage")
-    @Value("${pds.storage.s3.bucketname:" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
+    @Value("${" + PDS_STORAGE_S3_BUCKETNAME + ":" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
     private String bucketName;
 
     @PDSMustBeDocumented(value = "Defines the s3 endpoint - e.g. https://play.min.io", scope = "storage")
-    @Value("${pds.storage.s3.endpoint:" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
+    @Value("${" + PDS_STORAGE_S3_ENDPOINT + ":" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
     private String endpoint;
 
     /* timeout */
 
     @PDSMustBeDocumented(value = "S3 client timeout (in milliseconds) for creating new connections.", scope = "storage")
-    @Value("${pds.storage.s3.timeout.connection.milliseconds:" + S3Setup.DEFAULT_CONNECTION_TIMEOUT + "}")
+    @Value("${" + PDS_STORAGE_S3_TIMEOUT_CONNECTION_MILLISECONDS + ":" + S3Setup.DEFAULT_CONNECTION_TIMEOUT + "}")
     private int connectionTimeoutInMilliseconds;
 
     @PDSMustBeDocumented(value = "S3 client timeout (in milliseconds) for reading from a connected socket.", scope = "storage")
-    @Value("${pds.storage.s3.timeout.socket.milliseconds:" + S3Setup.DEFAULT_SOCKET_TIMEOUT + "}")
+    @Value("${" + PDS_STORAGE_S3_TIMEOUT_SOCKET_MILLISECONDS + ":" + S3Setup.DEFAULT_SOCKET_TIMEOUT + "}")
     private int socketTimeoutInMilliseconds;
 
     @PDSMustBeDocumented(value = "S3 client timeout (in milliseconds) for a request. 0 means it is disabled.", scope = "storage")
-    @Value("${pds.storage.s3.timeout.request.milliseconds:" + S3Setup.DEFAULT_REQUEST_TIMEOUT + "}")
+    @Value("${" + PDS_STORAGE_S3_TIMEOUT_REQUEST_MILLISECONDS + ":" + S3Setup.DEFAULT_REQUEST_TIMEOUT + "}")
     private int requestTimeoutInMilliseconds;
 
     @PDSMustBeDocumented(value = "S3 client timeout (in milliseconds) for execution. 0 means it is disabled.", scope = "storage")
-    @Value("${pds.storage.s3.timeout.execution.milliseconds:" + S3Setup.DEFAULT_CLIENT_EXECUTION_TIMEOUT + "}")
+    @Value("${" + PDS_STORAGE_S3_TIMEOUT_EXECUTION_MILLISECONDS + ":" + S3Setup.DEFAULT_CLIENT_EXECUTION_TIMEOUT + "}")
     private int clientExecutionTimeoutInMilliseconds;
 
     /* connections */
 
     @PDSMustBeDocumented(value = "S3 client max connection pool size.", scope = "storage")
-    @Value("${pds.storage.s3.connection.max.poolsize:" + S3Setup.DEFAULT_MAX_CONNECTIONS + "}")
+    @Value("${" + PDS_STORAGE_S3_CONNECTION_MAX_POOLSIZE + ":" + S3Setup.DEFAULT_MAX_CONNECTIONS + "}")
     private int maximumAllowedConnections;
 
     @PDSMustBeDocumented(value = "S3 client expiration time (in milliseconds) for a connection in the connection pool. -1 means deactivated", scope = "storage")
-    @Value("${pds.storage.s3.connection.ttl.milliseconds:" + S3Setup.DEFAULT_CONNECTION_TTL + "}")
+    @Value("${" + PDS_STORAGE_S3_CONNECTION_TTL_MILLISECONDS + ":" + S3Setup.DEFAULT_CONNECTION_TTL + "}")
     private long connectionTTLInMilliseconds;
 
     @PDSMustBeDocumented(value = "S3 client maximum idle time (in milliseconds) for a connection in the connection pool.", scope = "storage")
-    @Value("${pds.storage.s3.connection.idle.max.milliseconds:" + S3Setup.DEFAULT_CONNECTION_MAX_IDLE_MILLIS + "}")
+    @Value("${" + PDS_STORAGE_S3_CONNECTION_IDLE_MAX_MILLISECONDS + ":" + S3Setup.DEFAULT_CONNECTION_MAX_IDLE_MILLIS + "}")
     private long connectionMaxIdleInMilliseconds;
 
     @PDSMustBeDocumented(value = "S3 client time (in milliseconds) a connection can be idle in the connection pool before it must be validated that it's still open.", scope = "storage")
-    @Value("${pds.storage.s3.connection.idle.validate.milliseconds:" + S3Setup.DEFAULT_VALIDATE_AFTER_INACTIVITY_MILLIS + "}")
+    @Value("${" + PDS_STORAGE_S3_CONNECTION_IDLE_VALIDATE_MILLISECONDS + ":" + S3Setup.DEFAULT_VALIDATE_AFTER_INACTIVITY_MILLIS + "}")
     private int validateAfterInactivityInMilliseconds;
 
     /* signer */
 
     @PDSMustBeDocumented(value = "Can be used to override the default name of the signature algorithm used to sign requests.", scope = "storage")
-    @Value("${pds.storage.s3.signer.override:" + S3Setup.DEFAULT_SIGNER_OVERRIDE + "}")
+    @Value("${" + PDS_STORAGE_S3_SIGNER_OVERRIDE + ":" + S3Setup.DEFAULT_SIGNER_OVERRIDE + "}")
     private String signerOverride;
 
     @Override

sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/storage/PDSSharedVolumePropertiesSetup.java

@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.pds.storage;
 
+import static com.mercedesbenz.sechub.pds.commons.core.config.PDSStorageConstants.PDS_STORAGE_SHAREDVOLUME_UPLOAD_DIR;
+
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
@@ -15,7 +17,7 @@ public class PDSSharedVolumePropertiesSetup extends AbstractSharedVolumeProperti
      * server side will be used
      */
     @PDSMustBeDocumented(value = "Defines the root path for shared volume uploads - e.g. for sourcecode.zip etc. When using keyword *temp* as path, this will create a temporary directory (for testing).", scope = "storage")
-    @Value("${pds.storage.sharedvolume.upload.dir:" + UNDEFINED_UPLOAD_DIR + "}") // we use undefined here. Will be used in #isValid()
+    @Value("${" + PDS_STORAGE_SHAREDVOLUME_UPLOAD_DIR + ":" + UNDEFINED_UPLOAD_DIR + "}") // we use undefined here. Will be used in #isValid()
     private String configuredUploadDir;
 
     @Override

sechub-pds/src/test/java/com/mercedesbenz/sechub/pds/config/PDSConfigServiceTest.java

@@ -14,11 +14,11 @@
 import org.junit.jupiter.params.provider.CsvSource;
 import org.mockito.ArgumentCaptor;
 
-import com.mercedesbenz.sechub.pds.LogSanitizer;
 import com.mercedesbenz.sechub.pds.PDSNotAcceptableException;
 import com.mercedesbenz.sechub.pds.autocleanup.PDSAutoCleanupConfig;
 import com.mercedesbenz.sechub.pds.autocleanup.PDSAutoCleanupConfig.CleanupTime;
 import com.mercedesbenz.sechub.pds.autocleanup.PDSAutoCleanupDaysCalculator;
+import com.mercedesbenz.sechub.pds.commons.core.PDSLogSanitizer;
 import com.mercedesbenz.sechub.sharedkernel.CountableInDaysTimeUnit;
 
 class PDSConfigServiceTest {
@@ -27,7 +27,7 @@ class PDSConfigServiceTest {
     private PDSConfigRepository repository;
     private PDSConfigTransactionService transactionService;
     private PDSAutoCleanupDaysCalculator calculator;
-    private LogSanitizer logSanitizer;
+    private PDSLogSanitizer PDSLogSanitizer;
 
     @BeforeEach
     void beforeEach() {
@@ -36,12 +36,12 @@ void beforeEach() {
         repository = mock(PDSConfigRepository.class);
         transactionService = mock(PDSConfigTransactionService.class);
         calculator = mock(PDSAutoCleanupDaysCalculator.class);
-        logSanitizer = mock(LogSanitizer.class);
+        PDSLogSanitizer = mock(PDSLogSanitizer.class);
 
         serviceToTest.repository = repository;
         serviceToTest.transactionService = transactionService;
         serviceToTest.calculator = calculator;
-        serviceToTest.logSanitizer = logSanitizer;
+        serviceToTest.PDSLogSanitizer = PDSLogSanitizer;
     }
 
     @Test

sechub-wrapper-prepare/build.gradle

@@ -10,7 +10,12 @@ dependencies {
     implementation project(':sechub-commons-model')
     implementation project(':sechub-commons-core')
     implementation project(':sechub-commons-pds')
+    implementation project(':sechub-commons-archive')
     implementation project(':sechub-adapter')
+    implementation project(':sechub-storage-core')
+    implementation project(':sechub-storage-sharedvolume-spring')
+    implementation project(':sechub-storage-s3-aws')
+    implementation project(':sechub-pds-commons-core')
 
     implementation spring_boot_dependency.logback_classic
 

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/cli/PrepareWrapperCLI.java

@@ -15,8 +15,10 @@
 import com.mercedesbenz.sechub.commons.core.prepare.PrepareStatus;
 import com.mercedesbenz.sechub.commons.model.SecHubMessage;
 import com.mercedesbenz.sechub.commons.model.SecHubMessageType;
+import com.mercedesbenz.sechub.wrapper.prepare.modules.PrepareWrapperUsageException;
 import com.mercedesbenz.sechub.wrapper.prepare.prepare.PrepareWrapperPreparationService;
-import com.mercedesbenz.sechub.wrapper.prepare.prepare.PrepareWrapperStorageService;
+import com.mercedesbenz.sechub.wrapper.prepare.prepare.PrepareWrapperResultStorageService;
+import com.mercedesbenz.sechub.wrapper.prepare.upload.PrepareWrapperUploadException;
 
 @Component
 public class PrepareWrapperCLI implements CommandLineRunner {
@@ -27,25 +29,37 @@ public class PrepareWrapperCLI implements CommandLineRunner {
     PrepareWrapperPreparationService preparationService;
 
     @Autowired
-    PrepareWrapperStorageService storageService;
+    PrepareWrapperResultStorageService storageService;
 
     @Override
     public void run(String... args) {
-        LOG.debug("Prepare wrapper starting");
+        LOG.debug("Prepare wrapper is starting.");
         AdapterExecutionResult result;
+
         try {
             result = preparationService.startPreparation();
+
+        } catch (PrepareWrapperUploadException e) {
+            LOG.error("Preparation of remote data has failed. Could not upload data to shared storage. ExitCode: {}", e.getExitCode(), e);
+            result = getAdapterExecutionResultFailed("Could not prepare remote data, because of an internal storage error.");
+
+        } catch (PrepareWrapperUsageException e) {
+            /* Usage exception messages will be added to sechub messages */
+            LOG.error("Preparation of remote data has failed, because of wrong usage. ExitCode: {}", e.getExitCode(), e);
+            result = getAdapterExecutionResultFailed(e.getMessage());
+
         } catch (Exception e) {
-            result = getAdapterExecutionResultFailed(e);
             LOG.error("Preparation of remote data has failed.", e);
+            result = getAdapterExecutionResultFailed("Could not prepare remote data, because of an internal error.");
         }
+
         storeResultOrFail(result);
     }
 
-    private static AdapterExecutionResult getAdapterExecutionResultFailed(Exception e) {
+    private static AdapterExecutionResult getAdapterExecutionResultFailed(String message) {
         PrepareResult prepareResult = new PrepareResult(PrepareStatus.FAILED);
         Collection<SecHubMessage> messages = new ArrayList<>();
-        messages.add(new SecHubMessage(SecHubMessageType.ERROR, e.getMessage()));
+        messages.add(new SecHubMessage(SecHubMessageType.ERROR, message));
         return new AdapterExecutionResult(prepareResult.toString(), messages);
     }