Skip to content

Commit

Permalink
Refactored PDSStorageConstants #3026
Browse files Browse the repository at this point in the history
  • Loading branch information
@lorriborri
lorriborri committed May 24, 2024
1 parent 00a4ff3 commit e8ea8e6
Show file tree
Hide file tree
Showing 10 changed files with 98 additions and 66 deletions.
35 changes: 35 additions & 0 deletions ...re/src/main/java/com/mercedesbenz/sechub/pds/commons/core/config/PDSStorageConstants.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package com.mercedesbenz.sechub.pds.commons.core.config;

public class PDSStorageConstants {

/* shared volume */
public static final String PDS_STORAGE_SHAREDVOLUME_UPLOAD_DIR = "pds.storage.sharedvolume.upload.dir";

/* s3 */
public static final String PDS_STORAGE_S3_ACCESSKEY = "pds.storage.s3.accesskey";

public static final String PDS_STORAGE_S3_SECRETKEY = "pds.storage.s3.secretkey";

public static final String PDS_STORAGE_S3_BUCKETNAME = "pds.storage.s3.bucketname";

public static final String PDS_STORAGE_S3_ENDPOINT = "pds.storage.s3.endpoint";

public static final String PDS_STORAGE_S3_TIMEOUT_CONNECTION_MILLISECONDS = "pds.storage.s3.timeout.connection.milliseconds";

public static final String PDS_STORAGE_S3_TIMEOUT_SOCKET_MILLISECONDS = "pds.storage.s3.timeout.socket.milliseconds";

public static final String PDS_STORAGE_S3_TIMEOUT_REQUEST_MILLISECONDS = "pds.storage.s3.timeout.request.milliseconds";

public static final String PDS_STORAGE_S3_TIMEOUT_EXECUTION_MILLISECONDS = "pds.storage.s3.timeout.execution.milliseconds";

public static final String PDS_STORAGE_S3_CONNECTION_MAX_POOLSIZE = "pds.storage.s3.connection.max.poolsize";

public static final String PDS_STORAGE_S3_CONNECTION_TTL_MILLISECONDS = "pds.storage.s3.connection.ttl.milliseconds";

public static final String PDS_STORAGE_S3_CONNECTION_IDLE_MAX_MILLISECONDS = "pds.storage.s3.connection.idle.max.milliseconds";

public static final String PDS_STORAGE_S3_CONNECTION_IDLE_VALIDATE_MILLISECONDS = "pds.storage.s3.connection.idle.validate.milliseconds";

public static final String PDS_STORAGE_S3_SIGNER_OVERRIDE = "pds.storage.s3.signer.override";

}
29 changes: 16 additions & 13 deletions sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/storage/PDSS3PropertiesSetup.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
// SPDX-License-Identifier: MIT
package com.mercedesbenz.sechub.pds.storage;

import static com.mercedesbenz.sechub.pds.commons.core.config.PDSStorageConstants.*;

import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

Expand All @@ -11,62 +13,63 @@
public class PDSS3PropertiesSetup implements S3Setup {

private static final String UNDEFINED = "undefined";

@PDSMustBeDocumented(value = "Defines the access key for used s3 bucket", scope = "storage", secret = true)
@Value("${pds.storage.s3.accesskey:" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
@Value("${" + PDS_STORAGE_S3_ACCESSKEY + ":" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
private String accessKey;

@PDSMustBeDocumented(value = "Defines the secret key for used s3 bucket", scope = "storage", secret = true)
@Value("${pds.storage.s3.secretkey:" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
@Value("${" + PDS_STORAGE_S3_SECRETKEY + ":" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
private String secretKey;

@PDSMustBeDocumented(value = "Defines the s3 bucket name", scope = "storage")
@Value("${pds.storage.s3.bucketname:" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
@Value("${" + PDS_STORAGE_S3_BUCKETNAME + ":" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
private String bucketName;

@PDSMustBeDocumented(value = "Defines the s3 endpoint - e.g. https://play.min.io", scope = "storage")
@Value("${pds.storage.s3.endpoint:" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
@Value("${" + PDS_STORAGE_S3_ENDPOINT + ":" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
private String endpoint;

/* timeout */

@PDSMustBeDocumented(value = "S3 client timeout (in milliseconds) for creating new connections.", scope = "storage")
@Value("${pds.storage.s3.timeout.connection.milliseconds:" + S3Setup.DEFAULT_CONNECTION_TIMEOUT + "}")
@Value("${" + PDS_STORAGE_S3_TIMEOUT_CONNECTION_MILLISECONDS + ":" + S3Setup.DEFAULT_CONNECTION_TIMEOUT + "}")
private int connectionTimeoutInMilliseconds;

@PDSMustBeDocumented(value = "S3 client timeout (in milliseconds) for reading from a connected socket.", scope = "storage")
@Value("${pds.storage.s3.timeout.socket.milliseconds:" + S3Setup.DEFAULT_SOCKET_TIMEOUT + "}")
@Value("${" + PDS_STORAGE_S3_TIMEOUT_SOCKET_MILLISECONDS + ":" + S3Setup.DEFAULT_SOCKET_TIMEOUT + "}")
private int socketTimeoutInMilliseconds;

@PDSMustBeDocumented(value = "S3 client timeout (in milliseconds) for a request. 0 means it is disabled.", scope = "storage")
@Value("${pds.storage.s3.timeout.request.milliseconds:" + S3Setup.DEFAULT_REQUEST_TIMEOUT + "}")
@Value("${" + PDS_STORAGE_S3_TIMEOUT_REQUEST_MILLISECONDS + ":" + S3Setup.DEFAULT_REQUEST_TIMEOUT + "}")
private int requestTimeoutInMilliseconds;

@PDSMustBeDocumented(value = "S3 client timeout (in milliseconds) for execution. 0 means it is disabled.", scope = "storage")
@Value("${pds.storage.s3.timeout.execution.milliseconds:" + S3Setup.DEFAULT_CLIENT_EXECUTION_TIMEOUT + "}")
@Value("${" + PDS_STORAGE_S3_TIMEOUT_EXECUTION_MILLISECONDS + ":" + S3Setup.DEFAULT_CLIENT_EXECUTION_TIMEOUT + "}")
private int clientExecutionTimeoutInMilliseconds;

/* connections */

@PDSMustBeDocumented(value = "S3 client max connection pool size.", scope = "storage")
@Value("${pds.storage.s3.connection.max.poolsize:" + S3Setup.DEFAULT_MAX_CONNECTIONS + "}")
@Value("${" + PDS_STORAGE_S3_CONNECTION_MAX_POOLSIZE + ":" + S3Setup.DEFAULT_MAX_CONNECTIONS + "}")
private int maximumAllowedConnections;

@PDSMustBeDocumented(value = "S3 client expiration time (in milliseconds) for a connection in the connection pool. -1 means deactivated", scope = "storage")
@Value("${pds.storage.s3.connection.ttl.milliseconds:" + S3Setup.DEFAULT_CONNECTION_TTL + "}")
@Value("${" + PDS_STORAGE_S3_CONNECTION_TTL_MILLISECONDS + ":" + S3Setup.DEFAULT_CONNECTION_TTL + "}")
private long connectionTTLInMilliseconds;

@PDSMustBeDocumented(value = "S3 client maximum idle time (in milliseconds) for a connection in the connection pool.", scope = "storage")
@Value("${pds.storage.s3.connection.idle.max.milliseconds:" + S3Setup.DEFAULT_CONNECTION_MAX_IDLE_MILLIS + "}")
@Value("${" + PDS_STORAGE_S3_CONNECTION_IDLE_MAX_MILLISECONDS + ":" + S3Setup.DEFAULT_CONNECTION_MAX_IDLE_MILLIS + "}")
private long connectionMaxIdleInMilliseconds;

@PDSMustBeDocumented(value = "S3 client time (in milliseconds) a connection can be idle in the connection pool before it must be validated that it's still open.", scope = "storage")
@Value("${pds.storage.s3.connection.idle.validate.milliseconds:" + S3Setup.DEFAULT_VALIDATE_AFTER_INACTIVITY_MILLIS + "}")
@Value("${" + PDS_STORAGE_S3_CONNECTION_IDLE_VALIDATE_MILLISECONDS + ":" + S3Setup.DEFAULT_VALIDATE_AFTER_INACTIVITY_MILLIS + "}")
private int validateAfterInactivityInMilliseconds;

/* signer */

@PDSMustBeDocumented(value = "Can be used to override the default name of the signature algorithm used to sign requests.", scope = "storage")
@Value("${pds.storage.s3.signer.override:" + S3Setup.DEFAULT_SIGNER_OVERRIDE + "}")
@Value("${" + PDS_STORAGE_S3_SIGNER_OVERRIDE + ":" + S3Setup.DEFAULT_SIGNER_OVERRIDE + "}")
private String signerOverride;

@Override
Expand Down
4 changes: 3 additions & 1 deletion ...pds/src/main/java/com/mercedesbenz/sechub/pds/storage/PDSSharedVolumePropertiesSetup.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
// SPDX-License-Identifier: MIT
package com.mercedesbenz.sechub.pds.storage;

import static com.mercedesbenz.sechub.pds.commons.core.config.PDSStorageConstants.PDS_STORAGE_SHAREDVOLUME_UPLOAD_DIR;

import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

Expand All @@ -15,7 +17,7 @@ public class PDSSharedVolumePropertiesSetup extends AbstractSharedVolumeProperti
* server side will be used
*/
@PDSMustBeDocumented(value = "Defines the root path for shared volume uploads - e.g. for sourcecode.zip etc. When using keyword *temp* as path, this will create a temporary directory (for testing).", scope = "storage")
@Value("${pds.storage.sharedvolume.upload.dir:" + UNDEFINED_UPLOAD_DIR + "}") // we use undefined here. Will be used in #isValid()
@Value("${" + PDS_STORAGE_SHAREDVOLUME_UPLOAD_DIR + ":" + UNDEFINED_UPLOAD_DIR + "}") // we use undefined here. Will be used in #isValid()
private String configuredUploadDir;

@Override
Expand Down
3 changes: 1 addition & 2 deletions sechub-wrapper-prepare/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,11 @@ dependencies {
implementation project(':sechub-storage-core')
implementation project(':sechub-storage-sharedvolume-spring')
implementation project(':sechub-storage-s3-aws')
implementation project(':sechub-pds-commons-core')

implementation spring_boot_dependency.logback_classic

implementation library.jgit_core
implementation project(path: ':sechub-pds')


/* test */
testImplementation project(':sechub-testframework')
Expand Down
2 changes: 2 additions & 0 deletions ...src/main/java/com/mercedesbenz/sechub/wrapper/prepare/cli/PrepareWrapperKeyConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ public class PrepareWrapperKeyConstants {
* Folder for PDS prepare where remote data gets downloaded to be uploaded to
* the shared storage
*/
// TODO: 23.05.24 laura diese variable weg -> ersetzen durch WORKSPACE_DIRECTORY
// + xxx
public static final String KEY_PDS_PREPARE_UPLOAD_DIRECTORY = "pds.prepare.upload.directory";

/**
Expand Down
4 changes: 4 additions & 0 deletions ...rc/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/PrepareWrapperModuleGit.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ boolean isMatchingGitType(String type) {

boolean isDownloadSuccessful(PrepareWrapperContext context) {
// check if download folder contains git

String uploadFolder = context.getEnvironment().getPdsPrepareUploadFolderDirectory();
if (Files.isDirectory(Path.of(uploadFolder))) {
String gitRepo = filesSupport.getSubfolderFileNameFromDirectory(uploadFolder);
Expand Down Expand Up @@ -158,6 +159,8 @@ private void clonePrivateRepository(PrepareWrapperContext context, SecHubRemoteC

SecHubMessage message = new SecHubMessage(SecHubMessageType.INFO, "Cloned private repository: " + location);
context.getUserMessages().add(message);
// TODO: 23.05.24 laura isDownloadSuccessful check

}

private static void addSealedUserCredentials(SecHubRemoteCredentialUserData user, HashMap<String, SealedObject> credentialMap) {
Expand Down Expand Up @@ -185,6 +188,7 @@ private void clonePublicRepository(PrepareWrapperContext context, String locatio

SecHubMessage message = new SecHubMessage(SecHubMessageType.INFO, "Cloned public repository: " + location);
context.getUserMessages().add(message);
// TODO: 23.05.24 laura isDownloadSuccessful check
}

private void cleanup(PrepareWrapperContext context) throws IOException {
Expand Down
34 changes: 17 additions & 17 deletions .../java/com/mercedesbenz/sechub/wrapper/prepare/upload/PrepareWrapperFileUploadService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,20 +54,20 @@ private void assertUploadParams(String projectId, UUID jobUUID, File file, Strin
private void storeUploadFileAndSha256Checksum(String projectId, UUID jobUUID, File file, String checkSum) {
JobStorage jobStorage = storageService.getJobStorage(projectId, jobUUID);

UploadContext uploadContext;
UploadFileNameData uploadFileNameData;

if (file.getName().endsWith(".tar")) {
uploadContext = new UploadContext(FILENAME_BINARIES_TAR, FILENAME_BINARIES_TAR_FILESIZE, FILENAME_BINARIES_TAR_CHECKSUM);
uploadFileNameData = new UploadFileNameData(FILENAME_BINARIES_TAR, FILENAME_BINARIES_TAR_FILESIZE, FILENAME_BINARIES_TAR_CHECKSUM);
} else if (file.getName().endsWith(".zip")) {
uploadContext = new UploadContext(FILENAME_SOURCECODE_ZIP, FILENAME_SOURCECODE_ZIP_FILESIZE, FILENAME_SOURCECODE_ZIP_CHECKSUM);
uploadFileNameData = new UploadFileNameData(FILENAME_SOURCECODE_ZIP, FILENAME_SOURCECODE_ZIP_FILESIZE, FILENAME_SOURCECODE_ZIP_CHECKSUM);
} else {
throw new IllegalArgumentException("File must be a zip or tar file");
}

upload(file, checkSum, jobStorage, uploadContext);
upload(file, checkSum, jobStorage, uploadFileNameData);
}

private void upload(File file, String checkSum, JobStorage jobStorage, UploadContext uploadContext) {
private void upload(File file, String checkSum, JobStorage jobStorage, UploadFileNameData uploadFileNameData) {
try (InputStream inputStream = new FileInputStream(file)) {
long fileSize = file.length();

Expand All @@ -76,25 +76,25 @@ private void upload(File file, String checkSum, JobStorage jobStorage, UploadCon
String fileSizeAsString = "" + fileSize;
long fileSizeAsStringSizeInBytes = fileSizeAsString.getBytes().length;

jobStorage.store(uploadContext.filename, inputStream, fileSize);
jobStorage.store(uploadContext.filesize, new StringInputStream(fileSizeAsString), fileSizeAsStringSizeInBytes);
jobStorage.store(uploadContext.checksum, new StringInputStream(checkSum), checksumSizeInBytes);
jobStorage.store(uploadFileNameData.fileFilename, inputStream, fileSize);
jobStorage.store(uploadFileNameData.filesizeFilename, new StringInputStream(fileSizeAsString), fileSizeAsStringSizeInBytes);
jobStorage.store(uploadFileNameData.checksumFilename, new StringInputStream(checkSum), checksumSizeInBytes);

} catch (IOException e) {
LOG.error("Was not able to store file: " + uploadContext.filename, e);
LOG.error("Was not able to store file: " + uploadFileNameData.fileFilename, e);
throw new RuntimeException("Was not able to upload sources");
}
}

private static class UploadContext {
String filename;
String filesize;
String checksum;
private static class UploadFileNameData {
String fileFilename;
String filesizeFilename;
String checksumFilename;

public UploadContext(String filename, String filename_filesize, String filename_checksum) {
this.filename = filename;
this.filesize = filename_filesize;
this.checksum = filename_checksum;
public UploadFileNameData(String fileFilename, String filesizeFilename, String checksumFilename) {
this.fileFilename = fileFilename;
this.filesizeFilename = filesizeFilename;
this.checksumFilename = checksumFilename;
}
}
}
43 changes: 16 additions & 27 deletions .../java/com/mercedesbenz/sechub/wrapper/prepare/upload/PrepareWrapperS3PropertiesSetup.java
View file
Original file line number Diff line number Diff line change
@@ -1,72 +1,61 @@
// SPDX-License-Identifier: MIT
package com.mercedesbenz.sechub.wrapper.prepare.upload;

import static com.mercedesbenz.sechub.pds.commons.core.config.PDSStorageConstants.*;

import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import com.mercedesbenz.sechub.pds.PDSMustBeDocumented;
import com.mercedesbenz.sechub.storage.core.S3Setup;

@Component
public class PrepareWrapperS3PropertiesSetup implements S3Setup {

private static final String UNDEFINED = "undefined";
@PDSMustBeDocumented(value = "Defines the access key for used s3 bucket", scope = "storage", secret = true)
@Value("${pds.storage.s3.accesskey:" + UNDEFINED + "}") // we use undefined here. Will be used in isValid

@Value("${" + PDS_STORAGE_S3_ACCESSKEY + ":" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
private String accessKey;

@PDSMustBeDocumented(value = "Defines the secret key for used s3 bucket", scope = "storage", secret = true)
@Value("${pds.storage.s3.secretkey:" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
@Value("${" + PDS_STORAGE_S3_SECRETKEY + ":" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
private String secretKey;

@PDSMustBeDocumented(value = "Defines the s3 bucket name", scope = "storage")
@Value("${pds.storage.s3.bucketname:" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
@Value("${" + PDS_STORAGE_S3_BUCKETNAME + ":" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
private String bucketName;

@PDSMustBeDocumented(value = "Defines the s3 endpoint - e.g. https://play.min.io", scope = "storage")
@Value("${pds.storage.s3.endpoint:" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
@Value("${" + PDS_STORAGE_S3_ENDPOINT + ":" + UNDEFINED + "}") // we use undefined here. Will be used in isValid
private String endpoint;

/* timeout */

@PDSMustBeDocumented(value = "S3 client timeout (in milliseconds) for creating new connections.", scope = "storage")
@Value("${pds.storage.s3.timeout.connection.milliseconds:" + S3Setup.DEFAULT_CONNECTION_TIMEOUT + "}")
@Value("${" + PDS_STORAGE_S3_TIMEOUT_CONNECTION_MILLISECONDS + ":" + S3Setup.DEFAULT_CONNECTION_TIMEOUT + "}")
private int connectionTimeoutInMilliseconds;

@PDSMustBeDocumented(value = "S3 client timeout (in milliseconds) for reading from a connected socket.", scope = "storage")
@Value("${pds.storage.s3.timeout.socket.milliseconds:" + S3Setup.DEFAULT_SOCKET_TIMEOUT + "}")
@Value("${" + PDS_STORAGE_S3_TIMEOUT_SOCKET_MILLISECONDS + ":" + S3Setup.DEFAULT_SOCKET_TIMEOUT + "}")
private int socketTimeoutInMilliseconds;

@PDSMustBeDocumented(value = "S3 client timeout (in milliseconds) for a request. 0 means it is disabled.", scope = "storage")
@Value("${pds.storage.s3.timeout.request.milliseconds:" + S3Setup.DEFAULT_REQUEST_TIMEOUT + "}")
@Value("${" + PDS_STORAGE_S3_TIMEOUT_REQUEST_MILLISECONDS + ":" + S3Setup.DEFAULT_REQUEST_TIMEOUT + "}")
private int requestTimeoutInMilliseconds;

@PDSMustBeDocumented(value = "S3 client timeout (in milliseconds) for execution. 0 means it is disabled.", scope = "storage")
@Value("${pds.storage.s3.timeout.execution.milliseconds:" + S3Setup.DEFAULT_CLIENT_EXECUTION_TIMEOUT + "}")
@Value("${" + PDS_STORAGE_S3_TIMEOUT_EXECUTION_MILLISECONDS + ":" + S3Setup.DEFAULT_CLIENT_EXECUTION_TIMEOUT + "}")
private int clientExecutionTimeoutInMilliseconds;

/* connections */

@PDSMustBeDocumented(value = "S3 client max connection pool size.", scope = "storage")
@Value("${pds.storage.s3.connection.max.poolsize:" + S3Setup.DEFAULT_MAX_CONNECTIONS + "}")
@Value("${" + PDS_STORAGE_S3_CONNECTION_MAX_POOLSIZE + ":" + S3Setup.DEFAULT_MAX_CONNECTIONS + "}")
private int maximumAllowedConnections;

@PDSMustBeDocumented(value = "S3 client expiration time (in milliseconds) for a connection in the connection pool. -1 means deactivated", scope = "storage")
@Value("${pds.storage.s3.connection.ttl.milliseconds:" + S3Setup.DEFAULT_CONNECTION_TTL + "}")
@Value("${" + PDS_STORAGE_S3_CONNECTION_TTL_MILLISECONDS + ":" + S3Setup.DEFAULT_CONNECTION_TTL + "}")
private long connectionTTLInMilliseconds;

@PDSMustBeDocumented(value = "S3 client maximum idle time (in milliseconds) for a connection in the connection pool.", scope = "storage")
@Value("${pds.storage.s3.connection.idle.max.milliseconds:" + S3Setup.DEFAULT_CONNECTION_MAX_IDLE_MILLIS + "}")
@Value("${" + PDS_STORAGE_S3_CONNECTION_IDLE_MAX_MILLISECONDS + ":" + S3Setup.DEFAULT_CONNECTION_MAX_IDLE_MILLIS + "}")
private long connectionMaxIdleInMilliseconds;

@PDSMustBeDocumented(value = "S3 client time (in milliseconds) a connection can be idle in the connection pool before it must be validated that it's still open.", scope = "storage")
@Value("${pds.storage.s3.connection.idle.validate.milliseconds:" + S3Setup.DEFAULT_VALIDATE_AFTER_INACTIVITY_MILLIS + "}")
@Value("${" + PDS_STORAGE_S3_CONNECTION_IDLE_VALIDATE_MILLISECONDS + ":" + S3Setup.DEFAULT_VALIDATE_AFTER_INACTIVITY_MILLIS + "}")
private int validateAfterInactivityInMilliseconds;

/* signer */

@PDSMustBeDocumented(value = "Can be used to override the default name of the signature algorithm used to sign requests.", scope = "storage")
@Value("${pds.storage.s3.signer.override:" + S3Setup.DEFAULT_SIGNER_OVERRIDE + "}")
@Value("${" + PDS_STORAGE_S3_SIGNER_OVERRIDE + ":" + S3Setup.DEFAULT_SIGNER_OVERRIDE + "}")
private String signerOverride;

@Override
Expand Down
Loading

0 comments on commit e8ea8e6

Please sign in to comment.