Skip to content

Commit

Permalink
Merge branch 'develop' into feature-3233-improve-pds-solution-prepare
Browse files Browse the repository at this point in the history
# Conflicts:
#	sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/cli/PrepareWrapperCLI.java
  • Loading branch information
lorriborri committed Jul 2, 2024
2 parents f30cd11 + a2ea607 commit 24c330a
Show file tree
Hide file tree
Showing 77 changed files with 1,690 additions and 244 deletions.
30 changes: 17 additions & 13 deletions sechub-api-java/src/main/resources/reduced-openapi3.json
Original file line number Diff line number Diff line change
Expand Up @@ -970,19 +970,6 @@
"webScan": {
"type": "object",
"properties": {
"maxScanDuration": {
"type": "object",
"properties": {
"duration": {
"type": "number",
"description": "Duration of the scan as integer"
},
"unit": {
"type": "string",
"description": "Unit of the duration. Possible values are: millisecond(s), second(s), minute(s), hour(s), day(s)"
}
}
},
"headers": {
"type": "array",
"description": "List of HTTP headers. Can be used for authentication or anything else.",
Expand Down Expand Up @@ -1018,6 +1005,19 @@
}
}
},
"maxScanDuration": {
"type": "object",
"properties": {
"duration": {
"type": "number",
"description": "Duration of the scan as integer"
},
"unit": {
"type": "string",
"description": "Unit of the duration. Possible values are: millisecond(s), second(s), minute(s), hour(s), day(s)"
}
}
},
"clientCertificate": {
"type": "object",
"properties": {
Expand Down Expand Up @@ -1051,6 +1051,10 @@
"api": {
"type": "object",
"properties": {
"apiDefinitionUrl": {
"type": "string",
"description": "Specifies an URL to read the API definition from."
},
"use": {
"type": "array",
"description": "Reference to the data section containing the API definition files. Always use 'sources' with 'files' instead 'folders'.",
Expand Down
8 changes: 4 additions & 4 deletions sechub-commons-archive/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,10 @@ dependencies{

api project(':sechub-commons-core')
api project(':sechub-commons-model')

implementation library.apache_commons_io
implementation library.apache_commons_compress // we need TAR archive support
api library.apache_commons_io
api library.apache_commons_compress // we need TAR archive support


testImplementation project(':sechub-testframework')
testImplementation spring_boot_dependency.junit_jupiter

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
// SPDX-License-Identifier: MIT
package com.mercedesbenz.sechub.commons.archive;

import static java.util.Objects.requireNonNull;

import java.time.Duration;
import java.util.Objects;

/**
* The ArchiveExtractionConstraints class encapsulates the properties used to
* safeguard the extraction of an archive when using the
* {@link SafeArchiveInputStream}. These properties include maximum uncompressed
* file size, maximum number of entries, maximum directory depth, and timeout.
*
* <p>
* Each property is validated during the creation of an
* ArchiveExtractionConstraints object to ensure they meet the required
* conditions.
*
* <p>
* Example usage:
*
* <pre>
* FileSize maxFileSizeUncompressed = new FileSize("20MB");
* long maxEntries = 100;
* long maxDirectoryDepth = 5;
* Duration timeout = Duration.ofMinutes(1);
*
* ArchiveExtractionConstraints properties = new ArchiveExtractionConstraints(maxFileSizeUncompressed, maxEntries, maxDirectoryDepth, timeout);
* </pre>
*
* @author hamidonos
*/
public class ArchiveExtractionConstraints {
private final FileSize maxFileSizeUncompressed;
private final long maxEntries;
private final long maxDirectoryDepth;
private final Duration timeout;

public ArchiveExtractionConstraints(FileSize maxFileSizeUncompressed, long maxEntries, long maxDirectoryDepth, Duration timeout) {
this.maxFileSizeUncompressed = requireNonNull(maxFileSizeUncompressed, "Property maxFileSizeUncompressed must not be null");
this.maxEntries = maxEntries;
if (this.maxEntries <= 0) {
throw new IllegalArgumentException("Property maxEntries must be greater than 0");
}
this.maxDirectoryDepth = maxDirectoryDepth;
if (this.maxDirectoryDepth <= 0) {
throw new IllegalArgumentException("Property maxDirectoryDepth must be greater than 0");
}
this.timeout = requireNonNull(timeout, "Property timeout must not be null");
if (this.timeout.isNegative() || this.timeout.isZero()) {
throw new IllegalArgumentException("Property timeout must be greater than 0");
}
}

public FileSize getMaxFileSizeUncompressed() {
return maxFileSizeUncompressed;
}

public long getMaxEntries() {
return maxEntries;
}

public long getMaxDirectoryDepth() {
return maxDirectoryDepth;
}

public Duration getTimeout() {
return timeout;
}

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof ArchiveExtractionConstraints that)) {
return false;
}
return Objects.equals(maxFileSizeUncompressed, that.maxFileSizeUncompressed) && Objects.equals(maxEntries, that.maxEntries)
&& Objects.equals(maxDirectoryDepth, that.maxDirectoryDepth) && Objects.equals(timeout, that.timeout);
}

@Override
public int hashCode() {
return Objects.hash(maxFileSizeUncompressed, maxEntries, maxDirectoryDepth, timeout);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
// SPDX-License-Identifier: MIT
package com.mercedesbenz.sechub.commons.archive;

import java.io.IOException;

/**
* This exception is thrown when an error occurs during extraction of an archive
* through the {@link SafeArchiveInputStream}.
*
* @author hamidonos
*/
public class ArchiveExtractionException extends IOException {
private static final long serialVersionUID = 1L;

public ArchiveExtractionException(String msg) {
super(msg);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
public class ArchiveExtractionResult {
int extractedFilesCount;
int createdFoldersCount;
long size;

String sourceLocation;
String targetLocation;
Expand All @@ -16,6 +17,10 @@ public int getCreatedFoldersCount() {
return createdFoldersCount;
}

public long getSize() {
return size;
}

public String getSourceLocation() {
return sourceLocation;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: MIT
package com.mercedesbenz.sechub.commons.archive;

import static java.util.Objects.*;
import static java.util.Objects.requireNonNull;

import java.io.File;
import java.io.FileNotFoundException;
Expand Down Expand Up @@ -44,49 +44,53 @@ public class ArchiveSupport {
private static final KeepAsIsTransformationData DO_NOT_TRANSFORM = new KeepAsIsTransformationData();
private static final Logger LOG = LoggerFactory.getLogger(ArchiveSupport.class);

private ArchiveTransformationDataFactory archiveTransformationDataFactory;
private final ArchiveTransformationDataFactory archiveTransformationDataFactory;
private boolean createMissingFiles;

public ArchiveSupport() {
this.archiveTransformationDataFactory = new ArchiveTransformationDataFactory();
archiveTransformationDataFactory = new ArchiveTransformationDataFactory();
}

/**
* Extract given archive type to output directory by
*
* @param archiveType
* @param sourceInputStream
* @param sourceLocation the path for the given source input stream.
* This information is only used in case of
* failures to print out the location - means
* only for debugging/error handling
* @param sourceLocation the path for the given source input
* stream. This information is only used in
* case of failures to print out the
* location - means only for debugging/error
* handling
* @param outputDir
* @param fileStructureDataProvider used to transform/filter the extraction. If
* <code>null</code>, a fallback will be used
* which does no transformation or filtering
* @param fileStructureDataProvider used to transform/filter the extraction.
* If <code>null</code>, a fallback will be
* used which does no transformation or
* filtering
* @param archiveExtractionConstraints defines constraints to safeguard the
* extraction of the archive
*
* @return extraction result
*
* @throws IOException
*/
public ArchiveExtractionResult extract(ArchiveType archiveType, InputStream sourceInputStream, String sourceLocation, File outputDir,
SecHubFileStructureDataProvider fileStructureDataProvider) throws IOException {
SecHubFileStructureDataProvider fileStructureDataProvider, ArchiveExtractionConstraints archiveExtractionConstraints) throws IOException {
if (archiveType == null) {
throw new IllegalArgumentException("archive type must be defined!");
}
switch (archiveType) {
case TAR:
return extractTar(sourceInputStream, sourceLocation, outputDir, fileStructureDataProvider);
return extractTar(sourceInputStream, sourceLocation, outputDir, fileStructureDataProvider, archiveExtractionConstraints);
case ZIP:
return extractZip(sourceInputStream, sourceLocation, outputDir, fileStructureDataProvider);
return extractZip(sourceInputStream, sourceLocation, outputDir, fileStructureDataProvider, archiveExtractionConstraints);
default:
throw new IllegalArgumentException("archive type " + archiveType + " is not supported");

}
}

public void setCreateMissingFiles(boolean createPseudoFilesForMissingFiles) {
this.createMissingFiles = createPseudoFilesForMissingFiles;
createMissingFiles = createPseudoFilesForMissingFiles;
}

/**
Expand Down Expand Up @@ -164,9 +168,7 @@ private File createArchive(ArchiveType archiveType, SecHubConfigurationModel con
try (ArchiveOutputStream outputStream = new ArchiveStreamFactory().createArchiveOutputStream(archiveType.getType(),
new FileOutputStream(archiveFile))) {

if (outputStream instanceof TarArchiveOutputStream) {
@SuppressWarnings("resource")
TarArchiveOutputStream tarOutputStream = (TarArchiveOutputStream) outputStream;
if (outputStream instanceof @SuppressWarnings("resource") TarArchiveOutputStream tarOutputStream) {
tarOutputStream.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
tarOutputStream.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX);
}
Expand Down Expand Up @@ -249,8 +251,7 @@ private void collectBaseFoldersAndFiles(List<? extends SecHubFileSystemContainer
continue;
}
String uniqueName = ArchiveCreationContext.LEGACY_IDENTIFIER_UNIQUE_NAME;
if (fileSystemContainer instanceof SecHubDataConfigurationObject) {
SecHubDataConfigurationObject configObject = (SecHubDataConfigurationObject) fileSystemContainer;
if (fileSystemContainer instanceof SecHubDataConfigurationObject configObject) {
uniqueName = configObject.getUniqueName();
} else {
LOG.warn("No unique name found inside a data section entry - should not happen! Will use legacy identifier as fallback!");
Expand Down Expand Up @@ -312,10 +313,8 @@ public void compressFolder(ArchiveType type, File folder, File targetArchiveFile
try (ArchiveOutputStream outputStream = new ArchiveStreamFactory().createArchiveOutputStream(type.getType(), new FileOutputStream(targetArchiveFile))) {
String basePath = folder.toPath().toRealPath().toString();

if (outputStream instanceof TarArchiveOutputStream) {
if (outputStream instanceof @SuppressWarnings("resource") TarArchiveOutputStream tarArchiveOutputStream) {
/* in this case we activate long file support */
@SuppressWarnings("resource")
TarArchiveOutputStream tarArchiveOutputStream = (TarArchiveOutputStream) outputStream;
tarArchiveOutputStream.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
}
compressRecursively(basePath, outputStream, folder, type, null, null);
Expand Down Expand Up @@ -403,30 +402,31 @@ private void compressRecursively(String basePath, ArchiveOutputStream outputStre
IOUtils.copy(inputStream, outputStream);
}
outputStream.closeArchiveEntry();

}

}

private ArchiveExtractionResult extractTar(InputStream sourceInputStream, String sourceLocation, File outputDir,
SecHubFileStructureDataProvider fileStructureProvider) throws IOException {
try (ArchiveInputStream archiveInputStream = new ArchiveStreamFactory().createArchiveInputStream("tar", sourceInputStream)) {
SecHubFileStructureDataProvider fileStructureProvider, ArchiveExtractionConstraints archiveExtractionConstraints) throws IOException {
try (ArchiveInputStream archiveInputStream = new ArchiveStreamFactory().createArchiveInputStream(ArchiveType.TAR.getType(), sourceInputStream)) {
if (!(archiveInputStream instanceof TarArchiveInputStream)) {
throw new IOException("Cannot extract: " + sourceLocation + " because it is not a tar tar");
}
return extract(archiveInputStream, sourceLocation, outputDir, fileStructureProvider);

try (SafeArchiveInputStream safeArchiveInputStream = new SafeArchiveInputStream(archiveInputStream, archiveExtractionConstraints)) {
return extract(safeArchiveInputStream, sourceLocation, outputDir, fileStructureProvider);
}
} catch (ArchiveException e) {
throw new IOException("Was not able to extract tar:" + sourceLocation + " at " + outputDir, e);
}

}

private ArchiveExtractionResult extractZip(InputStream sourceInputStream, String sourceLocation, File outputDir,
SecHubFileStructureDataProvider configuration) throws IOException {
try (ArchiveInputStream archiveInputStream = new ArchiveStreamFactory().createArchiveInputStream("zip", sourceInputStream)) {
SecHubFileStructureDataProvider configuration, ArchiveExtractionConstraints archiveExtractionConstraints) throws IOException {
try (ArchiveInputStream archiveInputStream = new ArchiveStreamFactory().createArchiveInputStream(ArchiveType.ZIP.getType(), sourceInputStream);
SafeArchiveInputStream safeArchiveInputStream = new SafeArchiveInputStream(archiveInputStream, archiveExtractionConstraints)) {

return extract(archiveInputStream, sourceLocation, outputDir, configuration);
return extract(safeArchiveInputStream, sourceLocation, outputDir, configuration);

} catch (ArchiveException e) {
throw new IOException("Was not able to extract tar:" + sourceLocation + " at " + outputDir, e);
Expand Down Expand Up @@ -459,15 +459,15 @@ public boolean isZipFileStream(InputStream inputStream) {
}
}

private ArchiveExtractionResult extract(ArchiveInputStream sourceArchiveInputStream, String sourceLocation, File outputDir,
private ArchiveExtractionResult extract(SafeArchiveInputStream safeArchiveInputStream, String sourceLocation, File outputDir,
SecHubFileStructureDataProvider fileStructureProvider) throws ArchiveException, IOException {

ArchiveExtractionResult result = new ArchiveExtractionResult();
result.targetLocation = outputDir.getAbsolutePath();
result.sourceLocation = sourceLocation;

ArchiveEntry entry = null;
while ((entry = sourceArchiveInputStream.getNextEntry()) != null) {
while ((entry = safeArchiveInputStream.getNextEntry()) != null) {
String name = entry.getName();
if (name == null) {
throw new IllegalStateException("Entry path is null - cannot be handled!");
Expand Down Expand Up @@ -508,11 +508,12 @@ private ArchiveExtractionResult extract(ArchiveInputStream sourceArchiveInputStr
continue;
}
try (OutputStream outputFileStream = new FileOutputStream(outputFile)) {
IOUtils.copy(sourceArchiveInputStream, outputFileStream);
IOUtils.copy(safeArchiveInputStream, outputFileStream);
result.extractedFilesCount++;
}
}
}
result.size = safeArchiveInputStream.getBytesRead();
return result;
}

Expand Down Expand Up @@ -556,9 +557,9 @@ public enum ArchiveType {

TAR("tar");

private String type;
private final String type;

private ArchiveType(String type) {
ArchiveType(String type) {
this.type = type;
}

Expand Down
Loading

0 comments on commit 24c330a

Please sign in to comment.