Skip to content

Commit

Permalink
Moved existing PDS tool logic into systemtest #2092
Browse files Browse the repository at this point in the history
- Added upload definitions
- github workflow integration test gradle calls do now set
  SecHub build stage to "all"  which will execute system
  integration test as well
- Moved archive support into systemtest and used this one
  now in PDS tools
- added internal test for illegal states
- improved health check
- system test document now inside techdoc
  • Loading branch information
de-jcup committed Apr 26, 2023
1 parent 5e731f1 commit 1fc52dd
Show file tree
Hide file tree
Showing 40 changed files with 854 additions and 201 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/create-releases.yml
Expand Up @@ -154,7 +154,7 @@ jobs:
# Integration test
# ----------------------
- name: Integration test
run: ./gradlew integrationtest
run: ./gradlew integrationtest -Dsechub.build.stage=all

- name: Create combined test report
if: always()
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/gradle.yml
Expand Up @@ -51,7 +51,7 @@ jobs:

# Integration test
- name: Integration test
run: ./gradlew integrationtest
run: ./gradlew integrationtest -Dsechub.build.stage=all

# We use 'if: always()' to run a step even if a previous step failed
- name: Create combined test report
Expand Down
@@ -0,0 +1,20 @@
package com.mercedesbenz.sechub.commons.core;

public class IgnoreOutputHandler implements OutputHandler {

@Override
public void info(String text) {

}

@Override
public void warn(String message) {

}

@Override
public void error(String message, Throwable t) {

}

}
@@ -0,0 +1,27 @@
package com.mercedesbenz.sechub.commons.core;

/**
* Sometimes it is necessary to switch between different output locations. <br>
* <br>
* An example:<br>
* A library is used from a CLI tool and also from a server application. The CLI
* wants to send the output directly to command line (without using a logging
* framework) but the server application wants to use slf4j logger. The library
* code does use the output handler to separate the logic to the caller side.
*
* @author Albert Tregnaghi
*
*/
public interface OutputHandler {

public void info(String message);

public void warn(String message);

public default void error(String message) {
error(message, null);
}

public void error(String message, Throwable t);

}
5 changes: 5 additions & 0 deletions sechub-doc/src/docs/asciidoc/sechub-techdoc.adoc
Expand Up @@ -69,4 +69,9 @@ include::documents/techdoc/09_howtos.adoc[]
// 13. Reporting
include::documents/shared/report/sechub_reporting.adoc[]

<<<<
// 14. system tests
include::documents/shared/systemtests/systemtests.adoc[]

<<<

Expand Up @@ -52,6 +52,8 @@ public enum AdoptedSystemTestDefaultFallback {

FALLBACK_SECHUB_WAIT_FOR_AVAILABLE("true", "SecHub wait for available"),

FALLBACK_UPLOAD_REF_ID("default-ref", "Upload reference id"),

;

private String scope;
Expand Down
Expand Up @@ -11,9 +11,10 @@
* sechub-systemtest --> sechub-api-java --> openApiGenerator --> openapi3.json
* </pre>
*
* Equality is checked by <code>AdoptedSystemRuntimeVariablesTest.java</code> If
* it fails, please copy content system test RuntimeVariables at this location
* (class comments are ignored means can be custom)
* Equality to RuntimeVariable is checked by
* <code>AdoptedSystemTestRuntimeVariableTest.java</code> If it fails, please
* copy content system test RuntimeVariables at this location (class comments
* are ignored means can be custom)
*
* @return
*/
Expand Down
Expand Up @@ -5,8 +5,8 @@
import java.io.File;
import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.google.re2j.Pattern;
import com.mercedesbenz.sechub.commons.TextFileReader;

public class AdoptionChecker {
Expand Down
2 changes: 0 additions & 2 deletions sechub-pds-tools/build.gradle
Expand Up @@ -20,8 +20,6 @@ dependencies {
implementation project(':sechub-systemtest')

implementation library.apache_commons_io
implementation library.apache_commons_compress // we need TAR archive support


testImplementation spring_boot_dependency.junit_jupiter
testImplementation spring_boot_dependency.mockito_core
Expand Down
Expand Up @@ -36,7 +36,7 @@ void start(String... args) throws Exception {
exitHandler.exit(0);
case PDSToolsCLiConstants.CMD_GENERATE:
PDSSolutionTestFilesGenerator generator = new PDSSolutionTestFilesGenerator();
generator.setOutputHandler(consoleHandler);
generator.setConsoleHandler(consoleHandler);

if (args.length < 3 || args.length > 4) {
showHelpAndExit("Generate command needs 2 additional parameters: 1. config file path, 2.scan type", 3);
Expand Down
Expand Up @@ -5,71 +5,54 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;

import org.apache.commons.io.FileUtils;

import com.mercedesbenz.sechub.commons.TextFileReader;
import com.mercedesbenz.sechub.commons.TextFileWriter;
import com.mercedesbenz.sechub.commons.archive.ArchiveConstants;
import com.mercedesbenz.sechub.commons.archive.SecHubFileStructureDataProvider;
import com.mercedesbenz.sechub.commons.core.CommonConstants;
import com.mercedesbenz.sechub.commons.model.JSONConverter;
import com.mercedesbenz.sechub.commons.model.JSONConverterException;
import com.mercedesbenz.sechub.commons.model.ScanType;
import com.mercedesbenz.sechub.commons.model.SecHubCodeScanConfiguration;
import com.mercedesbenz.sechub.commons.model.SecHubConfigurationModel;
import com.mercedesbenz.sechub.commons.model.SecHubConfigurationModelReducedCloningSupport;
import com.mercedesbenz.sechub.commons.model.SecHubDataConfiguration;
import com.mercedesbenz.sechub.commons.model.SecHubDataConfigurationObject;
import com.mercedesbenz.sechub.commons.model.SecHubFileSystemConfiguration;
import com.mercedesbenz.sechub.commons.model.SecHubFileSystemContainer;
import com.mercedesbenz.sechub.commons.pds.PDSDefaultParameterKeyConstants;
import com.mercedesbenz.sechub.commons.pds.data.PDSJobData;
import com.mercedesbenz.sechub.commons.pds.data.PDSJobParameterEntry;
import com.mercedesbenz.sechub.pds.tools.archive.DeveloperArchiveSupport;
import com.mercedesbenz.sechub.pds.tools.handler.ConsoleHandler;
import com.mercedesbenz.sechub.pds.tools.handler.ConsoleOutputHandler;
import com.mercedesbenz.sechub.pds.tools.handler.PrintStreamConsoleHandler;
import com.mercedesbenz.sechub.systemtest.util.SystemTestUploadFilesSupport;

public class PDSSolutionTestFilesGenerator {

private TextFileReader reader = new TextFileReader();
private TextFileWriter writer = new TextFileWriter();
private DeveloperArchiveSupport developerArchiveSupport = new DeveloperArchiveSupport();

private File targetFolder;
private File originConfigFile;
private boolean createPseudoFilesWhenNotExisting = true;

public static void main(String[] args) throws Exception {
PDSSolutionTestFilesGenerator geno = new PDSSolutionTestFilesGenerator();
geno.generate(args);
}

private ConsoleHandler outputHandler;
private ConsoleHandler consoleHandler;
private ScanType scanType;
private SecHubConfigurationModel config;

ConsoleHandler getOutputHandler() {
if (outputHandler == null) {
outputHandler = new PrintStreamConsoleHandler();
ConsoleHandler getConsoleHandler() {
if (consoleHandler == null) {
consoleHandler = new PrintStreamConsoleHandler();
}
return outputHandler;
}

public void setCreatePseudoFilesWhenNotExisting(boolean create) {
this.createPseudoFilesWhenNotExisting = create;
return consoleHandler;
}

public void setOutputHandler(ConsoleHandler outputHandler) {
this.outputHandler = outputHandler;
public void setConsoleHandler(ConsoleHandler consoleHandler) {
this.consoleHandler = consoleHandler;
}

/* only for command line call - so private */
private File generate(String[] args) throws Exception {
if (args.length != 2) {
getOutputHandler().error("please call with ${pathToSechubConfigFile} ${scanType}");
getConsoleHandler().error("please call with ${pathToSechubConfigFile} ${scanType}");
throw new IllegalArgumentException("wrong number of parameters");
}
return generate(args[0], args[1], null);
Expand Down Expand Up @@ -100,12 +83,15 @@ public File generate(String pathToSecHubConfigFile, String wantedScanType, File
String recucedSecHubConfigJson = writeReducedConfigFile();
writePDSJobDataFile(recucedSecHubConfigJson);

writeTarAndZipFilesAsConfiguredInSecHubConfigFile();
SystemTestUploadFilesSupport uploadSupport = new SystemTestUploadFilesSupport(originConfigFile.getParentFile(), targetFolder);
uploadSupport.setOutputHandler(new ConsoleOutputHandler(getConsoleHandler()));

uploadSupport.createUploadFilesAsConfiguredInSecHubConfigFile(config, scanType);

getOutputHandler().output("Written files to:" + targetFolder.getAbsolutePath());
getConsoleHandler().output("Written files to:" + targetFolder.getAbsolutePath());
return targetFolder;
} catch (Exception e) {
getOutputHandler().error("Generation failed:" + e.getMessage());
getConsoleHandler().error("Generation failed:" + e.getMessage());
throw e;
}

Expand All @@ -116,104 +102,6 @@ private void writeSecHubConfigurationToTempFolder() throws JSONConverterExceptio

}

private void writeTarAndZipFilesAsConfiguredInSecHubConfigFile() throws IOException {

File extracted = new File(targetFolder, "extracted");

File binaryCopy = new File(extracted, "binaries");
File sourceCopy = new File(extracted, "source");

copyOriginFilesToExtractedFolder(binaryCopy, sourceCopy);

developerArchiveSupport.compressToTar(binaryCopy, new File(targetFolder, CommonConstants.FILENAME_BINARIES_TAR));
developerArchiveSupport.compressToZip(sourceCopy, new File(targetFolder, CommonConstants.FILENAME_SOURCECODE_ZIP));

SecHubFileStructureDataProvider.builder().setModel(config).setScanType(scanType).build();
}

private void copyOriginFilesToExtractedFolder(File binaryCopy, File sourceCopy) throws FileNotFoundException, IOException {
binaryCopy.mkdirs();
sourceCopy.mkdirs();

/* copy data parts */
if (config.getData().isPresent()) {
SecHubDataConfiguration data = config.getData().get();

copy(binaryCopy, data.getBinaries());

copy(sourceCopy, data.getSources());
}

/* copy embedded parts */
Optional<SecHubCodeScanConfiguration> codeScanOpt = config.getCodeScan();
if (codeScanOpt.isPresent()) {
SecHubCodeScanConfiguration codeScan = codeScanOpt.get();
copy(sourceCopy, Arrays.asList(codeScan));
}
}

private void copy(File binaryCopy, List<? extends SecHubFileSystemContainer> fileSystemContainers) throws FileNotFoundException, IOException {
for (SecHubFileSystemContainer config : fileSystemContainers) {
Optional<SecHubFileSystemConfiguration> fileSystemOpt = config.getFileSystem();
if (!fileSystemOpt.isPresent()) {
continue;
}
File baseTarget = null;
if ((config instanceof SecHubDataConfigurationObject)) {
SecHubDataConfigurationObject dataConfigObject = (SecHubDataConfigurationObject) config;
String uniqueName = dataConfigObject.getUniqueName();
baseTarget = new File(binaryCopy, ArchiveConstants.DATA_SECTION_FOLDER + uniqueName);
} else {
baseTarget = binaryCopy; // root folder, no data section
}

SecHubFileSystemConfiguration fileSystem = fileSystemOpt.get();
copyFileOrFolder(baseTarget, fileSystem.getFiles(), false);
copyFileOrFolder(baseTarget, fileSystem.getFolders(), true);

}
}

private void copyFileOrFolder(File baseTarget, List<String> files, boolean targetIsDirectory) throws FileNotFoundException, IOException {
for (String fileName : files) {
File file = new File(originConfigFile.getParentFile(), fileName);
boolean notExisting = !file.exists();

File targetFile = new File(baseTarget, fileName);
if (targetIsDirectory) {
targetFile.mkdirs();
} else {
targetFile.getParentFile().mkdirs();
}
if (notExisting) {
if (createPseudoFilesWhenNotExisting) {
if (targetIsDirectory) {
File pseudoFile = new File(targetFile, "gen_placeholder_because_origin_not_existing.txt");
pseudoFile.createNewFile();
getOutputHandler().output("WARN: origin folder:" + file.getAbsolutePath() + " did not exist. So created folder at "
+ targetFile.getAbsolutePath() + " with place holder file.");

} else {
targetFile.createNewFile();
getOutputHandler().output(
"WARN: origin file:" + file.getAbsolutePath() + " did not exist. So created pseudo file:" + targetFile.getAbsolutePath());
}
// we cannot create but have created pseudo-files/empty directories, so stop
// here
continue;
} else {
throw new FileNotFoundException("Cannot copy file/folder because it does not exist:" + file.getAbsolutePath());
}

}
if (targetIsDirectory) {
FileUtils.copyDirectory(file, targetFile);
} else {
FileUtils.copyFile(file, targetFile);
}
}
}

private String writeReducedConfigFile() throws IOException {
String recucedSecHubConfigJson = SecHubConfigurationModelReducedCloningSupport.DEFAULT.createReducedScanConfigurationCloneJSON(config, scanType);
SecHubConfigurationModel reducedConfig = JSONConverter.get().fromJSON(SecHubConfigurationModel.class, recucedSecHubConfigJson);
Expand Down
@@ -0,0 +1,32 @@
package com.mercedesbenz.sechub.pds.tools.handler;

import com.mercedesbenz.sechub.commons.core.OutputHandler;

public class ConsoleOutputHandler implements OutputHandler {

private ConsoleHandler consoleHandler;

public ConsoleOutputHandler(ConsoleHandler consoleHandler) {
this.consoleHandler = consoleHandler;
}

@Override
public void warn(String message) {
getConsoleHandler().output("WARN:" + message);
}

@Override
public void info(String message) {
getConsoleHandler().output(message);

}

@Override
public void error(String message, Throwable t) {
getConsoleHandler().error(message, t);
}

private ConsoleHandler getConsoleHandler() {
return consoleHandler;
}
}
Expand Up @@ -80,6 +80,8 @@ void generate_with_config_path_and_scan_type_and_target_folder_set_works() throw
cliToTest.start(new String[] { "--generate", testConfigFile.getAbsolutePath(), scanType, tmpFolder.getAbsolutePath() });

/* test */
assertFileExists(tmpFolder, "extracted");

assertFileExists(tmpFolder, "binaries.tar");
assertFileExists(tmpFolder, "sourcecode.zip");
assertFileExists(tmpFolder, "pdsJobData.json");
Expand Down
6 changes: 6 additions & 0 deletions sechub-systemtest/build.gradle
Expand Up @@ -16,12 +16,18 @@ dependencies {
implementation project(':sechub-pds-commons-core')
implementation project(':sechub-api-java')
implementation project(':sechub-commons-pds')
implementation project(':sechub-commons-archive')

implementation library.apache_commons_io
implementation library.apache_commons_compress // we need TAR archive support

implementation spring_boot_dependency.logback_classic

testImplementation spring_boot_dependency.junit_jupiter
testImplementation spring_boot_dependency.mockito_core
testImplementation project(':sechub-testframework')




}

0 comments on commit 1fc52dd

Please sign in to comment.