Skip to content

Commit

Permalink
Introduce system test api #2092 (#2119)
Browse files Browse the repository at this point in the history
- Provide initial system test suite #2092
- introduce build stage because of build lifecycle problems
- Introduce 'sechub-pds-commons-core' #2118
- provide wrapper api in sechub-api-java to stabilize usage #2184
- fix wrong OpenAPI generation for adminAssignsUserToProject #2191
- tests are not marked as integration tests but they are #2204
- Archive support is not able to compress multiple folders or single files #2359
- SecHubConfigurationModel has no excludes/includes #2363
- Improve log output for PDS execution failures #2398
  • Loading branch information
de-jcup committed Aug 11, 2023
1 parent 65a04b0 commit 5d7f446
Show file tree
Hide file tree
Showing 440 changed files with 23,277 additions and 1,518 deletions.
14 changes: 7 additions & 7 deletions .github/workflows/gradle.yml
Expand Up @@ -47,19 +47,19 @@ jobs:
run: ./gradlew :sechub-cli:buildGo :sechub-cli:testGo

- name: Build Server, DAUI and generate OpenAPI file
run: ./gradlew ensureLocalhostCertificate build generateOpenapi buildDeveloperAdminUI buildPDSToolsCLI -x :sechub-integrationtest:test -x :sechub-cli:build

- name: Generate and build API Java
run: ./gradlew :sechub-api-java:buildAPIJava
run: ./gradlew ensureLocalhostCertificate build generateOpenapi buildDeveloperAdminUI -x :sechub-cli:build
- name: Generate and build Java projects related to SecHub Java API
run: ./gradlew :sechub-api-java:build :sechub-systemtest:build :sechub-pds-tools:buildPDSToolsCLI -Dsechub.build.stage=api-necessary

# Integration test
- name: Integration test
run: ./gradlew integrationtest
run: ./gradlew :sechub-integrationtest:startIntegrationTestInstances :sechub-systemtest:integrationtest :sechub-integrationtest:integrationtest :sechub-integrationtest:stopIntegrationTestInstances -Dsechub.build.stage=all --console=plain

# We use 'if: always()' to run a step even if a previous step failed
- name: Create combined test report
if: always()
run: ./gradlew createCombinedTestReport
run: ./gradlew createCombinedTestReport -Dsechub.build.stage=all

# -----------------------------------------
# Upload Build Artifacts
Expand Down Expand Up @@ -117,7 +117,7 @@ jobs:
# Build Documentation
# -----------------------------------------
- name: Create documentation
run: ./gradlew documentation
run: ./gradlew documentation -Dsechub.build.stage=all

# -----------------------------------------
# Upload documentation
Expand Down
12 changes: 6 additions & 6 deletions .github/workflows/release-client-server-pds.yml
Expand Up @@ -163,25 +163,25 @@ jobs:
# Build SecHub Server + PDS
# ----------------------
- name: Build Server and PDS artifacts
run: ./gradlew ensureLocalhostCertificate build generateOpenapi buildDeveloperAdminUI -x :sechub-integrationtest:test -x :sechub-cli:build

run: ./gradlew ensureLocalhostCertificate build generateOpenapi buildDeveloperAdminUI -x :sechub-cli:build
# ----------------------
# Build API Java publish
# ----------------------
- name: Generate, build (and publish on server release) Java API
run: ./gradlew :sechub-api-java:buildAPIJava
- name: Generate and build Java projects related to SecHub Java API
run: ./gradlew :sechub-api-java:build :sechub-systemtest:build :sechub-pds-tools:buildPDSToolsCLI -Dsechub.build.stage=api-necessary
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token

# ----------------------
# Integration test
# ----------------------
- name: Integration test
run: ./gradlew integrationtest
run: ./gradlew :sechub-integrationtest:startIntegrationTestInstances :sechub-systemtest:integrationtest :sechub-integrationtest:integrationtest :sechub-integrationtest:stopIntegrationTestInstances -Dsechub.build.stage=all

- name: Create combined test report
if: always()
run: ./gradlew createCombinedTestReport
run: ./gradlew createCombinedTestReport -Dsechub.build.stage=all

# To identifiy parts not in git history and leading to "-dirty-$commitId" markern in documentation
- name: Inspect GIT status
Expand Down
7 changes: 5 additions & 2 deletions .github/workflows/release-pds-tools.yml
Expand Up @@ -84,8 +84,11 @@ jobs:
# ----------------------
# SecHub PDS-Tools
# ----------------------
- name: Build PDS-Tools
run: ./gradlew buildPDSToolsCLI
- name: Build Server, DAUI and generate OpenAPI file
run: ./gradlew ensureLocalhostCertificate build generateOpenapi buildDeveloperAdminUI -x :sechub-integrationtest:test -x :sechub-cli:build

- name: Generate and build Java projects related to SecHub Java API
run: ./gradlew :sechub-api-java:build :sechub-systemtest:build :sechub-pds-tools:buildPDSToolsCLI -Dsechub.build.stage=api-necessary

# To identifiy parts not in git history and leading to "-dirty-$commitId" markern in documentation
- name: Inspect GIT status
Expand Down
2 changes: 1 addition & 1 deletion build.gradle
@@ -1,6 +1,5 @@
// SPDX-License-Identifier: MIT
buildscript{

apply from: "${rootProject.projectDir}/gradle/libraries.gradle"
apply from: "${rootProject.projectDir}/gradle/projects.gradle"

Expand Down Expand Up @@ -139,3 +138,4 @@ apply from: "${rootProject.projectDir}/gradle/build-spring.gradle"
apply from: "${rootProject.projectDir}/gradle/build-maven.gradle"
apply from: "${rootProject.projectDir}/gradle/build-eclipse.gradle"
apply from: "${rootProject.projectDir}/gradle/build-report.gradle"
apply from: "${rootProject.projectDir}/gradle/build-integrationtest.gradle"
48 changes: 48 additions & 0 deletions buildSrc/src/main/groovy/BuildStage.groovy
@@ -0,0 +1,48 @@
/**
* Special build stage class. Because we need compiled java code to generate our open api file,
* the java api generation - which needs the open api file + a java compile - cannot happen
* on same "stage".
* To provide this, we have introduced the term sechub build stage - when stage "api-necessary" is
* used (or no stage is set), the parts which need a generated open api file will be included
* as well.
*/
class BuildStage{

private static final String STAGE_ALL = "all";
private static final String STAGE_WITHOUT_API = "without-api";
private static final String STAGE_API_NECESSARY = "api-necessary";

private String stage;
private boolean openApiFileMustExist;
private boolean acceptAll;

BuildStage(){
stage = System.getProperty("sechub.build.stage");
if(stage==null|| stage.isEmpty()){
// Per default we do not support API parts to avoid build life cycle problems
stage = STAGE_WITHOUT_API;
}

switch(stage){
case STAGE_ALL:
// We just do not define any constraints here
// Meaning: this stage can be imported by IDEs
acceptAll=true;
break;
case STAGE_WITHOUT_API:
openApiFileMustExist=false;
break;
case STAGE_API_NECESSARY:
openApiFileMustExist=true;
break;
default:
throw new IllegalArgumentException("Unknown build stage: '"+ stage+"'");
}

}

public boolean providesGeneratedOpenApiFile(){
return acceptAll || openApiFileMustExist;
}

}
63 changes: 63 additions & 0 deletions buildSrc/src/main/groovy/IOUtil.java
@@ -0,0 +1,63 @@
// SPDX-License-Identifier: MIT

import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.io.IOException;

public class IOUtil {

/**
* Creates a backup copy for the given file (if the file does exist) which
* can be restored by IOUtil. The location of the backup is handled by IOUtil internally.
*
* @param filePath the path for the file to backup
* @param backupPostFix a special post fix for the backup file, the backup file has
* the same name as the origin one, but with the post fix.
*/
public static final void createBackupFile(String filePath, String backupPostFix) throws IOException{
Path sourcePath = Paths.get(filePath);
Path targetPath = Paths.get(filePath + "_" + backupPostFix);

if (!Files.exists(sourcePath)) {
return;
}
System.out.println("Create backup file: "+targetPath + "\nfrom: "+sourcePath);

Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING);
}

/**
* Restores a previously created backup to the wanted file path (if a backup exists).
* The location of the backup is handled by IOUtil internally.
*
* @param filePath the path for the file to restore (not the backup file!)
* @backupPostFix a special post fix for the backup file
*/
public static final void restoreBackupFile(String filePath, String backupPostFix) throws IOException{
Path targetPath = Paths.get(filePath);
Path sourcePath = Paths.get(filePath + "_" + backupPostFix);
if (!Files.exists(sourcePath)) {
return;
}
System.out.println("Restore: "+targetPath + "\nfrom backup file: "+sourcePath);
Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING);
}

/**
* Copy a file to another location
* @sourcePath source path as string
* @targetPath target path as string
*/
public static final void copyFile(String sourcePath, String targetPath) throws IOException{

Path source = Paths.get(sourcePath);
Path target = Paths.get(targetPath);

target.toFile().getParentFile().mkdirs();

System.out.println("Copy: "+source + "\nto : "+target);
Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING);
}
}
4 changes: 2 additions & 2 deletions continous-integration-multibranch-pipeline.jenkins
Expand Up @@ -58,7 +58,7 @@ pipeline {
// Reason: because we do NOT want to have the integration tests executed, otherwise gradle will not execute them
// on integration phase again (because nothing has changed, so gradle will cache the results which are ignored ...
callGradleWrapper("ensureLocalhostCertificate build generateOpenapi buildDeveloperAdminUI -x :sechub-integrationtest:test -x :sechub-cli:build -Psechub.test.wiremock.https_port=${env.SECHUB_TEST_WIREMOCK_HTTPS_PORT} -Psechub.test.wiremock.http_port=${env.SECHUB_TEST_WIREMOCK_HTTP_PORT}")
callGradleWrapper(":sechub-api-java:buildAPIJava")
callGradleWrapper(":sechub-api-java:build :sechub-systemtest:build :sechub-pds-tools:buildPDSToolsCLI -Dsechub.build.stage=api-necessary")
}
}
}
Expand Down Expand Up @@ -131,7 +131,7 @@ pipeline {
stage('Build Documentation') {
steps {
script{
callGradleWrapper('documentation')
callGradleWrapper('documentation -Dsechub.build.stage=all')
}
}
}
Expand Down
23 changes: 23 additions & 0 deletions gradle/build-integrationtest.gradle
@@ -0,0 +1,23 @@
subprojects {

if (! projectType.integrationTestProjects.contains(project)){
return;
}

task copyIntegrationTestResults(type: Copy) {
with{
from "${project.projectDir}/build/test-results/integrationtest"
into "${combined_report_junit_tmp}"
include "*.xml"
}
}
task copyIntegrationServerLogs(type: Copy) {
with{
from "${project.projectDir}"
into "${combined_report_target}"
include "integrationtest-*.log"
}
}
copyTestResults.dependsOn copyIntegrationTestResults
copyTestResults.dependsOn copyIntegrationServerLogs
}
12 changes: 10 additions & 2 deletions gradle/build-java.gradle
Expand Up @@ -76,6 +76,11 @@ subprojects{
}

test {
filter {
excludeTestsMatching "*ManualTest"
excludeTestsMatching "*IntTest"
}

/* Per default GRADLE stops the build if one single test fails. We want to have all tests executed. */
ignoreFailures = true

Expand All @@ -92,7 +97,7 @@ subprojects{
'sechub.test.s3mock.https.port' ])

systemProperty 'sechub.build.gradle', 'true' // we can use this to check for gradle build inside java tests

// add a collection to track failedTests
ext.failedTests = []

Expand Down Expand Up @@ -142,9 +147,12 @@ subprojects{
archives sourcesJar
archives javadocJar
}

}




allprojects {
if(! project.name.equals(rootProject.name) ) {
return
Expand Down
2 changes: 1 addition & 1 deletion gradle/build-versioning.gradle
Expand Up @@ -222,7 +222,7 @@ task assertReleaseable {
subprojects{
/**
* Get the current version for the sub project.
* Usage: ./gradlew :eclipse-commons-model:getCurrentVersion
* Usage: ./gradlew :$subProjectNamel:getCurrentVersion
*/
task getCurrentVersion {
doLast {
Expand Down
34 changes: 30 additions & 4 deletions gradle/projects.gradle
@@ -1,5 +1,8 @@
// SPDX-License-Identifier: MIT


ext {
secHubBuildStage = new BuildStage()
projectType = [

goProjects: [
Expand All @@ -8,15 +11,14 @@ projectType = [
javaProjects: [
project(':sechub-testframework'),
project(':sechub-analyzer-cli'),
project(':sechub-api-java'),
project(':sechub-commons-core'),
project(':sechub-commons-model'),
project(':sechub-commons-model-testframework'),
project(':sechub-commons-pds'),
project(':sechub-commons-archive'),
project(':sechub-storage-core'),
project(':sechub-wrapper-owasp-zap'),
project(':sechub-pds-tools'),
project(':sechub-pds-commons-core'),

],

Expand Down Expand Up @@ -108,16 +110,40 @@ projectType = [
eclipseProjects: [
project(':sechub-doc'),
project(':sechub-other'),
project(':sechub-api-java'),
project(':sechub-pds-solutions'),
],

asciiDoctorProjects: [
project(':sechub-doc'),
]
],

noSpotless : [

],

integrationTestProjects: [
project(':sechub-integrationtest'),
],
]
}

if (secHubBuildStage.providesGeneratedOpenApiFile()){
/* add the java projects which need a open api file / compiled java api */
projectType.javaProjects.add(project(':sechub-api-java'))

projectType.javaProjects.add(project(':sechub-systemtest'))
projectType.javaProjects.add(project(':sechub-pds-tools'))

projectType.javaProjects.add(project(':sechub-examples:example-sechub-api-java'))

/* avoid spotless duplication problem in gradle build */
projectType.noSpotless.add(project(':sechub-examples:example-sechub-api-java'))

/* make it possible to use integration test parts for systemtest (unit tests) as well */
projectType.integrationTestProjects.add(project(':sechub-systemtest'))
}


/* dynamically define java projects */
projectType.springBootProjects.addAll(projectType.springBootSecHubServerProjects)
projectType.springBootProjects.addAll(projectType.springBootPDSProjects)
Expand Down

0 comments on commit 5d7f446

Please sign in to comment.