Merge pull request #1773 from mercedes-benz/feature-1172-possibility-…

…to-define-supported-pds-upload-types

Improve PDS auto ZIP/TAR handling

sechub-adapter-pds/src/main/java/com/mercedesbenz/sechub/adapter/pds/PDSAdapterV1.java

@@ -290,7 +290,7 @@ private void handleUploadWhenRequired(PDSContext context, SecHubDataConfiguratio
         String secHubTraceId = context.getTraceID();
         AdapterMetaData metaData = context.getRuntimeContext().getMetaData();
 
-        boolean required = checkRequired(data, type);
+        boolean required = checkUploadRequired(data, type);
 
         if (!required) {
             LOG.debug("Skipped {} file upload for pds job:{}, because not required", type, pdsJobUUID);
@@ -321,8 +321,10 @@ private void handleUploadWhenRequired(PDSContext context, SecHubDataConfiguratio
         context.getRuntimeContext().getCallback().persist(metaData);
     }
 
-    private boolean checkRequired(PDSAdapterConfigData data, SecHubDataConfigurationType type) {
+    private boolean checkUploadRequired(PDSAdapterConfigData data, SecHubDataConfigurationType type) {
         switch (type) {
+        case NONE:
+            return false;
         case BINARY:
             return data.isBinaryTarFileRequired();
         case SOURCE:
@@ -334,6 +336,8 @@ private boolean checkRequired(PDSAdapterConfigData data, SecHubDataConfiguration
 
     private String fetchChecksumOrNull(PDSAdapterConfigData data, SecHubDataConfigurationType type) {
         switch (type) {
+        case NONE:
+            return null;
         case BINARY:
             return data.getBinariesTarFileChecksumOrNull();
         case SOURCE:
@@ -345,6 +349,8 @@ private String fetchChecksumOrNull(PDSAdapterConfigData data, SecHubDataConfigur
 
     private Long fetchFileSizeOrNull(PDSAdapterConfigData data, SecHubDataConfigurationType type) {
         switch (type) {
+        case NONE:
+            return null;
         case BINARY:
             return data.getBinariesTarFileSizeInBytesOrNull();
         case SOURCE:
@@ -356,6 +362,8 @@ private Long fetchFileSizeOrNull(PDSAdapterConfigData data, SecHubDataConfigurat
 
     private String createUploadMetaDataKey(UUID pdsJobUUID, SecHubDataConfigurationType type) {
         switch (type) {
+        case NONE:
+            return null;
         case BINARY:
             return PDSMetaDataID.createBinaryUploadDoneKey(pdsJobUUID);
         case SOURCE:

sechub-adapter-pds/src/main/java/com/mercedesbenz/sechub/adapter/pds/PDSUploadSupport.java

@@ -55,6 +55,8 @@ private Resource fetchResource(SecHubDataConfigurationType dataType, PDSContext
     private String resolveUploadFileName(SecHubDataConfigurationType type) {
         String fileName = null;
         switch (type) {
+        case NONE:
+            return null;
         case BINARY:
             fileName = FILENAME_BINARIES_TAR;
             break;
@@ -68,21 +70,23 @@ private String resolveUploadFileName(SecHubDataConfigurationType type) {
     }
 
     private InputStream resolveInputStream(SecHubDataConfigurationType dataType, PDSContext context, PDSAdapterConfigData data) throws AdapterException {
-        InputStream zipInputstream = null;
+        InputStream inputStream = null;
         switch (dataType) {
+        case NONE:
+            throw new IllegalStateException("There cannot be an inputstream for: " + dataType + ". Illegal situation - should not be called!");
         case BINARY:
-            zipInputstream = data.getBinaryTarFileInputStreamOrNull();
+            inputStream = data.getBinaryTarFileInputStreamOrNull();
             break;
         case SOURCE:
-            zipInputstream = data.getSourceCodeZipFileInputStreamOrNull();
+            inputStream = data.getSourceCodeZipFileInputStreamOrNull();
             break;
         default:
             throw new IllegalStateException("unsupported data type:" + dataType);
         }
-        if (zipInputstream == null) {
+        if (inputStream == null) {
             throw context.asAdapterException("Input stream for " + dataType + " file is null!");
         }
-        return zipInputstream;
+        return inputStream;
     }
 
 }

sechub-commons-model/src/main/java/com/mercedesbenz/sechub/commons/model/SecHubConfigurationModelSupport.java

@@ -12,10 +12,26 @@ public class SecHubConfigurationModelSupport {
 
     private static final Logger LOG = LoggerFactory.getLogger(SecHubConfigurationModelSupport.class);
 
+    /**
+     * Inspects the given model and checks if for given scan type source content is
+     * available/required for the scan
+     *
+     * @param scanType
+     * @param model
+     * @return <code>true</code> when source reference found for given scan type
+     */
     public boolean isSourceRequired(ScanType scanType, SecHubConfigurationModel model) {
         return isRequired(scanType, model, SecHubDataConfigurationType.SOURCE);
     }
 
+    /**
+     * Inspects the given model and checks if for given scan type binary content is
+     * available/required for the scan
+     *
+     * @param scanType
+     * @param model
+     * @return <code>true</code> when binary reference found for given scan type
+     */
     public boolean isBinaryRequired(ScanType scanType, SecHubConfigurationModel model) {
         return isRequired(scanType, model, SecHubDataConfigurationType.BINARY);
     }
@@ -96,6 +112,8 @@ private boolean isDataTypeContainedOrReferenced(SecHubDataConfigurationType data
             return atLeastOneNameReferencesOneElementInGivenDataConfiguration(names, data.getBinaries(), dataType);
         case SOURCE:
             return atLeastOneNameReferencesOneElementInGivenDataConfiguration(names, data.getSources(), dataType);
+        case NONE:
+            return false;
         default:
             LOG.error("Datatype {} unknown, so never contained", dataType);
             return false;

sechub-commons-model/src/main/java/com/mercedesbenz/sechub/commons/model/SecHubDataConfigurationType.java

@@ -3,7 +3,23 @@
 
 public enum SecHubDataConfigurationType {
 
+    /**
+     * Data contains binaries
+     */
     BINARY,
 
-    SOURCE
+    /**
+     * Data contains sources
+     */
+    SOURCE,
+
+    /**
+     * No data - this is a type which can be used when we have a scan where we do
+     * not need data from configuration. E.g. when doing a simple web scan without
+     * defining an OpenAPI file.
+     */
+    NONE,
+
+    ;
+
 }

sechub-commons-model/src/main/java/com/mercedesbenz/sechub/commons/model/SecHubDataConfigurationTypeListParser.java

@@ -0,0 +1,49 @@
+package com.mercedesbenz.sechub.commons.model;
+
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.mercedesbenz.sechub.commons.core.util.SimpleStringUtils;
+
+public class SecHubDataConfigurationTypeListParser {
+
+    private static final Logger LOG = LoggerFactory.getLogger(SecHubDataConfigurationTypeListParser.class);
+
+    /**
+     * Parses a given string and returns a result set or <code>null</code>.
+     *
+     * @param commaSeparatedList
+     * @return set or <code>null</code> if the comma separated list did contain
+     *         wrong values or was empty
+     */
+    public Set<SecHubDataConfigurationType> fetchTypesAsSetOrNull(String commaSeparatedList) {
+        Set<SecHubDataConfigurationType> set = new LinkedHashSet<>();
+
+        List<String> list = SimpleStringUtils.createListForCommaSeparatedValues(commaSeparatedList);
+        for (String entry : list) {
+            boolean found = false;
+            for (SecHubDataConfigurationType type : SecHubDataConfigurationType.values()) {
+                if (type.name().equalsIgnoreCase(entry)) {
+                    found = true;
+                    set.add(type);
+                    break;
+                }
+            }
+
+            if (!found) {
+                LOG.debug("Found invalid data type entry:{}.", entry);
+                return null;
+            }
+        }
+        if (set.size() == 0) {
+            LOG.debug("No data types found.");
+            return null;
+        }
+
+        return set;
+    }
+}

sechub-commons-model/src/test/java/com/mercedesbenz/sechub/commons/model/SecHubDataConfigurationTypeListParserTest.java

@@ -0,0 +1,76 @@
+package com.mercedesbenz.sechub.commons.model;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.util.Set;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.NullAndEmptySource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+class SecHubDataConfigurationTypeListParserTest {
+
+    private SecHubDataConfigurationTypeListParser parserToTest;
+
+    @BeforeEach
+    void beforeEach() {
+        parserToTest = new SecHubDataConfigurationTypeListParser();
+    }
+
+    @ParameterizedTest
+    @NullAndEmptySource
+    @ValueSource(strings = { "   ", ",", " ,", " , ", ",,, , ,,", "x", "x,y", "binary,illegal" })
+    void illegal_args_will_return_null_as_result(String string) {
+
+        /* execute */
+        Set<SecHubDataConfigurationType> parsed = parserToTest.fetchTypesAsSetOrNull(string);
+
+        /* test */
+        assertNull(parsed);
+    }
+
+    @ParameterizedTest
+    @EnumSource(SecHubDataConfigurationType.class)
+    void one_type_name_as_uppercased_results_in_set_with_one_entry(SecHubDataConfigurationType type) {
+
+        /* execute */
+        Set<SecHubDataConfigurationType> parsed = parserToTest.fetchTypesAsSetOrNull(type.name().toUpperCase());
+
+        /* test */
+        assertNotNull(parsed);
+        assertEquals(parsed.size(), 1);
+        assertEquals(type, parsed.iterator().next());
+    }
+
+    @ParameterizedTest
+    @EnumSource(SecHubDataConfigurationType.class)
+    void one_type_name_as_lowercased_results_in_set_with_one_entry(SecHubDataConfigurationType type) {
+
+        /* execute */
+        Set<SecHubDataConfigurationType> parsed = parserToTest.fetchTypesAsSetOrNull(type.name().toLowerCase());
+
+        /* test */
+        assertNotNull(parsed);
+        assertEquals(parsed.size(), 1);
+        assertEquals(type, parsed.iterator().next());
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = { "1:binary", "1:BINARY", "3:NONE,BINARY,source", "2:none,source", "2:source,binary", "2:source , BINARY",
+            "3:source,binary,NONE,binary" })
+    void correct_args_will_return_args_as_set(String data) {
+        /* prepare */
+        String[] splitted = data.split(":");
+        String string = splitted[1];
+        int expectedAmount = Integer.parseInt(splitted[0]);
+
+        /* execute */
+        Set<SecHubDataConfigurationType> parsed = parserToTest.fetchTypesAsSetOrNull(string);
+
+        /* test */
+        assertNotNull(parsed);
+        assertEquals(expectedAmount, parsed.size());
+    }
+}

sechub-commons-pds/src/main/java/com/mercedesbenz/sechub/commons/pds/PDSConfigDataKeyProvider.java

@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.commons.pds;
 
+import com.mercedesbenz.sechub.commons.model.SecHubDataConfigurationType;
+
 /**
  * Provides keys which are interpreted <strong>at PDS side</strong>. When the
  * given key (ExecutionPDSKey) is marked as available inside script, the
@@ -74,6 +76,13 @@ public enum PDSConfigDataKeyProvider implements PDSKeyProvider<ExecutionPDSKey>
             PDSDefaultParameterKeyConstants.PARAM_KEY_PDS_CONFIG_SCRIPT_TRUSTALL_CERTIFICATES_ENABLED,
             "When 'true' the PDS adapter script used by the job will have the information and can use this information when it comes to remote operations.")
                     .markSendToPDS().markAsAvailableInsideScript().markDefaultRecommended().withDefault(false)),
+    /**
+     * Can describe supported data types.
+     */
+    PDS_CONFIG_SUPPORTED_DATATYPES(new ExecutionPDSKey(PDSDefaultParameterKeyConstants.PARAM_KEY_PDS_CONFIG_SUPPORTED_DATATYPES,
+            "Can be " + SecHubDataConfigurationType.SOURCE + ", " + SecHubDataConfigurationType.BINARY + ", " + SecHubDataConfigurationType.NONE
+                    + " or a combination as a comma separated list. This data should"
+                    + "normally not be defined via a default value of an optional PDS configuration parameter.").markSendToPDS()),
 
     /**
      * Contains product timeout information

sechub-commons-pds/src/main/java/com/mercedesbenz/sechub/commons/pds/PDSDefaultParameterKeyConstants.java

@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.commons.pds;
 
+import com.mercedesbenz.sechub.commons.model.SecHubDataConfigurationType;
+
 /**
  * All default parameter keys supported by PDS. A PDS can support optional
  * parameters (via its configuration) but these ones are always supported and be
@@ -62,6 +64,12 @@ public class PDSDefaultParameterKeyConstants {
      */
     public static final String PARAM_KEY_PDS_CONFIG_CANCEL_EVENT_CHECKINTERVAL_MILLISECONDS = "pds.config.cancel.event.checkinterval.milliseconds";
 
+    /**
+     * Define the supported data types. Can contain an entry or a comma separated
+     * list of {@link SecHubDataConfigurationType}
+     */
+    public static final String PARAM_KEY_PDS_CONFIG_SUPPORTED_DATATYPES = "pds.config.supported.datatypes";
+
     /* ---------------------- */
     /* Integration tests only */
     /* ---------------------- */

sechub-doc/src/docs/asciidoc/documents/pds/pds_config.adoc

@@ -91,18 +91,18 @@ include::product_delegation_server_config_example1.json[]
     as environment variable `PRODUCT1_QUALITYCHECK_ENABLED`.
     the `value`, will be set by {sechub} job call and available in former described ENV entry at execution time.
     {sechub} will 
-<9> a 'description' of the 'key'. Should contain information 
+<9> a description of the `key` or the used `default` setup. This is optional and only for information/better understanding.
 
 <10> optional parameters
-
-<11> defines if `PDS` will automatically extract archive files - default is `true`. When `true` the content
-   of `zip` source files will be available inside `$PDS_JOB_EXTRACTED_SOURCES_FOLDER`, `tar` files containing binaries can be found at `$PDS_JOB_EXTRACTED_BINARIES_FOLDER`.
-   The extraction will automatically filter and transform archive content as described at <<data-structure-tar-and-zip-files,data structure definition>>.
-   Also the environment variables `$PDS_JOB_HAS_EXTRACTED_SOURCES` and `$PDS_JOB_HAS_EXTRACTED_BINARIES` are automatically defined. 
-
-NOTE: When no {sechub} configuration model is defined for the `PDS` job, the transformation and filtering described at <11> will accept all binaries and all sources which are uploaded.
-      But this can happen only when PDS is used without SecHub - normally only for testing PDS solutions.  
-   
+<11> By defining supported data type `source` we ensure sources are downloaded from storage and automatically extracted.
+     If the extraction target folder is empty (e.g. filtering or no source available) the launcher script will NOT be called. +
+     Valid entries are `source`, `binary`, `none` or a combination of them as a comma separated list. When not defined as a parameter, the 
+     {pds} configuration default will be used. If available, the content of `zip` source files will be extracted into `$PDS_JOB_EXTRACTED_SOURCES_FOLDER`, 
+     `tar` archive content containing binaries can be found at `$PDS_JOB_EXTRACTED_BINARIES_FOLDER`. +
+     The extraction will automatically filter and transform archive content as described at <<data-structure-tar-and-zip-files,data structure definition>>.
+     Also the environment variables `$PDS_JOB_HAS_EXTRACTED_SOURCES` and `$PDS_JOB_HAS_EXTRACTED_BINARIES` are automatically defined. 
+
+<12> Using `none` will not try to download any ZIP or TAR from storage but does call the caller script even when no data is available. 
 
 === Launcher scripts
 

sechub-doc/src/docs/asciidoc/documents/pds/product_delegation_server_config_example1.json

@@ -25,7 +25,10 @@
                     {
                         "key" : "product1.add.tipoftheday",
                         "description" : "boolean as string, when 'true' we add always a tip of the day as info to results"
-                    }  
+                    }, {
+                        "key" : "pds.config.supported.datatypes", //<11>
+                        "default" : "source" 
+                    } 
                 ]
             }
 
@@ -34,8 +37,15 @@
         {
             "id" : "PRODUCT_2",
             "path" : "/srv/security/scanner2.sh",
-            "scanType" : "infraScan" ,
-            "extractUploads" : false //<11>
+            "scanType" : "infraScan",
+            "parameters" : {
+                "mandatory" : [
+                    {
+                        "key" : "pds.config.supported.datatypes", //<12>
+                        "default" : "none" 
+                    }  
+                ]
+            }
         }
     ]
 }