Added integration test for script env variable filtering #3142

- added integration tests
- fixed some problems
- some improvements for testing
- added asterisks support

sechub-integrationtest/integrationtest-pds.sh

@@ -12,6 +12,17 @@ PDS_DEFAULT_PORT=8444
 PDS_DEFAULT_VERSION="0.0.0"
 PDS_DEFAULT_TEMPFOLDER="temp-shared"
 
+# --------------------------------------------------------------------------------
+# Export special variables to test script environment cleanup works as expected
+# - we just start the PDS with those variables
+# - only INTEGRATIONTEST_SCRIPT_ENV_ACCEPTED is whitelisted and must be available
+#   inside integration tests variable dump, the forbidden one not, because not
+#   whitelisted...
+# --------------------------------------------------------------------------------
+export INTEGRATIONTEST_PDS_STARTED_BY_SCRIPT="true" # is checked in integration tests to check if server started by script
+export INTEGRATIONTEST_SCRIPT_ENV_ACCEPTED="accepted"
+export INTEGRATIONTEST_SCRIPT_ENV_FORBIDDEN="forbidden"
+
 function log() {
     echo "$1"
     echo "`date +%Y-%m-%d\ %H:%M:%S` $1" >> "$LOGFILE"

sechub-integrationtest/pds/product-scripts/integrationtest-codescan.sh

@@ -48,6 +48,12 @@ fi
 if [[ "$PDS_TEST_KEY_VARIANTNAME" = "j" ]]; then
     dumpVariable "TEST_MAPPING1_REPLACE_PROJECT1"
     dumpVariable "TEST_MAPPING2_NOT_EXISTING_IN_SECHUB"
+
+    # we dump here for PDSUseSecHubCentralMappingInJobScenario16IntTest to check that
+    # environment variables which are not white listed are not available here
+    dumpVariable "INTEGRATIONTEST_SCRIPT_ENV_ACCEPTED" # is white listed variable - must be not empty
+    dumpVariable "INTEGRATIONTEST_SCRIPT_ENV_FORBIDDEN" # not white listed - must be empty in test
+    dumpVariable "PATH" # check that this as a default white listed environment variable is available
 fi
 
 #

sechub-integrationtest/src/main/java/com/mercedesbenz/sechub/integrationtest/api/TestAPI.java

@@ -1365,6 +1365,12 @@ public static List<UUID> fetchAllPDSJobUUIDsForSecHubJob(UUID sechubJobUUID) {
         return jobUUIDS;
     }
 
+    public static String getPDSServerEnvironmentVariableValue(String environmentVariableName) {
+        String url = getPDSURLBuilder().buildIntegrationTestFetchEnvironmentVariableValue(environmentVariableName);
+        String value = getPDSAdminRestHelper().getStringFromURL(url);
+        return value;
+    }
+
     public static void dumpAllPDSJobOutputsForSecHubJob(UUID sechubJobUUID) {
         dumpAllPDSJobOutputsForSecHubJob(sechubJobUUID, null);
     }

sechub-integrationtest/src/test/java/com/mercedesbenz/sechub/integrationtest/scenario16/PDSUseSecHubCentralMappingInJobScenario16IntTest.java

@@ -4,22 +4,37 @@
 import static com.mercedesbenz.sechub.integrationtest.api.IntegrationTestMockMode.*;
 import static com.mercedesbenz.sechub.integrationtest.api.TestAPI.*;
 import static com.mercedesbenz.sechub.integrationtest.scenario16.Scenario16.*;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
+import java.util.Map;
 import java.util.UUID;
 
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.mercedesbenz.sechub.commons.mapping.MappingData;
 import com.mercedesbenz.sechub.commons.mapping.MappingEntry;
 import com.mercedesbenz.sechub.integrationtest.api.IntegrationTestSetup;
 import com.mercedesbenz.sechub.integrationtest.api.TestAPI;
 import com.mercedesbenz.sechub.integrationtest.api.TestProject;
 import com.mercedesbenz.sechub.integrationtest.internal.IntegrationTestExampleConstants;
+import com.mercedesbenz.sechub.integrationtest.scenario22.PDSPrepareIntegrationScenario22IntTest;
 
 public class PDSUseSecHubCentralMappingInJobScenario16IntTest {
 
+    private static final String TRUE = "true";
+    private static final String FORBIDDEN = "forbidden";
+    private static final String ACCEPTED = "accepted";
+    private static final String INTEGRATIONTEST_PDS_STARTED_BY_SCRIPT = "INTEGRATIONTEST_PDS_STARTED_BY_SCRIPT";
+    private static final String INTEGRATIONTEST_SCRIPT_ENV_ACCEPTED = "INTEGRATIONTEST_SCRIPT_ENV_ACCEPTED";
+    private static final String INTEGRATIONTEST_SCRIPT_ENV_FORBIDDEN = "INTEGRATIONTEST_SCRIPT_ENV_FORBIDDEN";
+
+    private static final Logger LOG = LoggerFactory.getLogger(PDSPrepareIntegrationScenario22IntTest.class);
+
     @Rule
     public IntegrationTestSetup setup = IntegrationTestSetup.forScenario(Scenario16.class);
 
@@ -54,9 +69,42 @@ public void pds_script_does_have_the_mappings_from_sechub_injected_as_environmen
         String expectedMapping2Json = "{}";
 
         // check the script has the mappings injected :
-        assertPDSJob(TestAPI.assertAndFetchPDSJobUUIDForSecHubJob(jobUUID)).
+        UUID pdsJobUUID = TestAPI.assertAndFetchPDSJobUUIDForSecHubJob(jobUUID);
+        assertPDSJob(pdsJobUUID).
             containsVariableTestOutput(IntegrationTestExampleConstants.PDS_ENV_NAME_MAPPING_ID_1_REPLACE_ANY_PROJECT1, expectedMapping1Json).
             containsVariableTestOutput(IntegrationTestExampleConstants.PDS_ENV_NAME_MAPPING_ID_2_NOT_EXISTING_IN_SECHUB, expectedMapping2Json);
+
+        // additional test: here we test that the script environment has only white listed parts from
+        // parent process (the variables were defined and exported to PDS on startup by integrationtest-pds.sh)
+        String pdsStartedByScriptValue = TestAPI.getPDSServerEnvironmentVariableValue(INTEGRATIONTEST_PDS_STARTED_BY_SCRIPT);
+        if (TRUE.equals(pdsStartedByScriptValue)) {
+            Map<String, String> variables = fetchPDSVariableTestOutputMap(pdsJobUUID);
+
+            // precondition check
+            assertEquals(ACCEPTED, TestAPI.getPDSServerEnvironmentVariableValue(INTEGRATIONTEST_SCRIPT_ENV_ACCEPTED));
+            assertEquals(FORBIDDEN, TestAPI.getPDSServerEnvironmentVariableValue(INTEGRATIONTEST_SCRIPT_ENV_FORBIDDEN));
+
+            assertEquals(ACCEPTED, variables.get(INTEGRATIONTEST_SCRIPT_ENV_ACCEPTED)); // defined + white listed
+            assertEquals("", variables.get(INTEGRATIONTEST_SCRIPT_ENV_FORBIDDEN));// was defined, but not white listed, means dump returns empty
+
+            assertNotNull(variables.get("PATH")); // one of the default PDS white list entries for script environments
+
+        }else {
+            LOG.error("#".repeat(120));
+            LOG.error("### ERROR - local PDS server cannot be tested without environment variables set!");
+            LOG.error("#".repeat(120));
+            LOG.error("The integration test usese a PDS server which is running not from script but locally (from an IDE).");
+            LOG.error("Means the environment variables are not set on PDS startup process and cannot be tested!");
+            LOG.error("The test will just skip the pds script cleanup test part in this case because otherwise always failing");
+            LOG.error("");
+            LOG.error("If you want to test this locally,you have to set the env variables on PDS start locally:");
+            LOG.error(" {}={} ",INTEGRATIONTEST_PDS_STARTED_BY_SCRIPT, TRUE);
+            LOG.error(" {}={} ",INTEGRATIONTEST_SCRIPT_ENV_ACCEPTED, ACCEPTED);
+            LOG.error(" {}={} ",INTEGRATIONTEST_SCRIPT_ENV_FORBIDDEN, FORBIDDEN);
+            LOG.error("#".repeat(120));
+        }
+
+
         /* @formatter:on */
     }
 

sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/IntegrationTestPDSRestController.java

@@ -98,6 +98,15 @@ public String getWorkspaceUploadFolder(@PathVariable("jobUUID") UUID pdsJobUUID)
         return uploadFolder;
     }
 
+    @RequestMapping(path = PDSAPIConstants.API_ANONYMOUS + "integrationtest/env/{envVariable}", method = RequestMethod.GET, produces = {
+            MediaType.TEXT_PLAIN_VALUE })
+    public String getEnvironmentVariableValue(@PathVariable("envVariable") String envVariable) {
+        String value = System.getenv(envVariable);
+
+        LOG.info("Integration test checks environment variable:{} - value:{}", envVariable, value);
+        return value;
+    }
+
     @RequestMapping(path = PDSAPIConstants.API_ANONYMOUS + "integrationtest/last/started/job/uuid", method = RequestMethod.GET, produces = {
             MediaType.APPLICATION_JSON_VALUE })
     public String fetchLastStartedPDSJobUUID() {

sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/execution/PDSDefaulScriptEnvironmentVariableWhitelist.java

@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: MIT
+package com.mercedesbenz.sechub.pds.execution;
+
+/**
+ * These enumeration represents variable names which are white listed per
+ * default by {@link PDSScriptEnvironmentCleaner#clean(java.util.Map)}.
+ *
+ * @author Albert Tregnaghi
+ *
+ */
+public enum PDSDefaulScriptEnvironmentVariableWhitelist {
+
+    HOME,
+
+    HOSTNAME,
+
+    PATH,
+
+    PWD,
+
+    TERM,
+
+    UID,
+
+    USER,
+
+    ;
+
+}

sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/execution/PDSExecutionEnvironmentService.java

@@ -43,6 +43,7 @@ public class PDSExecutionEnvironmentService {
 
     @PostConstruct
     void postConstruct() {
+        LOG.info("PDS script environment variable white list: '{}'", pdsScriptEnvWhitelist);
         cleaner.setWhiteListCommaSeparated(pdsScriptEnvWhitelist);
     }
 

sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/execution/PDSScriptEnvironmentCleaner.java

@@ -11,6 +11,14 @@ public class PDSScriptEnvironmentCleaner {
 
     private Set<String> whiteList = new HashSet<>();
 
+    /**
+     * Removes all environment variables from given environment map, except default
+     * and explicit white listed variable names. Default variable names are inside
+     * {@link PDSDefaulScriptEnvironmentVariableWhitelist}, set explicit white
+     * listed names are set via {@link #setWhiteListCommaSeparated(String)}
+     *
+     * @param environment the environment map to clean
+     */
     public void clean(Map<String, String> environment) {
         /* create backup */
         Map<String, String> backup = new HashMap<>();
@@ -25,12 +33,22 @@ public void clean(Map<String, String> environment) {
         while (variableNameIt.hasNext()) {
             String variableName = variableNameIt.next();
             if (isWhitelistedEnvironmentVariable(variableName)) {
-                environment.put(variableName, backup.get(variableName));
+                String backupValue = backup.get(variableName);
+                environment.put(variableName, backupValue);
             }
         }
 
     }
 
+    /**
+     * Sets the white list as a comma separated list of variable names to exclude
+     * from cleaning. If a variable name ends with an asterisk every variable which
+     * begins with such prefix will be accepted. For example: PDS_STORAGE_* will
+     * white list any kind of environment variable which starts with PDS_STORAGE_
+     * (e.g. PDS_STORAGE_S3_USER).
+     *
+     * @param commaSeparatedWhiteList comma separated list of white list entries.
+     */
     public void setWhiteListCommaSeparated(String commaSeparatedWhiteList) {
         whiteList.clear();
 
@@ -40,28 +58,28 @@ public void setWhiteListCommaSeparated(String commaSeparatedWhiteList) {
 
         String[] splitted = commaSeparatedWhiteList.split(",");
         for (String whiteListEntry : splitted) {
-            String trimmed = whiteListEntry.trim();
-            if (trimmed.isBlank()) {
+            String trimmedWhiteListEntry = whiteListEntry.trim();
+            if (trimmedWhiteListEntry.isBlank()) {
                 continue;
             }
-            whiteList.add(trimmed);
+            whiteList.add(trimmedWhiteListEntry);
         }
     }
 
-    private boolean isWhitelistedEnvironmentVariable(String key) {
-        if (key == null) {
+    private boolean isWhitelistedEnvironmentVariable(String variableName) {
+        if (variableName == null) {
             return false;
         }
 
         /* handle default white list entries */
-        for (PDSScriptEnvironmentVariableWhitelistDefault defaultKey : PDSScriptEnvironmentVariableWhitelistDefault.values()) {
-            if (defaultKey.name().equals(key)) {
+        for (PDSDefaulScriptEnvironmentVariableWhitelist defaultWhitelistVariable : PDSDefaulScriptEnvironmentVariableWhitelist.values()) {
+            if (defaultWhitelistVariable.name().equals(variableName)) {
                 return true;
             }
         }
 
         /* handle explicit white list entries */
-        if (whiteList.contains(key)) {
+        if (whiteList.contains(variableName)) {
             return true;
         }
 
@@ -71,7 +89,7 @@ private boolean isWhitelistedEnvironmentVariable(String key) {
 
             if (whiteListEntry.endsWith("*") && length > 2) {
                 String prefix = whiteListEntry.substring(0, length - 1);
-                if (key.startsWith(prefix)) {
+                if (variableName.startsWith(prefix)) {
                     return true;
                 }
             }

sechub-pds/src/main/resources/application-pds_integrationtest.yml

@@ -30,7 +30,10 @@ pds:
   admin:
     userid: pds-inttest-admin
     apitoken: '{noop}pds-inttest-apitoken'
-
+
+  script:
+    env:
+       whitelist: 'PDS_STORAGE_*,INTEGRATIONTEST_SCRIPT_ENV_ACCEPTED'
 logging:
   level:
     com.mercedesbenz.sechub: DEBUG

sechub-pds/src/test/java/com/mercedesbenz/sechub/pds/execution/PDSScriptEnvironmentCleanerTest.java

@@ -39,9 +39,10 @@ void a_key_which_was_in_environment_and_is_not_whitelisted_will_be_removed(Strin
     }
 
     @ParameterizedTest
-    @EnumSource(PDSScriptEnvironmentVariableWhitelistDefault.class)
-    void a_key_which_was_in_environment_and_is_whitelisted_per_default_will_not_be_removed(PDSScriptEnvironmentVariableWhitelistDefault whitelistDefault) {
-        String key = whitelistDefault.name();
+    @EnumSource(PDSDefaulScriptEnvironmentVariableWhitelist.class)
+    void a_key_which_was_in_environment_and_is_whitelisted_per_default_will_not_be_removed(
+            PDSDefaulScriptEnvironmentVariableWhitelist defaultWhiteListedVariable) {
+        String key = defaultWhiteListedVariable.name();
         String value = "value-" + key;
         /* prepare */
         environment.put(key, value);

sechub-testframework/src/main/java/com/mercedesbenz/sechub/test/PDSTestURLBuilder.java

@@ -117,6 +117,10 @@ public String buildIntegrationTestResetAutoCleanupInspectionUrl() {
         return buildUrl(API_ANONYMOUS, "integrationtest/autocleanup/inspection/reset");
     }
 
+    public String buildIntegrationTestFetchEnvironmentVariableValue(String environmentVariableName) {
+        return buildUrl(API_ANONYMOUS, "integrationtest/env", environmentVariableName);
+    }
+
     public String buildCheckIsAliveUrl() {
         return buildUrl(API_ANONYMOUS, "check/alive");
     }