Added tests and tested skopeo moduel #3028

mercedes-benz · May 8, 2024 · a0faba4 · a0faba4
1 parent bf9eac4
commit a0faba4
Show file tree

Hide file tree

Showing 10 changed files with 458 additions and 48 deletions.
diff --git a/...rc/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/PrepareWrapperModuleGit.java b/...rc/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/PrepareWrapperModuleGit.java
@@ -50,10 +50,11 @@ public boolean isAbleToPrepare(PrepareWrapperContext context) {
 
         for (SecHubRemoteDataConfiguration secHubRemoteDataConfiguration : context.getRemoteDataConfigurationList()) {
             String location = secHubRemoteDataConfiguration.getLocation();
+            String type = secHubRemoteDataConfiguration.getType();
 
             gitInputValidator.validateLocationCharacters(location, null);
 
-            if (isMatchingGitType(secHubRemoteDataConfiguration.getType())) {
+            if (isMatchingGitType(type)) {
                 LOG.debug("Type is git");
                 if (!gitInputValidator.validateLocation(location)) {
                     context.getUserMessages().add(new SecHubMessage(SecHubMessageType.WARNING, "Type is git but location does not match git URL pattern"));
@@ -63,6 +64,11 @@ public boolean isAbleToPrepare(PrepareWrapperContext context) {
                 return true;
             }
 
+            if (!isTypeNullOrEmpty(type)) {
+                // type was explicitly defined but is not matching
+                return false;
+            }
+
             if (gitInputValidator.validateLocation(location)) {
                 LOG.debug("Location is a git URL");
                 return true;
@@ -106,6 +112,10 @@ boolean isDownloadSuccessful(PrepareWrapperContext context) {
         return false;
     }
 
+    private boolean isTypeNullOrEmpty(String type) {
+        return type == null || type.isBlank();
+    }
+
     private void prepareRemoteConfiguration(PrepareWrapperContext context, SecHubRemoteDataConfiguration secHubRemoteDataConfiguration) throws IOException {
         String location = secHubRemoteDataConfiguration.getLocation();
         Optional<SecHubRemoteCredentialConfiguration> credentials = secHubRemoteDataConfiguration.getCredentials();

diff --git a/...main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/PrepareWrapperModuleSkopeo.java b/...main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/PrepareWrapperModuleSkopeo.java
@@ -11,6 +11,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Optional;
+import java.util.stream.Stream;
 
 import javax.crypto.SealedObject;
 
@@ -50,10 +51,11 @@ public boolean isAbleToPrepare(PrepareWrapperContext context) {
 
         for (SecHubRemoteDataConfiguration secHubRemoteDataConfiguration : context.getRemoteDataConfigurationList()) {
             String location = secHubRemoteDataConfiguration.getLocation();
+            String type = secHubRemoteDataConfiguration.getType();
 
             skopeoInputValidator.validateLocationCharacters(location, null);
 
-            if (isMatchingSkopeoType(secHubRemoteDataConfiguration.getType())) {
+            if (isMatchingSkopeoType(type)) {
                 LOG.debug("Type is: " + TYPE);
                 if (!skopeoInputValidator.validateLocation(location)) {
                     context.getUserMessages().add(new SecHubMessage(SecHubMessageType.WARNING, "Type is " + TYPE + " but location does not match URL pattern"));
@@ -63,18 +65,22 @@ public boolean isAbleToPrepare(PrepareWrapperContext context) {
                 return true;
             }
 
+            if (!isTypeNullOrEmpty(type)) {
+                // type was explicitly defined but is not matching
+                return false;
+            }
+
             if (skopeoInputValidator.validateLocation(location)) {
                 LOG.debug("Location is a " + TYPE + " URL");
                 return true;
             }
-
         }
         return false;
     }
 
     @Override
     public void prepare(PrepareWrapperContext context) throws IOException {
-        LOG.debug("Start remote data preparation for GIT repository");
+        LOG.debug("Start remote data preparation for Docker repository");
 
         List<SecHubRemoteDataConfiguration> remoteDataConfigurationList = context.getRemoteDataConfigurationList();
 
@@ -83,21 +89,43 @@ public void prepare(PrepareWrapperContext context) throws IOException {
         }
 
         if (!isDownloadSuccessful(context)) {
-            throw new IOException("Download of git repository was not successful.");
+            throw new IOException("Download of docker image was not successful.");
         }
+        cleanup(context);
     }
 
     boolean isDownloadSuccessful(PrepareWrapperContext context) {
-        // check if download folder contains docker archive
+        // check if download folder contains a .tar archive
         Path path = Paths.get(context.getEnvironment().getPdsPrepareUploadFolderDirectory());
         if (Files.isDirectory(path)) {
-            String gitFile = "image.tar";
-            Path gitPath = Paths.get(path + "/" + gitFile);
-            return Files.exists(gitPath);
+            try (Stream<Path> walk = Files.walk(path)) {
+                List<String> result = walk.filter(p -> !Files.isDirectory(p)) // not a directory
+                        .map(p -> p.toString().toLowerCase()) // convert path to string
+                        .filter(f -> f.endsWith(".tar")) // check end with
+                        .toList(); // collect all matched to a List
+                return !result.isEmpty();
+            } catch (IOException e) {
+                throw new RuntimeException("Error while checking download of docker image", e);
+            }
         }
         return false;
     }
 
+    boolean isMatchingSkopeoType(String type) {
+        if (type == null || type.isBlank()) {
+            return false;
+        }
+        return TYPE.equalsIgnoreCase(type);
+    }
+
+    private boolean isTypeNullOrEmpty(String type) {
+        return type == null || type.isBlank();
+    }
+
+    private void cleanup(PrepareWrapperContext context) throws IOException {
+        skopeo.cleanUploadDirectory(context.getEnvironment().getPdsPrepareUploadFolderDirectory());
+    }
+
     private void prepareRemoteConfiguration(PrepareWrapperContext context, SecHubRemoteDataConfiguration secHubRemoteDataConfiguration) throws IOException {
         String location = secHubRemoteDataConfiguration.getLocation();
         Optional<SecHubRemoteCredentialConfiguration> credentials = secHubRemoteDataConfiguration.getCredentials();
@@ -162,11 +190,4 @@ private void downloadPublicImage(PrepareWrapperContext context, String location)
         SecHubMessage message = new SecHubMessage(SecHubMessageType.INFO, "Cloned public repository: " + location);
         context.getUserMessages().add(message);
     }
-
-    private boolean isMatchingSkopeoType(String type) {
-        if (type == null || type.isBlank()) {
-            return false;
-        }
-        return TYPE.equalsIgnoreCase(type);
-    }
 }
diff --git a/...e/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/SkopeoInputValidator.java b/...e/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/SkopeoInputValidator.java
@@ -13,7 +13,7 @@ public class SkopeoInputValidator implements InputValidator {
     private static final Pattern SKOPEO_LOCATION_PATTERN = Pattern.compile(SKOPEO_LOCATION_REGEX);
     private static final String SKOPEO_USERNAME_REGEX = "^[a-zA-Z0-9-_\\d](?:[a-zA-Z0-9-_\\d]|(?=[a-zA-Z0-9-_\\d])){0,38}$";
     private static final Pattern SKOPEO_USERNAME_PATTERN = Pattern.compile(SKOPEO_USERNAME_REGEX);
-    private static final String SKOPEO_PASSWORD_REGEX = "^[a-zA-Z0-9-_\\d]{0,72}$";
+    private static final String SKOPEO_PASSWORD_REGEX = "^[a-zA-Z0-9-_\\d]{0,80}$";
     private static final Pattern SKOPEO_PASSWORD_PATTERN = Pattern.compile(SKOPEO_PASSWORD_REGEX);
     private final List<String> defaultForbiddenCharacters = Arrays.asList(">", "<", "!", "?", "*", "'", "\"", ";", "&", "|", "`", "$", "{", "}");
 

diff --git a/...-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/WrapperSkopeo.java b/...-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/WrapperSkopeo.java
@@ -21,15 +21,17 @@ public class WrapperSkopeo extends WrapperTool {
     String pdsPrepareAuthenticationFileSkopeo;
 
     void download(SkopeoContext context) throws IOException {
-        login(context);
+        if (!context.getCredentialMap().isEmpty()) {
+            login(context);
+        }
 
         ProcessBuilder builder = buildProcessDownload(context);
         ProcessAdapter process = null;
 
         try {
             process = processAdapterFactory.startProcess(builder);
         } catch (IOException e) {
-            throw new IOException("Error download with Skopeo from: " + context.getLocation(), e);
+            throw new IOException("Error while download with Skopeo from: " + context.getLocation(), e);
         }
 
         waitForProcessToFinish(process);
@@ -73,15 +75,16 @@ private ProcessBuilder buildProcessLogin(SkopeoContext context) {
 
         commands.add("/bin/bash");
         commands.add("-c");
-        commands.add("skopeo");
-        commands.add("login");
-        commands.add(location);
-        commands.add("--username");
-        commands.add("$" + PDS_PREPARE_CREDENTIAL_USERNAME);
-        commands.add("--password");
-        commands.add("$" + PDS_PREPARE_CREDENTIAL_PASSWORD);
-        commands.add("--authfile");
-        commands.add(pdsPrepareAuthenticationFileSkopeo);
+        commands.add("skopeo login " + location + " --username $" + PDS_PREPARE_CREDENTIAL_USERNAME + " --password $" + PDS_PREPARE_CREDENTIAL_PASSWORD
+                + " --authfile " + pdsPrepareAuthenticationFileSkopeo);
+
+        /*
+         * commands.add("skopeo"); commands.add("login"); commands.add(location);
+         * commands.add("--username"); commands.add("$" +
+         * PDS_PREPARE_CREDENTIAL_USERNAME); commands.add("--password");
+         * commands.add("$" + PDS_PREPARE_CREDENTIAL_PASSWORD);
+         * commands.add("--authfile"); commands.add(pdsPrepareAuthenticationFileSkopeo);
+         */
 
         ProcessBuilder builder = new ProcessBuilder(commands);
         builder.directory(uploadDir);
@@ -98,14 +101,23 @@ private ProcessBuilder buildProcessDownload(SkopeoContext context) {
         String location = transformLocationForDownload(context.getLocation());
         File uploadDir = Paths.get(context.getUploadDirectory()).toAbsolutePath().toFile();
 
-        commands.add("/bin/bash");
-        commands.add("-c");
+        /*
+         * commands.add("/bin/bash"); commands.add("-c"); if
+         * (!context.getCredentialMap().isEmpty()){ commands.add("skopeo copy " +
+         * location + " docker-archive:" + context.getFilename()); }else{
+         * commands.add("skopeo copy " + location + " docker-archive:" +
+         * context.getFilename() + " --authfile " + pdsPrepareAuthenticationFileSkopeo);
+         * }
+         */
+
         commands.add("skopeo");
         commands.add("copy");
         commands.add(location);
         commands.add("docker-archive:" + context.getFilename());
-        commands.add("--authfile");
-        commands.add(pdsPrepareAuthenticationFileSkopeo);
+        if (!context.getCredentialMap().isEmpty()) {
+            commands.add("--authfile");
+            commands.add(pdsPrepareAuthenticationFileSkopeo);
+        }
 
         ProcessBuilder builder = new ProcessBuilder(commands);
         builder.directory(uploadDir);
@@ -115,6 +127,7 @@ private ProcessBuilder buildProcessDownload(SkopeoContext context) {
     }
 
     private ProcessBuilder buildProcessClean(String pdsPrepareUploadFolderDirectory) {
+        // removes authentication file
         List<String> commands = new ArrayList<>();
 
         File uploadDir = Paths.get(pdsPrepareUploadFolderDirectory).toAbsolutePath().toFile();
@@ -151,6 +164,6 @@ private String transformLocationForLogin(String location) {
         if (location.startsWith("https://")) {
             location = location.replace("https://", "");
         }
-        return location;
+        return location.split("/")[0];
     }
 }
diff --git a/...er-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/WrapperTool.java b/...er-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/WrapperTool.java
@@ -46,15 +46,17 @@ void waitForProcessToFinish(ProcessAdapter process) {
         try {
             exitDoneInTime = process.waitFor(seconds, TimeUnit.SECONDS);
         } catch (InterruptedException e) {
-            throw new RuntimeException("GIT wrapper could not finish process.", e);
+            throw new RuntimeException("Wrapper for executed modul " + this.getClass().getSimpleName() + " could not finish process.", e);
         }
 
         if (!exitDoneInTime) {
-            throw new RuntimeException("GIT wrapper could not finish process. Waited " + pdsPrepareProcessTimeoutSeconds + " seconds.");
+            throw new RuntimeException("Wrapper for executed modul " + this.getClass().getSimpleName() + " could not finish process. Waited "
+                    + pdsPrepareProcessTimeoutSeconds + " seconds.");
         }
 
         if (process.exitValue() != 0) {
-            throw new RuntimeException("GIT wrapper process failed with exit code: " + process.exitValue());
+            throw new RuntimeException(
+                    "Wrapper for executed modul " + this.getClass().getSimpleName() + " process failed with exit code: " + process.exitValue());
         }
     }
 

diff --git a/...java/com/mercedesbenz/sechub/wrapper/prepare/PrepareWrapperApplicationSpringBootTest.java b/...java/com/mercedesbenz/sechub/wrapper/prepare/PrepareWrapperApplicationSpringBootTest.java
@@ -23,7 +23,8 @@
 
 @SpringBootTest(classes = { PrepareWrapperContextFactory.class, PrepareWrapperPreparationService.class, PrepareWrapperPojoFactory.class,
         PrepareWrapperEnvironment.class, PrepareWrapperPDSUserMessageSupportPojoFactory.class, PrepareWrapperRemoteConfigurationExtractor.class,
-        PrepareWrapperModuleGit.class, PrepareWrapperModule.class, WrapperGit.class, GitInputValidator.class, JGitAdapter.class })
+        PrepareWrapperModuleGit.class, PrepareWrapperModule.class, WrapperGit.class, GitInputValidator.class, JGitAdapter.class,
+        PrepareWrapperModuleSkopeo.class, WrapperSkopeo.class, SkopeoInputValidator.class })
 @ExtendWith(SpringExtension.class)
 @TestPropertySource(locations = "classpath:application-test-fail.properties")
 class PrepareWrapperApplicationSpringBootTest {

diff --git a/...est/java/com/mercedesbenz/sechub/wrapper/prepare/modules/PrepareWrapperModuleGitTest.java b/...est/java/com/mercedesbenz/sechub/wrapper/prepare/modules/PrepareWrapperModuleGitTest.java
@@ -147,7 +147,6 @@ void prepare_throws_exception_when_credentials_are_empty() {
         remoteDataConfiguration.setType("git");
         remoteDataConfigurationList.add(remoteDataConfiguration);
         context.setRemoteDataConfigurationList(remoteDataConfigurationList);
-        ReflectionTestUtils.setField(moduleToTest, "pdsPrepareModuleGitEnabled", true);
 
         /* execute */
         IllegalStateException exception = assertThrows(IllegalStateException.class, () -> moduleToTest.prepare(context));