Prepare Module Skopeo

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/cli/PrepareWrapperKeyConstants.java

@@ -20,8 +20,18 @@ public class PrepareWrapperKeyConstants {
      */
     public static final String KEY_PDS_PREPARE_MODULE_GIT_ENABLED = "pds.prepare.module.git.enabled";
 
+    /**
+     * Flag to enable the skopeo prepare module
+     */
+    public static final String KEY_PDS_PREPARE_MODULE_SKOPEO_ENABLED = "pds.prepare.module.skopeo.enabled";
+
     /**
      * Flag to clean the git folder from git files and clone without history
      */
     public static final String KEY_PDS_PREPARE_AUTO_CLEANUP_GIT_FOLDER = "pds.prepare.auto.cleanup.git.folder";
+
+    /**
+     * Filename for skopeo authentication file
+     */
+    public static final String KEY_PDS_PREPARE_AUTHENTICATION_FILE_SKOPEO = "pds.prepare.authentication.file.skopeo";
 }

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/PrepareWrapperModuleGit.java

@@ -50,10 +50,11 @@ public boolean isAbleToPrepare(PrepareWrapperContext context) {
 
         for (SecHubRemoteDataConfiguration secHubRemoteDataConfiguration : context.getRemoteDataConfigurationList()) {
             String location = secHubRemoteDataConfiguration.getLocation();
+            String type = secHubRemoteDataConfiguration.getType();
 
             gitInputValidator.validateLocationCharacters(location, null);
 
-            if (isMatchingGitType(secHubRemoteDataConfiguration.getType())) {
+            if (isMatchingGitType(type)) {
                 LOG.debug("Type is git");
                 if (!gitInputValidator.validateLocation(location)) {
                     context.getUserMessages().add(new SecHubMessage(SecHubMessageType.WARNING, "Type is git but location does not match git URL pattern"));
@@ -63,6 +64,11 @@ public boolean isAbleToPrepare(PrepareWrapperContext context) {
                 return true;
             }
 
+            if (!isTypeNullOrEmpty(type)) {
+                // type was explicitly defined but is not matching
+                return false;
+            }
+
             if (gitInputValidator.validateLocation(location)) {
                 LOG.debug("Location is a git URL");
                 return true;
@@ -106,6 +112,10 @@ boolean isDownloadSuccessful(PrepareWrapperContext context) {
         return false;
     }
 
+    private boolean isTypeNullOrEmpty(String type) {
+        return type == null || type.isBlank();
+    }
+
     private void prepareRemoteConfiguration(PrepareWrapperContext context, SecHubRemoteDataConfiguration secHubRemoteDataConfiguration) throws IOException {
         String location = secHubRemoteDataConfiguration.getLocation();
         Optional<SecHubRemoteCredentialConfiguration> credentials = secHubRemoteDataConfiguration.getCredentials();
@@ -134,8 +144,8 @@ private void clonePrivateRepository(PrepareWrapperContext context, SecHubRemoteC
         /* @formatter:off */
         GitContext gitContext = (GitContext) new GitContext.GitContextBuilder().
                 setCloneWithoutHistory(pdsPrepareAutoCleanupGitFolder).
-                setLocation(location)
-                .setCredentialMap(credentialMap).
+                setLocation(location).
+                setCredentialMap(credentialMap).
                 setUploadDirectory(context.getEnvironment().getPdsPrepareUploadFolderDirectory()).
                 build();
         /* @formatter:on */

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/PrepareWrapperModuleSkopeo.java

@@ -0,0 +1,193 @@
+package com.mercedesbenz.sechub.wrapper.prepare.modules;
+
+import static com.mercedesbenz.sechub.wrapper.prepare.cli.PrepareWrapperEnvironmentVariables.PDS_PREPARE_CREDENTIAL_PASSWORD;
+import static com.mercedesbenz.sechub.wrapper.prepare.cli.PrepareWrapperEnvironmentVariables.PDS_PREPARE_CREDENTIAL_USERNAME;
+import static com.mercedesbenz.sechub.wrapper.prepare.cli.PrepareWrapperKeyConstants.KEY_PDS_PREPARE_MODULE_SKOPEO_ENABLED;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Stream;
+
+import javax.crypto.SealedObject;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import com.mercedesbenz.sechub.commons.core.security.CryptoAccess;
+import com.mercedesbenz.sechub.commons.model.*;
+import com.mercedesbenz.sechub.wrapper.prepare.prepare.PrepareWrapperContext;
+
+@Service
+public class PrepareWrapperModuleSkopeo implements PrepareWrapperModule {
+
+    Logger LOG = LoggerFactory.getLogger(PrepareWrapperModuleSkopeo.class);
+
+    private static final String TYPE = "docker";
+
+    @Value("${" + KEY_PDS_PREPARE_MODULE_SKOPEO_ENABLED + ":true}")
+    private boolean pdsPrepareModuleSkopeoEnabled;
+
+    @Autowired
+    SkopeoInputValidator skopeoInputValidator;
+
+    @Autowired
+    WrapperSkopeo skopeo;
+
+    @Override
+    public boolean isAbleToPrepare(PrepareWrapperContext context) {
+
+        if (!pdsPrepareModuleSkopeoEnabled) {
+            LOG.debug("Skopeo module is disabled");
+            return false;
+        }
+
+        for (SecHubRemoteDataConfiguration secHubRemoteDataConfiguration : context.getRemoteDataConfigurationList()) {
+            String location = secHubRemoteDataConfiguration.getLocation();
+            String type = secHubRemoteDataConfiguration.getType();
+
+            skopeoInputValidator.validateLocationCharacters(location, null);
+
+            if (isMatchingSkopeoType(type)) {
+                LOG.debug("Type is: " + TYPE);
+                if (!skopeoInputValidator.validateLocation(location)) {
+                    context.getUserMessages().add(new SecHubMessage(SecHubMessageType.WARNING, "Type is " + TYPE + " but location does not match URL pattern"));
+                    LOG.warn("User defined type as " + TYPE + ", but the defined location was not a valid location: {}", location);
+                    return false;
+                }
+                return true;
+            }
+
+            if (!isTypeNullOrEmpty(type)) {
+                // type was explicitly defined but is not matching
+                return false;
+            }
+
+            if (skopeoInputValidator.validateLocation(location)) {
+                LOG.debug("Location is a " + TYPE + " URL");
+                return true;
+            }
+        }
+        return false;
+    }
+
+    @Override
+    public void prepare(PrepareWrapperContext context) throws IOException {
+        LOG.debug("Start remote data preparation for Docker repository");
+
+        List<SecHubRemoteDataConfiguration> remoteDataConfigurationList = context.getRemoteDataConfigurationList();
+
+        for (SecHubRemoteDataConfiguration secHubRemoteDataConfiguration : remoteDataConfigurationList) {
+            prepareRemoteConfiguration(context, secHubRemoteDataConfiguration);
+        }
+
+        if (!isDownloadSuccessful(context)) {
+            throw new IOException("Download of docker image was not successful.");
+        }
+        cleanup(context);
+    }
+
+    boolean isDownloadSuccessful(PrepareWrapperContext context) {
+        // check if download folder contains a .tar archive
+        Path path = Paths.get(context.getEnvironment().getPdsPrepareUploadFolderDirectory());
+        if (Files.isDirectory(path)) {
+            try (Stream<Path> walk = Files.walk(path)) {
+                List<String> result = walk.filter(p -> !Files.isDirectory(p)) // not a directory
+                        .map(p -> p.toString().toLowerCase()) // convert path to string
+                        .filter(f -> f.endsWith(".tar")) // check end with
+                        .toList(); // collect all matched to a List
+                return !result.isEmpty();
+            } catch (IOException e) {
+                throw new RuntimeException("Error while checking download of docker image", e);
+            }
+        }
+        return false;
+    }
+
+    boolean isMatchingSkopeoType(String type) {
+        if (type == null || type.isBlank()) {
+            return false;
+        }
+        return TYPE.equalsIgnoreCase(type);
+    }
+
+    private boolean isTypeNullOrEmpty(String type) {
+        return type == null || type.isBlank();
+    }
+
+    private void cleanup(PrepareWrapperContext context) throws IOException {
+        skopeo.cleanUploadDirectory(context.getEnvironment().getPdsPrepareUploadFolderDirectory());
+    }
+
+    private void prepareRemoteConfiguration(PrepareWrapperContext context, SecHubRemoteDataConfiguration secHubRemoteDataConfiguration) throws IOException {
+        String location = secHubRemoteDataConfiguration.getLocation();
+        Optional<SecHubRemoteCredentialConfiguration> credentials = secHubRemoteDataConfiguration.getCredentials();
+
+        if (!credentials.isPresent()) {
+            downloadPublicImage(context, location);
+            return;
+        }
+
+        Optional<SecHubRemoteCredentialUserData> optUser = credentials.get().getUser();
+        if (optUser.isPresent()) {
+            SecHubRemoteCredentialUserData user = optUser.get();
+            downloadPrivateImage(context, user, location);
+            return;
+        }
+
+        throw new IllegalStateException("Defined credentials have no credential user data for location: " + location);
+    }
+
+    private void downloadPrivateImage(PrepareWrapperContext context, SecHubRemoteCredentialUserData user, String location) throws IOException {
+        assertUserCredentials(user);
+
+        HashMap<String, SealedObject> credentialMap = new HashMap<>();
+        addSealedUserCredentials(user, credentialMap);
+
+        /* @formatter:off */
+        SkopeoContext skopeoContext = (SkopeoContext) new SkopeoContext.SkopeoContextBuilder().
+                setLocation(location).
+                setCredentialMap(credentialMap).
+                setUploadDirectory(context.getEnvironment().getPdsPrepareUploadFolderDirectory()).
+                build();
+        /* @formatter:on */
+
+        skopeo.download(skopeoContext);
+
+        SecHubMessage message = new SecHubMessage(SecHubMessageType.INFO, "Cloned private repository: " + location);
+        context.getUserMessages().add(message);
+    }
+
+    private static void addSealedUserCredentials(SecHubRemoteCredentialUserData user, HashMap<String, SealedObject> credentialMap) {
+        SealedObject sealedUsername = CryptoAccess.CRYPTO_STRING.seal(user.getName());
+        SealedObject sealedPassword = CryptoAccess.CRYPTO_STRING.seal(user.getPassword());
+        credentialMap.put(PDS_PREPARE_CREDENTIAL_USERNAME, sealedUsername);
+        credentialMap.put(PDS_PREPARE_CREDENTIAL_PASSWORD, sealedPassword);
+    }
+
+    private void assertUserCredentials(SecHubRemoteCredentialUserData user) {
+        skopeoInputValidator.validateUsername(user.getName());
+        skopeoInputValidator.validatePassword(user.getPassword());
+    }
+
+    private void downloadPublicImage(PrepareWrapperContext context, String location) throws IOException {
+        /* @formatter:off */
+        SkopeoContext skopeoContext = (SkopeoContext) new SkopeoContext.SkopeoContextBuilder().
+                setLocation(location).
+                setUploadDirectory(context.getEnvironment().getPdsPrepareUploadFolderDirectory()).
+                build();
+        /* @formatter:on */
+
+        skopeo.download(skopeoContext);
+
+        SecHubMessage message = new SecHubMessage(SecHubMessageType.INFO, "Cloned public repository: " + location);
+        context.getUserMessages().add(message);
+    }
+}

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/SkopeoContext.java

@@ -0,0 +1,36 @@
+package com.mercedesbenz.sechub.wrapper.prepare.modules;
+
+public class SkopeoContext extends ToolContext {
+
+    private final String filename;
+
+    private SkopeoContext(SkopeoContextBuilder builder) {
+        super(builder);
+        this.filename = builder.filename;
+    }
+
+    public String getFilename() {
+        return filename;
+    }
+
+    public static class SkopeoContextBuilder extends ToolContextBuilder {
+
+        private String filename = "SkopeoDownloadFile.tar";
+
+        @Override
+        public SkopeoContext build() {
+            return new SkopeoContext(this);
+        }
+
+        public SkopeoContextBuilder filename(String filename) {
+            if (filename == null || filename.isBlank()) {
+                return this;
+            }
+            if (!filename.endsWith(".tar")) {
+                throw new IllegalArgumentException("Filename must end with .tar");
+            }
+            this.filename = filename;
+            return this;
+        }
+    }
+}

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/SkopeoInputValidator.java

@@ -0,0 +1,62 @@
+package com.mercedesbenz.sechub.wrapper.prepare.modules;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class SkopeoInputValidator implements InputValidator {
+
+    private static final String SKOPEO_LOCATION_REGEX = "((docker://|https://)?([a-zA-Z0-9-_.].[a-zA-Z0-9-_.]/)?[a-zA-Z0-9-_.]+(:[a-zA-Z0-9-_.]+)?(/)?)+(@sha256:[a-f0-9]{64})?";
+    private static final Pattern SKOPEO_LOCATION_PATTERN = Pattern.compile(SKOPEO_LOCATION_REGEX);
+    private static final String SKOPEO_USERNAME_REGEX = "^[a-zA-Z0-9-_\\d](?:[a-zA-Z0-9-_\\d]|(?=[a-zA-Z0-9-_\\d])){0,38}$";
+    private static final Pattern SKOPEO_USERNAME_PATTERN = Pattern.compile(SKOPEO_USERNAME_REGEX);
+    private static final String SKOPEO_PASSWORD_REGEX = "^[a-zA-Z0-9-_\\d]{0,80}$";
+    private static final Pattern SKOPEO_PASSWORD_PATTERN = Pattern.compile(SKOPEO_PASSWORD_REGEX);
+    private final List<String> defaultForbiddenCharacters = Arrays.asList(">", "<", "!", "?", "*", "'", "\"", ";", "&", "|", "`", "$", "{", "}");
+
+    @Override
+    public boolean validateLocation(String location) {
+        if (location == null || location.isBlank()) {
+            throw new IllegalStateException("Defined location must not be null or empty.");
+        }
+        return SKOPEO_LOCATION_PATTERN.matcher(location).matches();
+    }
+
+    @Override
+    public void validateUsername(String username) {
+        if (username == null || username.isBlank()) {
+            throw new IllegalStateException("Defined username must not be null or empty.");
+        }
+        if (!SKOPEO_USERNAME_PATTERN.matcher(username).matches()) {
+            throw new IllegalStateException("Defined username must match the modules pattern.");
+        }
+    }
+
+    @Override
+    public void validatePassword(String password) {
+        if (password == null || password.isBlank()) {
+            throw new IllegalStateException("Defined password must not be null or empty.");
+        }
+        if (!SKOPEO_PASSWORD_PATTERN.matcher(password).matches()) {
+            throw new IllegalStateException("Defined password must match the Skopeo Api token pattern.");
+        }
+    }
+
+    @Override
+    public void validateLocationCharacters(String url, List<String> forbiddenCharacters) {
+        if (forbiddenCharacters == null) {
+            forbiddenCharacters = defaultForbiddenCharacters;
+        }
+        if (url.contains(" ")) {
+            throw new IllegalArgumentException("Defined URL must not contain whitespaces.");
+        }
+        for (String forbiddenCharacter : forbiddenCharacters) {
+            if (url.contains(forbiddenCharacter)) {
+                throw new IllegalArgumentException("Defined URL must not contain forbidden characters: " + forbiddenCharacter);
+            }
+        }
+    }
+}