PDS prepare upload remote data to shared storage

sechub-wrapper-prepare/build.gradle

@@ -10,11 +10,17 @@ dependencies {
     implementation project(':sechub-commons-model')
     implementation project(':sechub-commons-core')
     implementation project(':sechub-commons-pds')
+    implementation project(':sechub-commons-archive')
     implementation project(':sechub-adapter')
+    implementation project(':sechub-storage-core')
+    implementation project(':sechub-storage-sharedvolume-spring')
+    implementation project(':sechub-storage-s3-aws')
 
     implementation spring_boot_dependency.logback_classic
 
     implementation library.jgit_core
+    implementation project(path: ':sechub-pds')
+
 
     /* test */
     testImplementation project(':sechub-testframework')

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/cli/PrepareWrapperCLI.java

@@ -16,7 +16,7 @@
 import com.mercedesbenz.sechub.commons.model.SecHubMessage;
 import com.mercedesbenz.sechub.commons.model.SecHubMessageType;
 import com.mercedesbenz.sechub.wrapper.prepare.prepare.PrepareWrapperPreparationService;
-import com.mercedesbenz.sechub.wrapper.prepare.prepare.PrepareWrapperStorageService;
+import com.mercedesbenz.sechub.wrapper.prepare.prepare.PrepareWrapperResultStorageService;
 
 @Component
 public class PrepareWrapperCLI implements CommandLineRunner {
@@ -27,7 +27,7 @@ public class PrepareWrapperCLI implements CommandLineRunner {
     PrepareWrapperPreparationService preparationService;
 
     @Autowired
-    PrepareWrapperStorageService storageService;
+    PrepareWrapperResultStorageService storageService;
 
     @Override
     public void run(String... args) {

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/cli/PrepareWrapperEnvironment.java

@@ -24,6 +24,12 @@ public class PrepareWrapperEnvironment {
     @Value("${" + RT_KEY_PDS_JOB_USER_MESSAGES_FOLDER + "}")
     private String pdsUserMessagesFolder;
 
+    @Value("${" + PARAM_KEY_PDS_CONFIG_SECHUB_STORAGE_PATH + "}")
+    private String sechubStoragePath;
+
+    @Value("${" + PARAM_KEY_SECHUB_JOB_UUID + "}")
+    private String sechubJobUUID;
+
     /*********************************/
     /* PDS prepare environment setup */
     /*********************************/
@@ -47,4 +53,11 @@ public String getPdsPrepareUploadFolderDirectory() {
         return pdsPrepareUploadFolderDirectory;
     }
 
+    public String getSechubStoragePath() {
+        return sechubStoragePath;
+    }
+
+    public String getSechubJobUUID() {
+        return sechubJobUUID;
+    }
 }

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/factory/PrepareWrapperPojoFactory.java

@@ -5,6 +5,8 @@
 import org.springframework.stereotype.Component;
 
 import com.mercedesbenz.sechub.commons.TextFileWriter;
+import com.mercedesbenz.sechub.commons.archive.ArchiveSupport;
+import com.mercedesbenz.sechub.commons.core.security.CheckSumSupport;
 import com.mercedesbenz.sechub.commons.pds.PDSProcessAdapterFactory;
 
 @Component
@@ -20,4 +22,14 @@ PDSProcessAdapterFactory createPDSProcessAdapterFactory() {
         return new PDSProcessAdapterFactory();
     }
 
+    @Bean
+    ArchiveSupport createArchiveSupport() {
+        return new ArchiveSupport();
+    }
+
+    @Bean
+    CheckSumSupport createCheckSumSupport() {
+        return new CheckSumSupport();
+    }
+
 }

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/GitContext.java

@@ -3,23 +3,37 @@
 public class GitContext extends ToolContext {
     private boolean cloneWithoutHistory;
 
+    private String filename;
+
     private GitContext(GitContextBuilder builder) {
         super(builder);
         this.cloneWithoutHistory = builder.cloneWithoutHistory;
+        this.filename = builder.filename;
     }
 
     public boolean isCloneWithoutHistory() {
         return cloneWithoutHistory;
     }
 
+    public String getFilename() {
+        return filename;
+    }
+
     public static class GitContextBuilder extends ToolContextBuilder {
         private boolean cloneWithoutHistory = true;
 
+        private String filename = "GitDownloadDirectory";
+
         public GitContextBuilder setCloneWithoutHistory(boolean cloneWithoutHistory) {
             this.cloneWithoutHistory = cloneWithoutHistory;
             return this;
         }
 
+        public GitContextBuilder setFilename(String filename) {
+            this.filename = filename;
+            return this;
+        }
+
         @Override
         public GitContext build() {
             return new GitContext(this);

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/JGitAdapter.java

@@ -26,18 +26,19 @@ public class JGitAdapter {
 
     public void clone(GitContext gitContext) {
         String location = transformLocationToURL(gitContext.getLocation());
+        String uploadDirectory = gitContext.getUploadDirectory() + "/" + gitContext.getFilename();
         Map<String, SealedObject> credentialMap = gitContext.getCredentialMap();
 
         String username = getUserNameFromMap(credentialMap);
         String password = getPasswordFromMap(credentialMap);
 
-        CloneCommand command = Git.cloneRepository().setURI(location).setDirectory(Paths.get(gitContext.getUploadDirectory()).toFile());
+        CloneCommand command = Git.cloneRepository().setURI(location).setDirectory(Paths.get(uploadDirectory).toFile());
 
         if (username != null && password != null) {
-            LOG.debug("Cloning private repository: " + location + " with username and password to: " + gitContext.getUploadDirectory());
+            LOG.debug("Cloning private repository: " + location + " with username and password to: " + uploadDirectory);
             command = command.setCredentialsProvider(new UsernamePasswordCredentialsProvider(username, password));
         } else {
-            LOG.debug("Cloning public repository: " + location + " to: " + gitContext.getUploadDirectory());
+            LOG.debug("Cloning public repository: " + location + " to: " + uploadDirectory);
         }
 
         if (gitContext.isCloneWithoutHistory()) {
@@ -47,7 +48,7 @@ public void clone(GitContext gitContext) {
 
         try (Git git = command.call()) {
         } catch (GitAPIException e) {
-            throw new RuntimeException("Error while cloning from repository: " + location + " with " + username + " " + password, e);
+            throw new RuntimeException("Error while cloning from repository: " + location, e);
         }
     }
 

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/modules/PrepareWrapperModuleGit.java

@@ -21,6 +21,8 @@
 import com.mercedesbenz.sechub.commons.core.security.CryptoAccess;
 import com.mercedesbenz.sechub.commons.model.*;
 import com.mercedesbenz.sechub.wrapper.prepare.prepare.PrepareWrapperContext;
+import com.mercedesbenz.sechub.wrapper.prepare.upload.FileNameSupport;
+import com.mercedesbenz.sechub.wrapper.prepare.upload.PrepareWrapperUploadService;
 
 @Service
 public class PrepareWrapperModuleGit implements PrepareWrapperModule {
@@ -41,6 +43,12 @@ public class PrepareWrapperModuleGit implements PrepareWrapperModule {
     @Autowired
     GitInputValidator gitInputValidator;
 
+    @Autowired
+    PrepareWrapperUploadService uploadService;
+
+    @Autowired
+    FileNameSupport filesSupport;
+
     public boolean isAbleToPrepare(PrepareWrapperContext context) {
 
         if (!pdsPrepareModuleGitEnabled) {
@@ -86,6 +94,8 @@ public void prepare(PrepareWrapperContext context) throws IOException {
             throw new IOException("Download of git repository was not successful.");
         }
         cleanup(context);
+        // TODO: 15.05.24 laura proper upload
+        uploadService.upload(context);
     }
 
     boolean isMatchingGitType(String type) {
@@ -97,11 +107,15 @@ boolean isMatchingGitType(String type) {
 
     boolean isDownloadSuccessful(PrepareWrapperContext context) {
         // check if download folder contains git
-        Path path = Paths.get(context.getEnvironment().getPdsPrepareUploadFolderDirectory());
-        if (Files.isDirectory(path)) {
-            String gitFile = ".git";
-            Path gitPath = Paths.get(path + "/" + gitFile);
-            return Files.exists(gitPath);
+        String uploadFolder = context.getEnvironment().getPdsPrepareUploadFolderDirectory();
+        if (Files.isDirectory(Path.of(uploadFolder))) {
+            String gitRepo = filesSupport.getSubfolderFileNameFromDirectory(uploadFolder);
+            Path path = Paths.get(uploadFolder + "/" + gitRepo).toAbsolutePath();
+            if (Files.isDirectory(path)) {
+                String gitFile = ".git";
+                Path gitPath = Paths.get(path + "/" + gitFile);
+                return Files.exists(gitPath);
+            }
         }
         return false;
     }

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/prepare/PrepareWrapperStorageService.java → sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/prepare/PrepareWrapperResultStorageService.java

@@ -13,7 +13,7 @@
 import com.mercedesbenz.sechub.wrapper.prepare.cli.PrepareWrapperEnvironment;
 
 @Service
-public class PrepareWrapperStorageService {
+public class PrepareWrapperResultStorageService {
 
     @Autowired
     PrepareWrapperEnvironment environment;

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/upload/FileNameSupport.java

@@ -0,0 +1,60 @@
+package com.mercedesbenz.sechub.wrapper.prepare.upload;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class FileNameSupport {
+    public String getTarFileNameFromDirectory(String folder) {
+        // TODO: 22.05.24 laura we will use this method in skopeo modul for download
+        // check
+        Path path = Path.of(folder);
+        if (Files.isDirectory(path)) {
+            try (Stream<Path> walk = Files.walk(path)) {
+                List<String> result = walk.filter(p -> !Files.isDirectory(p)) // not a directory
+                        .map(p -> p.toString().toLowerCase()) // convert path to string
+                        .filter(f -> f.endsWith(".tar")) // check end with
+                        .toList(); // collect all matched to a List
+                if (result.size() == 1) {
+                    return result.get(0);
+                } else {
+                    throw new RuntimeException("Error while try to find .tar file.");
+                }
+            } catch (IOException e) {
+                throw new RuntimeException("Error while try to find .tar file.", e);
+            }
+        } else {
+            throw new IllegalArgumentException("Parameter " + folder + " is not a directory");
+        }
+    }
+
+    public String getSubfolderFileNameFromDirectory(String folder) {
+        Set<String> files = listFilesUsingJavaIO(new File(folder).getAbsolutePath());
+        if (files.size() == 1) {
+            return files.iterator().next();
+        } else if (files.isEmpty()) {
+            throw new IllegalArgumentException("Download directory is empty: " + folder);
+        } else {
+            throw new IllegalArgumentException("Download directory contains more than one subfolder: " + folder);
+        }
+    }
+
+    private Set<String> listFilesUsingJavaIO(String dir) {
+        /* @formatter:off */
+        return Stream.of(Objects.requireNonNull(new File(dir).listFiles())).
+                filter(File::isDirectory).
+                map(File::getName).
+                collect(Collectors.toSet());
+        /* @formatter:on */
+    }
+
+}

sechub-wrapper-prepare/src/main/java/com/mercedesbenz/sechub/wrapper/prepare/upload/PrepareWrapperArchiveCreator.java

@@ -0,0 +1,31 @@
+package com.mercedesbenz.sechub.wrapper.prepare.upload;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.mercedesbenz.sechub.commons.archive.ArchiveSupport;
+import com.mercedesbenz.sechub.commons.model.SecHubConfigurationModel;
+import com.mercedesbenz.sechub.wrapper.prepare.prepare.PrepareWrapperContext;
+
+@Service
+public class PrepareWrapperArchiveCreator {
+
+    @Autowired
+    ArchiveSupport archiveSupport;
+
+    @Autowired
+    PrepareWrapperSechubConfigurationSupport sechubConfigurationSupport;
+
+    public void create(PrepareWrapperContext context) throws IOException {
+        // replace remote with filesystem entry in configuration model
+        SecHubConfigurationModel model = sechubConfigurationSupport.replaceRemoteDataWithFilesystem(context);
+        String uploadDirectory = context.getEnvironment().getPdsPrepareUploadFolderDirectory();
+
+        final Path workingDirectory = Path.of(uploadDirectory);
+        archiveSupport.createArchives(model, workingDirectory, workingDirectory);
+    }
+
+}