fixed container deletion bug and added git caching

TrueWinter · Dec 7, 2023 · f9c99b7 · f9c99b7
1 parent 20f50cf
commit f9c99b7
Show file tree

Hide file tree

Showing 8 changed files with 92 additions and 29 deletions.
diff --git a/common/pom.xml b/common/pom.xml
@@ -7,7 +7,7 @@
     <parent>
         <groupId>dev.truewinter.simofa</groupId>
         <artifactId>SimofaProject</artifactId>
-        <version>0.0.8</version>
+        <version>0.0.9</version>
     </parent>
 
     <groupId>dev.truewinter.simofa.common</groupId>

diff --git a/deploy/pom.xml b/deploy/pom.xml
@@ -7,7 +7,7 @@
     <parent>
         <groupId>dev.truewinter.simofa</groupId>
         <artifactId>SimofaProject</artifactId>
-        <version>0.0.8</version>
+        <version>0.0.9</version>
     </parent>
 
     <groupId>dev.truewinter</groupId>

diff --git a/pom.xml b/pom.xml
@@ -8,7 +8,7 @@
     <artifactId>SimofaProject</artifactId>
     <packaging>pom</packaging>
     <!-- Do not manually modify the version. Use versions:set instead -->
-    <version>0.0.8</version>
+    <version>0.0.9</version>
 
     <modules>
         <module>simofa</module>

diff --git a/simofa/pom.xml b/simofa/pom.xml
@@ -7,7 +7,7 @@
     <parent>
         <groupId>dev.truewinter.simofa</groupId>
         <artifactId>SimofaProject</artifactId>
-        <version>0.0.8</version>
+        <version>0.0.9</version>
     </parent>
 
     <groupId>dev.truewinter</groupId>

diff --git a/simofa/src/main/java/dev/truewinter/simofa/GitFetcher.java b/simofa/src/main/java/dev/truewinter/simofa/GitFetcher.java
@@ -0,0 +1,70 @@
+package dev.truewinter.simofa;
+
+import dev.truewinter.simofa.common.LogType;
+import dev.truewinter.simofa.common.SimofaLog;
+import dev.truewinter.simofa.common.Util;
+import dev.truewinter.simofa.docker.WebsiteBuild;
+import org.apache.commons.io.FileUtils;
+import org.eclipse.jgit.api.*;
+import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
+
+import java.io.File;
+import java.io.IOException;
+
+public class GitFetcher {
+    public static void fetch(WebsiteBuild build, File tmpInDir, File websiteCacheDir) throws GitAPIException, IOException {
+        File gitCacheDir = new File(websiteCacheDir, "git");
+
+        if (gitCacheDir.exists()) {
+            build.addLog(new SimofaLog(LogType.INFO, "Using git cache"));
+            pull(build, tmpInDir, websiteCacheDir);
+        } else {
+            clone(build, tmpInDir, websiteCacheDir);
+        }
+    }
+
+    @SuppressWarnings("rawtypes")
+    private static void setCredentials(WebsiteBuild build, TransportCommand cloneCommand) {
+        GitCredential gitCredential = build.getWebsite().getGitCredential();
+        if (gitCredential != null) {
+            cloneCommand.setCredentialsProvider(new UsernamePasswordCredentialsProvider(
+                    gitCredential.getUsername(),
+                    gitCredential.getPassword()
+            ));
+        }
+    }
+
+    private static void clone(WebsiteBuild build, File tmpInDir, File websiteCacheDir) throws GitAPIException, IOException {
+        CloneCommand cloneCommand = Git.cloneRepository()
+                .setURI(build.getWebsite().getGitUrl())
+                .setBranch(build.getWebsite().getGitBranch())
+                .setCloneAllBranches(false)
+                .setDirectory(tmpInDir);
+
+        setCredentials(build, cloneCommand);
+        cloneCommand.call().close();
+
+        if (!Util.isBlank(build.getCacheDir())) {
+            System.out.println("will copy git");
+            File gitCacheDir = new File(websiteCacheDir, "git");
+            if (!gitCacheDir.exists() && !gitCacheDir.mkdir()) {
+                build.addLog(new SimofaLog(LogType.WARN, "Unable to create git cache directory for website " + build.getWebsite().getId()));
+            } else {
+                System.out.println("copying git");
+                FileUtils.copyDirectory(tmpInDir, gitCacheDir);
+            }
+        }
+    }
+
+    private static void pull(WebsiteBuild build, File tmpInDir, File websiteCacheDir) throws IOException, GitAPIException {
+        File gitCacheDir = new File(websiteCacheDir, "git");
+        try (Git git = Git.open(gitCacheDir)) {
+            PullCommand pullCommand = git.pull();
+            setCredentials(build, pullCommand);
+            pullCommand.call();
+
+            FileUtils.copyDirectory(gitCacheDir, tmpInDir);
+        }
+    }
+}
diff --git a/simofa/src/main/java/dev/truewinter/simofa/docker/BuildQueue.java b/simofa/src/main/java/dev/truewinter/simofa/docker/BuildQueue.java
@@ -100,12 +100,11 @@ public synchronized void remove(int id) {
                 if (w.getStatus().equals(BuildStatus.QUEUED.toString()) ||
                         w.getStatus().equals(BuildStatus.BUILDING.toString())) {
                     w.setStatus(BuildStatus.STOPPED);
+                }
 
-                    System.out.printf("Stopping build %s%n", w.getId());
-                    if (!Util.isBlank(w.getContainerId())) {
-                        System.out.printf("Build %s has container %s%n", w.getId(), w.getContainerId());
-                        Simofa.getDockerManager().deleteContainer(w.getContainerId());
-                    }
+                if (!Util.isBlank(w.getContainerId())) {
+                    System.out.printf("Deleting container %s for build %s%n", w.getContainerId(), w.getId());
+                    Simofa.getDockerManager().deleteContainer(w.getContainerId());
                 }
             });
         }

diff --git a/simofa/src/main/java/dev/truewinter/simofa/docker/DockerManager.java b/simofa/src/main/java/dev/truewinter/simofa/docker/DockerManager.java
@@ -131,7 +131,13 @@ public List<dev.truewinter.simofa.docker.Container> getContainers() {
     }
 
     public void deleteContainer(String id) {
-        dockerClient.removeContainerCmd(id).withForce(true).withRemoveVolumes(true).exec();
+        List<com.github.dockerjava.api.model.Container> containers = dockerClient.listContainersCmd().withShowAll(true).exec();
+        for (com.github.dockerjava.api.model.Container container : containers) {
+            if (container.getId().equals(id)) {
+                dockerClient.removeContainerCmd(container.getId()).withForce(true).withRemoveVolumes(true).exec();
+                break;
+            }
+        }
     }
 
     @SuppressWarnings("RedundantThrows")

diff --git a/simofa/src/main/java/dev/truewinter/simofa/docker/WebsiteBuilder.java b/simofa/src/main/java/dev/truewinter/simofa/docker/WebsiteBuilder.java
@@ -5,6 +5,7 @@
 import com.github.dockerjava.api.model.WaitResponse;
 import dev.truewinter.simofa.DeploymentServer;
 import dev.truewinter.simofa.GitCredential;
+import dev.truewinter.simofa.GitFetcher;
 import dev.truewinter.simofa.Simofa;
 import dev.truewinter.simofa.common.BuildStatus;
 import dev.truewinter.simofa.common.LogType;
@@ -37,33 +38,21 @@ protected WebsiteBuilder(WebsiteBuild build) {
     public void run() {
         Simofa.getLogger().info(String.format("Running build %s for website %d", build.getId(), build.getWebsite().getId()));
         build.setStatus(BuildStatus.BUILDING);
+
         try {
             File tmpDir = Util.createTempDir(build.getId());
             File tmpInDir = new File(tmpDir, "in");
 
-            CloneCommand cloneCommand = Git.cloneRepository()
-                    .setURI(build.getWebsite().getGitUrl())
-                    .setBranch(build.getWebsite().getGitBranch())
-                    .setCloneAllBranches(false)
-                    .setDirectory(tmpInDir);
-
-            GitCredential gitCredential = build.getWebsite().getGitCredential();
-            if (gitCredential != null) {
-                cloneCommand.setCredentialsProvider(new UsernamePasswordCredentialsProvider(
-                        gitCredential.getUsername(),
-                        gitCredential.getPassword()
-                ));
-            }
-            Git git = cloneCommand.call();
+            File tmpCacheDir = new File(tmpDir, "cache");
+            File websiteCacheDir = new File(build.getCacheDir(), "website-" + build.getWebsite().getId());
+
+            GitFetcher.fetch(build, tmpInDir, websiteCacheDir);
 
             File tmpScriptDir = new File(tmpDir, "scripts");
             FileOutputStream fileOutputStream = new FileOutputStream(new File(tmpScriptDir, "build.sh"));
             fileOutputStream.write(build.getWebsite().getBuildCommand().getBytes());
             fileOutputStream.close();
-            git.close();
 
-            File tmpCacheDir = new File(tmpDir, "cache");
-            File websiteCacheDir = new File(build.getCacheDir(), "website-" + build.getWebsite().getId());
             File websiteCache = new File(websiteCacheDir, "cache.zip");
             if (websiteCache.exists()) {
                 FileInputStream fileInputStream = new FileInputStream(websiteCache);
@@ -109,8 +98,7 @@ void onExit(WaitResponse waitResponse) {
                         }
                     }
 
-                    Simofa.getBuildQueueManager().getBuildQueue().getCurrentBuilds()
-                        .removeIf(w -> w.getId().equals(build.getId()));
+                    Simofa.getBuildQueueManager().getBuildQueue().remove(build.getWebsite());
 
                     if (waitResponse.getStatusCode() == 0) {
                         File tempDir = null;