Write all failed projects to file - #145

 * fetch docker container name
 * write processed projects to file
 * remove sucessfully processed projects from file
 * add failed projects file to config

data-collection/src/main/java/refactoringml/RunQueue.java

@@ -3,23 +3,27 @@
 import com.rabbitmq.client.*;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.docker.DockerLookup;
 import refactoringml.db.Database;
 import refactoringml.db.HibernateConfig;
 import refactoringml.util.PropertiesUtils;
+import java.io.File;
 import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.concurrent.TimeoutException;
-
 import static refactoringml.util.FilePathUtils.enforceUnixPaths;
 import static refactoringml.util.FileUtils.appendToFile;
+import static refactoringml.util.FileUtils.removeFromFile;
 import static refactoringml.util.PropertiesUtils.getProperty;
 
 public class RunQueue {
 	private static final Logger log = LogManager.getLogger(RunQueue.class);
 	//total count of queue failures during this run, e.g. a redelivery of a already delivered message or an empty message
 	private static int queueFailures = 0;
+	//name of the docker container if any exists, otherwise null
+	private String containerName;
+	//Store all failed projects of this worker in this file
+	private File failedProjectsFile;
 
 	public final static String QUEUE_NAME = "refactoring";
 	private final Database db;
@@ -32,7 +36,12 @@ public RunQueue(String host, String url, String user, String pwd, String storage
 		this.storagePath = storagePath;
 		this.storeFullSourceCode = storeFullSourceCode;
 		db = new Database(new HibernateConfig().getSessionFactory(url, user, pwd));
+		DockerLookup dockerLookup = new DockerLookup();
 
+		containerName = dockerLookup.lookup(null, "containerName");
+		if(containerName == null)
+			containerName = "null";
+		failedProjectsFile = new File(enforceUnixPaths(PropertiesUtils.getProperty("failedProjectsFile") + "_" + containerName));
 		log.debug(toString());
 	}
 
@@ -106,38 +115,36 @@ private void shutdown(Channel channel) throws IOException {
 		System.exit(0);
 	}
 
-	private void processRepository(String message) {
+	private void processRepository(String message) throws IOException {
 		log.debug("Got a new element from rabbitmq queue: " + message);
 
 		String[] msg = message.split(",");
 		String dataset = msg[2];
 		String gitUrl = msg[1];
-
+		String projectInfo = gitUrl + ", " + dataset + "\n";
+		appendToFile(failedProjectsFile, projectInfo);
 		try {
 			new App(dataset, gitUrl, storagePath, db, storeFullSourceCode).run();
 		} catch (org.eclipse.jgit.api.errors.TransportException te){
 			storeFailedProject(gitUrl, "Repository not available", te);
-		}
-		catch (Exception e) {
+		} catch (Exception e) {
 			log.fatal(e.getClass().getCanonicalName() + " while processing " + gitUrl, e);
 			storeFailedProject(gitUrl, e.getClass().getCanonicalName(), e);
 		}
+		removeFromFile(failedProjectsFile, projectInfo);
 	}
 
-	private void storeFailedProject(String gitUrl, String failureReason, Exception exception){
+	private void storeFailedProject(String gitUrl, String failureReason, Exception exception) throws IOException {
 		String failedProject = gitUrl + ", " + failureReason + ", " + exception.toString()  + "\n";
-		String filePath = enforceUnixPaths(PropertiesUtils.getProperty("failedProjectsFile"));
-		try {
-			appendToFile(filePath, failedProject);
-		} catch (IOException e) {
-			log.fatal("Failed to write to file: " + filePath, e);
-		}
+		appendToFile(failedProjectsFile, failedProject);
 	}
 
 	@Override
 	public String toString(){
 		return "RunQueue{\n"+
 				"QUEUE_NAME = " + QUEUE_NAME + "\n" +
+				"containerName=" + containerName + "\n" +
+				"failedProjectsFile=" + failedProjectsFile.getAbsolutePath() + "\n" +
 				"host = " + host + "\n" +
 				"storagePath = " + storagePath + "\n" +
 				"storeFullSourceCode = " + storeFullSourceCode +

data-collection/src/main/java/refactoringml/util/FileUtils.java

@@ -2,10 +2,10 @@
 
 import java.io.*;
 import java.nio.file.Files;
-import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.List;
 import java.util.Scanner;
-
+import java.util.stream.Collectors;
 import static refactoringml.util.FilePathUtils.*;
 
 public class FileUtils {
@@ -69,14 +69,20 @@ public static void writeFile(String filePath, Object content) throws FileNotFoun
 	}
 
 	//Write the toString of an object to a file at the given path. Creates a new file and directory at the path if necessary.
-	public static void appendToFile(String filePath, String content) throws IOException {
-		new File(dirsOnly(filePath)).mkdirs();
-		File file = new File(filePath);
+	public static void appendToFile(File file, String content) throws IOException {
 		FileWriter fr = new FileWriter(file, true);
 		fr.write(content);
 		fr.close();
 	}
 
+	//remove all matching lines from the file
+	public static void removeFromFile(File file, String condition) throws IOException {
+		List<String> lines = org.apache.commons.io.FileUtils.readLines(file);
+		List<String> cleanedLines = lines.stream().filter(line ->
+				!line.contains(condition)).collect(Collectors.toList());
+		org.apache.commons.io.FileUtils.writeLines(file, cleanedLines, false);
+	}
+
 	//Write the content to a new file at the given path. Creates a new directory at the path if necessary.
 	public static String readFile(String filePath) throws FileNotFoundException {
 		File myObj = new File(filePath);

data-collection/src/main/resources/config.properties

@@ -17,4 +17,7 @@ stableCommitThresholds=15,20,25,30,35,40,45,50
 queueWaitTime=45
 
 #Import the Queue after this time in seconds
-queueImportWaitTime=30
+queueImportWaitTime=30
+
+#Store all failed projects in this file
+failedProjectsFile=./logs/failed-projects