diff --git a/syncany-lib/src/main/java/org/syncany/operations/cleanup/CleanupOperation.java b/syncany-lib/src/main/java/org/syncany/operations/cleanup/CleanupOperation.java index 1f781e5ee..ae67919cd 100644 --- a/syncany-lib/src/main/java/org/syncany/operations/cleanup/CleanupOperation.java +++ b/syncany-lib/src/main/java/org/syncany/operations/cleanup/CleanupOperation.java @@ -80,7 +80,9 @@ public class CleanupOperation extends AbstractTransferOperation { private static final Logger logger = Logger.getLogger(CleanupOperation.class.getSimpleName()); public static final String ACTION_ID = "cleanup"; + // Minimal number of database versions per client public static final int MIN_KEEP_DATABASE_VERSIONS = 5; + // Maximal number of database versions per client public static final int MAX_KEEP_DATABASE_VERSIONS = 15; private static final int BEFORE_DOUBLE_CHECK_TIME = 1200; @@ -327,81 +329,111 @@ private boolean hasRemoteChanges() throws Exception { private void mergeRemoteFiles() throws IOException, StorageException { // Retrieve and sort machine's database versions - TreeMap ownDatabaseFilesMap = retrieveOwnRemoteDatabaseFiles(); - - if (ownDatabaseFilesMap.size() <= MAX_KEEP_DATABASE_VERSIONS) { - logger.log(Level.INFO, "- Merge remote files: Not necessary ({0} database files, max. {1})", new Object[] { ownDatabaseFilesMap.size(), - MAX_KEEP_DATABASE_VERSIONS }); + Map> allDatabaseFilesMap = retrieveAllRemoteDatabaseFiles(); + + List allToDeleteDatabaseFiles = new ArrayList(); + Map allMergedDatabaseFiles = new TreeMap(); + + // A client will merge databases if their own machine exceeds the maximum number of database version to be kept + if (allDatabaseFilesMap.get(config.getMachineName()).size() <= MAX_KEEP_DATABASE_VERSIONS) { + logger.log(Level.INFO, "- Merge remote files: Not necessary for client {2} ({0} database files, max. {1})", new Object[] { + allDatabaseFilesMap.get(config.getMachineName()).size(), MAX_KEEP_DATABASE_VERSIONS }); return; } - - // // Now do the merge! - logger.log(Level.INFO, "- Merge remote files: Merging necessary ({0} database files, max. {1}) ...", - new Object[] { ownDatabaseFilesMap.size(), MAX_KEEP_DATABASE_VERSIONS }); - - // 1. Determine files to delete remotely - List toDeleteDatabaseFiles = new ArrayList(); - int numOfDatabaseFilesToDelete = ownDatabaseFilesMap.size() - MIN_KEEP_DATABASE_VERSIONS; - - for (DatabaseRemoteFile ownDatabaseFile : ownDatabaseFilesMap.values()) { - if (toDeleteDatabaseFiles.size() < numOfDatabaseFilesToDelete) { - toDeleteDatabaseFiles.add(ownDatabaseFile); + + for (String client : allDatabaseFilesMap.keySet()) { + List clientDatabaseFiles = allDatabaseFilesMap.get(client); + + // Now do the merge! + logger.log(Level.INFO, "- Merge remote files: Merging necessary ({0} database files, max. {1}) ...", + new Object[] { clientDatabaseFiles.size(), MAX_KEEP_DATABASE_VERSIONS }); + + // 1. Determine files to delete remotely + List toDeleteDatabaseFiles = new ArrayList(); + int numOfDatabaseFilesToDelete = Math.max(clientDatabaseFiles.size() - MIN_KEEP_DATABASE_VERSIONS, 0); + + // This client needs no merging + if (numOfDatabaseFilesToDelete == 0) { + continue; } - } - - // 2. Write merge file - DatabaseRemoteFile lastRemoteMergeDatabaseFile = toDeleteDatabaseFiles.get(toDeleteDatabaseFiles.size() - 1); - File lastLocalMergeDatabaseFile = config.getCache().getDatabaseFile(lastRemoteMergeDatabaseFile.getName()); + + for (DatabaseRemoteFile clientDatabaseFile : clientDatabaseFiles) { + if (toDeleteDatabaseFiles.size() < numOfDatabaseFilesToDelete) { + toDeleteDatabaseFiles.add(clientDatabaseFile); + } + } + + // 2. Write merge file + DatabaseRemoteFile lastRemoteMergeDatabaseFile = toDeleteDatabaseFiles.get(toDeleteDatabaseFiles.size() - 1); + File lastLocalMergeDatabaseFile = config.getCache().getDatabaseFile(lastRemoteMergeDatabaseFile.getName()); - logger.log(Level.INFO, " + Writing new merge file (from {0}, to {1}) to {2} ...", new Object[] { - toDeleteDatabaseFiles.get(0).getClientVersion(), lastRemoteMergeDatabaseFile.getClientVersion(), lastLocalMergeDatabaseFile }); + logger.log(Level.INFO, " + Writing new merge file (from {0}, to {1}) to {2} ...", new Object[] { + toDeleteDatabaseFiles.get(0).getClientVersion(), lastRemoteMergeDatabaseFile.getClientVersion(), lastLocalMergeDatabaseFile }); - long lastLocalClientVersion = lastRemoteMergeDatabaseFile.getClientVersion(); - Iterator lastNDatabaseVersions = localDatabase.getDatabaseVersionsTo(config.getMachineName(), lastLocalClientVersion); + long lastLocalClientVersion = lastRemoteMergeDatabaseFile.getClientVersion(); + Iterator lastNDatabaseVersions = localDatabase.getDatabaseVersionsTo(client, lastLocalClientVersion); - DatabaseXmlSerializer databaseDAO = new DatabaseXmlSerializer(config.getTransformer()); - databaseDAO.save(lastNDatabaseVersions, lastLocalMergeDatabaseFile); + DatabaseXmlSerializer databaseDAO = new DatabaseXmlSerializer(config.getTransformer()); + databaseDAO.save(lastNDatabaseVersions, lastLocalMergeDatabaseFile); + + // Queue files for uploading and deletion + allToDeleteDatabaseFiles.addAll(toDeleteDatabaseFiles); + allMergedDatabaseFiles.put(lastLocalMergeDatabaseFile, lastRemoteMergeDatabaseFile); + } + // 3. Uploading merge file // And delete others - for (RemoteFile toDeleteRemoteFile : toDeleteDatabaseFiles) { + for (RemoteFile toDeleteRemoteFile : allToDeleteDatabaseFiles) { logger.log(Level.INFO, " + Deleting remote file " + toDeleteRemoteFile + " ..."); transferManager.delete(toDeleteRemoteFile); } - - // TODO [high] Issue #64: TM cannot overwrite, might lead to chaos if operation does not finish, uploading the new merge file, this might - // happen often if - // new file is bigger! - - logger.log(Level.INFO, " + Uploading new file {0} from local file {1} ...", new Object[] { lastRemoteMergeDatabaseFile, - lastLocalMergeDatabaseFile }); - - try { - // Make sure it's deleted - transferManager.delete(lastRemoteMergeDatabaseFile); - } - catch (StorageException e) { - // Don't care! + + for (File lastLocalMergeDatabaseFile : allMergedDatabaseFiles.keySet()) { + // TODO [high] Issue #64: TM cannot overwrite, might lead to chaos if operation does not finish, uploading the new merge file, this might + // happen often if + // new file is bigger! + RemoteFile lastRemoteMergeDatabaseFile = allMergedDatabaseFiles.get(lastLocalMergeDatabaseFile); + + logger.log(Level.INFO, " + Uploading new file {0} from local file {1} ...", new Object[] { lastRemoteMergeDatabaseFile, + lastLocalMergeDatabaseFile }); + + try { + // Make sure it's deleted + transferManager.delete(lastRemoteMergeDatabaseFile); + } + catch (StorageException e) { + // Don't care! + } + + transferManager.upload(lastLocalMergeDatabaseFile, lastRemoteMergeDatabaseFile); } - transferManager.upload(lastLocalMergeDatabaseFile, lastRemoteMergeDatabaseFile); - // Update stats - result.setMergedDatabaseFilesCount(toDeleteDatabaseFiles.size()); + result.setMergedDatabaseFilesCount(allToDeleteDatabaseFiles.size()); + + + } - private TreeMap retrieveOwnRemoteDatabaseFiles() throws StorageException { - TreeMap ownDatabaseRemoteFiles = new TreeMap(); + /** + * retrieveAllRemoteDatabaseFiles returns a Map with clientNames as keys and + * lists of corresponding DatabaseRemoteFiles as values. + */ + private Map> retrieveAllRemoteDatabaseFiles() throws StorageException { + TreeMap> allDatabaseRemoteFilesMap = new TreeMap>(); Map allDatabaseRemoteFiles = transferManager.list(DatabaseRemoteFile.class); for (Map.Entry entry : allDatabaseRemoteFiles.entrySet()) { - if (config.getMachineName().equals(entry.getValue().getClientName())) { - ownDatabaseRemoteFiles.put(entry.getKey(), entry.getValue()); + String clientName = entry.getValue().getClientName(); + if (allDatabaseRemoteFilesMap.get(clientName) == null) { + allDatabaseRemoteFilesMap.put(clientName, new ArrayList()); } + allDatabaseRemoteFilesMap.get(clientName).add(entry.getValue()); } - return ownDatabaseRemoteFiles; + return allDatabaseRemoteFilesMap; } }