Skip to content
This repository has been archived by the owner on Jul 26, 2022. It is now read-only.

Commit

Permalink
Switch cleanup operation to full
Browse files Browse the repository at this point in the history
  • Loading branch information
@pimotte
pimotte committed Aug 8, 2014
1 parent bd2f68a commit 4680449
Showing 1 changed file with 83 additions and 51 deletions.
134 changes: 83 additions & 51 deletions syncany-lib/src/main/java/org/syncany/operations/cleanup/CleanupOperation.java
View file
Expand Up @@ -80,7 +80,9 @@ public class CleanupOperation extends AbstractTransferOperation {
private static final Logger logger = Logger.getLogger(CleanupOperation.class.getSimpleName());

public static final String ACTION_ID = "cleanup";
// Minimal number of database versions per client
public static final int MIN_KEEP_DATABASE_VERSIONS = 5;
// Maximal number of database versions per client
public static final int MAX_KEEP_DATABASE_VERSIONS = 15;

private static final int BEFORE_DOUBLE_CHECK_TIME = 1200;
Expand Down Expand Up @@ -327,81 +329,111 @@ private boolean hasRemoteChanges() throws Exception {

private void mergeRemoteFiles() throws IOException, StorageException {
// Retrieve and sort machine's database versions
TreeMap<String, DatabaseRemoteFile> ownDatabaseFilesMap = retrieveOwnRemoteDatabaseFiles();

if (ownDatabaseFilesMap.size() <= MAX_KEEP_DATABASE_VERSIONS) {
logger.log(Level.INFO, "- Merge remote files: Not necessary ({0} database files, max. {1})", new Object[] { ownDatabaseFilesMap.size(),
MAX_KEEP_DATABASE_VERSIONS });
Map<String, List<DatabaseRemoteFile>> allDatabaseFilesMap = retrieveAllRemoteDatabaseFiles();

List<DatabaseRemoteFile> allToDeleteDatabaseFiles = new ArrayList<DatabaseRemoteFile>();
Map<File, DatabaseRemoteFile> allMergedDatabaseFiles = new TreeMap<File, DatabaseRemoteFile>();

// A client will merge databases if their own machine exceeds the maximum number of database version to be kept
if (allDatabaseFilesMap.get(config.getMachineName()).size() <= MAX_KEEP_DATABASE_VERSIONS) {
logger.log(Level.INFO, "- Merge remote files: Not necessary for client {2} ({0} database files, max. {1})", new Object[] {
allDatabaseFilesMap.get(config.getMachineName()).size(), MAX_KEEP_DATABASE_VERSIONS });

return;
}

// // Now do the merge!
logger.log(Level.INFO, "- Merge remote files: Merging necessary ({0} database files, max. {1}) ...",
new Object[] { ownDatabaseFilesMap.size(), MAX_KEEP_DATABASE_VERSIONS });

// 1. Determine files to delete remotely
List<DatabaseRemoteFile> toDeleteDatabaseFiles = new ArrayList<DatabaseRemoteFile>();
int numOfDatabaseFilesToDelete = ownDatabaseFilesMap.size() - MIN_KEEP_DATABASE_VERSIONS;

for (DatabaseRemoteFile ownDatabaseFile : ownDatabaseFilesMap.values()) {
if (toDeleteDatabaseFiles.size() < numOfDatabaseFilesToDelete) {
toDeleteDatabaseFiles.add(ownDatabaseFile);

for (String client : allDatabaseFilesMap.keySet()) {
List<DatabaseRemoteFile> clientDatabaseFiles = allDatabaseFilesMap.get(client);

// Now do the merge!
logger.log(Level.INFO, "- Merge remote files: Merging necessary ({0} database files, max. {1}) ...",
new Object[] { clientDatabaseFiles.size(), MAX_KEEP_DATABASE_VERSIONS });

// 1. Determine files to delete remotely
List<DatabaseRemoteFile> toDeleteDatabaseFiles = new ArrayList<DatabaseRemoteFile>();
int numOfDatabaseFilesToDelete = Math.max(clientDatabaseFiles.size() - MIN_KEEP_DATABASE_VERSIONS, 0);

// This client needs no merging
if (numOfDatabaseFilesToDelete == 0) {
continue;
}
}

// 2. Write merge file
DatabaseRemoteFile lastRemoteMergeDatabaseFile = toDeleteDatabaseFiles.get(toDeleteDatabaseFiles.size() - 1);
File lastLocalMergeDatabaseFile = config.getCache().getDatabaseFile(lastRemoteMergeDatabaseFile.getName());

for (DatabaseRemoteFile clientDatabaseFile : clientDatabaseFiles) {
if (toDeleteDatabaseFiles.size() < numOfDatabaseFilesToDelete) {
toDeleteDatabaseFiles.add(clientDatabaseFile);
}
}

// 2. Write merge file
DatabaseRemoteFile lastRemoteMergeDatabaseFile = toDeleteDatabaseFiles.get(toDeleteDatabaseFiles.size() - 1);
File lastLocalMergeDatabaseFile = config.getCache().getDatabaseFile(lastRemoteMergeDatabaseFile.getName());

logger.log(Level.INFO, " + Writing new merge file (from {0}, to {1}) to {2} ...", new Object[] {
toDeleteDatabaseFiles.get(0).getClientVersion(), lastRemoteMergeDatabaseFile.getClientVersion(), lastLocalMergeDatabaseFile });
logger.log(Level.INFO, " + Writing new merge file (from {0}, to {1}) to {2} ...", new Object[] {
toDeleteDatabaseFiles.get(0).getClientVersion(), lastRemoteMergeDatabaseFile.getClientVersion(), lastLocalMergeDatabaseFile });

long lastLocalClientVersion = lastRemoteMergeDatabaseFile.getClientVersion();
Iterator<DatabaseVersion> lastNDatabaseVersions = localDatabase.getDatabaseVersionsTo(config.getMachineName(), lastLocalClientVersion);
long lastLocalClientVersion = lastRemoteMergeDatabaseFile.getClientVersion();
Iterator<DatabaseVersion> lastNDatabaseVersions = localDatabase.getDatabaseVersionsTo(client, lastLocalClientVersion);

DatabaseXmlSerializer databaseDAO = new DatabaseXmlSerializer(config.getTransformer());
databaseDAO.save(lastNDatabaseVersions, lastLocalMergeDatabaseFile);
DatabaseXmlSerializer databaseDAO = new DatabaseXmlSerializer(config.getTransformer());
databaseDAO.save(lastNDatabaseVersions, lastLocalMergeDatabaseFile);

// Queue files for uploading and deletion
allToDeleteDatabaseFiles.addAll(toDeleteDatabaseFiles);
allMergedDatabaseFiles.put(lastLocalMergeDatabaseFile, lastRemoteMergeDatabaseFile);

}

// 3. Uploading merge file

// And delete others
for (RemoteFile toDeleteRemoteFile : toDeleteDatabaseFiles) {
for (RemoteFile toDeleteRemoteFile : allToDeleteDatabaseFiles) {
logger.log(Level.INFO, " + Deleting remote file " + toDeleteRemoteFile + " ...");
transferManager.delete(toDeleteRemoteFile);
}

// TODO [high] Issue #64: TM cannot overwrite, might lead to chaos if operation does not finish, uploading the new merge file, this might
// happen often if
// new file is bigger!

logger.log(Level.INFO, " + Uploading new file {0} from local file {1} ...", new Object[] { lastRemoteMergeDatabaseFile,
lastLocalMergeDatabaseFile });

try {
// Make sure it's deleted
transferManager.delete(lastRemoteMergeDatabaseFile);
}
catch (StorageException e) {
// Don't care!

for (File lastLocalMergeDatabaseFile : allMergedDatabaseFiles.keySet()) {
// TODO [high] Issue #64: TM cannot overwrite, might lead to chaos if operation does not finish, uploading the new merge file, this might
// happen often if
// new file is bigger!
RemoteFile lastRemoteMergeDatabaseFile = allMergedDatabaseFiles.get(lastLocalMergeDatabaseFile);

logger.log(Level.INFO, " + Uploading new file {0} from local file {1} ...", new Object[] { lastRemoteMergeDatabaseFile,
lastLocalMergeDatabaseFile });

try {
// Make sure it's deleted
transferManager.delete(lastRemoteMergeDatabaseFile);
}
catch (StorageException e) {
// Don't care!
}

transferManager.upload(lastLocalMergeDatabaseFile, lastRemoteMergeDatabaseFile);
}

transferManager.upload(lastLocalMergeDatabaseFile, lastRemoteMergeDatabaseFile);

// Update stats
result.setMergedDatabaseFilesCount(toDeleteDatabaseFiles.size());
result.setMergedDatabaseFilesCount(allToDeleteDatabaseFiles.size());



}

private TreeMap<String, DatabaseRemoteFile> retrieveOwnRemoteDatabaseFiles() throws StorageException {
TreeMap<String, DatabaseRemoteFile> ownDatabaseRemoteFiles = new TreeMap<String, DatabaseRemoteFile>();
/**
* retrieveAllRemoteDatabaseFiles returns a Map with clientNames as keys and
* lists of corresponding DatabaseRemoteFiles as values.
*/
private Map<String, List<DatabaseRemoteFile>> retrieveAllRemoteDatabaseFiles() throws StorageException {
TreeMap<String, List<DatabaseRemoteFile>> allDatabaseRemoteFilesMap = new TreeMap<String, List<DatabaseRemoteFile>>();
Map<String, DatabaseRemoteFile> allDatabaseRemoteFiles = transferManager.list(DatabaseRemoteFile.class);

for (Map.Entry<String, DatabaseRemoteFile> entry : allDatabaseRemoteFiles.entrySet()) {
if (config.getMachineName().equals(entry.getValue().getClientName())) {
ownDatabaseRemoteFiles.put(entry.getKey(), entry.getValue());
String clientName = entry.getValue().getClientName();
if (allDatabaseRemoteFilesMap.get(clientName) == null) {
allDatabaseRemoteFilesMap.put(clientName, new ArrayList<DatabaseRemoteFile>());
}
allDatabaseRemoteFilesMap.get(clientName).add(entry.getValue());
}

return ownDatabaseRemoteFiles;
return allDatabaseRemoteFilesMap;
}
}

0 comments on commit 4680449

Please sign in to comment.