Skip to content
This repository has been archived by the owner on Jul 26, 2022. It is now read-only.

Commit

Permalink
Enabling atomic cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@pimotte
pimotte committed Aug 13, 2014
1 parent aa856b4 commit c85ae4b
Show file tree
Hide file tree
Showing 4 changed files with 66 additions and 53 deletions.
47 changes: 11 additions & 36 deletions syncany-lib/src/main/java/org/syncany/operations/cleanup/CleanupOperation.java
View file
Expand Up @@ -57,6 +57,7 @@
import org.syncany.operations.status.StatusOperationResult;
import org.syncany.operations.up.UpOperation;
import org.syncany.plugins.StorageException;
import org.syncany.plugins.transfer.RemoteTransaction;
import org.syncany.plugins.transfer.files.DatabaseRemoteFile;
import org.syncany.plugins.transfer.files.MultiChunkRemoteFile;
import org.syncany.plugins.transfer.files.RemoteFile;
Expand Down Expand Up @@ -91,6 +92,7 @@ public class CleanupOperation extends AbstractTransferOperation {
private CleanupOperationResult result;

private SqlDatabase localDatabase;
private RemoteTransaction remoteTransaction;

public CleanupOperation(Config config) {
this(config, new CleanupOperationOptions());
Expand All @@ -103,6 +105,7 @@ public CleanupOperation(Config config, CleanupOperationOptions options) {
this.result = new CleanupOperationResult();

this.localDatabase = new SqlDatabase(config);
this.remoteTransaction = new RemoteTransaction(config, transferManager);
}

@Override
Expand Down Expand Up @@ -141,34 +144,14 @@ public CleanupOperationResult execute() throws Exception {
if (options.isMergeRemoteFiles()) {
mergeRemoteFiles();
}

removeLostMultiChunks();
remoteTransaction.commit();

setLastTimeCleaned(System.currentTimeMillis()/1000);
finishOperation();
return updateResultCode(result);
}

private void removeLostMultiChunks() throws StorageException {
Map<String, MultiChunkRemoteFile> remoteMultiChunks = transferManager.list(MultiChunkRemoteFile.class);

Map<MultiChunkId, MultiChunkEntry> locallyKnownMultiChunks = new HashMap<>();

locallyKnownMultiChunks.putAll(localDatabase.getMultiChunks());
locallyKnownMultiChunks.putAll(localDatabase.getMuddyMultiChunks());

for (MultiChunkRemoteFile remoteMultiChunk : remoteMultiChunks.values()) {
MultiChunkId remoteMultiChunkId = new MultiChunkId(remoteMultiChunk.getMultiChunkId());
boolean multiChunkExistsLocally = locallyKnownMultiChunks.containsKey(remoteMultiChunkId);

if (!multiChunkExistsLocally) {
logger.log(Level.WARNING, "- Deleting lost/unknown remote multichunk " + remoteMultiChunk + " ...");
transferManager.delete(remoteMultiChunk);

result.getRemovedMultiChunks().put(remoteMultiChunkId, new MultiChunkEntry(remoteMultiChunkId, -1));
}
}
}

private CleanupOperationResult updateResultCode(CleanupOperationResult result) {
if (result.getMergedDatabaseFilesCount() > 0 || result.getRemovedMultiChunks().size() > 0 || result.getRemovedOldVersionsCount() > 0) {
Expand Down Expand Up @@ -254,17 +237,17 @@ private void removeOldVersions() throws Exception {
DatabaseRemoteFile newPurgeRemoteFile = findNewPurgeRemoteFile(purgeDatabaseVersion.getHeader());
File tempLocalPurgeDatabaseFile = writePurgeFile(purgeDatabaseVersion, newPurgeRemoteFile);

uploadPurgeFile(tempLocalPurgeDatabaseFile, newPurgeRemoteFile);
addPurgeFileToTransaction(tempLocalPurgeDatabaseFile, newPurgeRemoteFile);
remoteDeleteUnusedMultiChunks(unusedMultiChunks);

// Update stats
result.setRemovedOldVersionsCount(purgeFileVersions.size());
result.setRemovedMultiChunks(unusedMultiChunks);
}

private void uploadPurgeFile(File tempPurgeFile, DatabaseRemoteFile newPurgeRemoteFile) throws StorageException {
private void addPurgeFileToTransaction(File tempPurgeFile, DatabaseRemoteFile newPurgeRemoteFile) throws StorageException {
logger.log(Level.INFO, "- Uploading PURGE database file " + newPurgeRemoteFile + " ...");
transferManager.upload(tempPurgeFile, newPurgeRemoteFile);
remoteTransaction.add(tempPurgeFile, newPurgeRemoteFile);
}

private DatabaseVersion createPurgeDatabaseVersion(Map<FileHistoryId, FileVersion> mostRecentPurgeFileVersions) {
Expand Down Expand Up @@ -314,7 +297,7 @@ private void remoteDeleteUnusedMultiChunks(Map<MultiChunkId, MultiChunkEntry> un

for (MultiChunkEntry multiChunkEntry : unusedMultiChunks.values()) {
logger.log(Level.FINE, " + Deleting remote multichunk " + multiChunkEntry + " ...");
transferManager.delete(new MultiChunkRemoteFile(multiChunkEntry.getId()));
remoteTransaction.delete(new MultiChunkRemoteFile(multiChunkEntry.getId()));
}
}

Expand Down Expand Up @@ -391,7 +374,7 @@ private void mergeRemoteFiles() throws IOException, StorageException {
// And delete others
for (RemoteFile toDeleteRemoteFile : allToDeleteDatabaseFiles) {
logger.log(Level.INFO, " + Deleting remote file " + toDeleteRemoteFile + " ...");
transferManager.delete(toDeleteRemoteFile);
remoteTransaction.delete(toDeleteRemoteFile);
}

for (File lastLocalMergeDatabaseFile : allMergedDatabaseFiles.keySet()) {
Expand All @@ -403,15 +386,7 @@ private void mergeRemoteFiles() throws IOException, StorageException {
logger.log(Level.INFO, " + Uploading new file {0} from local file {1} ...", new Object[] { lastRemoteMergeDatabaseFile,
lastLocalMergeDatabaseFile });

try {
// Make sure it's deleted
transferManager.delete(lastRemoteMergeDatabaseFile);
}
catch (StorageException e) {
// Don't care!
}

transferManager.upload(lastLocalMergeDatabaseFile, lastRemoteMergeDatabaseFile);
remoteTransaction.add(lastLocalMergeDatabaseFile, lastRemoteMergeDatabaseFile);
}

// Update stats
Expand Down
1 change: 1 addition & 0 deletions syncany-lib/src/main/java/org/syncany/plugins/local/LocalTransferManager.java
View file
Expand Up @@ -30,6 +30,7 @@
import java.util.logging.Level;
import java.util.logging.Logger;

import org.apache.commons.io.FileExistsException;
import org.apache.commons.io.FileUtils;
import org.syncany.plugins.StorageException;
import org.syncany.plugins.transfer.AbstractTransferManager;
Expand Down
60 changes: 49 additions & 11 deletions syncany-lib/src/main/java/org/syncany/plugins/transfer/RemoteTransaction.java
View file
Expand Up @@ -58,7 +58,7 @@ public RemoteTransaction(Config config, TransferManager transferManager) {
* Adds a file to this transaction. Generates a temporary file to store it.
*/
public void add(File localFile, RemoteFile remoteFile) throws StorageException {
TempRemoteFile temporaryRemoteFile = new TempRemoteFile(localFile);
TempRemoteFile temporaryRemoteFile = new TempRemoteFile();

logger.log(Level.INFO, "Adding file to transaction: " + localFile);
logger.log(Level.INFO, " -> Temp. remote file: " + temporaryRemoteFile + ", final location: " + remoteFile);
Expand All @@ -71,6 +71,26 @@ public void add(File localFile, RemoteFile remoteFile) throws StorageException {
transactionTO.addAction(action);
}

/**
* Adds the deletion of a file to this transaction. Generates a temporary file
* to store it while the transaction is being finalized.
*/
public void delete(RemoteFile remoteFile) throws StorageException {
TempRemoteFile temporaryRemoteFile = new TempRemoteFile();

logger.log(Level.INFO, "Adding file to transaction for deletion: " + remoteFile);
logger.log(Level.INFO, " -> Temp. remote file: " + temporaryRemoteFile);

ActionTO action = new ActionTO();
action.setType(ActionTO.TYPE_DELETE);
action.setRemoteLocation(remoteFile);
action.setRemoteTempLocation(temporaryRemoteFile);
transactionTO.addAction(action);

}



/**
* Moves all files to the temporary remote location. If
* no errors occur, all files are moved to their final location.
Expand All @@ -84,21 +104,39 @@ public void commit() throws StorageException {
transferManager.upload(localTransactionFile, remoteTransactionFile);

for (ActionTO action : transactionTO.getActions()) {
File localFile = action.getLocalTempLocation();
RemoteFile tempRemoteFile = action.getTempRemoteFile();
logger.log(Level.INFO, "- Uploading {0} to temp. file {1} ...", new Object[] { localFile, tempRemoteFile });
transferManager.upload(localFile, tempRemoteFile);
if (action.getType().equals(ActionTO.TYPE_UPLOAD)) {
File localFile = action.getLocalTempLocation();
RemoteFile tempRemoteFile = action.getTempRemoteFile();
logger.log(Level.INFO, "- Uploading {0} to temp. file {1} ...", new Object[] { localFile, tempRemoteFile });
transferManager.upload(localFile, tempRemoteFile);
}
else if (action.getType().equals(ActionTO.TYPE_DELETE)) {
RemoteFile remoteFile = action.getRemoteFile();
RemoteFile tempRemoteFile = action.getTempRemoteFile();
logger.log(Level.INFO, "- Moving {0} to temp. file {1} ...", new Object[] { remoteFile, tempRemoteFile });
transferManager.move(remoteFile, tempRemoteFile);
}
}

// After this deletion, the transaction is final!
logger.log(Level.INFO, "- Deleting remote transaction file {0} ...", remoteTransactionFile);
transferManager.delete(remoteTransactionFile);

for (ActionTO action : transactionTO.getActions()) {
RemoteFile tempRemoteFile = action.getTempRemoteFile();
RemoteFile finalRemoteFile = action.getRemoteFile();
logger.log(Level.INFO, "- Moving temp. file {0} to final location {1} ...", new Object[] { tempRemoteFile, finalRemoteFile });
transferManager.move(tempRemoteFile, finalRemoteFile);
if (action.getType().equals(ActionTO.TYPE_UPLOAD)) {
RemoteFile tempRemoteFile = action.getTempRemoteFile();
RemoteFile finalRemoteFile = action.getRemoteFile();
logger.log(Level.INFO, "- Moving temp. file {0} to final location {1} ...", new Object[] { tempRemoteFile, finalRemoteFile });
transferManager.move(tempRemoteFile, finalRemoteFile);
}
else if (action.getType().equals(ActionTO.TYPE_DELETE)){
RemoteFile tempRemoteFile = action.getTempRemoteFile();
logger.log(Level.INFO, "- Moving deleting temp. file {0} ...", new Object[] { tempRemoteFile });
transferManager.delete(tempRemoteFile);
}
}

logger.log(Level.INFO, "- Deleting remote transaction file {0} ...", remoteTransactionFile);
transferManager.delete(remoteTransactionFile);

localTransactionFile.delete();

logger.log(Level.INFO, "Succesfully committed transaction.");
Expand Down
11 changes: 5 additions & 6 deletions syncany-lib/src/main/java/org/syncany/plugins/transfer/files/TempRemoteFile.java
View file
Expand Up @@ -33,7 +33,7 @@
* @author Pim Otte
*/
public class TempRemoteFile extends RemoteFile {
private static final Pattern NAME_PATTERN = Pattern.compile("temp-([a-f0-9]+)");
private static final Pattern NAME_PATTERN = Pattern.compile("temp-([A-Za-z]+)");
private static final String NAME_FORMAT = "temp-%s";

/**
Expand All @@ -47,13 +47,12 @@ public TempRemoteFile(String name) throws StorageException {
}

/**
* Initializes a new temp file, given the local file which will be moved there.
* Initializes a new randomly named temp file.
*
* @param localFile the local file that will be moved to this temporary location.
* @throws StorageException If the name is not match the name pattern
* @throws StorageException
*/
public TempRemoteFile(File localFile) throws StorageException {
super(String.format(NAME_FORMAT, Integer.toHexString(CipherUtil.createRandomAlphabeticString(20).hashCode())));
public TempRemoteFile() throws StorageException {
super("temp-" + CipherUtil.createRandomAlphabeticString(20));
}

@Override
Expand Down

0 comments on commit c85ae4b

Please sign in to comment.