Skip to content
Permalink
Browse files
[NO ISSUE][REP] Ensure files delete order from replicas
- user model changes: no
- storage format changes: no
- interface changes: no

Details:

- When deleting files from a replica, delete the resource
  files last to ensure no orphaned component files if
  the replica fails after deleting the resource file.
- When replicating a resource file, ensure the index
  directory is empty.

Change-Id: I7dbb9960cb644c063de49dd0920a1709964de2f8
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13224
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
  • Loading branch information
mhubail committed Sep 14, 2021
1 parent 8f278c0 commit d8a1a93f99ce7c789dcc78afa1df3398d833cb15
Show file tree
Hide file tree
Showing 3 changed files with 10 additions and 1 deletion.
@@ -46,7 +46,6 @@ public void perform(CcId ccId, IControllerService cs) throws HyracksDataExceptio
deleteInvalidMetadataIndexes(localResourceRepository);
final Set<Integer> nodePartitions = appContext.getReplicaManager().getPartitions();
localResourceRepository.deleteCorruptedResources();
//TODO optimize this to cleanup all active partitions at once
for (Integer partition : nodePartitions) {
localResourceRepository.cleanup(partition);
}
@@ -39,6 +39,7 @@
import org.apache.asterix.replication.api.IReplicaTask;
import org.apache.asterix.replication.api.IReplicationWorker;
import org.apache.asterix.replication.management.NetworkingUtil;
import org.apache.commons.io.FileUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
@@ -72,6 +73,10 @@ public void perform(INcApplicationContext appCtx, IReplicationWorker worker) {
// resolve path
final FileReference localPath = ioManager.resolve(file);
final Path resourceDir = Files.createDirectories(localPath.getFile().getParentFile().toPath());
if (indexMetadata) {
// ensure clean index directory
FileUtils.cleanDirectory(resourceDir.toFile());
}
// create mask
final Path maskPath = Paths.get(resourceDir.toString(),
StorageConstants.MASK_FILE_PREFIX + localPath.getFile().getName());
@@ -21,6 +21,7 @@
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -127,6 +128,10 @@ private void replicateMissingFiles(List<String> files) {

private void deleteInvalidFiles(List<String> files) {
final FileSynchronizer sync = new FileSynchronizer(appCtx, replica);
// sort files to ensure index metadata files starting with "." are deleted last
files.sort(String::compareTo);
Collections.reverse(files);
LOGGER.info("deleting {}", files);
files.forEach(sync::delete);
}

0 comments on commit d8a1a93

Please sign in to comment.