Skip to content

Commit

Permalink
HDDS-4742. Make trash work with FS Optimised Buckets. (#1915)
Browse files Browse the repository at this point in the history
  • Loading branch information
@sadanand48 @rakeshadr
sadanand48 authored and rakeshadr committed Apr 7, 2021
1 parent 81a3e81 commit 4cb5091
Show file tree
Hide file tree
Showing 4 changed files with 135 additions and 59 deletions.
25 changes: 25 additions & 0 deletions hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,12 @@
package org.apache.hadoop.ozone.om.helpers;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.util.StringUtils;

import javax.annotation.Nonnull;
import java.nio.file.Paths;
import java.util.Map;

import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
Expand Down Expand Up @@ -205,4 +207,27 @@ public static int getFileCount(String keyName) {
java.nio.file.Path keyPath = Paths.get(keyName);
return keyPath.getNameCount();
}


/**
* Returns true if the bucket is FS Optimised.
* @param bucketMetadata
* @return
*/
public static boolean isFSOptimizedBucket(
Map<String, String> bucketMetadata) {
// layout version V1 represents optimized FS path
boolean layoutVersionEnabled =
org.apache.commons.lang3.StringUtils.equalsIgnoreCase(
OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1,
bucketMetadata
.get(OMConfigKeys.OZONE_OM_LAYOUT_VERSION));

boolean fsEnabled =
Boolean.parseBoolean(bucketMetadata
.get(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS));

return layoutVersionEnabled && fsEnabled;
}

}
12 changes: 0 additions & 12 deletions ...zone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -380,18 +380,6 @@ public void testRenameDestinationParentDoesntExist() throws Exception {
}
}

@Override
@Test
@Ignore("TODO:HDDS-2939")
public void testTrash() throws Exception {
}

@Override
@Test
@Ignore("TODO:HDDS-2939")
public void testRenameToTrashEnabled() throws Exception {
}

@Override
@Test
@Ignore("TODO:HDDS-2939")
Expand Down
142 changes: 108 additions & 34 deletions ...op-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,11 @@
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.helpers.OMRatisHelper;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.om.helpers.OMRatisHelper;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Progressable;
import org.apache.ratis.protocol.ClientId;
Expand Down Expand Up @@ -154,6 +155,11 @@ public boolean rename(Path src, Path dst) throws IOException {
// check whether the src and dst belong to the same bucket & trashroot.
OFSPath srcPath = new OFSPath(src);
OFSPath dstPath = new OFSPath(dst);
OmBucketInfo bucket = ozoneManager.getBucketInfo(srcPath.getVolumeName(),
srcPath.getBucketName());
if (OzoneFSUtils.isFSOptimizedBucket(bucket.getMetadata())) {
return renameV1(srcPath, dstPath);
}
Preconditions.checkArgument(srcPath.getBucketName().
equals(dstPath.getBucketName()));
Preconditions.checkArgument(srcPath.getTrashRoot().
Expand All @@ -163,14 +169,50 @@ public boolean rename(Path src, Path dst) throws IOException {
return true;
}

private boolean renameV1(OFSPath srcPath, OFSPath dstPath) {
OzoneManagerProtocolProtos.OMRequest omRequest =
getRenameKeyRequest(srcPath, dstPath);
try {
if(omRequest != null) {
submitRequest(omRequest);
return true;
}
return false;
} catch (Exception e){
LOG.error("couldnt send rename requestV1", e);
return false;
}
}

@Override
public boolean delete(Path path, boolean b) throws IOException {
ozoneManager.getMetrics().incNumTrashDeletes();
OFSPath srcPath = new OFSPath(path);
OmBucketInfo bucket = ozoneManager.getBucketInfo(srcPath.getVolumeName(),
srcPath.getBucketName());
if (OzoneFSUtils.isFSOptimizedBucket(bucket.getMetadata())) {
return deleteV1(srcPath);
}
DeleteIterator iterator = new DeleteIterator(path, true);
iterator.iterate();
return true;
}

private boolean deleteV1(OFSPath srcPath) {
OzoneManagerProtocolProtos.OMRequest omRequest =
getDeleteKeyRequest(srcPath);
try {
if(omRequest != null) {
submitRequest(omRequest);
return true;
}
return false;
} catch (Throwable e) {
LOG.error("Couldn't send delete request.", e);
return false;
}
}

@Override
public FileStatus[] listStatus(Path path) throws IOException {
ozoneManager.getMetrics().incNumTrashListStatus();
Expand Down Expand Up @@ -377,6 +419,41 @@ FileStatus getStatus() {
}
}


private OzoneManagerProtocolProtos.OMRequest
getRenameKeyRequest(
OFSPath src, OFSPath dst) {
String volumeName = src.getVolumeName();
String bucketName = src.getBucketName();
String keyName = src.getKeyName();

OzoneManagerProtocolProtos.KeyArgs keyArgs =
OzoneManagerProtocolProtos.KeyArgs.newBuilder()
.setKeyName(keyName)
.setVolumeName(volumeName)
.setBucketName(bucketName)
.build();
String toKeyName = dst.getKeyName();
OzoneManagerProtocolProtos.RenameKeyRequest renameKeyRequest =
OzoneManagerProtocolProtos.RenameKeyRequest.newBuilder()
.setKeyArgs(keyArgs)
.setToKeyName(toKeyName)
.build();
OzoneManagerProtocolProtos.OMRequest omRequest =
null;
try {
omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder()
.setClientId(CLIENT_ID.toString())
.setUserInfo(getUserInfo())
.setRenameKeyRequest(renameKeyRequest)
.setCmdType(OzoneManagerProtocolProtos.Type.RenameKey)
.build();
} catch (IOException e) {
LOG.error("Couldn't get userinfo", e);
}
return omRequest;
}

private class RenameIterator extends OzoneListingIterator {
private final String srcPath;
private final String dstPath;
Expand Down Expand Up @@ -408,40 +485,37 @@ boolean processKeyPath(List<String> keyPathList) {
}
return true;
}
}

private OzoneManagerProtocolProtos.OMRequest
getRenameKeyRequest(
OFSPath src, OFSPath dst) {
String volumeName = src.getVolumeName();
String bucketName = src.getBucketName();
String keyName = src.getKeyName();

OzoneManagerProtocolProtos.KeyArgs keyArgs =
OzoneManagerProtocolProtos.KeyArgs.newBuilder()
.setKeyName(keyName)
.setVolumeName(volumeName)
.setBucketName(bucketName)
.build();
String toKeyName = dst.getKeyName();
OzoneManagerProtocolProtos.RenameKeyRequest renameKeyRequest =
OzoneManagerProtocolProtos.RenameKeyRequest.newBuilder()
.setKeyArgs(keyArgs)
.setToKeyName(toKeyName)
.build();
OzoneManagerProtocolProtos.OMRequest omRequest =
null;
try {
omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder()
.setClientId(CLIENT_ID.toString())
.setUserInfo(getUserInfo())
.setRenameKeyRequest(renameKeyRequest)
.setCmdType(OzoneManagerProtocolProtos.Type.RenameKey)
private OzoneManagerProtocolProtos.OMRequest getDeleteKeyRequest(
OFSPath srcPath) {
String volume = srcPath.getVolumeName();
String bucket = srcPath.getBucketName();
String key = srcPath.getKeyName();
OzoneManagerProtocolProtos.KeyArgs keyArgs =
OzoneManagerProtocolProtos.KeyArgs.newBuilder()
.setKeyName(key)
.setVolumeName(volume)
.setBucketName(bucket)
.setRecursive(true)
.build();
} catch (IOException e) {
LOG.error("Couldn't get userinfo", e);
}
return omRequest;
OzoneManagerProtocolProtos.DeleteKeyRequest deleteKeyRequest =
OzoneManagerProtocolProtos.DeleteKeyRequest.newBuilder()
.setKeyArgs(keyArgs).build();
OzoneManagerProtocolProtos.OMRequest omRequest =
null;
try {
omRequest =
OzoneManagerProtocolProtos.OMRequest.newBuilder()
.setClientId(CLIENT_ID.toString())
.setUserInfo(getUserInfo())
.setDeleteKeyRequest(deleteKeyRequest)
.setCmdType(OzoneManagerProtocolProtos.Type.DeleteKey)
.build();
} catch (IOException e) {
LOG.error("Couldn't get userinfo", e);
}
return omRequest;
}

private class DeleteIterator extends OzoneListingIterator {
Expand All @@ -467,7 +541,7 @@ boolean processKeyPath(List<String> keyPathList) {
for (String keyPath : keyPathList) {
OFSPath path = new OFSPath(keyPath);
OzoneManagerProtocolProtos.OMRequest omRequest =
getDeleteKeyRequest(path);
getDeleteKeysRequest(path);
try {
ozoneManager.getMetrics().incNumTrashFilesDeletes();
submitRequest(omRequest);
Expand All @@ -479,7 +553,7 @@ boolean processKeyPath(List<String> keyPathList) {
}

private OzoneManagerProtocolProtos.OMRequest
getDeleteKeyRequest(
getDeleteKeysRequest(
OFSPath keyPath) {
String volumeName = keyPath.getVolumeName();
String bucketName = keyPath.getBucketName();
Expand Down
15 changes: 2 additions & 13 deletions .../ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,12 +48,12 @@
import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenRenewer;
Expand Down Expand Up @@ -548,17 +548,6 @@ private BlockLocation[] getBlockLocations(OzoneFileStatus fileStatus) {

@Override
public boolean isFSOptimizedBucket() {
// layout version V1 represents optimized FS path
boolean layoutVersionEnabled =
StringUtils.equalsIgnoreCase(
OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1,
bucket.getMetadata()
.get(OMConfigKeys.OZONE_OM_LAYOUT_VERSION));

boolean fsEnabled =
Boolean.parseBoolean(bucket.getMetadata()
.get(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS));

return layoutVersionEnabled && fsEnabled;
return OzoneFSUtils.isFSOptimizedBucket(bucket.getMetadata());
}
}

0 comments on commit 4cb5091

Please sign in to comment.