HDDS-5362. [FSO] Support bucket layouts in OM (#2357)

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java

@@ -21,6 +21,7 @@
 import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.ozone.OzoneAcl;
 import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 
 import java.util.HashMap;
 import java.util.List;
@@ -61,6 +62,11 @@ public final class BucketArgs {
   private long quotaInBytes;
   private long quotaInNamespace;
 
+  /**
+   * Bucket Type.
+   */
+  private BucketLayout bucketLayout = BucketLayout.DEFAULT;
+
   /**
    * Private constructor, constructed via builder.
    * @param versioning Bucket version flag.
@@ -72,12 +78,13 @@ public final class BucketArgs {
    * @param sourceBucket
    * @param quotaInBytes Bucket quota in bytes.
    * @param quotaInNamespace Bucket quota in counts.
+   * @param bucketLayout Bucket Layouts.
    */
   @SuppressWarnings("parameternumber")
   private BucketArgs(Boolean versioning, StorageType storageType,
       List<OzoneAcl> acls, Map<String, String> metadata,
       String bucketEncryptionKey, String sourceVolume, String sourceBucket,
-      long quotaInBytes, long quotaInNamespace) {
+      long quotaInBytes, long quotaInNamespace, BucketLayout bucketLayout) {
     this.acls = acls;
     this.versioning = versioning;
     this.storageType = storageType;
@@ -87,6 +94,7 @@ private BucketArgs(Boolean versioning, StorageType storageType,
     this.sourceBucket = sourceBucket;
     this.quotaInBytes = quotaInBytes;
     this.quotaInNamespace = quotaInNamespace;
+    this.bucketLayout = bucketLayout;
   }
 
   /**
@@ -163,6 +171,13 @@ public long getQuotaInNamespace() {
     return quotaInNamespace;
   }
 
+  /**
+   * Returns the Bucket Type.
+   */
+  public BucketLayout getBucketLayout() {
+    return bucketLayout;
+  }
+
   /**
    * Builder for OmBucketInfo.
    */
@@ -176,6 +191,7 @@ public static class Builder {
     private String sourceBucket;
     private long quotaInBytes;
     private long quotaInNamespace;
+    private BucketLayout bucketLayout;
 
     public Builder() {
       metadata = new HashMap<>();
@@ -228,6 +244,11 @@ public BucketArgs.Builder setQuotaInNamespace(long quota) {
       return this;
     }
 
+    public BucketArgs.Builder setBucketLayout(BucketLayout type) {
+      bucketLayout = type;
+      return this;
+    }
+
 
     /**
      * Constructs the BucketArgs.
@@ -236,7 +257,7 @@ public BucketArgs.Builder setQuotaInNamespace(long quota) {
     public BucketArgs build() {
       return new BucketArgs(versioning, storageType, acls, metadata,
           bucketEncryptionKey, sourceVolume, sourceBucket, quotaInBytes,
-          quotaInNamespace);
+          quotaInNamespace, bucketLayout);
     }
   }
 }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java

@@ -41,6 +41,7 @@
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
 import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
 import org.apache.hadoop.ozone.om.helpers.WithMetadata;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.security.acl.OzoneObj;
 import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
 import org.apache.hadoop.util.Time;
@@ -135,6 +136,10 @@ public class OzoneBucket extends WithMetadata {
    * Quota of key count allocated for the bucket.
    */
   private long quotaInNamespace;
+  /**
+   * Bucket Layout.
+   */
+  private BucketLayout bucketLayout = BucketLayout.DEFAULT;
 
   private OzoneBucket(ConfigurationSource conf, String volumeName,
       String bucketName, ClientProtocol proxy) {
@@ -201,6 +206,21 @@ public OzoneBucket(ConfigurationSource conf, ClientProtocol proxy,
     this.quotaInNamespace = quotaInNamespace;
   }
 
+  @SuppressWarnings("parameternumber")
+  public OzoneBucket(ConfigurationSource conf, ClientProtocol proxy,
+      String volumeName, String bucketName, StorageType storageType,
+      Boolean versioning, long creationTime, long modificationTime,
+      Map<String, String> metadata, String encryptionKeyName,
+      String sourceVolume, String sourceBucket, long usedBytes,
+      long usedNamespace, long quotaInBytes, long quotaInNamespace,
+      BucketLayout bucketLayout) {
+    this(conf, proxy, volumeName, bucketName, storageType, versioning,
+        creationTime, modificationTime, metadata, encryptionKeyName,
+        sourceVolume, sourceBucket, usedBytes, usedNamespace, quotaInBytes,
+        quotaInNamespace);
+    this.bucketLayout = bucketLayout;
+  }
+
   /**
    * Constructs OzoneBucket instance.
    * @param conf Configuration object.
@@ -576,13 +596,11 @@ public Iterator<? extends OzoneKey> listKeys(String keyPrefix)
    * @param prevKey Keys will be listed after this key name
    * @return {@code Iterator<OzoneKey>}
    */
-  public Iterator<? extends OzoneKey> listKeys(String keyPrefix,
-      String prevKey) throws IOException {
+  public Iterator<? extends OzoneKey> listKeys(String keyPrefix, String prevKey)
+      throws IOException {
 
-    if(OzoneFSUtils.isFSOptimizedBucket(getMetadata())){
-      return new KeyIteratorWithFSO(keyPrefix, prevKey);
-    }
-    return new KeyIterator(keyPrefix, prevKey);
+    return new KeyIteratorFactory()
+        .getKeyIterator(keyPrefix, prevKey, bucketLayout);
   }
 
   /**
@@ -1186,4 +1204,19 @@ private void addKeyPrefixInfoToResultList(String keyPrefix,
     }
 
   }
+
+  private class KeyIteratorFactory {
+    KeyIterator getKeyIterator(String keyPrefix, String prevKey,
+        BucketLayout bType) throws IOException {
+      if (bType.equals(BucketLayout.FILE_SYSTEM_OPTIMIZED)) {
+        return new KeyIteratorWithFSO(keyPrefix, prevKey);
+      } else {
+        return new KeyIterator(keyPrefix, prevKey);
+      }
+    }
+  }
+
+  public BucketLayout getBucketLayout() {
+    return bucketLayout;
+  }
 }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -492,7 +492,8 @@ public void createBucket(
         .setSourceBucket(bucketArgs.getSourceBucket())
         .setQuotaInBytes(bucketArgs.getQuotaInBytes())
         .setQuotaInNamespace(bucketArgs.getQuotaInNamespace())
-        .setAcls(listOfAcls.stream().distinct().collect(Collectors.toList()));
+        .setAcls(listOfAcls.stream().distinct().collect(Collectors.toList()))
+        .setBucketLayout(bucketArgs.getBucketLayout());
 
     if (bek != null) {
       builder.setBucketEncryptionKey(bek);
@@ -715,7 +716,8 @@ public OzoneBucket getBucketDetails(
         bucketInfo.getUsedBytes(),
         bucketInfo.getUsedNamespace(),
         bucketInfo.getQuotaInBytes(),
-        bucketInfo.getQuotaInNamespace()
+        bucketInfo.getQuotaInNamespace(),
+        bucketInfo.getBucketLayout()
     );
   }
 
@@ -910,7 +912,6 @@ public List<OzoneKey> listKeys(String volumeName, String bucketName,
       throws IOException {
     List<OmKeyInfo> keys = ozoneManagerClient.listKeys(
         volumeName, bucketName, prevKey, keyPrefix, maxListResult);
-
     return keys.stream().map(key -> new OzoneKey(
         key.getVolumeName(),
         key.getBucketName(),

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java

@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+
+/**
+ * BucketLayout enum
+ * We have 3 types of bucket layouts - FSO, OBJECT_STORE, and LEGACY
+ * LEGACY is used to represent the old buckets which are already
+ * present in DB while user can create new buckets as FSO or OBJECT_STORE.
+ */
+public enum BucketLayout {
+  FILE_SYSTEM_OPTIMIZED, OBJECT_STORE, LEGACY;
+  public static final BucketLayout DEFAULT = LEGACY;
+  public static BucketLayout fromProto(
+      OzoneManagerProtocolProtos.BucketLayoutProto bucketLayout) {
+    if (bucketLayout == null) {
+      return BucketLayout.LEGACY;
+    }
+    switch (bucketLayout) {
+    case FILE_SYSTEM_OPTIMIZED:
+      return BucketLayout.FILE_SYSTEM_OPTIMIZED;
+    case LEGACY:
+      return BucketLayout.LEGACY;
+    case OBJECT_STORE:
+      return BucketLayout.OBJECT_STORE;
+    default:
+      return DEFAULT;
+    }
+  }
+
+  public OzoneManagerProtocolProtos.BucketLayoutProto toProto() {
+    switch (this) {
+    case FILE_SYSTEM_OPTIMIZED:
+      return OzoneManagerProtocolProtos.BucketLayoutProto.FILE_SYSTEM_OPTIMIZED;
+    case OBJECT_STORE:
+      return OzoneManagerProtocolProtos.BucketLayoutProto.OBJECT_STORE;
+    case LEGACY:
+      return OzoneManagerProtocolProtos.BucketLayoutProto.LEGACY;
+    default:
+      throw new IllegalArgumentException(
+          "Error: BucketLayout not found, type=" + this);
+    }
+  }
+}