From 9053e6e0b556ca7f2b5542ce382bb99bf66198fd Mon Sep 17 00:00:00 2001 From: Ritesh H Shukla Date: Wed, 18 Dec 2024 17:34:59 -0800 Subject: [PATCH 1/4] Improve ozone cli Change-Id: If802ac163c5c74a7c1372f9e4a5af298cf1abd1e --- .../main/smoketest/compatibility/write.robot | 2 +- .../smoketest/debug/ozone-debug-ldb.robot | 2 +- .../shell/bucket/CreateBucketHandler.java | 28 ++++++++++++++++--- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/compatibility/write.robot b/hadoop-ozone/dist/src/main/smoketest/compatibility/write.robot index 0497edaca168..c9fddb14386d 100644 --- a/hadoop-ozone/dist/src/main/smoketest/compatibility/write.robot +++ b/hadoop-ozone/dist/src/main/smoketest/compatibility/write.robot @@ -68,7 +68,7 @@ File Can Be Deleted FSO Bucket Can Be Created and Used Pass Execution If '${CLIENT_VERSION}' < '${FSO_VERSION}' Client does not support FSO Pass Execution If '${CLUSTER_VERSION}' < '${FSO_VERSION}' Cluster does not support FSO - Execute ozone sh bucket create --layout FILE_SYSTEM_OPTIMIZED /vol1/fso-bucket-${CLIENT_VERSION} + Execute ozone sh bucket create --layout fso /vol1/fso-bucket-${CLIENT_VERSION} Execute ozone fs -mkdir -p ofs://om/vol1/fso-bucket-${CLIENT_VERSION}/dir/subdir Execute ozone fs -put ${TESTFILE} ofs://om/vol1/fso-bucket-${CLIENT_VERSION}/dir/subdir/file diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-ldb.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-ldb.robot index 0fa43dee6c0b..4380a3cf94da 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-ldb.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-ldb.robot @@ -31,7 +31,7 @@ ${TESTFILE} testfile Write keys Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab Execute ozone sh volume create ${VOLUME} - Execute ozone sh bucket create ${VOLUME}/${BUCKET} -l OBJECT_STORE + Execute ozone sh bucket create ${VOLUME}/${BUCKET} -l obs Execute dd if=/dev/urandom of=${TEMP_DIR}/${TESTFILE}1 bs=100 count=10 Execute ozone sh key put ${VOLUME}/${BUCKET}/${TESTFILE}1 ${TEMP_DIR}/${TESTFILE}1 Execute dd if=/dev/urandom of=${TEMP_DIR}/${TESTFILE}2 bs=100 count=15 diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java index 277c5afff217..79bf31661927 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java @@ -59,11 +59,31 @@ public class CreateBucketHandler extends BucketHandler { " user if not specified") private String ownerName; - enum AllowedBucketLayouts { FILE_SYSTEM_OPTIMIZED, OBJECT_STORE, LEGACY } + enum AllowedBucketLayouts { + FILE_SYSTEM_OPTIMIZED, + fso, + OBJECT_STORE, + obs, + LEGACY; + + public static AllowedBucketLayouts fromString(String value) { + if (value.equals("FILE_SYSTEM_OPTIMIZED") || value.equalsIgnoreCase("fso")) { + return FILE_SYSTEM_OPTIMIZED; + } + if (value.equals("OBJECT_STORE") || value.equalsIgnoreCase("obs")) { + return OBJECT_STORE; + } + if (value.equals("LEGACY") || value.equalsIgnoreCase("legacy")) { + return LEGACY; + } + return valueOf(value); // throws IllegalArgumentException if not mapped to an enum, better than returning null + } + } @Option(names = { "--layout", "-l" }, - description = "Allowed Bucket Layouts: ${COMPLETION-CANDIDATES}") - private AllowedBucketLayouts allowedBucketLayout; + description = "Allowed Bucket Layouts: fso (for file system optimized buckets FILE_SYSTEM_OPTIMIZED), " + + "obs (for object store optimized OBJECT_STORE) and legacy (LEGACY is Deprecated)") + private String allowedBucketLayout; @CommandLine.Mixin private ShellReplicationOptions replication; @@ -87,7 +107,7 @@ public void execute(OzoneClient client, OzoneAddress address) .setVersioning(false).setOwner(ownerName); if (allowedBucketLayout != null) { BucketLayout bucketLayout = - BucketLayout.fromString(allowedBucketLayout.toString()); + BucketLayout.fromString(AllowedBucketLayouts.fromString(allowedBucketLayout).toString()); bb.setBucketLayout(bucketLayout); } // TODO: New Client talking to old server, will it create a LEGACY bucket? From c9691dece7f0347491296adb0bb25242b7e38829 Mon Sep 17 00:00:00 2001 From: Ritesh H Shukla Date: Thu, 19 Dec 2024 11:19:59 -0800 Subject: [PATCH 2/4] address review comments Change-Id: If9920ecb39f4f43745bfa9708815e3027b366128 --- .../main/smoketest/compatibility/write.robot | 2 +- .../dist/src/main/smoketest/createmrenv.robot | 2 +- .../shell/bucket/CreateBucketHandler.java | 41 +++++++++---------- 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/compatibility/write.robot b/hadoop-ozone/dist/src/main/smoketest/compatibility/write.robot index c9fddb14386d..0497edaca168 100644 --- a/hadoop-ozone/dist/src/main/smoketest/compatibility/write.robot +++ b/hadoop-ozone/dist/src/main/smoketest/compatibility/write.robot @@ -68,7 +68,7 @@ File Can Be Deleted FSO Bucket Can Be Created and Used Pass Execution If '${CLIENT_VERSION}' < '${FSO_VERSION}' Client does not support FSO Pass Execution If '${CLUSTER_VERSION}' < '${FSO_VERSION}' Cluster does not support FSO - Execute ozone sh bucket create --layout fso /vol1/fso-bucket-${CLIENT_VERSION} + Execute ozone sh bucket create --layout FILE_SYSTEM_OPTIMIZED /vol1/fso-bucket-${CLIENT_VERSION} Execute ozone fs -mkdir -p ofs://om/vol1/fso-bucket-${CLIENT_VERSION}/dir/subdir Execute ozone fs -put ${TESTFILE} ofs://om/vol1/fso-bucket-${CLIENT_VERSION}/dir/subdir/file diff --git a/hadoop-ozone/dist/src/main/smoketest/createmrenv.robot b/hadoop-ozone/dist/src/main/smoketest/createmrenv.robot index 7fcf3619effa..2d68be00e655 100644 --- a/hadoop-ozone/dist/src/main/smoketest/createmrenv.robot +++ b/hadoop-ozone/dist/src/main/smoketest/createmrenv.robot @@ -32,7 +32,7 @@ Create volume ${result} = Execute ozone sh volume create /${volume} --user hadoop --space-quota 100TB --namespace-quota 100 Should not contain ${result} Failed Create bucket - Execute ozone sh bucket create /${volume}/${bucket} --space-quota 1TB --layout FILE_SYSTEM_OPTIMIZED + Execute ozone sh bucket create /${volume}/${bucket} --space-quota 1TB --layout fso *** Test Cases *** Create test volume, bucket and key diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java index 79bf31661927..cf5c3ee6a99d 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java @@ -59,31 +59,32 @@ public class CreateBucketHandler extends BucketHandler { " user if not specified") private String ownerName; - enum AllowedBucketLayouts { - FILE_SYSTEM_OPTIMIZED, - fso, - OBJECT_STORE, - obs, - LEGACY; - - public static AllowedBucketLayouts fromString(String value) { - if (value.equals("FILE_SYSTEM_OPTIMIZED") || value.equalsIgnoreCase("fso")) { - return FILE_SYSTEM_OPTIMIZED; + private static class LayoutConverter implements CommandLine.ITypeConverter { + @Override + public BucketLayout convert(String value) { + if (value == null) { + return null; } - if (value.equals("OBJECT_STORE") || value.equalsIgnoreCase("obs")) { - return OBJECT_STORE; + switch (value) { + case "fso": + return BucketLayout.FILE_SYSTEM_OPTIMIZED; + case "obs": + return BucketLayout.OBJECT_STORE; + default: + for (BucketLayout candidate : BucketLayout.values()) { + if (candidate.name().equalsIgnoreCase(value)) { + return candidate; + } + } + throw new IllegalArgumentException("Unknown bucket layout: " + value); } - if (value.equals("LEGACY") || value.equalsIgnoreCase("legacy")) { - return LEGACY; - } - return valueOf(value); // throws IllegalArgumentException if not mapped to an enum, better than returning null } } - @Option(names = { "--layout", "-l" }, + @Option(names = { "--layout", "-l" }, converter = LayoutConverter.class, description = "Allowed Bucket Layouts: fso (for file system optimized buckets FILE_SYSTEM_OPTIMIZED), " + "obs (for object store optimized OBJECT_STORE) and legacy (LEGACY is Deprecated)") - private String allowedBucketLayout; + private BucketLayout allowedBucketLayout; @CommandLine.Mixin private ShellReplicationOptions replication; @@ -106,9 +107,7 @@ public void execute(OzoneClient client, OzoneAddress address) new BucketArgs.Builder().setStorageType(StorageType.DEFAULT) .setVersioning(false).setOwner(ownerName); if (allowedBucketLayout != null) { - BucketLayout bucketLayout = - BucketLayout.fromString(AllowedBucketLayouts.fromString(allowedBucketLayout).toString()); - bb.setBucketLayout(bucketLayout); + bb.setBucketLayout(allowedBucketLayout); } // TODO: New Client talking to old server, will it create a LEGACY bucket? From 523842cb4ed5b2a38f84cbfc0c1001702034b8a7 Mon Sep 17 00:00:00 2001 From: Ritesh H Shukla Date: Thu, 19 Dec 2024 11:36:27 -0800 Subject: [PATCH 3/4] checkstyle fixes Change-Id: Iaf7183a2dfb41b78a0f74cf39b30e0aafd7d449e --- .../apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java index cf5c3ee6a99d..373e6bfd9f59 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java @@ -82,8 +82,8 @@ public BucketLayout convert(String value) { } @Option(names = { "--layout", "-l" }, converter = LayoutConverter.class, - description = "Allowed Bucket Layouts: fso (for file system optimized buckets FILE_SYSTEM_OPTIMIZED), " + - "obs (for object store optimized OBJECT_STORE) and legacy (LEGACY is Deprecated)") + description = "Allowed Bucket Layouts: fso (for file system optimized buckets FILE_SYSTEM_OPTIMIZED), " + + "obs (for object store optimized OBJECT_STORE) and legacy (LEGACY is Deprecated)") private BucketLayout allowedBucketLayout; @CommandLine.Mixin From 52dc19f95d5aac54ab6ff24c8f21b2ce47747ca1 Mon Sep 17 00:00:00 2001 From: Ritesh H Shukla Date: Thu, 19 Dec 2024 11:42:20 -0800 Subject: [PATCH 4/4] checkstyle fixes Change-Id: I3b2c120415a85c5f0b318e5310f0ac3a0c414398 --- .../shell/bucket/CreateBucketHandler.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java index 373e6bfd9f59..02f5e80f6eb7 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java @@ -66,17 +66,17 @@ public BucketLayout convert(String value) { return null; } switch (value) { - case "fso": - return BucketLayout.FILE_SYSTEM_OPTIMIZED; - case "obs": - return BucketLayout.OBJECT_STORE; - default: - for (BucketLayout candidate : BucketLayout.values()) { - if (candidate.name().equalsIgnoreCase(value)) { - return candidate; - } + case "fso": + return BucketLayout.FILE_SYSTEM_OPTIMIZED; + case "obs": + return BucketLayout.OBJECT_STORE; + default: + for (BucketLayout candidate : BucketLayout.values()) { + if (candidate.name().equalsIgnoreCase(value)) { + return candidate; } - throw new IllegalArgumentException("Unknown bucket layout: " + value); + } + throw new IllegalArgumentException("Unknown bucket layout: " + value); } } }