From 20b51e8530e1d2092f7249155fb70e94b3e19941 Mon Sep 17 00:00:00 2001 From: navinko Date: Fri, 29 May 2026 03:49:48 +0530 Subject: [PATCH 1/3] HDDS-15370. Change sequenceIdTable to use SequenceIdType --- .../hadoop/hdds/scm/ha/SequenceIdType.java | 3 +- .../hdds/scm/metadata/SCMMetadataStore.java | 3 +- .../hdds/scm/ha/TestSequenceIdType.java | 0 .../hdds/scm/ha/SequenceIdGenerator.java | 66 +++++++-------- .../hdds/scm/metadata/SCMDBDefinition.java | 5 +- .../scm/metadata/SCMMetadataStoreImpl.java | 5 +- .../scm/metadata/SequenceIdTypeCodec.java | 80 +++++++++++++++++++ .../hdds/scm/ha/TestSequenceIDGenerator.java | 4 +- .../scm/metadata/TestSequenceIdTypeCodec.java | 78 ++++++++++++++++++ 9 files changed, 203 insertions(+), 41 deletions(-) rename hadoop-hdds/{server-scm => framework}/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdType.java (91%) rename hadoop-hdds/{server-scm => framework}/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSequenceIdType.java (100%) create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SequenceIdTypeCodec.java create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/metadata/TestSequenceIdTypeCodec.java diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdType.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdType.java similarity index 91% rename from hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdType.java rename to hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdType.java index 4b80e3feef49..4279b0bd832a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdType.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdType.java @@ -18,7 +18,8 @@ package org.apache.hadoop.hdds.scm.ha; /** - * Represents the sequence ID types managed by {@link SequenceIdGenerator} + * Represents the sequence ID types managed by + * {@code org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator} * The enum constant names are kept exactly as their persisted RocksDB keys. */ public enum SequenceIdType { diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java index 9d109c32b0b7..37322ee2c135 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStore.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.common.helpers.MoveDataNodePair; +import org.apache.hadoop.hdds.scm.ha.SequenceIdType; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.utils.DBStoreHAManager; @@ -102,7 +103,7 @@ public interface SCMMetadataStore extends DBStoreHAManager { /** * Table that maintains sequence id information. */ - Table getSequenceIdTable(); + Table getSequenceIdTable(); /** * Table that maintains move information. diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSequenceIdType.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSequenceIdType.java similarity index 100% rename from hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSequenceIdType.java rename to hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSequenceIdType.java diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdGenerator.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdGenerator.java index 92659d2f3a06..08916b65dc50 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdGenerator.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdGenerator.java @@ -77,7 +77,7 @@ public class SequenceIdGenerator { * @param sequenceIdTable : sequenceIdTable */ public SequenceIdGenerator(ConfigurationSource conf, - SCMHAManager scmhaManager, Table sequenceIdTable) { + SCMHAManager scmhaManager, Table sequenceIdTable) { this.sequenceIdToBatchMap = newSequenceIdToBatchMap(); this.lock = new ReentrantLock(); this.batchSize = conf.getInt(OZONE_SCM_SEQUENCE_ID_BATCH_SIZE, @@ -96,7 +96,7 @@ static Map newSequenceIdToBatchMap() { } public StateManager createStateManager(SCMHAManager scmhaManager, - Table sequenceIdTable) { + Table sequenceIdTable) { Objects.requireNonNull(scmhaManager, "scmhaManager == null"); return new StateManagerImpl.Builder() .setRatisServer(scmhaManager.getRatisServer()) @@ -168,7 +168,7 @@ private void invalidateBatchInternal() { * Reinitialize the SequenceIdGenerator with the latest sequenceIdTable * during SCM reload. */ - public void reinitialize(Table sequenceIdTable) + public void reinitialize(Table sequenceIdTable) throws IOException { LOG.info("reinitialize SequenceIdGenerator."); lock.lock(); @@ -208,7 +208,7 @@ Boolean allocateBatch(String sequenceIdName, * Reinitialize the SequenceIdGenerator with the latest sequenceIdTable * during SCM reload. */ - void reinitialize(Table sequenceIdTable) throws IOException; + void reinitialize(Table sequenceIdTable) throws IOException; @Override default RequestType getType() { @@ -221,11 +221,11 @@ default RequestType getType() { * DBTransactionBuffer until a snapshot is taken. */ static final class StateManagerImpl implements StateManager { - private Table sequenceIdTable; + private Table sequenceIdTable; private final DBTransactionBuffer transactionBuffer; private final Map sequenceIdToLastIdMap; - private StateManagerImpl(Table sequenceIdTable, + private StateManagerImpl(Table sequenceIdTable, DBTransactionBuffer trxBuffer) { this.sequenceIdTable = sequenceIdTable; this.transactionBuffer = trxBuffer; @@ -240,7 +240,7 @@ public Boolean allocateBatch(String sequenceIdName, Long lastId = sequenceIdToLastIdMap.computeIfAbsent(idType, key -> { try { - Long idInDb = this.sequenceIdTable.get(key.name()); + Long idInDb = this.sequenceIdTable.get(key); return idInDb != null ? idInDb : INVALID_SEQUENCE_ID; } catch (IOException ioe) { throw new RuntimeException("Failed to get lastId from db", ioe); @@ -255,7 +255,7 @@ public Boolean allocateBatch(String sequenceIdName, try { transactionBuffer - .addToBuffer(sequenceIdTable, idType.name(), newLastId); + .addToBuffer(sequenceIdTable, idType, newLastId); } catch (IOException ioe) { throw new RuntimeException("Failed to put lastId to Batch", ioe); } @@ -270,7 +270,7 @@ public Long getLastId(SequenceIdType idType) { } @Override - public void reinitialize(Table seqIdTable) + public void reinitialize(Table seqIdTable) throws IOException { this.sequenceIdTable = seqIdTable; this.sequenceIdToLastIdMap.clear(); @@ -278,17 +278,17 @@ public void reinitialize(Table seqIdTable) } private void initialize() throws IOException { - try (Table.KeyValueIterator iterator = sequenceIdTable.iterator()) { + try (Table.KeyValueIterator iterator = sequenceIdTable.iterator()) { while (iterator.hasNext()) { - Table.KeyValue kv = iterator.next(); - final String sequenceIdName = kv.getKey(); + Table.KeyValue kv = iterator.next(); + final SequenceIdType idType = kv.getKey(); final Long lastId = kv.getValue(); - Objects.requireNonNull(sequenceIdName, - "sequenceIdName should not be null"); + Objects.requireNonNull(idType, + "idType should not be null"); Objects.requireNonNull(lastId, "lastId should not be null"); - sequenceIdToLastIdMap.put(SequenceIdType.valueOf(sequenceIdName), lastId); + sequenceIdToLastIdMap.put(idType, lastId); } } } @@ -297,7 +297,7 @@ private void initialize() throws IOException { * Builder for Ratis based StateManager. */ public static class Builder { - private Table table; + private Table table; private DBTransactionBuffer buffer; private SCMRatisServer ratisServer; @@ -307,7 +307,7 @@ public Builder setRatisServer(final SCMRatisServer scmRatisServer) { } public Builder setSequenceIdTable( - final Table sequenceIdTable) { + final Table sequenceIdTable) { table = sequenceIdTable; return this; } @@ -337,7 +337,7 @@ public StateManager build() { */ public static void upgradeToSequenceId(SCMMetadataStore scmMetadataStore) throws IOException { - Table sequenceIdTable = scmMetadataStore.getSequenceIdTable(); + Table sequenceIdTable = scmMetadataStore.getSequenceIdTable(); // upgrade localId // Short-term solution: when setup multi SCM from scratch, they need @@ -345,29 +345,29 @@ public static void upgradeToSequenceId(SCMMetadataStore scmMetadataStore) // Long-term solution: the bootstrapped SCM will explicitly download // scm.db from leader SCM, and drop its own scm.db. Thus the upgrade // operations can take effect exactly once in a SCM HA cluster. - if (sequenceIdTable.get(SequenceIdType.localId.name()) == null) { + if (sequenceIdTable.get(SequenceIdType.localId) == null) { long millisSinceEpoch = TimeUnit.DAYS.toMillis( LocalDate.of(LocalDate.now().getYear() + 1, 1, 1).toEpochDay()); long localId = millisSinceEpoch << Short.SIZE; Preconditions.checkArgument(localId > UniqueId.next()); - sequenceIdTable.put(SequenceIdType.localId.name(), localId); - LOG.info("upgrade {} to {}", SequenceIdType.localId, sequenceIdTable.get(SequenceIdType.localId.name())); + sequenceIdTable.put(SequenceIdType.localId, localId); + LOG.info("upgrade {} to {}", SequenceIdType.localId, sequenceIdTable.get(SequenceIdType.localId)); } // upgrade delTxnId - if (sequenceIdTable.get(SequenceIdType.delTxnId.name()) == null) { + if (sequenceIdTable.get(SequenceIdType.delTxnId) == null) { // fetch delTxnId from DeletedBlocksTXTable // check HDDS-4477 for details. DeletedBlocksTransaction txn = scmMetadataStore.getDeletedBlocksTXTable().get(0L); - sequenceIdTable.put(SequenceIdType.delTxnId.name(), txn != null ? txn.getTxID() : 0L); - LOG.info("upgrade {} to {}", SequenceIdType.delTxnId, sequenceIdTable.get(SequenceIdType.delTxnId.name())); + sequenceIdTable.put(SequenceIdType.delTxnId, txn != null ? txn.getTxID() : 0L); + LOG.info("upgrade {} to {}", SequenceIdType.delTxnId, sequenceIdTable.get(SequenceIdType.delTxnId)); } // upgrade containerId - if (sequenceIdTable.get(SequenceIdType.containerId.name()) == null) { + if (sequenceIdTable.get(SequenceIdType.containerId) == null) { long largestContainerId = 0; try (TableIterator iterator = scmMetadataStore.getContainerTable().valueIterator()) { @@ -378,9 +378,9 @@ public static void upgradeToSequenceId(SCMMetadataStore scmMetadataStore) } } - sequenceIdTable.put(SequenceIdType.containerId.name(), largestContainerId); + sequenceIdTable.put(SequenceIdType.containerId, largestContainerId); LOG.info("upgrade {} to {}", - SequenceIdType.containerId, sequenceIdTable.get(SequenceIdType.containerId.name())); + SequenceIdType.containerId, sequenceIdTable.get(SequenceIdType.containerId)); } upgradeToCertificateSequenceId(scmMetadataStore, false); @@ -388,10 +388,10 @@ public static void upgradeToSequenceId(SCMMetadataStore scmMetadataStore) public static void upgradeToCertificateSequenceId( SCMMetadataStore scmMetadataStore, boolean force) throws IOException { - Table sequenceIdTable = scmMetadataStore.getSequenceIdTable(); + Table sequenceIdTable = scmMetadataStore.getSequenceIdTable(); // upgrade certificate ID table - if (sequenceIdTable.get(SequenceIdType.CertificateId.name()) == null || force) { + if (sequenceIdTable.get(SequenceIdType.CertificateId) == null || force) { // Start from ID 2. // ID 1 - root certificate, ID 2 - first SCM certificate. long largestCertId = BigInteger.ONE.add(BigInteger.ONE).longValueExact(); @@ -413,15 +413,15 @@ public static void upgradeToCertificateSequenceId( } } - sequenceIdTable.put(SequenceIdType.CertificateId.name(), largestCertId); + sequenceIdTable.put(SequenceIdType.CertificateId, largestCertId); LOG.info("upgrade {} to {}", SequenceIdType.CertificateId, - sequenceIdTable.get(SequenceIdType.CertificateId.name())); + sequenceIdTable.get(SequenceIdType.CertificateId)); } // delete the ROOT_CERTIFICATE_ID record if exists // ROOT_CERTIFICATE_ID is replaced with CERTIFICATE_ID now - if (sequenceIdTable.get(SequenceIdType.rootCertificateId.name()) != null) { - sequenceIdTable.delete(SequenceIdType.rootCertificateId.name()); + if (sequenceIdTable.get(SequenceIdType.rootCertificateId) != null) { + sequenceIdTable.delete(SequenceIdType.rootCertificateId); } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java index 4aae413c0c2b..e7e771d40509 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.common.helpers.MoveDataNodePair; +import org.apache.hadoop.hdds.scm.ha.SequenceIdType; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.utils.TransactionInfo; @@ -82,11 +83,11 @@ public class SCMDBDefinition extends DBDefinition.WithMap { StringCodec.get(), TransactionInfo.getCodec()); - public static final DBColumnFamilyDefinition + public static final DBColumnFamilyDefinition SEQUENCE_ID = new DBColumnFamilyDefinition<>( "sequenceId", - StringCodec.get(), + SequenceIdTypeCodec.get(), LongCodec.get()); public static final DBColumnFamilyDefinition transactionInfoTable; - private Table sequenceIdTable; + private Table sequenceIdTable; private Table moveTable; @@ -214,7 +215,7 @@ public Table getContainerTable() { } @Override - public Table getSequenceIdTable() { + public Table getSequenceIdTable() { return sequenceIdTable; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SequenceIdTypeCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SequenceIdTypeCodec.java new file mode 100644 index 000000000000..026c98719d1f --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SequenceIdTypeCodec.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.metadata; + +import jakarta.annotation.Nonnull; +import org.apache.hadoop.hdds.scm.ha.SequenceIdType; +import org.apache.hadoop.hdds.utils.db.Codec; +import org.apache.hadoop.hdds.utils.db.CodecBuffer; +import org.apache.hadoop.hdds.utils.db.CodecException; +import org.apache.hadoop.hdds.utils.db.StringCodec; + +/** + * Codec to serialize/deserialize {@link SequenceIdType}. + * It delegates directly to {@link StringCodec} to leverage its encoding/decoding + * optimizations and fallback logic, ensuring disk format continuity. + */ +public final class SequenceIdTypeCodec implements Codec { + + private static final Codec INSTANCE = new SequenceIdTypeCodec(); + private static final Codec STRING_CODEC = StringCodec.get(); + + public static Codec get() { + return INSTANCE; + } + + private SequenceIdTypeCodec() { + // singleton + } + + @Override + public Class getTypeClass() { + return SequenceIdType.class; + } + + @Override + public byte[] toPersistedFormat(SequenceIdType object) throws CodecException { + return STRING_CODEC.toPersistedFormat(object.name()); + } + + @Override + public SequenceIdType fromPersistedFormat(byte[] rawData) throws CodecException { + return SequenceIdType.valueOf(STRING_CODEC.fromPersistedFormat(rawData)); + } + + @Override + public boolean supportCodecBuffer() { + return STRING_CODEC.supportCodecBuffer(); + } + + @Override + public CodecBuffer toCodecBuffer(@Nonnull SequenceIdType object, + CodecBuffer.Allocator allocator) throws CodecException { + return STRING_CODEC.toCodecBuffer(object.name(), allocator); + } + + @Override + public SequenceIdType fromCodecBuffer(@Nonnull CodecBuffer buffer) throws CodecException { + return SequenceIdType.valueOf(STRING_CODEC.fromCodecBuffer(buffer)); + } + + @Override + public SequenceIdType copyObject(SequenceIdType object) { + return object; // Enums are singletons; safe to return direct reference + } +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSequenceIDGenerator.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSequenceIDGenerator.java index f7d8b1c499a3..1f07927a49dc 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSequenceIDGenerator.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSequenceIDGenerator.java @@ -156,7 +156,7 @@ public void testSequenceIDGenUponRatisWhenCurrentScmIsNotALeader() conf, scmHAManager, scmMetadataStore.getSequenceIdTable()) { @Override public StateManager createStateManager( - SCMHAManager scmhaManager, Table sequenceIdTable) { + SCMHAManager scmhaManager, Table sequenceIdTable) { Objects.requireNonNull(scmhaManager, "scmhaManager == null"); return stateManager; } @@ -227,7 +227,7 @@ public void testReinitializePopulatesSequenceIdMapFromDB() throws Exception { SequenceIdType idType = SequenceIdType.containerId; // Simulate an SCM restart by writing a raw String directly to the database. - scmMetadataStore.getSequenceIdTable().put(idType.name(), 100L); + scmMetadataStore.getSequenceIdTable().put(idType, 100L); // Create the StateManager directly using its Builder SequenceIdGenerator.StateManager stateManager = diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/metadata/TestSequenceIdTypeCodec.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/metadata/TestSequenceIdTypeCodec.java new file mode 100644 index 000000000000..7bd8110b1cb2 --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/metadata/TestSequenceIdTypeCodec.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.metadata; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.apache.hadoop.hdds.scm.ha.SequenceIdType; +import org.apache.hadoop.hdds.utils.db.Codec; +import org.apache.hadoop.hdds.utils.db.CodecTestUtil; +import org.apache.hadoop.hdds.utils.db.StringCodec; +import org.junit.jupiter.api.Test; + +/** + * Testing serialization and deserialization of SequenceIdType objects to/from RocksDB. + */ +public class TestSequenceIdTypeCodec { + + private final Codec enumCodec = SequenceIdTypeCodec.get(); + private final Codec stringCodec = StringCodec.get(); + + @Test + public void testCodecBuffersWithOzoneTestUtil() throws Exception { + for (SequenceIdType type : SequenceIdType.values()) { + // Verify codec compatibility with heap and direct byte buffers. + CodecTestUtil.runTest(enumCodec, type, null, null); + } + } + + @Test + public void testSerializedBytesMatchStringCodec() throws Exception { + for (SequenceIdType type : SequenceIdType.values()) { + byte[] expectedStringBytes = stringCodec.toPersistedFormat(type.name()); + byte[] computedEnumBytes = enumCodec.toPersistedFormat(type); + + // Verify exact match for on-disk binary format representation. + assertArrayEquals(expectedStringBytes, computedEnumBytes, + "Serialized bytes must match the StringCodec exactly"); + } + } + + @Test + public void testSequenceIdTypeCodecCanReadStringCodecBytes() throws Exception { + for (SequenceIdType type : SequenceIdType.values()) { + byte[] legacyBytes = stringCodec.toPersistedFormat(type.name()); + + // Verify deserialization compatibility for cluster upgrade path. + SequenceIdType decodedEnum = enumCodec.fromPersistedFormat(legacyBytes); + assertEquals(type, decodedEnum, "SequenceIdTypeCodec failed to read legacy string bytes"); + } + } + + @Test + public void testStringCodecCanReadSequenceIdTypeCodecBytes() throws Exception { + for (SequenceIdType type : SequenceIdType.values()) { + byte[] newBytes = enumCodec.toPersistedFormat(type); + + // Verify deserialization compatibility for cluster downgrade path. + String decodedString = stringCodec.fromPersistedFormat(newBytes); + assertEquals(type.name(), decodedString, "StringCodec failed to read new enum bytes"); + } + } +} From 61c4e0c84626ab89076ac1f620fed75759625310 Mon Sep 17 00:00:00 2001 From: navinko Date: Fri, 29 May 2026 04:21:22 +0530 Subject: [PATCH 2/3] HDDS-15370. updated comments --- .../apache/hadoop/hdds/scm/metadata/SequenceIdTypeCodec.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SequenceIdTypeCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SequenceIdTypeCodec.java index 026c98719d1f..93cae6145a42 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SequenceIdTypeCodec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SequenceIdTypeCodec.java @@ -26,8 +26,6 @@ /** * Codec to serialize/deserialize {@link SequenceIdType}. - * It delegates directly to {@link StringCodec} to leverage its encoding/decoding - * optimizations and fallback logic, ensuring disk format continuity. */ public final class SequenceIdTypeCodec implements Codec { @@ -75,6 +73,6 @@ public SequenceIdType fromCodecBuffer(@Nonnull CodecBuffer buffer) throws CodecE @Override public SequenceIdType copyObject(SequenceIdType object) { - return object; // Enums are singletons; safe to return direct reference + return object; } } From 971d2197d98ed5c69935912237d6f4e5f305236a Mon Sep 17 00:00:00 2001 From: navinko Date: Sat, 30 May 2026 02:54:17 +0530 Subject: [PATCH 3/3] HDDS-15370. Addressed review comments --- .../hadoop/hdds/scm/ha/SequenceIdType.java | 101 ++++++++++++++++++ .../hdds/scm/metadata/SCMDBDefinition.java | 2 +- .../scm/metadata/SequenceIdTypeCodec.java | 78 -------------- .../scm/metadata/TestSequenceIdTypeCodec.java | 4 +- 4 files changed, 104 insertions(+), 81 deletions(-) delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SequenceIdTypeCodec.java diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdType.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdType.java index 4279b0bd832a..13e4255e24d8 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdType.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/ha/SequenceIdType.java @@ -17,6 +17,18 @@ package org.apache.hadoop.hdds.scm.ha; +import jakarta.annotation.Nonnull; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.apache.hadoop.hdds.StringUtils; +import org.apache.hadoop.hdds.utils.db.Codec; +import org.apache.hadoop.hdds.utils.db.CodecBuffer; +import org.apache.hadoop.hdds.utils.db.CodecException; +import org.apache.hadoop.hdds.utils.db.StringCodec; + /** * Represents the sequence ID types managed by * {@code org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator} @@ -39,4 +51,93 @@ public enum SequenceIdType { @Deprecated rootCertificateId; + private static final Codec INSTANCE = new Codec() { + @Override + public Class getTypeClass() { + return SequenceIdType.class; + } + + @Override + public boolean supportCodecBuffer() { + return true; + } + + @Override + public byte[] toPersistedFormat(SequenceIdType type) { + return type.getByteArray(); + } + + @Override + public SequenceIdType fromPersistedFormat(byte[] bytes) throws CodecException { + final SequenceIdType type = SEQUENCE_ID_TYPES.get(bytes[0]); + if (type != null && Arrays.equals(type.getByteArray(), bytes)) { + return type; + } + throw new CodecException("Failed to decode " + StringUtils.bytes2Hex(ByteBuffer.wrap(bytes), 20)); + } + + @Override + public CodecBuffer toCodecBuffer(@Nonnull SequenceIdType object, CodecBuffer.Allocator allocator) { + final ByteBuffer buffer = object.getByteBuffer(); + final CodecBuffer cb = allocator.apply(buffer.remaining()); + cb.put(buffer); + return cb; + } + + @Override + public SequenceIdType fromCodecBuffer(@Nonnull CodecBuffer bytes) throws CodecException { + final ByteBuffer buffer = bytes.asReadOnlyByteBuffer(); + final SequenceIdType type = SEQUENCE_ID_TYPES.get(buffer.get(buffer.position())); + if (type != null && type.getByteBuffer().equals(buffer)) { + return type; + } + throw new CodecException("Failed to decode " + StringUtils.bytes2Hex(buffer, 20)); + + } + + @Override + public SequenceIdType copyObject(SequenceIdType object) { + return object; + } + }; + + /** Only use the first byte in the name since they are all distinct. */ + private static final Map SEQUENCE_ID_TYPES; + + private final byte[] byteArray; + private final ByteBuffer byteBuffer; + + SequenceIdType() { + try { + this.byteArray = StringCodec.getCodecNoFallback().toPersistedFormat(name()); + } catch (CodecException e) { + throw new IllegalStateException("Failed to construct " + this, e); + } + + this.byteBuffer = ByteBuffer.wrap(byteArray).asReadOnlyBuffer(); + } + + public byte[] getByteArray() { + return byteArray.clone(); + } + + public ByteBuffer getByteBuffer() { + return byteBuffer.duplicate(); + } + + static { + final Map map = new HashMap<>(); + for (SequenceIdType type : SequenceIdType.values()) { + final byte first = type.getByteArray()[0]; + final SequenceIdType previous = map.put(first, type); + if (previous != null) { + throw new IllegalStateException("Duplicated first byte: " + type + " and " + previous); + } + } + SEQUENCE_ID_TYPES = Collections.unmodifiableMap(map); + } + + public static Codec getCodec() { + return INSTANCE; + } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java index e7e771d40509..3b22a9f528e9 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java @@ -87,7 +87,7 @@ public class SCMDBDefinition extends DBDefinition.WithMap { SEQUENCE_ID = new DBColumnFamilyDefinition<>( "sequenceId", - SequenceIdTypeCodec.get(), + SequenceIdType.getCodec(), LongCodec.get()); public static final DBColumnFamilyDefinition { - - private static final Codec INSTANCE = new SequenceIdTypeCodec(); - private static final Codec STRING_CODEC = StringCodec.get(); - - public static Codec get() { - return INSTANCE; - } - - private SequenceIdTypeCodec() { - // singleton - } - - @Override - public Class getTypeClass() { - return SequenceIdType.class; - } - - @Override - public byte[] toPersistedFormat(SequenceIdType object) throws CodecException { - return STRING_CODEC.toPersistedFormat(object.name()); - } - - @Override - public SequenceIdType fromPersistedFormat(byte[] rawData) throws CodecException { - return SequenceIdType.valueOf(STRING_CODEC.fromPersistedFormat(rawData)); - } - - @Override - public boolean supportCodecBuffer() { - return STRING_CODEC.supportCodecBuffer(); - } - - @Override - public CodecBuffer toCodecBuffer(@Nonnull SequenceIdType object, - CodecBuffer.Allocator allocator) throws CodecException { - return STRING_CODEC.toCodecBuffer(object.name(), allocator); - } - - @Override - public SequenceIdType fromCodecBuffer(@Nonnull CodecBuffer buffer) throws CodecException { - return SequenceIdType.valueOf(STRING_CODEC.fromCodecBuffer(buffer)); - } - - @Override - public SequenceIdType copyObject(SequenceIdType object) { - return object; - } -} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/metadata/TestSequenceIdTypeCodec.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/metadata/TestSequenceIdTypeCodec.java index 7bd8110b1cb2..846c4e3a4314 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/metadata/TestSequenceIdTypeCodec.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/metadata/TestSequenceIdTypeCodec.java @@ -31,14 +31,14 @@ */ public class TestSequenceIdTypeCodec { - private final Codec enumCodec = SequenceIdTypeCodec.get(); + private final Codec enumCodec = SequenceIdType.getCodec(); private final Codec stringCodec = StringCodec.get(); @Test public void testCodecBuffersWithOzoneTestUtil() throws Exception { for (SequenceIdType type : SequenceIdType.values()) { // Verify codec compatibility with heap and direct byte buffers. - CodecTestUtil.runTest(enumCodec, type, null, null); + CodecTestUtil.runTest(enumCodec, type, type.getByteArray().length, null); } }