From 4ea6c0a3794f1188a59ad1e7f2ea70a3b54c9049 Mon Sep 17 00:00:00 2001 From: rich7420 Date: Sat, 1 Aug 2026 15:29:19 +0800 Subject: [PATCH 1/7] HDDS-12688. Set test-friendly defaults in mini-cluster Move the test-friendly space/memory settings that only lived in integration-test's ozone-site.xml into MiniOzoneCluster.Builder so every mini-cluster consumer (integration-test, -s3, -recon, and external users of the ozone-mini-cluster artifact) gets them. setIfUnset could not be reused directly: after HDDS-12777 a key present in ozone-default.xml always returns non-null from get(), so the stock setIfUnset never applied. OzoneConfiguration now overrides setIfUnset to treat only non *-default.xml sources (programmatic, command line, *-site.xml, custom resources) as explicitly set, so Builder defaults can override default-resource values while preserving anything the caller set. ClientConfigForTesting moves into ozone-mini-cluster main and gains an applyTo(conf, onlyIfUnset) overload backed by MutableConfigurationSource.ifUnsetWrapper. Redundant settings are dropped from the three integration-test ozone-site.xml files; module-specific ones (MockSpaceUsage, transport class, 128MB container size, 5GB min free space) are kept. --- .../hadoop/hdds/conf/OzoneConfiguration.java | 30 +++++++ .../hdds/conf/TestOzoneConfiguration.java | 60 +++++++++++++ .../hdds/conf/MutableConfigurationSource.java | 57 ++++++++++++ .../src/test/resources/ozone-site.xml | 77 +--------------- .../src/test/resources/ozone-site.xml | 86 +----------------- .../src/test/resources/ozone-site.xml | 90 +------------------ .../hadoop/ozone/ClientConfigForTesting.java | 42 ++++++++- .../apache/hadoop/ozone/MiniOzoneCluster.java | 34 +++++++ 8 files changed, 226 insertions(+), 250 deletions(-) rename hadoop-ozone/{integration-test/src/test => mini-cluster/src/main}/java/org/apache/hadoop/ozone/ClientConfigForTesting.java (77%) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java index 69c4c029ce10..5b22a0332059 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java @@ -446,6 +446,36 @@ public synchronized void reloadConfiguration() { delegatingProps = null; } + /** + * Sets {@code value} unless the property was already set explicitly + * (programmatically, from the command line, or from a {@code *-site.xml}). + * Values that come only from default resources ({@code *-default.xml}) are overridden. + *

+ * Hadoop {@link Configuration#setIfUnset(String, String)} uses {@code get(name) == null}, + * which never succeeds for keys present in default resources after HDDS-12777. + */ + @Override + public synchronized void setIfUnset(String name, String value) { + if (!isExplicitlySet(name)) { + set(name, value); + } + } + + private boolean isExplicitlySet(String name) { + String[] sources = getPropertySources(name); + if (sources == null) { + return false; + } + for (String source : sources) { + // Any source other than a *-default.xml (programmatically, command line, + // a *-site.xml, or a custom resource) counts as explicitly set. + if (source != null && !source.endsWith("-default.xml")) { + return true; + } + } + return false; + } + @Override protected final synchronized Properties getProps() { if (delegatingProps == null) { diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java index 8046deafea64..386ee3ff634f 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java @@ -25,6 +25,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -301,6 +302,65 @@ public void testInstantiationWithInputConfiguration(@TempDir File tempDir) assertNotEquals(val, new OzoneConfiguration().get(key)); } + @Test + public void setIfUnsetOverridesDefaultButKeepsExplicitValue() { + final String key = OZONE_SCM_HANDLER_COUNT_KEY; + OzoneConfiguration subject = new OzoneConfiguration(); + + // Default resources provide a value, so Hadoop's get(key) is non-null. + assertNotNull(subject.get(key)); + String fromDefaults = subject.get(key); + + subject.setIfUnset(key, "20"); + assertEquals("20", subject.get(key)); + assertNotEquals(fromDefaults, subject.get(key)); + + subject.set(key, "42"); + subject.setIfUnset(key, "20"); + assertEquals("42", subject.get(key)); + } + + @Test + public void setIfUnsetPreservesSiteXmlValue(@TempDir File tempDir) + throws IOException { + final String key = OZONE_SCM_HANDLER_COUNT_KEY; + File ozoneSite = new File(tempDir, "ozone-site.xml"); + try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter( + Files.newOutputStream(ozoneSite.toPath()), StandardCharsets.UTF_8))) { + startConfig(out); + appendProperty(out, key, "99"); + endConfig(out); + } + + OzoneConfiguration subject = new OzoneConfiguration(); + subject.addResource(new Path(ozoneSite.getAbsolutePath())); + assertEquals("99", subject.get(key)); + + subject.setIfUnset(key, "20"); + assertEquals("99", subject.get(key)); + } + + @Test + public void setIfUnsetPreservesCustomResourceValue(@TempDir File tempDir) + throws IOException { + final String key = OZONE_SCM_HANDLER_COUNT_KEY; + File custom = new File(tempDir, "custom-config.xml"); + try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter( + Files.newOutputStream(custom.toPath()), StandardCharsets.UTF_8))) { + startConfig(out); + appendProperty(out, key, "77"); + endConfig(out); + } + + OzoneConfiguration subject = new OzoneConfiguration(); + subject.addResource(new Path(custom.getAbsolutePath())); + assertEquals("77", subject.get(key)); + + // A value from any non-default resource is explicit and must be preserved. + subject.setIfUnset(key, "20"); + assertEquals("77", subject.get(key)); + } + @Test public void setConfigFromObjectWithObjectDefaults() { // GIVEN diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java index 779a8e76417d..c328eca85f59 100644 --- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java @@ -17,9 +17,66 @@ package org.apache.hadoop.hdds.conf; +import java.io.IOException; +import java.util.Collection; + /** * Configuration that can be both read and written. */ public interface MutableConfigurationSource extends ConfigurationSource, ConfigurationTarget { + + /** + * Sets {@code value} for {@code key} only if the key is not already set. + * Default implementation treats any non-null {@link #get(String)} result as set. + * {@link org.apache.hadoop.hdds.conf.OzoneConfiguration} overrides this to allow + * overriding values that come only from default resources. + */ + default void setIfUnset(String key, String value) { + if (get(key) == null) { + set(key, value); + } + } + + /** + * Creates a wrapper config that changes {@link #set(String, String)} to + * {@link #setIfUnset(String, String)}. In other words, value is stored only if + * no existing value is explicitly set. + */ + static MutableConfigurationSource ifUnsetWrapper(MutableConfigurationSource wrapped) { + return new IfUnsetWrapper(wrapped); + } + + /** + * Delegates all calls to another configuration object, but changes semantics of + * {@link #set(String, String)} to {@link #setIfUnset(String, String)}. + */ + class IfUnsetWrapper implements MutableConfigurationSource { + + private final MutableConfigurationSource wrapped; + + private IfUnsetWrapper(MutableConfigurationSource wrapped) { + this.wrapped = wrapped; + } + + @Override + public String get(String key) { + return wrapped.get(key); + } + + @Override + public Collection getConfigKeys() { + return wrapped.getConfigKeys(); + } + + @Override + public char[] getPassword(String key) throws IOException { + return wrapped.getPassword(key); + } + + @Override + public void set(String key, String value) { + wrapped.setIfUnset(key, value); + } + } } diff --git a/hadoop-ozone/integration-test-recon/src/test/resources/ozone-site.xml b/hadoop-ozone/integration-test-recon/src/test/resources/ozone-site.xml index 95a78dd5f38d..c21b140814a3 100644 --- a/hadoop-ozone/integration-test-recon/src/test/resources/ozone-site.xml +++ b/hadoop-ozone/integration-test-recon/src/test/resources/ozone-site.xml @@ -26,90 +26,15 @@ ozone.om.s3.grpc.server_enabled false - - hdds.container.ratis.num.write.chunk.threads.per.volume - 4 - - - ozone.scm.handler.count.key - 20 - - - ozone.om.handler.count.key - 20 - - - hdds.container.ratis.datastream.enabled - true - - - hdds.heartbeat.interval - 1s - - - ozone.scm.heartbeat.thread.interval - 100ms - ozone.scm.ratis.pipeline.limit 3 - - ozone.scm.close.container.wait.duration - 1s - - - ozone.om.snapshot.diff.job.default.wait.time - 1s - - - hdds.container.ratis.log.appender.queue.byte-limit - 32MB - - - ozone.om.ratis.log.appender.queue.byte-limit - 4MB - - - ozone.scm.ha.ratis.log.appender.queue.byte-limit - 4MB - - - ozone.scm.chunk.size - 1MB - - - ozone.scm.block.size - 4MB - + ozone.scm.container.size 128MB - - ozone.client.stream.buffer.flush.size - 1MB - - - ozone.client.stream.buffer.max.size - 2MB - - - ozone.client.stream.buffer.size - 1MB - - - ozone.client.datastream.buffer.flush.size - 4MB - - - ozone.client.datastream.min.packet.size - 256KB - - - ozone.client.datastream.window.size - 8MB - hdds.datanode.volume.min.free.space 5GB diff --git a/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml b/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml index da0ea9ab8c39..d400c4ef49c5 100644 --- a/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml +++ b/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml @@ -31,100 +31,16 @@ false - - hdds.container.ratis.num.write.chunk.threads.per.volume - 4 - - - - ozone.scm.handler.count.key - 20 - - - - ozone.om.handler.count.key - 20 - - - - hdds.container.ratis.datastream.enabled - true - - - - - hdds.heartbeat.interval - 1s - - - ozone.scm.heartbeat.thread.interval - 100ms - - ozone.scm.ratis.pipeline.limit 3 - - ozone.scm.close.container.wait.duration - 1s - - - - ozone.om.snapshot.diff.job.default.wait.time - 1s - - - - hdds.container.ratis.log.appender.queue.byte-limit - 32MB - - - ozone.om.ratis.log.appender.queue.byte-limit - 4MB - - - ozone.scm.ha.ratis.log.appender.queue.byte-limit - 4MB - - - - ozone.scm.chunk.size - 1MB - - - ozone.scm.block.size - 4MB - + ozone.scm.container.size 128MB - - ozone.client.stream.buffer.flush.size - 1MB - - - ozone.client.stream.buffer.max.size - 2MB - - - ozone.client.stream.buffer.size - 1MB - - - ozone.client.datastream.buffer.flush.size - 4MB - - - ozone.client.datastream.min.packet.size - 256KB - - - ozone.client.datastream.window.size - 8MB - hdds.datanode.volume.min.free.space 5GB diff --git a/hadoop-ozone/integration-test/src/test/resources/ozone-site.xml b/hadoop-ozone/integration-test/src/test/resources/ozone-site.xml index 2b07b1d060dc..f669c4c60c5b 100644 --- a/hadoop-ozone/integration-test/src/test/resources/ozone-site.xml +++ b/hadoop-ozone/integration-test/src/test/resources/ozone-site.xml @@ -36,97 +36,11 @@ false - - hdds.container.ratis.num.write.chunk.threads.per.volume - 4 - - - - ozone.scm.handler.count.key - 20 - - - - ozone.om.handler.count.key - 20 - - - - hdds.container.ratis.datastream.enabled - true - - - - - hdds.heartbeat.interval - 1s - - - ozone.scm.heartbeat.thread.interval - 100ms - - ozone.scm.ratis.pipeline.limit 3 - - ozone.scm.close.container.wait.duration - 1s - - - - ozone.om.snapshot.diff.job.default.wait.time - 1s - - - - hdds.container.ratis.log.appender.queue.byte-limit - - 32MB - - - ozone.om.ratis.log.appender.queue.byte-limit - 4MB - - - ozone.scm.ha.ratis.log.appender.queue.byte-limit - 4MB - - - - ozone.scm.chunk.size - 1MB - - - ozone.scm.block.size - 4MB - - - ozone.client.stream.buffer.flush.size - 1MB - - - ozone.client.stream.buffer.max.size - 2MB - - - ozone.client.stream.buffer.size - 1MB - - - ozone.client.datastream.buffer.flush.size - 4MB - - - ozone.client.datastream.min.packet.size - 256KB - - - ozone.client.datastream.window.size - 8MB - ozone.readonly.administrators admin @@ -136,8 +50,8 @@ admin - ozone.directory.deleting.service.interval - 2m + ozone.directory.deleting.service.interval + 2m diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigForTesting.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/ClientConfigForTesting.java similarity index 77% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigForTesting.java rename to hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/ClientConfigForTesting.java index cbf06ec608b3..011c876bc0b4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigForTesting.java +++ b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/ClientConfigForTesting.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE; import org.apache.hadoop.hdds.conf.MutableConfigurationSource; @@ -31,6 +32,7 @@ public final class ClientConfigForTesting { private int chunkSize = 1024 * 1024; private Long blockSize; + private Long containerSize; private Integer streamBufferSize; private Long streamBufferFlushSize; private Long dataStreamBufferFlushSize; @@ -61,6 +63,11 @@ public ClientConfigForTesting setBlockSize(long size) { return this; } + public ClientConfigForTesting setContainerSize(long size) { + containerSize = toBytes(size); + return this; + } + @SuppressWarnings("unused") // kept for completeness public ClientConfigForTesting setStreamBufferSize(int size) { streamBufferSize = (int) toBytes(size); @@ -93,6 +100,24 @@ public ClientConfigForTesting setDataStreamWindowSize(long size) { } public void applyTo(MutableConfigurationSource conf) { + applyTo(conf, false); + } + + public void applyTo(MutableConfigurationSource conf, boolean onlyIfUnset) { + calculateUndefinedValues(); + + final MutableConfigurationSource target = + onlyIfUnset ? MutableConfigurationSource.ifUnsetWrapper(conf) : conf; + target.setFromObject(getClientConfig(conf)); + + if (onlyIfUnset) { + setIfUnset(conf); + } else { + set(conf); + } + } + + private void calculateUndefinedValues() { if (streamBufferSize == null) { streamBufferSize = chunkSize; } @@ -114,7 +139,12 @@ public void applyTo(MutableConfigurationSource conf) { if (blockSize == null) { blockSize = 2 * streamBufferMaxSize; } + if (containerSize == null) { + containerSize = 4 * blockSize; + } + } + private OzoneClientConfig getClientConfig(MutableConfigurationSource conf) { OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); clientConfig.setStreamBufferSize(streamBufferSize); clientConfig.setStreamBufferMaxSize(streamBufferMaxSize); @@ -122,10 +152,20 @@ public void applyTo(MutableConfigurationSource conf) { clientConfig.setDataStreamBufferFlushSize(dataStreamBufferFlushSize); clientConfig.setDataStreamMinPacketSize(dataStreamMinPacketSize); clientConfig.setStreamWindowSize(dataStreamWindowSize); + return clientConfig; + } - conf.setFromObject(clientConfig); + private void set(MutableConfigurationSource conf) { conf.setStorageSize(OZONE_SCM_CHUNK_SIZE_KEY, chunkSize, StorageUnit.BYTES); conf.setStorageSize(OZONE_SCM_BLOCK_SIZE, blockSize, StorageUnit.BYTES); + conf.setStorageSize(OZONE_SCM_CONTAINER_SIZE, containerSize, StorageUnit.BYTES); + } + + private void setIfUnset(MutableConfigurationSource conf) { + final String suffix = StorageUnit.BYTES.getShortName(); + conf.setIfUnset(OZONE_SCM_CHUNK_SIZE_KEY, chunkSize + suffix); + conf.setIfUnset(OZONE_SCM_BLOCK_SIZE, blockSize + suffix); + conf.setIfUnset(OZONE_SCM_CONTAINER_SIZE, containerSize + suffix); } private long toBytes(long value) { diff --git a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java index 8765c2aaaae6..fc96311e8f4b 100644 --- a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java +++ b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java @@ -17,6 +17,17 @@ package org.apache.hadoop.ozone; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_PER_VOLUME; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HA_RAFT_LOG_APPENDER_QUEUE_BYTE_LIMIT; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL; +import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_CLOSE_CONTAINER_WAIT_DURATION; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_JOB_DEFAULT_WAIT_TIME; + import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -26,6 +37,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; @@ -272,11 +284,33 @@ abstract class Builder { protected Builder(OzoneConfiguration conf) { this.conf = conf; setClusterId(); + setDefaultConfigs(); // Use default SCM configurations if no override is provided. setSCMConfigurator(new SCMConfigurator()); ExitUtils.disableSystemExit(); } + /** + * Applies test-friendly defaults that reduce memory/disk needs for MiniOzoneCluster. + * Explicit values from the caller or from {@code *-site.xml} are preserved. + */ + protected void setDefaultConfigs() { + ClientConfigForTesting.newBuilder(StorageUnit.MB) + .setChunkSize(1) + .applyTo(conf, true); + + conf.setIfUnset(HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED, "true"); + conf.setIfUnset(HDDS_HEARTBEAT_INTERVAL, "1s"); + conf.setIfUnset(OZONE_OM_SNAPSHOT_DIFF_JOB_DEFAULT_WAIT_TIME, "1s"); + conf.setIfUnset(OZONE_SCM_CLOSE_CONTAINER_WAIT_DURATION, "1s"); + conf.setIfUnset(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, "100ms"); + conf.setIfUnset(OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT, "4MB"); + conf.setIfUnset(OZONE_SCM_HA_RAFT_LOG_APPENDER_QUEUE_BYTE_LIMIT, "4MB"); + conf.setIfUnset(HDDS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_PER_VOLUME, "4"); + conf.setIfUnset(OZONE_OM_HANDLER_COUNT_KEY, "20"); + conf.setIfUnset(OZONE_SCM_HANDLER_COUNT_KEY, "20"); + } + /** Prepare the builder for another call to {@link #build()}, avoiding conflict * between the clusters created. */ protected void prepareForNextBuild() { From 157b0759dea6894366cf8840d018efa7906a7e90 Mon Sep 17 00:00:00 2001 From: rich7420 Date: Sat, 1 Aug 2026 15:47:19 +0800 Subject: [PATCH 2/7] HDDS-12688. Declare hdds-client dependency in mini-cluster ClientConfigForTesting (moved into mini-cluster main) uses org.apache.hadoop.hdds.scm.OzoneClientConfig from hdds-client, which was previously pulled in only transitively. Declare it explicitly so maven-dependency-plugin:analyze-only no longer fails the build with 'Used undeclared dependencies found: hdds-client'. --- hadoop-ozone/mini-cluster/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hadoop-ozone/mini-cluster/pom.xml b/hadoop-ozone/mini-cluster/pom.xml index a6a250be449d..cc51dbf52f2f 100644 --- a/hadoop-ozone/mini-cluster/pom.xml +++ b/hadoop-ozone/mini-cluster/pom.xml @@ -50,6 +50,10 @@ org.apache.hadoop hadoop-common + + org.apache.ozone + hdds-client + org.apache.ozone hdds-common From 00204436a87dffbccf667347c6a74452f5c18690 Mon Sep 17 00:00:00 2001 From: rich7420 Date: Sat, 1 Aug 2026 16:07:29 +0800 Subject: [PATCH 3/7] HDDS-12688. Avoid cross-module javadoc link to OzoneConfiguration MutableConfigurationSource is in hdds-config, which does not (and cannot, without a cycle) depend on hdds-common where OzoneConfiguration lives. The {@link} to it failed javadoc reference resolution in the compile check. Use {@code} instead. --- .../org/apache/hadoop/hdds/conf/MutableConfigurationSource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java index c328eca85f59..d05030b8d573 100644 --- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java @@ -29,7 +29,7 @@ public interface MutableConfigurationSource /** * Sets {@code value} for {@code key} only if the key is not already set. * Default implementation treats any non-null {@link #get(String)} result as set. - * {@link org.apache.hadoop.hdds.conf.OzoneConfiguration} overrides this to allow + * {@code OzoneConfiguration} (in hdds-common) overrides this to allow * overriding values that come only from default resources. */ default void setIfUnset(String key, String value) { From 47f3c56b6d2ddbed563306d0e322caec0e0e9fd8 Mon Sep 17 00:00:00 2001 From: rich7420 Date: Sun, 2 Aug 2026 15:18:14 +0800 Subject: [PATCH 4/7] HDDS-12688. Address review: restrict default-resource check and align wrapper - isExplicitlySet() now checks the property source against Ozone's registered built-in default resources (getConfigurationResourceFiles()) instead of a "*-default.xml" suffix. A user-provided resource is preserved even when named *-default.xml. - IfUnsetWrapper now overrides setIfUnset() to delegate to wrapped.setIfUnset(), so its set() and setIfUnset() behave consistently. - Strengthen the custom-resource test to use a *-default.xml name, guarding the edge case above. --- .../hadoop/hdds/conf/OzoneConfiguration.java | 18 +++++++++++++----- .../hdds/conf/TestOzoneConfiguration.java | 4 +++- .../hdds/conf/MutableConfigurationSource.java | 5 +++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java index 5b22a0332059..7f74e0ac09c3 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java @@ -34,6 +34,7 @@ import java.util.Map; import java.util.Objects; import java.util.Properties; +import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.TimeUnit; @@ -446,10 +447,16 @@ public synchronized void reloadConfiguration() { delegatingProps = null; } + /** Ozone's built-in default resources; a value coming only from these may be overridden. */ + private static final Set DEFAULT_RESOURCES = getConfigurationResourceFiles().stream() + .filter(resource -> resource.endsWith("-default.xml")) + .collect(Collectors.toSet()); + /** * Sets {@code value} unless the property was already set explicitly - * (programmatically, from the command line, or from a {@code *-site.xml}). - * Values that come only from default resources ({@code *-default.xml}) are overridden. + * (programmatically, from the command line, from a {@code *-site.xml}, or from a + * user-provided resource). Values that come only from Ozone's built-in default + * resources ({@code *-default.xml}) are overridden. *

* Hadoop {@link Configuration#setIfUnset(String, String)} uses {@code get(name) == null}, * which never succeeds for keys present in default resources after HDDS-12777. @@ -467,9 +474,10 @@ private boolean isExplicitlySet(String name) { return false; } for (String source : sources) { - // Any source other than a *-default.xml (programmatically, command line, - // a *-site.xml, or a custom resource) counts as explicitly set. - if (source != null && !source.endsWith("-default.xml")) { + // Explicit unless the value comes only from one of Ozone's built-in default + // resources. A user-provided resource is not in that set and is preserved, + // even if it happens to be named *-default.xml. + if (source != null && !DEFAULT_RESOURCES.contains(source)) { return true; } } diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java index 386ee3ff634f..96701dcd46c8 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java @@ -344,7 +344,9 @@ public void setIfUnsetPreservesSiteXmlValue(@TempDir File tempDir) public void setIfUnsetPreservesCustomResourceValue(@TempDir File tempDir) throws IOException { final String key = OZONE_SCM_HANDLER_COUNT_KEY; - File custom = new File(tempDir, "custom-config.xml"); + // Named *-default.xml on purpose: a user-provided resource is explicit even + // when its name matches the built-in default resource convention. + File custom = new File(tempDir, "custom-default.xml"); try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter( Files.newOutputStream(custom.toPath()), StandardCharsets.UTF_8))) { startConfig(out); diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java index d05030b8d573..d5578f1aa264 100644 --- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java @@ -78,5 +78,10 @@ public char[] getPassword(String key) throws IOException { public void set(String key, String value) { wrapped.setIfUnset(key, value); } + + @Override + public void setIfUnset(String key, String value) { + wrapped.setIfUnset(key, value); + } } } From 1d038e27f181b9e5a06622a77c801d8f952ef703 Mon Sep 17 00:00:00 2001 From: rich7420 Date: Tue, 4 Aug 2026 20:49:39 +0800 Subject: [PATCH 5/7] HDDS-12688. Move DEFAULT_RESOURCES field to class start to satisfy PMD --- .../apache/hadoop/hdds/conf/OzoneConfiguration.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java index 7f74e0ac09c3..ecbc9ced693d 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java @@ -73,6 +73,11 @@ public class OzoneConfiguration extends Configuration implements MutableConfigur activate(); } + /** Ozone's built-in default resources; a value coming only from these may be overridden. */ + private static final Set DEFAULT_RESOURCES = getConfigurationResourceFiles().stream() + .filter(resource -> resource.endsWith("-default.xml")) + .collect(Collectors.toSet()); + private Properties delegatingProps; public static OzoneConfiguration of(ConfigurationSource source) { @@ -447,11 +452,6 @@ public synchronized void reloadConfiguration() { delegatingProps = null; } - /** Ozone's built-in default resources; a value coming only from these may be overridden. */ - private static final Set DEFAULT_RESOURCES = getConfigurationResourceFiles().stream() - .filter(resource -> resource.endsWith("-default.xml")) - .collect(Collectors.toSet()); - /** * Sets {@code value} unless the property was already set explicitly * (programmatically, from the command line, from a {@code *-site.xml}, or from a From 988f1f87e718668d29ad9124a1aa017914505386 Mon Sep 17 00:00:00 2001 From: rich7420 Date: Thu, 6 Aug 2026 14:00:23 +0800 Subject: [PATCH 6/7] HDDS-12688. Fix mini-cluster test regressions --- .../client/rpc/TestFailureHandlingByClient.java | 5 +---- .../apache/hadoop/ozone/ClientConfigForTesting.java | 12 ------------ 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java index 5ccf9b98b185..da325811484e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java @@ -41,7 +41,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; -import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; @@ -130,9 +129,7 @@ public void init() throws Exception { raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(3)); conf.setFromObject(raftClientConfig); - OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); - clientConfig.setStreamBufferFlushDelay(false); - conf.setFromObject(clientConfig); + conf.setBoolean("ozone.client.stream.buffer.flush.delay", false); conf.setQuietMode(false); conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, diff --git a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/ClientConfigForTesting.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/ClientConfigForTesting.java index 011c876bc0b4..1a4c79a548df 100644 --- a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/ClientConfigForTesting.java +++ b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/ClientConfigForTesting.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE; import org.apache.hadoop.hdds.conf.MutableConfigurationSource; @@ -32,7 +31,6 @@ public final class ClientConfigForTesting { private int chunkSize = 1024 * 1024; private Long blockSize; - private Long containerSize; private Integer streamBufferSize; private Long streamBufferFlushSize; private Long dataStreamBufferFlushSize; @@ -63,11 +61,6 @@ public ClientConfigForTesting setBlockSize(long size) { return this; } - public ClientConfigForTesting setContainerSize(long size) { - containerSize = toBytes(size); - return this; - } - @SuppressWarnings("unused") // kept for completeness public ClientConfigForTesting setStreamBufferSize(int size) { streamBufferSize = (int) toBytes(size); @@ -139,9 +132,6 @@ private void calculateUndefinedValues() { if (blockSize == null) { blockSize = 2 * streamBufferMaxSize; } - if (containerSize == null) { - containerSize = 4 * blockSize; - } } private OzoneClientConfig getClientConfig(MutableConfigurationSource conf) { @@ -158,14 +148,12 @@ private OzoneClientConfig getClientConfig(MutableConfigurationSource conf) { private void set(MutableConfigurationSource conf) { conf.setStorageSize(OZONE_SCM_CHUNK_SIZE_KEY, chunkSize, StorageUnit.BYTES); conf.setStorageSize(OZONE_SCM_BLOCK_SIZE, blockSize, StorageUnit.BYTES); - conf.setStorageSize(OZONE_SCM_CONTAINER_SIZE, containerSize, StorageUnit.BYTES); } private void setIfUnset(MutableConfigurationSource conf) { final String suffix = StorageUnit.BYTES.getShortName(); conf.setIfUnset(OZONE_SCM_CHUNK_SIZE_KEY, chunkSize + suffix); conf.setIfUnset(OZONE_SCM_BLOCK_SIZE, blockSize + suffix); - conf.setIfUnset(OZONE_SCM_CONTAINER_SIZE, containerSize + suffix); } private long toBytes(long value) { From 43aef51d6d7976a762e6081b7d54ec4ef73cf39e Mon Sep 17 00:00:00 2001 From: rich7420 Date: Thu, 6 Aug 2026 15:32:53 +0800 Subject: [PATCH 7/7] HDDS-12688. Preserve mini-cluster client defaults in integration tests --- .../src/test/java/org/apache/ozone/test/ClusterForTests.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/ClusterForTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/ClusterForTests.java index f3823644cde1..372eb67b9a71 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/ClusterForTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/ClusterForTests.java @@ -26,7 +26,6 @@ import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; -import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.junit.jupiter.api.AfterAll; @@ -61,9 +60,7 @@ protected static OzoneConfiguration createBaseConfiguration() { raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(10)); conf.setFromObject(raftClientConfig); - OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); - clientConfig.setStreamBufferFlushDelay(false); - conf.setFromObject(clientConfig); + conf.setBoolean("ozone.client.stream.buffer.flush.delay", false); conf.setBoolean(OZONE_HBASE_ENHANCEMENTS_ALLOWED, true); conf.setBoolean("ozone.client.hbase.enhancements.allowed", true);