From 7f74964cc01c176c3a6962c2e2a1a9ce9ed8a5ed Mon Sep 17 00:00:00 2001 From: Alkis Evlogimenos Date: Mon, 22 May 2023 12:12:46 +0200 Subject: [PATCH 1/3] Optimize Configuration.handleDeprecation. --- .../org/apache/hadoop/conf/Configuration.java | 67 +++++++++---------- 1 file changed, 32 insertions(+), 35 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index ab7ff0bd40cc2..94a360bea2665 100755 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -724,54 +724,51 @@ private String[] handleDeprecation(DeprecationContext deprecations, if (null != name) { name = name.trim(); } - // Initialize the return value with requested name - String[] names = new String[]{name}; - // Deprecated keys are logged once and an updated names are returned - DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name); - if (keyInfo != null) { + + final DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name); + final String[] names; + if (keyInfo == null) { + names = new String[]{name}; + // Handling deprecations is rare so bail out early for the common case. + if (deprecations.getReverseDeprecatedKeyMap().get(name) == null) { + return names; + } + } else { + names = keyInfo.newKeys; if (!keyInfo.getAndSetAccessed()) { logDeprecation(keyInfo.getWarningMessage(name)); } - // Override return value for deprecated keys - names = keyInfo.newKeys; } - // Update properties with deprecated key if already loaded and new - // deprecation has been added - updatePropertiesWithDeprecatedKeys(deprecations, names); - - // If there are no overlay values we can return early - Properties overlayProperties = getOverlay(); - if (overlayProperties.isEmpty()) { - return names; - } - // Update properties and overlays with reverse lookup values - for (String n : names) { - String deprecatedKey = deprecations.getReverseDeprecatedKeyMap().get(n); - if (deprecatedKey != null && !overlayProperties.containsKey(n)) { - String deprecatedValue = overlayProperties.getProperty(deprecatedKey); - if (deprecatedValue != null) { - getProps().setProperty(n, deprecatedValue); - overlayProperties.setProperty(n, deprecatedValue); + final Map map = deprecations.getReverseDeprecatedKeyMap(); + for (String nk : names) { + String k = map.get(nk); + if (k != null && !getProps().containsKey(nk)) { + String v = getProps().getProperty(k); + if (v != null) { + getProps().setProperty(nk, v); } } } - return names; - } - private void updatePropertiesWithDeprecatedKeys( - DeprecationContext deprecations, String[] newNames) { - for (String newName : newNames) { - String deprecatedKey = deprecations.getReverseDeprecatedKeyMap().get(newName); - if (deprecatedKey != null && !getProps().containsKey(newName)) { - String deprecatedValue = getProps().getProperty(deprecatedKey); - if (deprecatedValue != null) { - getProps().setProperty(newName, deprecatedValue); + Properties overlayProps = getOverlay(); + if (!overlayProps.isEmpty()) { + for (String nk : names) { + String k = map.get(nk); + if (k != null && !overlayProps.containsKey(nk)) { + String v = overlayProps.getProperty(k); + if (v != null) { + // Update both properties and overlays. + getProps().setProperty(nk, v); + overlayProps.setProperty(nk, v); + } } } } + + return names; } - + private void handleDeprecation() { LOG.debug("Handling deprecation for all properties in config..."); DeprecationContext deprecations = deprecationContext.get(); From b822551a82f09321487da56b5de83bfb92cf5974 Mon Sep 17 00:00:00 2001 From: Alkis Evlogimenos Date: Wed, 7 Jun 2023 15:48:47 +0200 Subject: [PATCH 2/3] optimize property propagation with computeIfAbsent --- .../main/java/org/apache/hadoop/conf/Configuration.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 94a360bea2665..b78a49e3fd8a4 100755 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -743,11 +743,8 @@ private String[] handleDeprecation(DeprecationContext deprecations, final Map map = deprecations.getReverseDeprecatedKeyMap(); for (String nk : names) { String k = map.get(nk); - if (k != null && !getProps().containsKey(nk)) { - String v = getProps().getProperty(k); - if (v != null) { - getProps().setProperty(nk, v); - } + if (k != null) { + getProps().computeIfAbsent(nk, (x) -> getProps().getProperty(k)); } } From c44a6a5e793ab665c80da3b9c1f8fb82379dd486 Mon Sep 17 00:00:00 2001 From: Alkis Evlogimenos Date: Thu, 8 Jun 2023 10:10:03 +0200 Subject: [PATCH 3/3] use computeIfAbsent in overlay property propagation for consistency --- .../java/org/apache/hadoop/conf/Configuration.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index b78a49e3fd8a4..22d102cef3524 100755 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -752,13 +752,15 @@ private String[] handleDeprecation(DeprecationContext deprecations, if (!overlayProps.isEmpty()) { for (String nk : names) { String k = map.get(nk); - if (k != null && !overlayProps.containsKey(nk)) { - String v = overlayProps.getProperty(k); - if (v != null) { + if (k != null) { + overlayProps.computeIfAbsent(nk, (x) -> { + String v = overlayProps.getProperty(k); // Update both properties and overlays. - getProps().setProperty(nk, v); - overlayProps.setProperty(nk, v); - } + if (v != null) { + getProps().setProperty(nk, v); + } + return v; + }); } } }