From c1e78c1288f2ca9d0f640307b0ee52f18614ff50 Mon Sep 17 00:00:00 2001 From: Michael Iseli Date: Tue, 5 Jan 2021 13:22:27 +0100 Subject: [PATCH] Add support for environment variable property maps Previously, it was not possible, to define property maps using environment variables. The only viable workaround was to define the property map in the config.properties file and use variables for certain key values sourced from the environment to inject environment variable values into property maps. Although this workaround was functional in a certain way, it did not allow for the definition of property map keys not present in the config.properties file nor to override values of property map keys from the config.properties file that were hardcoded. This change introduces a mechanism to define property maps using environment variables with the help of JSON object strings: MAPPROPERTY={"keya": "valuea", "keyb": "valueb", "keyc": null} Using this approach it is possible to - introduce property map values not defined in any config.properties file: just create an environment variable with the appropriate name and specify the map as a JSON object string. - define new property map keys: as above, just create the environment variable like above with the new key defined. The mixing of property map values is explained below. - change the value of of a property map key already defined in the config.properties file: as above, just create an environment variable with the key to change and the new value. - delete a property map key previously defined in the config.propertiesfile: as above, just create an environment variable with the key to delete and a value of 'null' w/o the quotes. The last three options may occur within the same JSON object string defined in an environment variable. When property maps are defined a different levels, the usual precedence applies: config.properties < environment variable < system property. Change-Id: I2b2ee1fc472ad1ec29e414ce23a5101e37a7303b Signed-off-by: Michael Iseli Reviewed-on: https://git.eclipse.org/r/c/scout/org.eclipse.scout.rt/+/174274 Tested-by: Scout Bot Reviewed-by: Ivan Motsch --- .../platform/config/PropertiesHelperTest.java | 153 +++++++++++++++++- .../rt/platform/config/map-test.properties | 3 + org.eclipse.scout.rt.platform/pom.xml | 5 + .../rt/platform/config/PropertiesHelper.java | 78 +++++++-- 4 files changed, 217 insertions(+), 22 deletions(-) diff --git a/org.eclipse.scout.rt.platform.test/src/test/java/org/eclipse/scout/rt/platform/config/PropertiesHelperTest.java b/org.eclipse.scout.rt.platform.test/src/test/java/org/eclipse/scout/rt/platform/config/PropertiesHelperTest.java index c00a3da7201..9ff7a067d05 100644 --- a/org.eclipse.scout.rt.platform.test/src/test/java/org/eclipse/scout/rt/platform/config/PropertiesHelperTest.java +++ b/org.eclipse.scout.rt.platform.test/src/test/java/org/eclipse/scout/rt/platform/config/PropertiesHelperTest.java @@ -10,11 +10,9 @@ ******************************************************************************/ package org.eclipse.scout.rt.platform.config; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; import java.io.IOException; import java.net.MalformedURLException; @@ -27,9 +25,13 @@ import java.util.List; import java.util.Map; +import org.eclipse.scout.rt.platform.util.CollectionUtility; import org.eclipse.scout.rt.platform.util.IOUtility; +import org.eclipse.scout.rt.platform.util.ImmutablePair; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.mockito.Mockito; /** @@ -72,6 +74,9 @@ public class PropertiesHelperTest { private static final String MAP_KEY = "mapKey"; private static final String EMPTY_KEY = "emptyKey"; + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Test public void testPropertiesHelper() throws Exception { PropertiesHelper instance = new PropertiesHelper(SAMPLE_CONFIG_PROPS); @@ -91,6 +96,10 @@ public void testNamespaceProperty() { assertEquals(null, instance.getProperty(NAMESPACE_PROP + "-not-existing", null, NAMESPACE)); assertEquals(null, instance.getProperty(NAMESPACE_PROP, null, NAMESPACE + "-not-existing")); assertEquals("defaultval", instance.getProperty(NAMESPACE_PROP, "defaultval", NAMESPACE + "-not-existing")); + + PropertiesHelper spiedInstance = spy(instance); + when(spiedInstance.getEnvironmentVariable(NAMESPACE + "__" + NAMESPACE_PROP)).thenReturn(NAMESPACE_PROP_VAL + "-from-env"); + assertThat(spiedInstance.getProperty(NAMESPACE_PROP, null, NAMESPACE), is(NAMESPACE_PROP_VAL + "-from-env")); } @Test @@ -223,6 +232,140 @@ public void testPropertyList() { } } + @Test + public void testReadPropertyMapFromEnvironment() { + PropertiesHelper originalInstance = new PropertiesHelper(SAMPLE_CONFIG_PROPS); + PropertiesHelper spiedInstance = spy(originalInstance); + when(spiedInstance.getEnvironmentVariable("map_not_in_file")).thenReturn("{\"keya\": \"valuea\",\"keyb\": \"valueb\",\"keyc\": \"valuec\"}"); + + assertThat(spiedInstance.getPropertyMap("map.not.in.file"), is(CollectionUtility.hashMap( + new ImmutablePair<>("keya", "valuea"), + new ImmutablePair<>("keyb", "valueb"), + new ImmutablePair<>("keyc", "valuec")))); + } + + @Test + public void testReadPropertyMapFromEnvironmentWithVariableReference() throws Exception { + PropertiesHelper originalInstance = new PropertiesHelper(SAMPLE_CONFIG_PROPS); + PropertiesHelper spiedInstance = spy(originalInstance); + try { + System.setProperty("sysProp", "sysPropVal"); + System.setProperty("stringKey", "stringKeyValueFromSystemProperty"); + when(spiedInstance.getEnvironmentVariable("envProp")).thenReturn("envPropVal"); + when(spiedInstance.getEnvironmentVariable("intKey")).thenReturn("intKeyFromEnv"); + when(spiedInstance.getEnvironmentVariable("map_not_in_file")).thenReturn("{" + + "\"propFromConfigProperties\": \"${otherProp}\"," + + "\"propFromSystemProperties\": \"${sysProp}\"," + + "\"propFromEnvironment\": \"${envProp}\"," + + "\"propFromConfigPropertiesOverriddenByEnv\": \"${intKey}\"," + + "\"propFromConfigPropertiesOverriddenBySystemProperty\": \"${stringKey}\"," + + "\"propFromConfigPropertiesInString\": \"test${longKey}testtest\"" + + "}"); + + assertThat(spiedInstance.getPropertyMap("map.not.in.file"), is(CollectionUtility.hashMap( + new ImmutablePair<>("propFromConfigProperties", "otherVal"), + new ImmutablePair<>("propFromSystemProperties", "sysPropVal"), + new ImmutablePair<>("propFromEnvironment", "envPropVal"), + new ImmutablePair<>("propFromConfigPropertiesOverriddenByEnv", "intKeyFromEnv"), + new ImmutablePair<>("propFromConfigPropertiesOverriddenBySystemProperty", "stringKeyValueFromSystemProperty"), + new ImmutablePair<>("propFromConfigPropertiesInString", "test2testtest")))); + } + finally { + System.clearProperty("sysProp"); + System.clearProperty("stringKey"); + } + } + + @Test + public void testReadPropertyMapFromEnvironmentMixingWithOtherPropertyMapSources() throws Exception { + PropertiesHelper originalInstance = new PropertiesHelper(MAP_CONFIG_PROPS); + PropertiesHelper spiedInstance = spy(originalInstance); + when(spiedInstance.getEnvironmentVariable("mapKey")).thenReturn("{\"second\": \"zwei\", \"third\": null}"); + + assertThat(spiedInstance.getPropertyMap("mapKey"), is(CollectionUtility.hashMap( + new ImmutablePair<>("first", "one"), + new ImmutablePair<>("second", "zwei"), + new ImmutablePair<>("empty", null), + new ImmutablePair<>("last", "last")))); + } + + @Test + public void testReadPropertyMapRespectingPrecedence() throws Exception { + PropertiesHelper originalInstance = new PropertiesHelper(MAP_CONFIG_PROPS); + PropertiesHelper spiedInstance = spy(originalInstance); + try { + System.setProperty("mapKey[first]", "one-from-system-property"); + when(spiedInstance.getEnvironmentVariable("mapKey")).thenReturn("{\"first\": \"one-from-env\", \"second\": \"two-from-env\"}"); + + assertThat(spiedInstance.getPropertyMap("mapKey"), is(CollectionUtility.hashMap( + new ImmutablePair<>("first", "one-from-system-property"), + new ImmutablePair<>("second", "two-from-env"), + new ImmutablePair<>("third", "three"), + new ImmutablePair<>("empty", null), + new ImmutablePair<>("last", "last")))); + } + finally { + System.clearProperty("mapKey[first]"); + } + } + + @Test + public void testReadPropertyMapFromEnvironmentWithNamespace() throws Exception { + PropertiesHelper originalInstance = new PropertiesHelper(MAP_CONFIG_PROPS); + PropertiesHelper spiedInstance = spy(originalInstance); + when(spiedInstance.getEnvironmentVariable("mapKey")).thenReturn("{\"second\": \"zwei\", \"third\": null}"); + when(spiedInstance.getEnvironmentVariable("namespace1__mapKey")).thenReturn("{\"a\": null, \"b\": \"b\", \"c\": \"ce\"}"); + when(spiedInstance.getEnvironmentVariable("namespace2__mapKey")).thenReturn("{\"d\": \"de\", \"e\": \"ee\"}"); + + assertThat(spiedInstance.getPropertyMap("mapKey"), is(CollectionUtility.hashMap( + new ImmutablePair<>("first", "one"), + new ImmutablePair<>("second", "zwei"), + new ImmutablePair<>("empty", null), + new ImmutablePair<>("last", "last")))); + assertThat(spiedInstance.getPropertyMap("mapKey", "namespace1"), is(CollectionUtility.hashMap( + new ImmutablePair<>("b", "b"), + new ImmutablePair<>("c", "ce")))); + assertThat(spiedInstance.getPropertyMap("mapKey", "namespace2"), is(CollectionUtility.hashMap( + new ImmutablePair<>("d", "de"), + new ImmutablePair<>("e", "ee")))); + } + + @Test + public void testExpectedExceptionForMalformedJsonStringMissingComma() { + PropertiesHelper originalInstance = new PropertiesHelper(MAP_CONFIG_PROPS); + PropertiesHelper spiedInstance = spy(originalInstance); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Error parsing value of environment variable 'mapKey' as JSON value"); + + when(spiedInstance.getEnvironmentVariable("mapKey")).thenReturn("{\"second\": \"zwei\" \"third\": null}"); // missing comma after "zwei" + spiedInstance.getPropertyMap("mapKey"); + } + + @Test + public void testExpectedExceptionForMalformedJsonStringMissingClosingBraces() { + PropertiesHelper originalInstance = new PropertiesHelper(MAP_CONFIG_PROPS); + PropertiesHelper spiedInstance = spy(originalInstance); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Error parsing value of environment variable 'mapKey' as JSON value"); + + when(spiedInstance.getEnvironmentVariable("mapKey")).thenReturn("{\"second\": \"zwei\", \"third\": null"); // missing } at the end + spiedInstance.getPropertyMap("mapKey"); + } + + @Test + public void testReadPropertyListFromEnvironment() throws Exception { + PropertiesHelper originalInstance = new PropertiesHelper(LIST_PROPS); + PropertiesHelper spiedInstance = spy(originalInstance); + + when(spiedInstance.getEnvironmentVariable("list")).thenReturn("{\"0\": \"zero\", \"1\": \"one\", \"2\": \"two\", \"3\": \"three\"}"); + when(spiedInstance.getEnvironmentVariable("listWithValidIndices")).thenReturn("{\"2\": \"two\", \"4\": \"four\", \"5\": \"five\"}"); + + assertThat(spiedInstance.getPropertyList("list"), is(Arrays.asList("zero", "one", "two", "three"))); + assertThat(spiedInstance.getPropertyList("listWithValidIndices"), is(Arrays.asList("a", null, "two", "b", "four", "five"))); + } + @Test public void testPropertyString() { PropertiesHelper instance = new PropertiesHelper(SAMPLE_CONFIG_PROPS); diff --git a/org.eclipse.scout.rt.platform.test/src/test/resources/org/eclipse/scout/rt/platform/config/map-test.properties b/org.eclipse.scout.rt.platform.test/src/test/resources/org/eclipse/scout/rt/platform/config/map-test.properties index fd7131b7ecf..52467e4d0c5 100644 --- a/org.eclipse.scout.rt.platform.test/src/test/resources/org/eclipse/scout/rt/platform/config/map-test.properties +++ b/org.eclipse.scout.rt.platform.test/src/test/resources/org/eclipse/scout/rt/platform/config/map-test.properties @@ -19,3 +19,6 @@ namespace|mapKey[a]=b namespace|mapKey[c]=c namespace|mapKey[c=not-a-valid-map-property namespace|mapKey[]=not-a-valid-map-property-2 +namespace1|mapKey[a]=a +namespace1|mapKey[a]=b +namespace1|mapKey[c]=c diff --git a/org.eclipse.scout.rt.platform/pom.xml b/org.eclipse.scout.rt.platform/pom.xml index 120e2908094..dfb6b7ba3a3 100644 --- a/org.eclipse.scout.rt.platform/pom.xml +++ b/org.eclipse.scout.rt.platform/pom.xml @@ -26,6 +26,11 @@ ${project.groupId}:${project.artifactId} + + org.eclipse.scout.rt + org.eclipse.scout.json + + org.jboss diff --git a/org.eclipse.scout.rt.platform/src/main/java/org/eclipse/scout/rt/platform/config/PropertiesHelper.java b/org.eclipse.scout.rt.platform/src/main/java/org/eclipse/scout/rt/platform/config/PropertiesHelper.java index 7cc0ad927b8..4a9de9a930c 100644 --- a/org.eclipse.scout.rt.platform/src/main/java/org/eclipse/scout/rt/platform/config/PropertiesHelper.java +++ b/org.eclipse.scout.rt.platform/src/main/java/org/eclipse/scout/rt/platform/config/PropertiesHelper.java @@ -22,13 +22,17 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Properties; import java.util.ServiceLoader; import java.util.Set; +import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.eclipse.scout.rt.platform.util.StringUtility; +import org.json.JSONException; +import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,6 +93,7 @@ public class PropertiesHelper { public static final String CLASSPATH_PROTOCOL_NAME = "classpath"; public static final char PROTOCOL_DELIMITER = ':'; public static final char NAMESPACE_DELIMITER = '|'; + public static final String NAMESPACE_DELIMITER_FOR_ENV = "__"; public static final char COLLECTION_DELIMITER_START = '['; public static final char COLLECTION_DELIMITER_END = ']'; public static final String CLASSPATH_PREFIX = CLASSPATH_PROTOCOL_NAME + PROTOCOL_DELIMITER; @@ -206,27 +211,41 @@ public String getProperty(String key, String defaultValue, String namespace) { } String propKey = toPropertyKey(key, namespace).toString(); - String value = null; - // system config - value = System.getProperty(propKey); + List> propertyValueRetrievers = Arrays.asList( + this::getSystemPropertyValue, + this::getEnvironmentPropertyValue, + this::getConfigPropertyValue); + + return propertyValueRetrievers.stream() + .map(propertyValueRetriever -> propertyValueRetriever.apply(propKey)) + .filter(Objects::nonNull) + .findFirst() + .orElse(defaultValue); + } + + protected String getSystemPropertyValue(String propKey) { + String value = System.getProperty(propKey); if (StringUtility.hasText(value)) { return resolve(value, PLACEHOLDER_PATTERN); } + return null; + } - // environment config + protected String getEnvironmentPropertyValue(String propKey) { String envValue = lookupEnvironmentVariableValue(propKey); if (StringUtility.hasText(envValue)) { return resolve(envValue, PLACEHOLDER_PATTERN); } + return null; + } - // properties file - value = m_configProperties.get(propKey); + protected String getConfigPropertyValue(String propKey) { + String value = m_configProperties.get(propKey); if (StringUtility.hasText(value)) { return value; } - - return defaultValue; + return null; } /** @@ -239,10 +258,14 @@ public String getProperty(String key, String defaultValue, String namespace) { *
  • Original in uppercase: MY.PROPERTY
    • *
  • Periods replaced, in uppercase: MY_PROPERTY
    • * + * The standard namespace delimiter (|) will always be replaced by a delimiter more suited for use in environment + * variables (__). */ protected String lookupEnvironmentVariableValue(String propKey) { + String nsDelimiterReplacedPropKey = propKey.replace(String.valueOf(NAMESPACE_DELIMITER), NAMESPACE_DELIMITER_FOR_ENV); + // 1. Original - String value = getEnvironmentVariable(propKey); + String value = getEnvironmentVariable(nsDelimiterReplacedPropKey); if (value != null) { return value; } @@ -250,7 +273,7 @@ protected String lookupEnvironmentVariableValue(String propKey) { // Periods in environment variable names are not POSIX compliant (See IEEE Standard 1003.1-2017, Chapter 8.1 "Environment Variable Definition"), // but supported by some shells. To allow overriding via environment variables (Bugzilla 541099) in any shell, convert them to underscores. // 2. With periods replaced - String keyWithoutDots = propKey.replace('.', ENVIRONMENT_VARIABLE_DOT_REPLACEMENT); + String keyWithoutDots = nsDelimiterReplacedPropKey.replace('.', ENVIRONMENT_VARIABLE_DOT_REPLACEMENT); value = getEnvironmentVariable(keyWithoutDots); if (value != null) { logInexactEnvNameMatch(propKey, keyWithoutDots); @@ -260,7 +283,7 @@ protected String lookupEnvironmentVariableValue(String propKey) { // Applications may define environment variable names with lower case, but only upper case is POSIX compliant for the environment. // To override from a shell, we should also check for upper case. // 3. In Uppercase, original periods - String uppercasedKey = propKey.toUpperCase(); + String uppercasedKey = nsDelimiterReplacedPropKey.toUpperCase(); value = getEnvironmentVariable(uppercasedKey); if (value != null) { logInexactEnvNameMatch(propKey, uppercasedKey); @@ -507,9 +530,9 @@ public Map getPropertyMap(String key, Map defaul String keyPrefix = toCollectionKeyPrefix(key, namespace).toString(); Map result = new HashMap<>(); - collectMapEntriesWith(keyPrefix, System.getenv().keySet(), result); - collectMapEntriesWith(keyPrefix, m_configProperties.keySet(), result); - collectMapEntriesWith(keyPrefix, System.getProperties().keySet(), result); + collectMapEntriesWith(keyPrefix, m_configProperties.keySet(), this::getConfigPropertyValue, result); + collectMapEntriesFromEnvironment(key, namespace, result); + collectMapEntriesWith(keyPrefix, System.getProperties().keySet(), this::getSystemPropertyValue, result); if (result.isEmpty()) { return defaultValue; @@ -795,13 +818,34 @@ public boolean isInitialized() { return m_isInitialized; } - protected void collectMapEntriesWith(String keyPrefix, Set keySet, Map collector) { + protected void collectMapEntriesWith(String keyPrefix, Set keySet, Function propertyValueRetriever, Map collector) { for (Object propKey : keySet) { String k = propKey.toString(); String mapKey = toMapKey(k, keyPrefix); if (mapKey != null) { - // we can overwrite here because the old entry has already the same value - collector.put(mapKey, getProperty(k)); + collector.put(mapKey, propertyValueRetriever.apply(k)); + } + } + } + + protected void collectMapEntriesFromEnvironment(String key, String namespace, Map collector) { + String envKey = toPropertyKey(key, namespace).toString(); + String keyValueFromEnv = lookupEnvironmentVariableValue(envKey); + if (StringUtility.hasText(keyValueFromEnv)) { + try { + JSONObject jsonObject = new JSONObject(keyValueFromEnv); + for (String mapKey : jsonObject.keySet()) { + Object mapValue = jsonObject.get(mapKey); + if (JSONObject.NULL.equals(mapValue)) { + collector.remove(mapKey); + } + else { + collector.put(mapKey, resolve((String) mapValue, PLACEHOLDER_PATTERN)); + } + } + } + catch (JSONException e) { + throw new IllegalArgumentException(String.format("Error parsing value of environment variable '%s' as JSON value", envKey), e); } } }