Skip to content

Commit

Permalink
Add support for environment variable property maps
Browse files Browse the repository at this point in the history 
Previously, it was not possible, to define property maps using
environment variables. The only viable workaround was to define the
property map in the config.properties file and use variables for certain
key values sourced from the environment to inject environment variable
values into property maps.
Although this workaround was functional in a certain way, it did not
allow for the definition of property map keys not present in the
config.properties file nor to override values of property map keys from
the config.properties file that were hardcoded.

This change introduces a mechanism to define property maps using
environment variables with the help of JSON object strings:
MAPPROPERTY={"keya": "valuea", "keyb": "valueb", "keyc": null}
Using this approach it is possible to
- introduce property map values not defined in any config.properties
  file: just create an environment variable with the appropriate name
  and specify the map as a JSON object string.
- define new property map keys: as above, just create the environment
  variable like above with the new key defined. The mixing of property
  map values is explained below.
- change the value of of a property map key already defined in the
  config.properties file: as above, just create an environment variable
  with the key to change and the new value.
- delete a property map key previously defined in the
  config.propertiesfile: as above, just create an environment variable
  with the key to delete and a value of 'null' w/o the quotes.
  The last three options may occur within the same JSON object string
  defined in an environment variable.

When property maps are defined a different levels, the usual precedence
applies: config.properties < environment variable < system property.

Change-Id: I2b2ee1fc472ad1ec29e414ce23a5101e37a7303b
Signed-off-by: Michael Iseli <michael.iseli@bsi-software.com>
Reviewed-on: https://git.eclipse.org/r/c/scout/org.eclipse.scout.rt/+/174274
Tested-by: Scout Bot <scout-bot@eclipse.org>
Reviewed-by: Ivan Motsch <ivan.motsch@bsiag.com>
  • Loading branch information
@michaelze @imotsch
michaelze authored and imotsch committed Feb 10, 2021
1 parent ddd6efb commit c1e78c1
Show file tree
Hide file tree
Showing 4 changed files with 217 additions and 22 deletions.
153 changes: 148 additions & 5 deletions ...latform.test/src/test/java/org/eclipse/scout/rt/platform/config/PropertiesHelperTest.java
View file
Expand Up @@ -10,11 +10,9 @@
******************************************************************************/
package org.eclipse.scout.rt.platform.config;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.*;
import static org.mockito.Mockito.*;

import java.io.IOException;
import java.net.MalformedURLException;
Expand All @@ -27,9 +25,13 @@
import java.util.List;
import java.util.Map;

import org.eclipse.scout.rt.platform.util.CollectionUtility;
import org.eclipse.scout.rt.platform.util.IOUtility;
import org.eclipse.scout.rt.platform.util.ImmutablePair;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.mockito.Mockito;

/**
Expand Down Expand Up @@ -72,6 +74,9 @@ public class PropertiesHelperTest {
private static final String MAP_KEY = "mapKey";
private static final String EMPTY_KEY = "emptyKey";

@Rule
public ExpectedException expectedException = ExpectedException.none();

@Test
public void testPropertiesHelper() throws Exception {
PropertiesHelper instance = new PropertiesHelper(SAMPLE_CONFIG_PROPS);
Expand All @@ -91,6 +96,10 @@ public void testNamespaceProperty() {
assertEquals(null, instance.getProperty(NAMESPACE_PROP + "-not-existing", null, NAMESPACE));
assertEquals(null, instance.getProperty(NAMESPACE_PROP, null, NAMESPACE + "-not-existing"));
assertEquals("defaultval", instance.getProperty(NAMESPACE_PROP, "defaultval", NAMESPACE + "-not-existing"));

PropertiesHelper spiedInstance = spy(instance);
when(spiedInstance.getEnvironmentVariable(NAMESPACE + "__" + NAMESPACE_PROP)).thenReturn(NAMESPACE_PROP_VAL + "-from-env");
assertThat(spiedInstance.getProperty(NAMESPACE_PROP, null, NAMESPACE), is(NAMESPACE_PROP_VAL + "-from-env"));
}

@Test
Expand Down Expand Up @@ -223,6 +232,140 @@ public void testPropertyList() {
}
}

@Test
public void testReadPropertyMapFromEnvironment() {
PropertiesHelper originalInstance = new PropertiesHelper(SAMPLE_CONFIG_PROPS);
PropertiesHelper spiedInstance = spy(originalInstance);
when(spiedInstance.getEnvironmentVariable("map_not_in_file")).thenReturn("{\"keya\": \"valuea\",\"keyb\": \"valueb\",\"keyc\": \"valuec\"}");

assertThat(spiedInstance.getPropertyMap("map.not.in.file"), is(CollectionUtility.hashMap(
new ImmutablePair<>("keya", "valuea"),
new ImmutablePair<>("keyb", "valueb"),
new ImmutablePair<>("keyc", "valuec"))));
}

@Test
public void testReadPropertyMapFromEnvironmentWithVariableReference() throws Exception {
PropertiesHelper originalInstance = new PropertiesHelper(SAMPLE_CONFIG_PROPS);
PropertiesHelper spiedInstance = spy(originalInstance);
try {
System.setProperty("sysProp", "sysPropVal");
System.setProperty("stringKey", "stringKeyValueFromSystemProperty");
when(spiedInstance.getEnvironmentVariable("envProp")).thenReturn("envPropVal");
when(spiedInstance.getEnvironmentVariable("intKey")).thenReturn("intKeyFromEnv");
when(spiedInstance.getEnvironmentVariable("map_not_in_file")).thenReturn("{" +
"\"propFromConfigProperties\": \"${otherProp}\"," +
"\"propFromSystemProperties\": \"${sysProp}\"," +
"\"propFromEnvironment\": \"${envProp}\"," +
"\"propFromConfigPropertiesOverriddenByEnv\": \"${intKey}\"," +
"\"propFromConfigPropertiesOverriddenBySystemProperty\": \"${stringKey}\"," +
"\"propFromConfigPropertiesInString\": \"test${longKey}testtest\"" +
"}");

assertThat(spiedInstance.getPropertyMap("map.not.in.file"), is(CollectionUtility.hashMap(
new ImmutablePair<>("propFromConfigProperties", "otherVal"),
new ImmutablePair<>("propFromSystemProperties", "sysPropVal"),
new ImmutablePair<>("propFromEnvironment", "envPropVal"),
new ImmutablePair<>("propFromConfigPropertiesOverriddenByEnv", "intKeyFromEnv"),
new ImmutablePair<>("propFromConfigPropertiesOverriddenBySystemProperty", "stringKeyValueFromSystemProperty"),
new ImmutablePair<>("propFromConfigPropertiesInString", "test2testtest"))));
}
finally {
System.clearProperty("sysProp");
System.clearProperty("stringKey");
}
}

@Test
public void testReadPropertyMapFromEnvironmentMixingWithOtherPropertyMapSources() throws Exception {
PropertiesHelper originalInstance = new PropertiesHelper(MAP_CONFIG_PROPS);
PropertiesHelper spiedInstance = spy(originalInstance);
when(spiedInstance.getEnvironmentVariable("mapKey")).thenReturn("{\"second\": \"zwei\", \"third\": null}");

assertThat(spiedInstance.getPropertyMap("mapKey"), is(CollectionUtility.hashMap(
new ImmutablePair<>("first", "one"),
new ImmutablePair<>("second", "zwei"),
new ImmutablePair<>("empty", null),
new ImmutablePair<>("last", "last"))));
}

@Test
public void testReadPropertyMapRespectingPrecedence() throws Exception {
PropertiesHelper originalInstance = new PropertiesHelper(MAP_CONFIG_PROPS);
PropertiesHelper spiedInstance = spy(originalInstance);
try {
System.setProperty("mapKey[first]", "one-from-system-property");
when(spiedInstance.getEnvironmentVariable("mapKey")).thenReturn("{\"first\": \"one-from-env\", \"second\": \"two-from-env\"}");

assertThat(spiedInstance.getPropertyMap("mapKey"), is(CollectionUtility.hashMap(
new ImmutablePair<>("first", "one-from-system-property"),
new ImmutablePair<>("second", "two-from-env"),
new ImmutablePair<>("third", "three"),
new ImmutablePair<>("empty", null),
new ImmutablePair<>("last", "last"))));
}
finally {
System.clearProperty("mapKey[first]");
}
}

@Test
public void testReadPropertyMapFromEnvironmentWithNamespace() throws Exception {
PropertiesHelper originalInstance = new PropertiesHelper(MAP_CONFIG_PROPS);
PropertiesHelper spiedInstance = spy(originalInstance);
when(spiedInstance.getEnvironmentVariable("mapKey")).thenReturn("{\"second\": \"zwei\", \"third\": null}");
when(spiedInstance.getEnvironmentVariable("namespace1__mapKey")).thenReturn("{\"a\": null, \"b\": \"b\", \"c\": \"ce\"}");
when(spiedInstance.getEnvironmentVariable("namespace2__mapKey")).thenReturn("{\"d\": \"de\", \"e\": \"ee\"}");

assertThat(spiedInstance.getPropertyMap("mapKey"), is(CollectionUtility.hashMap(
new ImmutablePair<>("first", "one"),
new ImmutablePair<>("second", "zwei"),
new ImmutablePair<>("empty", null),
new ImmutablePair<>("last", "last"))));
assertThat(spiedInstance.getPropertyMap("mapKey", "namespace1"), is(CollectionUtility.hashMap(
new ImmutablePair<>("b", "b"),
new ImmutablePair<>("c", "ce"))));
assertThat(spiedInstance.getPropertyMap("mapKey", "namespace2"), is(CollectionUtility.hashMap(
new ImmutablePair<>("d", "de"),
new ImmutablePair<>("e", "ee"))));
}

@Test
public void testExpectedExceptionForMalformedJsonStringMissingComma() {
PropertiesHelper originalInstance = new PropertiesHelper(MAP_CONFIG_PROPS);
PropertiesHelper spiedInstance = spy(originalInstance);

expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("Error parsing value of environment variable 'mapKey' as JSON value");

when(spiedInstance.getEnvironmentVariable("mapKey")).thenReturn("{\"second\": \"zwei\" \"third\": null}"); // missing comma after "zwei"
spiedInstance.getPropertyMap("mapKey");
}

@Test
public void testExpectedExceptionForMalformedJsonStringMissingClosingBraces() {
PropertiesHelper originalInstance = new PropertiesHelper(MAP_CONFIG_PROPS);
PropertiesHelper spiedInstance = spy(originalInstance);

expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("Error parsing value of environment variable 'mapKey' as JSON value");

when(spiedInstance.getEnvironmentVariable("mapKey")).thenReturn("{\"second\": \"zwei\", \"third\": null"); // missing } at the end
spiedInstance.getPropertyMap("mapKey");
}

@Test
public void testReadPropertyListFromEnvironment() throws Exception {
PropertiesHelper originalInstance = new PropertiesHelper(LIST_PROPS);
PropertiesHelper spiedInstance = spy(originalInstance);

when(spiedInstance.getEnvironmentVariable("list")).thenReturn("{\"0\": \"zero\", \"1\": \"one\", \"2\": \"two\", \"3\": \"three\"}");
when(spiedInstance.getEnvironmentVariable("listWithValidIndices")).thenReturn("{\"2\": \"two\", \"4\": \"four\", \"5\": \"five\"}");

assertThat(spiedInstance.getPropertyList("list"), is(Arrays.asList("zero", "one", "two", "three")));
assertThat(spiedInstance.getPropertyList("listWithValidIndices"), is(Arrays.asList("a", null, "two", "b", "four", "five")));
}

@Test
public void testPropertyString() {
PropertiesHelper instance = new PropertiesHelper(SAMPLE_CONFIG_PROPS);
Expand Down
3 changes: 3 additions & 0 deletions ...platform.test/src/test/resources/org/eclipse/scout/rt/platform/config/map-test.properties
View file
Expand Up @@ -19,3 +19,6 @@ namespace|mapKey[a]=b
namespace|mapKey[c]=c
namespace|mapKey[c=not-a-valid-map-property
namespace|mapKey[]=not-a-valid-map-property-2
namespace1|mapKey[a]=a
namespace1|mapKey[a]=b
namespace1|mapKey[c]=c
5 changes: 5 additions & 0 deletions org.eclipse.scout.rt.platform/pom.xml
View file
Expand Up @@ -26,6 +26,11 @@

<name>${project.groupId}:${project.artifactId}</name>
<dependencies>
<dependency>
<groupId>org.eclipse.scout.rt</groupId>
<artifactId>org.eclipse.scout.json</artifactId>
</dependency>

<!-- Build Dependencies -->
<dependency>
<groupId>org.jboss</groupId>
Expand Down
78 changes: 61 additions & 17 deletions ...cout.rt.platform/src/main/java/org/eclipse/scout/rt/platform/config/PropertiesHelper.java
View file
Expand Up @@ -22,13 +22,17 @@
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Properties;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.eclipse.scout.rt.platform.util.StringUtility;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -89,6 +93,7 @@ public class PropertiesHelper {
public static final String CLASSPATH_PROTOCOL_NAME = "classpath";
public static final char PROTOCOL_DELIMITER = ':';
public static final char NAMESPACE_DELIMITER = '|';
public static final String NAMESPACE_DELIMITER_FOR_ENV = "__";
public static final char COLLECTION_DELIMITER_START = '[';
public static final char COLLECTION_DELIMITER_END = ']';
public static final String CLASSPATH_PREFIX = CLASSPATH_PROTOCOL_NAME + PROTOCOL_DELIMITER;
Expand Down Expand Up @@ -206,27 +211,41 @@ public String getProperty(String key, String defaultValue, String namespace) {
}

String propKey = toPropertyKey(key, namespace).toString();
String value = null;

// system config
value = System.getProperty(propKey);
List<Function<String, String>> propertyValueRetrievers = Arrays.asList(
this::getSystemPropertyValue,
this::getEnvironmentPropertyValue,
this::getConfigPropertyValue);

return propertyValueRetrievers.stream()
.map(propertyValueRetriever -> propertyValueRetriever.apply(propKey))
.filter(Objects::nonNull)
.findFirst()
.orElse(defaultValue);
}

protected String getSystemPropertyValue(String propKey) {
String value = System.getProperty(propKey);
if (StringUtility.hasText(value)) {
return resolve(value, PLACEHOLDER_PATTERN);
}
return null;
}

// environment config
protected String getEnvironmentPropertyValue(String propKey) {
String envValue = lookupEnvironmentVariableValue(propKey);
if (StringUtility.hasText(envValue)) {
return resolve(envValue, PLACEHOLDER_PATTERN);
}
return null;
}

// properties file
value = m_configProperties.get(propKey);
protected String getConfigPropertyValue(String propKey) {
String value = m_configProperties.get(propKey);
if (StringUtility.hasText(value)) {
return value;
}

return defaultValue;
return null;
}

/**
Expand All @@ -239,18 +258,22 @@ public String getProperty(String key, String defaultValue, String namespace) {
* <li>Original in uppercase: <code>MY.PROPERTY</code></li>
* <li>Periods replaced, in uppercase: <code>MY_PROPERTY</code></li>
* </ol>
* The standard namespace delimiter (|) will always be replaced by a delimiter more suited for use in environment
* variables (__).
*/
protected String lookupEnvironmentVariableValue(String propKey) {
String nsDelimiterReplacedPropKey = propKey.replace(String.valueOf(NAMESPACE_DELIMITER), NAMESPACE_DELIMITER_FOR_ENV);

// 1. Original
String value = getEnvironmentVariable(propKey);
String value = getEnvironmentVariable(nsDelimiterReplacedPropKey);
if (value != null) {
return value;
}

// Periods in environment variable names are not POSIX compliant (See IEEE Standard 1003.1-2017, Chapter 8.1 "Environment Variable Definition"),
// but supported by some shells. To allow overriding via environment variables (Bugzilla 541099) in any shell, convert them to underscores.
// 2. With periods replaced
String keyWithoutDots = propKey.replace('.', ENVIRONMENT_VARIABLE_DOT_REPLACEMENT);
String keyWithoutDots = nsDelimiterReplacedPropKey.replace('.', ENVIRONMENT_VARIABLE_DOT_REPLACEMENT);
value = getEnvironmentVariable(keyWithoutDots);
if (value != null) {
logInexactEnvNameMatch(propKey, keyWithoutDots);
Expand All @@ -260,7 +283,7 @@ protected String lookupEnvironmentVariableValue(String propKey) {
// Applications may define environment variable names with lower case, but only upper case is POSIX compliant for the environment.
// To override from a shell, we should also check for upper case.
// 3. In Uppercase, original periods
String uppercasedKey = propKey.toUpperCase();
String uppercasedKey = nsDelimiterReplacedPropKey.toUpperCase();
value = getEnvironmentVariable(uppercasedKey);
if (value != null) {
logInexactEnvNameMatch(propKey, uppercasedKey);
Expand Down Expand Up @@ -507,9 +530,9 @@ public Map<String, String> getPropertyMap(String key, Map<String, String> defaul
String keyPrefix = toCollectionKeyPrefix(key, namespace).toString();

Map<String, String> result = new HashMap<>();
collectMapEntriesWith(keyPrefix, System.getenv().keySet(), result);
collectMapEntriesWith(keyPrefix, m_configProperties.keySet(), result);
collectMapEntriesWith(keyPrefix, System.getProperties().keySet(), result);
collectMapEntriesWith(keyPrefix, m_configProperties.keySet(), this::getConfigPropertyValue, result);
collectMapEntriesFromEnvironment(key, namespace, result);
collectMapEntriesWith(keyPrefix, System.getProperties().keySet(), this::getSystemPropertyValue, result);

if (result.isEmpty()) {
return defaultValue;
Expand Down Expand Up @@ -795,13 +818,34 @@ public boolean isInitialized() {
return m_isInitialized;
}

protected void collectMapEntriesWith(String keyPrefix, Set<?> keySet, Map<String, String> collector) {
protected void collectMapEntriesWith(String keyPrefix, Set<?> keySet, Function<String, String> propertyValueRetriever, Map<String, String> collector) {
for (Object propKey : keySet) {
String k = propKey.toString();
String mapKey = toMapKey(k, keyPrefix);
if (mapKey != null) {
// we can overwrite here because the old entry has already the same value
collector.put(mapKey, getProperty(k));
collector.put(mapKey, propertyValueRetriever.apply(k));
}
}
}

protected void collectMapEntriesFromEnvironment(String key, String namespace, Map<String, String> collector) {
String envKey = toPropertyKey(key, namespace).toString();
String keyValueFromEnv = lookupEnvironmentVariableValue(envKey);
if (StringUtility.hasText(keyValueFromEnv)) {
try {
JSONObject jsonObject = new JSONObject(keyValueFromEnv);
for (String mapKey : jsonObject.keySet()) {
Object mapValue = jsonObject.get(mapKey);
if (JSONObject.NULL.equals(mapValue)) {
collector.remove(mapKey);
}
else {
collector.put(mapKey, resolve((String) mapValue, PLACEHOLDER_PATTERN));
}
}
}
catch (JSONException e) {
throw new IllegalArgumentException(String.format("Error parsing value of environment variable '%s' as JSON value", envKey), e);
}
}
}
Expand Down

0 comments on commit c1e78c1

Please sign in to comment.