Skip to content

Commit 60561ff

Browse files
rgoersrgoers
committed
FLUME-3433 - Limit the Lookups used by MapResolver
1 parent ad41383 commit 60561ff

File tree

5 files changed

+216
-1
lines changed

5 files changed

+216
-1
lines changed

‎flume-ng-node/src/main/java/org/apache/flume/node/MapResolver.java‎

Lines changed: 96 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,22 @@
1717
*/
1818
package org.apache.flume.node;
1919

20+
import java.io.FileInputStream;
21+
import java.io.IOException;
22+
import java.io.InputStream;
23+
import java.util.Arrays;
2024
import java.util.HashMap;
25+
import java.util.Locale;
2126
import java.util.Map;
27+
import java.util.Objects;
2228
import java.util.Properties;
2329

2430
import org.apache.commons.text.StringSubstitutor;
31+
import org.apache.commons.text.lookup.DefaultStringLookup;
2532
import org.apache.commons.text.lookup.StringLookup;
2633
import org.apache.commons.text.lookup.StringLookupFactory;
34+
import org.slf4j.Logger;
35+
import org.slf4j.LoggerFactory;
2736

2837
/**
2938
* Resolves replaceable tokens to create a Map.
@@ -32,22 +41,70 @@
3241
*/
3342
final class MapResolver {
3443

44+
private static final Logger LOGGER = LoggerFactory.getLogger(MapResolver.class);
45+
private static final String DEFAULT_LOOKUPS = "lookups.properties";
46+
private static final String CUSTOM_LOOKUPS_KEY = "lookups";
3547
private static final String PROPS_IMPL_KEY = "propertiesImplementation";
3648
private static final String ENV_VAR_PROPERTY = "org.apache.flume.node.EnvVarResolverProperties";
49+
private static final String LOOKUP = "org.apache.commons.text.lookup.DefaultStringLookup.";
50+
private static final LookupEntry[] LOOKUP_ENTRIES = {
51+
new LookupEntry("sys", DefaultStringLookup.SYSTEM_PROPERTIES.getStringLookup()),
52+
new LookupEntry("env", DefaultStringLookup.ENVIRONMENT.getStringLookup()) ,
53+
new LookupEntry("java", DefaultStringLookup.JAVA.getStringLookup()),
54+
new LookupEntry("date", DefaultStringLookup.DATE.getStringLookup())
55+
};
3756

3857
public static Map<String, String> resolveProperties(Properties properties) {
3958
Map<String, String> map = new HashMap<>();
4059
boolean useEnvVars = ENV_VAR_PROPERTY.equals(System.getProperty(PROPS_IMPL_KEY));
4160
StringLookup defaultLookup = useEnvVars ? new DefaultLookup(map) :
4261
StringLookupFactory.INSTANCE.mapStringLookup(map);
43-
StringLookup lookup = StringLookupFactory.INSTANCE.interpolatorStringLookup(defaultLookup);
62+
StringLookup lookup = StringLookupFactory.INSTANCE.interpolatorStringLookup(createLookupMap(),
63+
defaultLookup, false);
4464
StringSubstitutor substitutor = new StringSubstitutor(lookup);
4565
substitutor.setEnableSubstitutionInVariables(true);
4666
properties.stringPropertyNames().forEach((k) -> map.put(k,
4767
substitutor.replace(properties.getProperty(k))));
4868
return map;
4969
}
5070

71+
private static Map<String, StringLookup> createLookupMap() {
72+
Map<String, StringLookup> map = new HashMap<>();
73+
Properties properties = loadProperties();
74+
if (properties == null) {
75+
Arrays.stream(LOOKUP_ENTRIES).forEach((e) -> {
76+
map.put(e.key, e.lookup);
77+
});
78+
} else {
79+
properties.forEach((k, v) -> {
80+
String key = Objects.toString(k);
81+
String value = Objects.toString(v);
82+
if (value.startsWith(LOOKUP)) {
83+
String lookupEnum = value.substring(LOOKUP.length());
84+
try {
85+
StringLookup stringLookup = DefaultStringLookup.valueOf(lookupEnum).getStringLookup();
86+
map.put(key.toLowerCase(Locale.ROOT), stringLookup);
87+
} catch (IllegalArgumentException ex) {
88+
LOGGER.warn("{} is not a DefaultStringLookup enum value, ignoring", key);
89+
}
90+
} else {
91+
try {
92+
Class<?> clazz = Class.forName(Objects.toString(v));
93+
if (StringLookup.class.isAssignableFrom(clazz)) {
94+
StringLookup stringLookup = (StringLookup) clazz.newInstance();
95+
map.put(k.toString().toLowerCase(Locale.ROOT), stringLookup);
96+
} else {
97+
LOGGER.warn("{} is not a StringLookup, ignoring", v);
98+
}
99+
} catch (Exception ex) {
100+
LOGGER.warn("Unable to load {} due to {}, ignoring", v, ex.getMessage());
101+
}
102+
}
103+
});
104+
}
105+
return map;
106+
}
107+
51108
private static class DefaultLookup implements StringLookup {
52109
private final Map<String, String> properties;
53110

@@ -67,4 +124,42 @@ public String lookup(String key) {
67124
properties.get(key) : System.getenv(key);
68125
}
69126
}
127+
128+
private static class LookupEntry {
129+
private final String key;
130+
private final StringLookup lookup;
131+
132+
public LookupEntry(String key, StringLookup lookup) {
133+
this.key = key;
134+
this.lookup = lookup;
135+
}
136+
}
137+
138+
private static Properties loadProperties() {
139+
final Properties properties = new Properties();
140+
String fileName = System.getProperty(CUSTOM_LOOKUPS_KEY);
141+
if (fileName != null) {
142+
try (InputStream inputStream = new FileInputStream(fileName)) {
143+
properties.load(inputStream);
144+
} catch (final IOException e) {
145+
try (InputStream inputStream = ClassLoader.getSystemResourceAsStream(fileName)) {
146+
properties.load(inputStream);
147+
} catch (final IOException ex) {
148+
LOGGER.warn("Unable to load {} due to {}", fileName, ex.getMessage());
149+
}
150+
}
151+
}
152+
if (properties.size() == 0) {
153+
try (InputStream inputStream = ClassLoader.getSystemResourceAsStream(DEFAULT_LOOKUPS)) {
154+
if (inputStream != null) {
155+
properties.load(inputStream);
156+
} else {
157+
return null;
158+
}
159+
} catch (final IOException e) {
160+
return null;
161+
}
162+
}
163+
return properties;
164+
}
70165
}

‎flume-ng-node/src/test/java/org/apache/flume/node/TestMapResolver.java‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache license, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the license for the specific language governing permissions and
15+
* limitations under the license.
16+
*/
17+
package org.apache.flume.node;
18+
19+
import java.io.FileInputStream;
20+
import java.util.Map;
21+
import java.util.Properties;
22+
23+
import org.junit.After;
24+
import org.junit.Test;
25+
26+
import static org.junit.Assert.assertEquals;
27+
import static org.junit.Assert.assertFalse;
28+
import static org.junit.Assert.assertNotNull;
29+
import static org.junit.Assert.assertTrue;
30+
31+
/**
32+
* Tests the MapResolver.
33+
*/
34+
public class TestMapResolver {
35+
36+
public static final String TEST_CONST = "Apache Flume";
37+
private static final String TEST_PROPS = "target/test-classes/map-resolver.properties";
38+
private static final String NAME_VALUE = "FLUME";
39+
40+
@After
41+
public void after() {
42+
System.clearProperty("lookups");
43+
}
44+
45+
@Test
46+
public void testDefaultResolver() throws Exception {
47+
Properties props = new Properties();
48+
props.load(new FileInputStream(TEST_PROPS));
49+
System.setProperty("name", NAME_VALUE);
50+
Map<String, String> properties = MapResolver.resolveProperties(props);
51+
String name = properties.get("name");
52+
assertNotNull("No name property", name);
53+
assertEquals("Incorrect system property resolution", NAME_VALUE, name);
54+
String testStr = properties.get("const");
55+
assertNotNull("No const property", testStr);
56+
assertTrue("Constant was resolved", testStr.contains("${const:"));
57+
String version = properties.get("version");
58+
assertNotNull("No Java property", version);
59+
assertFalse("Java lookup was not resolved", version.contains("${java:"));
60+
}
61+
62+
@Test
63+
public void testCustomResolver() throws Exception {
64+
Properties props = new Properties();
65+
props.load(new FileInputStream(TEST_PROPS));
66+
System.setProperty("name", NAME_VALUE);
67+
System.setProperty("lookups", "test-lookups.properties");
68+
Map<String, String> properties = MapResolver.resolveProperties(props);
69+
String name = properties.get("name");
70+
assertNotNull("No name property", name);
71+
assertEquals("Incorrect system property resolution", NAME_VALUE, name);
72+
String testStr = properties.get("const");
73+
assertNotNull("No const property", testStr);
74+
assertTrue("Constant was resolved", testStr.contains("${const:"));
75+
String version = properties.get("version");
76+
assertNotNull("No Java property", version);
77+
assertFalse("Java lookup was not resolved", version.contains("${java:"));
78+
String test = properties.get("test");
79+
assertNotNull("No Test property", version);
80+
assertEquals("Test lookup was not resolved", "Value", test);
81+
}
82+
83+
}

‎flume-ng-node/src/test/java/org/apache/flume/node/lookup/TestLookup.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache license, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the license for the specific language governing permissions and
15+
* limitations under the license.
16+
*/
17+
package org.apache.flume.node.lookup;
18+
19+
import org.apache.commons.text.lookup.StringLookup;
20+
21+
/**
22+
* Test Lookup.
23+
*/
24+
public class TestLookup implements StringLookup {
25+
@Override
26+
public String lookup(String key) {
27+
return key;
28+
}
29+
}

‎flume-ng-node/src/test/resources/map-resolver.properties‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
name = ${sys:name}
2+
const = ${const:org.apache.flume.node.TestMapResolver.TEST_CONST}
3+
version = ${java:version}
4+
test = ${test:Value}

‎flume-ng-node/src/test/resources/test-lookups.properties‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
sys = org.apache.commons.text.lookup.DefaultStringLookup.SYSTEM_PROPERTIES
2+
env = org.apache.commons.text.lookup.DefaultStringLookup.ENVIRONMENT
3+
java = org.apache.commons.text.lookup.DefaultStringLookup.JAVA
4+
test = org.apache.flume.node.lookup.TestLookup

0 commit comments

Comments
 (0)