Skip to content

Commit 243edfe

Browse files
Hisoka-Xruanwenjun
Hisoka-X
and
ruanwenjun
authored
[Improve] Remove useless ReadonlyConfig flatten feature (#5612)
 --------- Co-authored-by: Wenjun Ruan <wenjun@apache.org>
1 parent 38764f1 commit 243edfe

File tree

13 files changed

+79
-569
lines changed

13 files changed

+79
-569
lines changed

‎seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/ConfigAdapter.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,10 @@ public interface ConfigAdapter {
3131
String[] extensionIdentifiers();
3232

3333
/**
34-
* Converter config file to path_key-value Map (FlattenedMap) in HOCON
34+
* Converter config file to path_key-value Map in HOCON
3535
*
36-
* @see org.apache.seatunnel.api.configuration.util.ConfigUtil#flatteningMap(Map)
3736
* @param configFilePath config file path.
38-
* @return FlattenedMap
37+
* @return Map
3938
*/
4039
Map<String, Object> loadConfig(Path configFilePath);
4140
}

‎seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/ReadonlyConfig.java‎

Lines changed: 22 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,6 @@
3434

3535
import static org.apache.seatunnel.api.configuration.util.ConfigUtil.convertToJsonString;
3636
import static org.apache.seatunnel.api.configuration.util.ConfigUtil.convertValue;
37-
import static org.apache.seatunnel.api.configuration.util.ConfigUtil.flatteningMap;
38-
import static org.apache.seatunnel.api.configuration.util.ConfigUtil.treeMap;
3937

4038
@Slf4j
4139
public class ReadonlyConfig implements Serializable {
@@ -50,7 +48,7 @@ private ReadonlyConfig(Map<String, Object> confData) {
5048
}
5149

5250
public static ReadonlyConfig fromMap(Map<String, Object> map) {
53-
return new ReadonlyConfig(treeMap(map));
51+
return new ReadonlyConfig(map);
5452
}
5553

5654
public static ReadonlyConfig fromConfig(Config config) {
@@ -65,68 +63,38 @@ public static ReadonlyConfig fromConfig(Config config) {
6563
}
6664

6765
public <T> T get(Option<T> option) {
68-
return get(option, true);
69-
}
70-
71-
public <T> T get(Option<T> option, boolean flatten) {
72-
return getOptional(option, flatten).orElseGet(option::defaultValue);
73-
}
74-
75-
public Map<String, String> toMap() {
76-
return toMap(true);
77-
}
78-
79-
public Config toConfig() {
80-
return toConfig(true);
66+
return getOptional(option).orElseGet(option::defaultValue);
8167
}
8268

8369
/**
8470
* Transform to Config todo: This method should be removed after we remove Config
8571
*
8672
* @return Config
8773
*/
88-
public Config toConfig(boolean flatten) {
89-
if (flatten) {
90-
return ConfigFactory.parseMap(flatteningMap(confData));
91-
}
74+
public Config toConfig() {
9275
return ConfigFactory.parseMap(confData);
9376
}
9477

95-
public Map<String, String> toMap(boolean flatten) {
78+
public Map<String, String> toMap() {
9679
if (confData.isEmpty()) {
9780
return Collections.emptyMap();
9881
}
9982

10083
Map<String, String> result = new LinkedHashMap<>();
101-
toMap(result, flatten);
84+
toMap(result);
10285
return result;
10386
}
10487

10588
public void toMap(Map<String, String> result) {
106-
toMap(result, true);
107-
}
108-
109-
public void toMap(Map<String, String> result, boolean flatten) {
11089
if (confData.isEmpty()) {
11190
return;
11291
}
113-
Map<String, Object> map;
114-
if (flatten) {
115-
map = flatteningMap(confData);
116-
} else {
117-
map = confData;
118-
}
119-
for (Map.Entry<String, Object> entry : map.entrySet()) {
92+
for (Map.Entry<String, Object> entry : confData.entrySet()) {
12093
result.put(entry.getKey(), convertToJsonString(entry.getValue()));
12194
}
12295
}
12396

12497
public <T> Optional<T> getOptional(Option<T> option) {
125-
return getOptional(option, true);
126-
}
127-
128-
@SuppressWarnings("unchecked")
129-
public <T> Optional<T> getOptional(Option<T> option, boolean flatten) {
13098
if (option == null) {
13199
throw new NullPointerException("Option not be null.");
132100
}
@@ -146,24 +114,28 @@ public <T> Optional<T> getOptional(Option<T> option, boolean flatten) {
146114
if (value == null) {
147115
return Optional.empty();
148116
}
149-
return Optional.of(convertValue(value, option, flatten));
117+
return Optional.of(convertValue(value, option));
150118
}
151119

152120
private Object getValue(String key) {
153-
String[] keys = key.split("\\.");
154-
Map<String, Object> data = this.confData;
155-
Object value = null;
156-
for (int i = 0; i < keys.length; i++) {
157-
value = data.get(keys[i]);
158-
if (i < keys.length - 1) {
159-
if (!(value instanceof Map)) {
160-
return null;
161-
} else {
162-
data = (Map<String, Object>) value;
121+
if (this.confData.containsKey(key)) {
122+
return this.confData.get(key);
123+
} else {
124+
String[] keys = key.split("\\.");
125+
Map<String, Object> data = this.confData;
126+
Object value = null;
127+
for (int i = 0; i < keys.length; i++) {
128+
value = data.get(keys[i]);
129+
if (i < keys.length - 1) {
130+
if (!(value instanceof Map)) {
131+
return null;
132+
} else {
133+
data = (Map<String, Object>) value;
134+
}
163135
}
164136
}
137+
return value;
165138
}
166-
return value;
167139
}
168140

169141
@Override

‎seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigUtil.java‎

Lines changed: 5 additions & 208 deletions
Original file line numberDiff line numberDiff line change
@@ -20,233 +20,27 @@
2020
import org.apache.seatunnel.shade.com.fasterxml.jackson.core.JsonProcessingException;
2121
import org.apache.seatunnel.shade.com.fasterxml.jackson.core.type.TypeReference;
2222
import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.ObjectMapper;
23-
import org.apache.seatunnel.shade.com.fasterxml.jackson.dataformat.javaprop.JavaPropsMapper;
2423
import org.apache.seatunnel.shade.com.typesafe.config.Config;
2524
import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
2625

2726
import org.apache.seatunnel.api.configuration.Option;
28-
import org.apache.seatunnel.api.table.catalog.schema.TableSchemaOptions;
29-
30-
import org.apache.commons.lang3.StringEscapeUtils;
31-
import org.apache.commons.lang3.StringUtils;
3227

3328
import lombok.extern.slf4j.Slf4j;
3429

3530
import java.lang.reflect.ParameterizedType;
36-
import java.util.ArrayList;
3731
import java.util.Arrays;
38-
import java.util.Collections;
39-
import java.util.LinkedHashMap;
4032
import java.util.List;
4133
import java.util.Locale;
42-
import java.util.Map;
4334
import java.util.stream.Collectors;
4435

4536
@Slf4j
4637
public class ConfigUtil {
47-
private static final JavaPropsMapper PROPERTIES_MAPPER = new JavaPropsMapper();
48-
private static final ObjectMapper JACKSON_MAPPER = new ObjectMapper();
49-
50-
/**
51-
*
52-
*
53-
* <pre>
54-
* poll.timeout = 1000
55-
* ==>> poll : {timeout = 1000, interval = 500}
56-
* poll.interval = 500
57-
* </pre>
58-
*/
59-
public static Map<String, Object> treeMap(Map<String, Object> rawMap) {
60-
try {
61-
Map<List<String>, String> properties =
62-
Arrays.stream(PROPERTIES_MAPPER.writeValueAsString(rawMap).split("\n"))
63-
.filter(StringUtils::isNoneEmpty)
64-
.map(line -> line.split("=", 2))
65-
.collect(
66-
Collectors.toMap(
67-
kv -> Arrays.asList(kv[0].split("\\.")),
68-
kv -> kv[1],
69-
(o, n) -> o,
70-
LinkedHashMap::new));
71-
Map<String, Object> result = loadPropertiesStyleMap(properties);
72-
// Special case, we shouldn't change key in schema config.
73-
// TODO we should not hard code it, it should be as a config.
74-
if (rawMap.containsKey(TableSchemaOptions.SCHEMA.key())) {
75-
result.put(
76-
TableSchemaOptions.SCHEMA.key(),
77-
rawMap.get(TableSchemaOptions.SCHEMA.key()));
78-
}
79-
return result;
80-
} catch (JsonProcessingException e) {
81-
throw new IllegalArgumentException("Json parsing exception.");
82-
}
83-
}
8438

85-
private static Map<String, Object> loadPropertiesStyleMap(
86-
Map<List<String>, String> properties) {
87-
Map<String, Object> propertiesMap = new LinkedHashMap<>();
88-
Map<List<String>, String> temp = new LinkedHashMap<>();
89-
String tempPrefix = null;
90-
for (Map.Entry<List<String>, String> entry : properties.entrySet()) {
91-
String key = entry.getKey().get(0);
92-
if (!key.equals(tempPrefix)) {
93-
putKeyValueToMapCheck(propertiesMap, temp, tempPrefix);
94-
tempPrefix = key;
95-
}
96-
if (entry.getKey().size() > 1) {
97-
temp.put(entry.getKey().subList(1, entry.getKey().size()), entry.getValue());
98-
} else if (!temp.isEmpty()) {
99-
temp.put(Collections.singletonList(""), entry.getValue());
100-
} else {
101-
temp.put(null, entry.getValue());
102-
}
103-
}
104-
putKeyValueToMapCheck(propertiesMap, temp, tempPrefix);
105-
return propertiesMap;
106-
}
107-
108-
private static void putKeyValueToMapCheck(
109-
Map<String, Object> propertiesMap, Map<List<String>, String> temp, String tempPrefix) {
110-
if (!temp.isEmpty()) {
111-
if (propertiesMap.containsKey(tempPrefix)) {
112-
if (temp.containsKey(null)) {
113-
((Map) propertiesMap.get(tempPrefix)).put("", temp.get(null));
114-
} else if (propertiesMap.get(tempPrefix) instanceof String) {
115-
loadPropertiesStyleMap(temp).put("", propertiesMap.get(tempPrefix));
116-
} else {
117-
mergeTwoMap((Map) propertiesMap.get(tempPrefix), loadPropertiesStyleMap(temp));
118-
}
119-
} else {
120-
propertiesMap.put(tempPrefix, loadPropertiesStyleObject(temp));
121-
}
122-
temp.clear();
123-
}
124-
}
125-
126-
private static void mergeTwoMap(Map<String, Object> base, Map<String, Object> merged) {
127-
for (Map.Entry<String, Object> entry : merged.entrySet()) {
128-
if (base.containsKey(entry.getKey())) {
129-
if (base.get(entry.getKey()) instanceof Map && entry.getValue() instanceof Map) {
130-
mergeTwoMap((Map) base.get(entry.getKey()), (Map) entry.getValue());
131-
} else if (base.get(entry.getKey()) instanceof Map) {
132-
((Map) base.get(entry.getKey())).put("", entry.getValue());
133-
} else if (entry.getValue() instanceof Map) {
134-
Map<String, Object> child = new LinkedHashMap<>();
135-
child.put("", base.get(entry.getKey()));
136-
child.putAll((Map) entry.getValue());
137-
base.put(entry.getKey(), child);
138-
} else {
139-
throw new IllegalArgumentException(
140-
String.format(
141-
"Duplicate key '%s' in config file, value '%s' and value '%s'",
142-
entry.getKey(), base.get(entry.getKey()), entry.getValue()));
143-
}
144-
} else {
145-
base.put(entry.getKey(), entry.getValue());
146-
}
147-
}
148-
}
149-
150-
private static List<Object> loadPropertiesStyleList(Map<List<String>, String> properties) {
151-
List<Object> propertiesList = new ArrayList<>();
152-
Map<List<String>, String> temp = new LinkedHashMap<>();
153-
int tempIndex = -1;
154-
for (Map.Entry<List<String>, String> entry : properties.entrySet()) {
155-
int index = Integer.parseInt(entry.getKey().get(0));
156-
if (index != tempIndex) {
157-
if (!temp.isEmpty()) {
158-
propertiesList.add(loadPropertiesStyleObject(temp));
159-
temp.clear();
160-
}
161-
tempIndex = index;
162-
}
163-
if (entry.getKey().size() == 1) {
164-
temp.put(null, entry.getValue());
165-
} else {
166-
temp.put(entry.getKey().subList(1, entry.getKey().size()), entry.getValue());
167-
}
168-
}
169-
if (!temp.isEmpty()) {
170-
propertiesList.add(loadPropertiesStyleObject(temp));
171-
}
172-
return propertiesList;
173-
}
174-
175-
private static Object loadPropertiesStyleObject(Map<List<String>, String> properties) {
176-
if (properties.containsKey(null) && properties.size() == 1) {
177-
return StringEscapeUtils.unescapeJava(properties.get(null));
178-
} else if (properties.containsKey(null)) {
179-
if (properties.containsKey(null)) {
180-
properties.put(Collections.singletonList(""), properties.get(null));
181-
properties.remove(null);
182-
}
183-
return loadPropertiesStyleMap(properties);
184-
} else if (properties.entrySet().stream().anyMatch(kv -> kv.getKey().get(0).equals("1"))) {
185-
return loadPropertiesStyleList(properties);
186-
} else {
187-
return loadPropertiesStyleMap(properties);
188-
}
189-
}
190-
191-
@SuppressWarnings("unchecked")
192-
static Object flatteningMap(
193-
Object rawValue, Map<String, Object> newMap, List<String> keys, boolean nestedMap) {
194-
if (rawValue == null) {
195-
return null;
196-
}
197-
if (!(rawValue instanceof List) && !(rawValue instanceof Map)) {
198-
if (newMap == null) {
199-
return rawValue;
200-
}
201-
newMap.put(String.join(".", keys), rawValue);
202-
return newMap;
203-
}
204-
205-
if (rawValue instanceof List) {
206-
List<Object> rawList = (List<Object>) rawValue;
207-
rawList.replaceAll(value -> flatteningMap(value, null, null, false));
208-
if (newMap != null) {
209-
newMap.put(String.join(".", keys), rawList);
210-
return newMap;
211-
}
212-
return rawList;
213-
} else {
214-
Map<String, Object> rawMap = (Map<String, Object>) rawValue;
215-
if (!nestedMap) {
216-
keys = new ArrayList<>();
217-
newMap = new LinkedHashMap<>(rawMap.size());
218-
}
219-
for (Map.Entry<String, Object> entry : rawMap.entrySet()) {
220-
keys.add(entry.getKey());
221-
flatteningMap(entry.getValue(), newMap, keys, true);
222-
keys.remove(keys.size() - 1);
223-
}
224-
return newMap;
225-
}
226-
}
227-
228-
/**
229-
*
230-
*
231-
* <pre>
232-
* poll.timeout = 1000
233-
* poll : {timeout = 1000, interval = 500} ==>>
234-
* poll.interval = 500
235-
* </pre>
236-
*/
237-
@SuppressWarnings("unchecked")
238-
public static Map<String, Object> flatteningMap(Map<String, Object> treeMap) {
239-
return (Map<String, Object>) flatteningMapWithObject(treeMap);
240-
}
241-
242-
static Object flatteningMapWithObject(Object rawValue) {
243-
return flatteningMap(rawValue, null, null, false);
244-
}
39+
private static final ObjectMapper JACKSON_MAPPER = new ObjectMapper();
24540

24641
@SuppressWarnings("unchecked")
247-
public static <T> T convertValue(Object rawValue, Option<T> option, boolean flatten) {
42+
public static <T> T convertValue(Object rawValue, Option<T> option) {
24843
TypeReference<T> typeReference = option.typeReference();
249-
rawValue = flatten ? flatteningMapWithObject(rawValue) : rawValue;
25044
if (typeReference.getType() instanceof Class) {
25145
// simple type
25246
Class<T> clazz = (Class<T>) typeReference.getType();
@@ -404,6 +198,9 @@ static <E extends Enum<?>> E convertToEnum(Object o, Class<E> clazz) {
404198
}
405199

406200
public static String convertToJsonString(Object o) {
201+
if (o == null) {
202+
return null;
203+
}
407204
if (o instanceof String) {
408205
return (String) o;
409206
}

0 commit comments

Comments
 (0)