-
Notifications
You must be signed in to change notification settings - Fork 985
/
FunctionRegistryHolder.java
417 lines (386 loc) · 17.4 KB
/
FunctionRegistryHolder.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.drill.exec.expr.fn.registry;
import org.apache.drill.shaded.guava.com.google.common.collect.ArrayListMultimap;
import org.apache.drill.shaded.guava.com.google.common.collect.ListMultimap;
import org.apache.drill.common.AutoCloseables.Closeable;
import org.apache.drill.common.concurrent.AutoCloseableLock;
import org.apache.drill.exec.expr.fn.DrillFuncHolder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* Function registry holder stores function implementations by jar name, function name.
* Contains two maps that hold data by jars and functions respectively.
* Jars map contains each jar as a key and map of all its functions with collection of function signatures as value.
* Functions map contains function name as key and map of its signatures and function holder as value.
* All maps and collections used are concurrent to guarantee memory consistency effects.
* Such structure is chosen to achieve maximum speed while retrieving data by jar or by function name,
* since we expect infrequent registry changes.
* Holder is designed to allow concurrent reads and single writes to keep data consistent.
* This is achieved by {@link ReadWriteLock} implementation usage.
* Holder has number version which indicates remote function registry version number it is in sync with.
* <p/>
* Structure example:
*
* <pre>
* JARS
* built-in -> upper -> upper(VARCHAR-REQUIRED)
* -> lower -> lower(VARCHAR-REQUIRED)
*
* First.jar -> upper -> upper(VARCHAR-OPTIONAL)
* -> custom_upper -> custom_upper(VARCHAR-REQUIRED)
* -> custom_upper(VARCHAR-OPTIONAL)
*
* Second.jar -> lower -> lower(VARCHAR-OPTIONAL)
* -> custom_upper -> custom_upper(VARCHAR-REQUIRED)
* -> custom_upper(VARCHAR-OPTIONAL)
*
* FUNCTIONS
* upper -> upper(VARCHAR-REQUIRED) -> function holder for upper(VARCHAR-REQUIRED)
* -> upper(VARCHAR-OPTIONAL) -> function holder for upper(VARCHAR-OPTIONAL)
*
* lower -> lower(VARCHAR-REQUIRED) -> function holder for lower(VARCHAR-REQUIRED)
* -> lower(VARCHAR-OPTIONAL) -> function holder for lower(VARCHAR-OPTIONAL)
*
* custom_upper -> custom_upper(VARCHAR-REQUIRED) -> function holder for custom_upper(VARCHAR-REQUIRED)
* -> custom_upper(VARCHAR-OPTIONAL) -> function holder for custom_upper(VARCHAR-OPTIONAL)
*
* custom_lower -> custom_lower(VARCHAR-REQUIRED) -> function holder for custom_lower(VARCHAR-REQUIRED)
* -> custom_lower(VARCHAR-OPTIONAL) -> function holder for custom_lower(VARCHAR-OPTIONAL)
* </pre>
* where
* <li><b>First.jar</b> is jar name represented by {@link String}.</li>
* <li><b>upper</b> is function name represented by {@link String}.</li>
* <li><b>upper(VARCHAR-REQUIRED)</b> is signature name represented by String which consist of function name, list of input parameters.</li>
* <li><b>function holder for upper(VARCHAR-REQUIRED)</b> is {@link DrillFuncHolder} initiated for each function.</li>
*
*/
public class FunctionRegistryHolder {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionRegistryHolder.class);
private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
private final AutoCloseableLock readLock = new AutoCloseableLock(readWriteLock.readLock());
private final AutoCloseableLock writeLock = new AutoCloseableLock(readWriteLock.writeLock());
// remote function registry number, it is in sync with
private int version;
// jar name, Map<function name, Queue<function signature>
private final Map<String, Map<String, Queue<String>>> jars;
// function name, Map<function signature, function holder>
private final Map<String, Map<String, DrillFuncHolder>> functions;
public FunctionRegistryHolder() {
this.functions = new ConcurrentHashMap<>();
this.jars = new ConcurrentHashMap<>();
}
/**
* This is read operation, so several users at a time can get this data.
* @return local function registry version number
*/
public int getVersion() {
try (@SuppressWarnings("unused") Closeable lock = readLock.open()) {
return version;
}
}
/**
* Adds jars to the function registry.
* If jar with the same name already exists, it and its functions will be removed.
* Then jar will be added to {@link #jars}
* and each function will be added using {@link #addFunctions(Map, List)}.
* Registry version is updated with passed version if all jars were added successfully.
* This is write operation, so one user at a time can call perform such action,
* others will wait till first user completes his action.
*
* @param newJars jars and list of their function holders, each contains function name, signature and holder
*/
public void addJars(Map<String, List<FunctionHolder>> newJars, int version) {
try (@SuppressWarnings("unused") Closeable lock = writeLock.open()) {
for (Map.Entry<String, List<FunctionHolder>> newJar : newJars.entrySet()) {
String jarName = newJar.getKey();
removeAllByJar(jarName);
Map<String, Queue<String>> jar = new ConcurrentHashMap<>();
jars.put(jarName, jar);
addFunctions(jar, newJar.getValue());
}
this.version = version;
}
}
/**
* Removes jar from {@link #jars} and all associated with jar functions from {@link #functions}
* This is write operation, so one user at a time can call perform such action,
* others will wait till first user completes his action.
*
* @param jarName jar name to be removed
*/
public void removeJar(String jarName) {
try (@SuppressWarnings("unused") Closeable lock = writeLock.open()) {
removeAllByJar(jarName);
}
}
/**
* Removes all jars from {@link #jars} map and all associated with jars functions
* from {@link #functions}.
* This is write operation, so one user at a time can call perform such action,
* others will wait till first user completes his action.
*/
public void removeAllJars() {
try (@SuppressWarnings("unused") Closeable lock = writeLock.open()) {
jars.keySet().stream()
.filter(jarName -> !jarName.equals(LocalFunctionRegistry.BUILT_IN))
.forEach(this::removeAllByJar);
}
}
/**
* Retrieves list of all jars name present in {@link #jars}
* This is read operation, so several users can get this data.
*
* @return list of all jar names
*/
public List<String> getAllJarNames() {
try (@SuppressWarnings("unused") Closeable lock = readLock.open()) {
return new ArrayList<>(jars.keySet());
}
}
/**
* Retrieves all functions (holders) associated with all the jars
* This is read operation, so several users can perform this operation at the same time.
* @return list of all functions, mapped by their sources
*/
public Map<String, List<FunctionHolder>> getAllJarsWithFunctionHolders() {
Map<String, List<FunctionHolder>> allFunctionHoldersByJar = new HashMap<>();
try (@SuppressWarnings("unused") Closeable lock = readLock.open()) {
for (String jarName : jars.keySet()) {
//Capture functionHolders here
List<FunctionHolder> drillFuncHolderList = new LinkedList<>();
Map<String, Queue<String>> functionsInJar = jars.get(jarName);
for (Map.Entry<String, Queue<String>> functionEntry : functionsInJar.entrySet()) {
String fnName = functionEntry.getKey();
Queue<String> fnSignatureList = functionEntry.getValue();
//Get all FunctionHolders (irrespective of source)
Map<String, DrillFuncHolder> functionHolders = functions.get(fnName);
//Iterate for matching entries and populate new Map
for (Map.Entry<String, DrillFuncHolder> entry : functionHolders.entrySet()) {
if (fnSignatureList.contains(entry.getKey())) {
drillFuncHolderList.add(new FunctionHolder(fnName, entry.getKey(), entry.getValue()));
}
}
}
allFunctionHoldersByJar.put(jarName, drillFuncHolderList);
}
}
return allFunctionHoldersByJar;
}
/**
* Retrieves all function names associated with the jar from {@link #jars}.
* Returns empty list if jar is not registered.
* This is read operation, so several users can perform this operation at the same time.
*
* @param jarName jar name
* @return list of functions names associated from the jar
*/
public List<String> getFunctionNamesByJar(String jarName) {
try (@SuppressWarnings("unused") Closeable lock = readLock.open()){
Map<String, Queue<String>> functions = jars.get(jarName);
return functions == null ? new ArrayList<>() : new ArrayList<>(functions.keySet());
}
}
/**
* Returns list of functions with list of function holders for each functions.
* Uses guava {@link ListMultimap} structure to return data.
* If no functions present, will return empty {@link ListMultimap}.
* If version holder is not null, updates it with current registry version number.
* This is read operation, so several users can perform this operation at the same time.
*
* @param version version holder
* @return all functions which their holders
*/
public ListMultimap<String, DrillFuncHolder> getAllFunctionsWithHolders(AtomicInteger version) {
try (@SuppressWarnings("unused") Closeable lock = readLock.open()) {
if (version != null) {
version.set(this.version);
}
ListMultimap<String, DrillFuncHolder> functionsWithHolders = ArrayListMultimap.create();
for (Map.Entry<String, Map<String, DrillFuncHolder>> function : functions.entrySet()) {
functionsWithHolders.putAll(function.getKey(), new ArrayList<>(function.getValue().values()));
}
return functionsWithHolders;
}
}
/**
* Returns list of functions with list of function holders for each functions without version number.
* This is read operation, so several users can perform this operation at the same time.
*
* @return all functions which their holders
*/
public ListMultimap<String, DrillFuncHolder> getAllFunctionsWithHolders() {
return getAllFunctionsWithHolders(null);
}
/**
* Returns list of functions with list of function signatures for each functions.
* Uses guava {@link ListMultimap} structure to return data.
* If no functions present, will return empty {@link ListMultimap}.
* This is read operation, so several users can perform this operation at the same time.
*
* @return all functions which their signatures
*/
public ListMultimap<String, String> getAllFunctionsWithSignatures() {
try (@SuppressWarnings("unused") Closeable lock = readLock.open()) {
ListMultimap<String, String> functionsWithSignatures = ArrayListMultimap.create();
for (Map.Entry<String, Map<String, DrillFuncHolder>> function : functions.entrySet()) {
functionsWithSignatures.putAll(function.getKey(), new ArrayList<>(function.getValue().keySet()));
}
return functionsWithSignatures;
}
}
/**
* Returns all function holders associated with function name.
* If function is not present, will return empty list.
* If version holder is not null, updates it with current registry version number.
* This is read operation, so several users can perform this operation at the same time.
*
* @param functionName function name
* @param version version holder
* @return list of function holders
*/
public List<DrillFuncHolder> getHoldersByFunctionName(String functionName, AtomicInteger version) {
try (@SuppressWarnings("unused") Closeable lock = readLock.open()) {
if (version != null) {
version.set(this.version);
}
Map<String, DrillFuncHolder> holders = functions.get(functionName);
return holders == null ? new ArrayList<>() : new ArrayList<>(holders.values());
}
}
/**
* Returns all function holders associated with function name without version number.
* This is read operation, so several users can perform this operation at the same time.
*
* @param functionName function name
* @return list of function holders
*/
public List<DrillFuncHolder> getHoldersByFunctionName(String functionName) {
return getHoldersByFunctionName(functionName, null);
}
/**
* Checks is jar is present in {@link #jars}.
* This is read operation, so several users can perform this operation at the same time.
*
* @param jarName jar name
* @return true if jar exists, else false
*/
public boolean containsJar(String jarName) {
try (@SuppressWarnings("unused") Closeable lock = readLock.open()) {
return jars.containsKey(jarName);
}
}
/**
* Returns quantity of functions stored in {@link #functions}.
* This is read operation, so several users can perform this operation at the same time.
*
* @return quantity of functions
*/
public int functionsSize() {
try (@SuppressWarnings("unused") Closeable lock = readLock.open()) {
return functions.size();
}
}
/**
* Looks which jar in {@link #jars} contains passed function signature.
* First looks by function name and if found checks if such function has passed function signature.
* Returns jar name if found matching function signature, else null.
* This is read operation, so several users can perform this operation at the same time.
*
* @param functionName function name
* @param functionSignature function signature
* @return jar name
*/
public String getJarNameByFunctionSignature(String functionName, String functionSignature) {
try (@SuppressWarnings("unused") Closeable lock = readLock.open()) {
for (Map.Entry<String, Map<String, Queue<String>>> jar : jars.entrySet()) {
Queue<String> functionSignatures = jar.getValue().get(functionName);
if (functionSignatures != null && functionSignatures.contains(functionSignature)) {
return jar.getKey();
}
}
}
return null;
}
/**
* Adds all function names and signatures to passed jar,
* adds all function names, their signatures and holders to {@link #functions}.
*
* @param jar jar where function to be added
* @param newFunctions collection of function holders, each contains function name, signature and holder.
*/
private void addFunctions(Map<String, Queue<String>> jar, List<FunctionHolder> newFunctions) {
for (FunctionHolder function : newFunctions) {
final String functionName = function.getName();
Queue<String> jarFunctions = jar.get(functionName);
if (jarFunctions == null) {
jarFunctions = new ConcurrentLinkedQueue<>();
jar.put(functionName, jarFunctions);
}
final String functionSignature = function.getSignature();
jarFunctions.add(functionSignature);
Map<String, DrillFuncHolder> signatures = functions.computeIfAbsent(functionName, k -> new ConcurrentHashMap<>());
signatures.put(functionSignature, function.getHolder());
}
}
/**
* Removes jar from {@link #jars} and all associated with jars functions from {@link #functions}
* Since each jar is loaded with separate class loader before
* removing we need to close class loader to release opened connection to jar.
* All jar functions have the same class loader, so we need to close only one time.
*
* @param jarName jar name to be removed
*/
private void removeAllByJar(String jarName) {
Map<String, Queue<String>> jar = jars.remove(jarName);
if (jar == null) {
return;
}
for (Map.Entry<String, Queue<String>> functionEntry : jar.entrySet()) {
final String function = functionEntry.getKey();
Map<String, DrillFuncHolder> functionHolders = functions.get(function);
Queue<String> functionSignatures = functionEntry.getValue();
for (Map.Entry<String, DrillFuncHolder> entry : functionHolders.entrySet()) {
if (functionSignatures.contains(entry.getKey())) {
ClassLoader classLoader = entry.getValue().getClassLoader();
if (classLoader instanceof AutoCloseable) {
try {
((AutoCloseable) classLoader).close();
} catch (Exception e) {
logger.warn("Problem during closing class loader", e);
}
}
break;
}
}
functionHolders.keySet().removeAll(functionSignatures);
if (functionHolders.isEmpty()) {
functions.remove(function);
}
}
}
}