Skip to content

Commit 53ff56b

Browse files
yanghuapan3793
yanghua
authored and
pan3793
committed
[KYUUBI #1525][FLINK] Implement GetCatalogs operation and do the further initialization
… <!-- Thanks for sending a pull request! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html 2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'. 3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'. --> ### _Why are the changes needed?_ <!-- Please clarify why the changes are needed. For instance, 1. If you add a feature, you can talk about the use case of it. 2. If you fix a bug, you can clarify why it is a bug. --> ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #1529 from yanghua/KYUUBI-1525. Closes #1525 e166965 [yanghua] Address review suggestion 9929aa2 [yanghua] Remove some comment eafb200 [yanghua] refactor code 58df34c [yanghua] refactor code 675e14e [yanghua] pruning unnecessary code 15e9728 [yanghua] pruning unnecessary code c6d4f54 [yanghua] pruning unnecessary code b099a45 [yanghua] pruning unnecessary code 316086b [yanghua] pruning unnecessary code fe7af8e [yanghua] pruning unnecessary code ecf3ccf [yanghua] pruning unnecessary code 4c17451 [yanghua] pruning unnecessary code b9226c3 [yanghua] pruning unnecessary code fb2f6a6 [yanghua] pruning unnecessary code 21e21a5 [yanghua] pruning unnecessary code a58076f [yanghua] pruning unnecessary code acd24fc [yanghua] pruning unnecessary code 333d427 [yanghua] pruning unnecessary code 7416b3d [yanghua] pruning unnecessary code 84d0db9 [yanghua] pruning unnecessary code and refactor code a55db41 [yanghua] pruning unnecessary code b990930 [yanghua] [KYUUBI #1525] Implement GetCatalogs operation and do the further initialization Authored-by: yanghua <yanghua1127@gmail.com> Signed-off-by: Cheng Pan <chengpan@apache.org>
1 parent 3594e08 commit 53ff56b

File tree

20 files changed

+2190
-0
lines changed

20 files changed

+2190
-0
lines changed

‎externals/kyuubi-flink-sql-engine/pom.xml

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,94 @@
4545
<version>${project.version}</version>
4646
</dependency>
4747

48+
<!-- flink -->
49+
<dependency>
50+
<groupId>org.apache.flink</groupId>
51+
<artifactId>flink-core</artifactId>
52+
<scope>provided</scope>
53+
</dependency>
54+
55+
<dependency>
56+
<groupId>org.apache.flink</groupId>
57+
<artifactId>flink-runtime_${scala.binary.version}</artifactId>
58+
<scope>provided</scope>
59+
</dependency>
60+
61+
<dependency>
62+
<groupId>org.apache.flink</groupId>
63+
<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
64+
<scope>provided</scope>
65+
</dependency>
66+
67+
<dependency>
68+
<groupId>org.apache.flink</groupId>
69+
<artifactId>flink-clients_${scala.binary.version}</artifactId>
70+
<scope>provided</scope>
71+
</dependency>
72+
73+
<dependency>
74+
<groupId>org.apache.flink</groupId>
75+
<artifactId>flink-table-common</artifactId>
76+
<scope>provided</scope>
77+
</dependency>
78+
79+
<dependency>
80+
<groupId>org.apache.flink</groupId>
81+
<artifactId>flink-table-api-java</artifactId>
82+
<scope>provided</scope>
83+
</dependency>
84+
85+
<dependency>
86+
<groupId>org.apache.flink</groupId>
87+
<artifactId>flink-table-api-java-bridge_${scala.binary.version}</artifactId>
88+
<scope>provided</scope>
89+
</dependency>
90+
91+
<dependency>
92+
<groupId>org.apache.flink</groupId>
93+
<artifactId>flink-table-planner-blink_${scala.binary.version}</artifactId>
94+
<scope>provided</scope>
95+
</dependency>
96+
97+
<dependency>
98+
<groupId>org.apache.flink</groupId>
99+
<artifactId>flink-table-runtime-blink_${scala.binary.version}</artifactId>
100+
<scope>provided</scope>
101+
</dependency>
102+
103+
<dependency>
104+
<groupId>org.apache.flink</groupId>
105+
<artifactId>flink-sql-parser</artifactId>
106+
<scope>provided</scope>
107+
</dependency>
108+
109+
<dependency>
110+
<groupId>org.apache.flink</groupId>
111+
<artifactId>flink-yarn_${scala.binary.version}</artifactId>
112+
<scope>provided</scope>
113+
</dependency>
114+
115+
<!-- tests -->
116+
<dependency>
117+
<groupId>org.apache.kyuubi</groupId>
118+
<artifactId>kyuubi-common_${scala.binary.version}</artifactId>
119+
<version>${project.version}</version>
120+
<type>test-jar</type>
121+
<scope>test</scope>
122+
</dependency>
123+
124+
<dependency>
125+
<groupId>org.apache.kyuubi</groupId>
126+
<artifactId>kyuubi-hive-jdbc-shaded</artifactId>
127+
<version>${project.version}</version>
128+
<scope>test</scope>
129+
</dependency>
130+
131+
<dependency>
132+
<groupId>org.slf4j</groupId>
133+
<artifactId>jul-to-slf4j</artifactId>
134+
<scope>test</scope>
135+
</dependency>
48136
</dependencies>
49137

50138
</project>

‎externals/kyuubi-flink-sql-engine/src/main/java/org/apache/kyuubi/engine/flink/config/EngineEnvironment.java

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.kyuubi.engine.flink.config;
20+
21+
import java.util.Collections;
22+
import java.util.Map;
23+
import org.apache.kyuubi.engine.flink.config.entries.ExecutionEntry;
24+
25+
/**
26+
* EngineEnvironment configuration that represents the content of an environment file.
27+
* EngineEnvironment files define engine, session, catalogs, tables, execution, and deployment
28+
* behavior. An environment might be defined by default or as part of a session. Environments can be
29+
* merged or enriched with properties.
30+
*/
31+
public class EngineEnvironment {
32+
33+
public static final String EXECUTION_ENTRY = "execution";
34+
35+
private ExecutionEntry execution;
36+
37+
public EngineEnvironment() {
38+
this.execution = ExecutionEntry.DEFAULT_INSTANCE;
39+
}
40+
41+
public ExecutionEntry getExecution() {
42+
return execution;
43+
}
44+
45+
@Override
46+
public String toString() {
47+
final StringBuilder sb = new StringBuilder();
48+
sb.append("=================== Execution ====================\n");
49+
execution.asTopLevelMap().forEach((k, v) -> sb.append(k).append(": ").append(v).append('\n'));
50+
return sb.toString();
51+
}
52+
53+
// --------------------------------------------------------------------------------------------
54+
55+
/**
56+
* Merges two environments. The properties of the first environment might be overwritten by the
57+
* second one.
58+
*/
59+
public static EngineEnvironment merge(EngineEnvironment env1, EngineEnvironment env2) {
60+
final EngineEnvironment mergedEnv = new EngineEnvironment();
61+
62+
// merge execution properties
63+
mergedEnv.execution = ExecutionEntry.merge(env1.getExecution(), env2.getExecution());
64+
65+
return mergedEnv;
66+
}
67+
68+
public EngineEnvironment clone() {
69+
return enrich(this, Collections.emptyMap());
70+
}
71+
72+
/** Enriches an environment with new/modified properties or views and returns the new instance. */
73+
public static EngineEnvironment enrich(EngineEnvironment env, Map<String, String> properties) {
74+
final EngineEnvironment enrichedEnv = new EngineEnvironment();
75+
76+
// enrich execution properties
77+
enrichedEnv.execution = ExecutionEntry.enrich(env.execution, properties);
78+
79+
return enrichedEnv;
80+
}
81+
}

‎externals/kyuubi-flink-sql-engine/src/main/java/org/apache/kyuubi/engine/flink/config/entries/ConfigEntry.java

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.kyuubi.engine.flink.config.entries;
20+
21+
import java.util.Map;
22+
import java.util.Objects;
23+
import org.apache.flink.table.api.ValidationException;
24+
import org.apache.flink.table.descriptors.DescriptorProperties;
25+
26+
/**
27+
* Describes an environment configuration entry (such as catalogs, table, functions, views). Config
28+
* entries are similar to {@link org.apache.flink.table.descriptors.Descriptor} but apply to SQL
29+
* Engine's environment files only.
30+
*/
31+
abstract class ConfigEntry {
32+
33+
protected final DescriptorProperties properties;
34+
35+
protected ConfigEntry(DescriptorProperties properties) {
36+
try {
37+
validate(properties);
38+
} catch (ValidationException e) {
39+
throw new RuntimeException("Invalid configuration entry.", e);
40+
}
41+
42+
this.properties = properties;
43+
}
44+
45+
/** Performs syntactic validation. */
46+
protected abstract void validate(DescriptorProperties properties);
47+
48+
public Map<String, String> asMap() {
49+
return properties.asMap();
50+
}
51+
52+
@Override
53+
public boolean equals(Object o) {
54+
if (this == o) {
55+
return true;
56+
}
57+
if (o == null || getClass() != o.getClass()) {
58+
return false;
59+
}
60+
ConfigEntry that = (ConfigEntry) o;
61+
return Objects.equals(properties, that.properties);
62+
}
63+
64+
@Override
65+
public int hashCode() {
66+
return Objects.hash(properties);
67+
}
68+
}

0 commit comments

Comments
 (0)