From 843e90d32ff551bf5f910a44e43c1d937484c697 Mon Sep 17 00:00:00 2001 From: xiaojie19852006 Date: Mon, 7 Feb 2022 14:30:00 +0800 Subject: [PATCH 1/6] add module linkis-metadata-manager-service-hive --- .../service/hive/pom.xml | 262 ++++++++++++++++++ .../linkis-datasource/pom.xml | 1 + 2 files changed, 263 insertions(+) create mode 100644 linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/pom.xml diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/pom.xml b/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/pom.xml new file mode 100644 index 0000000000..19c1c34dcf --- /dev/null +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/pom.xml @@ -0,0 +1,262 @@ + + + + + + linkis + org.apache.linkis + 1.0.3 + + 4.0.0 + linkis-metadata-manager-service-hive + + + UTF-8 + 2.3.3 + + 2.7.2 + 4.2.4 + + + + + org.apache.linkis + linkis-metadata-manager-common + ${linkis.version} + provided + + + org.apache.linkis + linkis-module + ${linkis.version} + provided + + + + org.datanucleus + datanucleus-api-jdo + ${datanucleus-api-jdo.version} + + + org.apache.hadoop + hadoop-client + + provided + ${hadoop.version} + + + org.slf4j + slf4j-log4j12 + + + servlet-api + javax.servlet + + + guava + com.google.guava + + + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + + + hadoop-yarn-common + org.apache.hadoop + + + + + org.apache.hive + hive-common + ${hive.version} + + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + + + org.apache.curator + curator-framework + + + org.apache.hadoop + hadoop-yarn-server-applicationhistoryservice + + + org.apache.hive + hive-shims + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-api + + + jsr311-api + javax.ws.rs + + + hadoop-common + org.apache.hadoop + + + dropwizard-metrics-hadoop-metrics2-reporter + com.github.joshelser + + + + + org.apache.hive + hive-exec + ${hive.version} + + + slf4j-log4j12 + org.slf4j + + + org.apache.ivy + ivy + + + guava + com.google.guava + + + commons-lang3 + org.apache.commons + + + hadoop-yarn-api + org.apache.hadoop + + + hadoop-yarn-common + org.apache.hadoop + + + hadoop-yarn-server-applicationhistoryservice + org.apache.hadoop + + + hadoop-yarn-server-common + org.apache.hadoop + + + hadoop-yarn-server-resourcemanager + org.apache.hadoop + + + hadoop-yarn-server-web-proxy + org.apache.hadoop + + + + + + org.apache.linkis + linkis-bml-client + ${linkis.version} + + + linkis-httpclient + com.webank.wedatasphere.linkis + + + + + + org.apache.httpcomponents + httpcore + 4.4.14 + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 3.2.0 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + + + src/main/java + + **/*.xml + + + + src/main/resources + + **/*.properties + **/application.yml + **/bootstrap.yml + **/log4j2.xml + + + + ${project.artifactId}-${project.version} + + \ No newline at end of file diff --git a/linkis-public-enhancements/linkis-datasource/pom.xml b/linkis-public-enhancements/linkis-datasource/pom.xml index b4c2ea60eb..e23f41cbb9 100644 --- a/linkis-public-enhancements/linkis-datasource/pom.xml +++ b/linkis-public-enhancements/linkis-datasource/pom.xml @@ -35,5 +35,6 @@ linkis-metadata-manager/common linkis-metadata-manager/server linkis-metadata-manager/service/elasticsearch + linkis-metadata-manager/service/hive \ No newline at end of file From 6aa58afdf2f7b0d23955301a11bde81f4270f7e4 Mon Sep 17 00:00:00 2001 From: xiaojie19852006 Date: Mon, 7 Feb 2022 14:31:32 +0800 Subject: [PATCH 2/6] add assembly plugin --- .../hive/src/main/assembly/distribution.xml | 239 ++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/src/main/assembly/distribution.xml diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/src/main/assembly/distribution.xml b/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/src/main/assembly/distribution.xml new file mode 100644 index 0000000000..29c49e08da --- /dev/null +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/src/main/assembly/distribution.xml @@ -0,0 +1,239 @@ + + + + + linkis-metadata-manager-service-hive + + zip + dir + + false + linkis-metadata-manager-service-hive + + + + + + lib + true + true + false + false + true + + antlr:antlr:jar + asm:asm:jar + cglib:cglib:jar + com.amazonaws:aws-java-sdk-autoscaling:jar + com.amazonaws:aws-java-sdk-core:jar + com.amazonaws:aws-java-sdk-ec2:jar + com.amazonaws:aws-java-sdk-route53:jar + com.amazonaws:aws-java-sdk-sts:jar + com.amazonaws:jmespath-java:jar + com.fasterxml.jackson.core:jackson-annotations:jar + com.fasterxml.jackson.core:jackson-core:jar + com.fasterxml.jackson.core:jackson-databind:jar + com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:jar + com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar + com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar + com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar + com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar + com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar + com.fasterxml.jackson.module:jackson-module-parameter-names:jar + com.fasterxml.jackson.module:jackson-module-paranamer:jar + com.fasterxml.jackson.module:jackson-module-scala_2.11:jar + com.github.andrewoma.dexx:dexx-collections:jar + com.github.vlsi.compactmap:compactmap:jar + com.google.code.findbugs:annotations:jar + com.google.code.gson:gson:jar + com.google.guava:guava:jar + com.google.protobuf:protobuf-java:jar + com.ning:async-http-client:jar + com.sun.jersey.contribs:jersey-apache-client4:jar + com.sun.jersey:jersey-client:jar + com.sun.jersey:jersey-core:jar + com.sun.jersey:jersey-json:jar + com.sun.jersey:jersey-server:jar + com.sun.jersey:jersey-servlet:jar + com.sun.xml.bind:jaxb-impl:jar + commons-beanutils:commons-beanutils:jar + commons-beanutils:commons-beanutils-core:jar + commons-cli:commons-cli:jar + commons-collections:commons-collections:jar + commons-configuration:commons-configuration:jar + commons-dbcp:commons-dbcp:jar + commons-digester:commons-digester:jar + commons-io:commons-io:jar + commons-logging:commons-logging:jar + commons-jxpath:commons-jxpath:jar + commons-lang:commons-lang:jar + commons-net:commons-net:jar + commons-pool:commons-pool:jar + io.micrometer:micrometer-core:jar + io.netty:netty-all:jar + io.netty:netty-buffer:jar + io.netty:netty-codec:jar + io.netty:netty-codec-http:jar + io.netty:netty-common:jar + io.netty:netty-handler:jar + io.netty:netty-transport:jar + io.netty:netty-transport-native-epoll:jar + io.reactivex:rxjava:jar + io.reactivex:rxnetty:jar + io.reactivex:rxnetty-contexts:jar + io.reactivex:rxnetty-servo:jar + javax.activation:activation:jar + javax.annotation:javax.annotation-api:jar + javax.inject:javax.inject:jar + javax.servlet:javax.servlet-api:jar + javax.servlet.jsp:jsp-api:jar + javax.validation:validation-api:jar + javax.websocket:javax.websocket-api:jar + javax.ws.rs:javax.ws.rs-api:jar + javax.xml.bind:jaxb-api:jar + javax.xml.stream:stax-api:jar + mysql:mysql-connector-java:jar + org.antlr:antlr-runtime:jar + org.antlr:stringtemplate:jar + org.apache.commons:commons-compress:jar + org.apache.commons:commons-math:jar + org.apache.commons:commons-math3:jar + org.apache.curator:curator-framework:jar + org.apache.curator:curator-recipes:jar + org.apache.directory.api:api-asn1-api:jar + org.apache.directory.api:api-util:jar + org.apache.directory.server:apacheds-i18n:jar + org.apache.directory.server:apacheds-kerberos-codec:jar + org.apache.hadoop:hadoop-annotations:jar + org.apache.hadoop:hadoop-auth:jar + org.apache.hadoop:hadoop-common:jar + org.apache.hadoop:hadoop-hdfs:jar + org.apache.htrace:htrace-core:jar + org.apache.logging.log4j:log4j-api:jar + org.apache.logging.log4j:log4j-core:jar + org.apache.logging.log4j:log4j-jul:jar + org.apache.logging.log4j:log4j-slf4j-impl:jar + log4j:log4j:jar + org.apache.zookeeper:zookeeper:jar + org.aspectj:aspectjweaver:jar + org.bouncycastle:bcpkix-jdk15on:jar + org.bouncycastle:bcprov-jdk15on:jar + org.codehaus.jackson:jackson-jaxrs:jar + org.codehaus.jackson:jackson-xc:jar + org.codehaus.jettison:jettison:jar + org.codehaus.woodstox:stax2-api:jar + org.codehaus.woodstox:woodstox-core-asl:jar + org.eclipse.jetty:jetty-annotations:jar + org.eclipse.jetty:jetty-client:jar + org.eclipse.jetty:jetty-continuation:jar + org.eclipse.jetty:jetty-http:jar + org.eclipse.jetty:jetty-io:jar + org.eclipse.jetty:jetty-jndi:jar + org.eclipse.jetty:jetty-plus:jar + org.eclipse.jetty:jetty-security:jar + org.eclipse.jetty:jetty-server:jar + org.eclipse.jetty:jetty-servlet:jar + org.eclipse.jetty:jetty-servlets:jar + org.eclipse.jetty:jetty-util:jar + org.eclipse.jetty:jetty-webapp:jar + org.eclipse.jetty:jetty-xml:jar + org.eclipse.jetty.websocket:javax-websocket-client-impl:jar + org.eclipse.jetty.websocket:javax-websocket-server-impl:jar + org.eclipse.jetty.websocket:websocket-api:jar + org.eclipse.jetty.websocket:websocket-client:jar + org.eclipse.jetty.websocket:websocket-common:jar + org.eclipse.jetty.websocket:websocket-server:jar + org.eclipse.jetty.websocket:websocket-servlet:jar + org.eclipse.jetty.orbit:javax.servlet:jar + org.eclipse.jetty.aggregate:jetty-all:jar + org.fusesource.leveldbjni:leveldbjni-all:jar + org.glassfish.hk2:class-model:jar + org.glassfish.hk2:config-types:jar + org.glassfish.hk2.external:aopalliance-repackaged:jar + org.glassfish.hk2.external:asm-all-repackaged:jar + org.glassfish.hk2.external:bean-validator:jar + org.glassfish.hk2.external:javax.inject:jar + org.glassfish.hk2:hk2:jar + org.glassfish.hk2:hk2-api:jar + org.glassfish.hk2:hk2-config:jar + org.glassfish.hk2:hk2-core:jar + org.glassfish.hk2:hk2-locator:jar + org.glassfish.hk2:hk2-runlevel:jar + org.glassfish.hk2:hk2-utils:jar + org.glassfish.hk2:osgi-resource-locator:jar + org.glassfish.hk2:spring-bridge:jar + org.glassfish.jersey.bundles:jaxrs-ri:jar + org.glassfish.jersey.bundles.repackaged:jersey-guava:jar + org.glassfish.jersey.containers:jersey-container-servlet:jar + org.glassfish.jersey.containers:jersey-container-servlet-core:jar + org.glassfish.jersey.core:jersey-client:jar + org.glassfish.jersey.core:jersey-common:jar + org.glassfish.jersey.core:jersey-server:jar + org.glassfish.jersey.ext:jersey-entity-filtering:jar + org.glassfish.jersey.ext:jersey-spring3:jar + org.glassfish.jersey.media:jersey-media-jaxb:jar + org.glassfish.jersey.media:jersey-media-json-jackson:jar + org.glassfish.jersey.media:jersey-media-multipart:jar + org.hdrhistogram:HdrHistogram:jar + org.javassist:javassist:jar + org.json4s:json4s-ast_2.11:jar + org.json4s:json4s-core_2.11:jar + org.json4s:json4s-jackson_2.11:jar + org.jsoup:jsoup:jar + org.jvnet.mimepull:mimepull:jar + org.jvnet:tiger-types:jar + org.latencyutils:LatencyUtils:jar + org.mortbay.jasper:apache-el:jar + org.mortbay.jetty:jetty-util:jar + org.mortbay.jetty:jetty:jar + tomcat:jasper-compiler:jar + tomcat:jasper-runtime:jar + org.ow2.asm:asm-analysis:jar + org.ow2.asm:asm-commons:jar + org.ow2.asm:asm-tree:jar + org.reflections:reflections:jar + org.slf4j:jul-to-slf4j:jar + org.slf4j:slf4j-api:jar + org.tukaani:xz:jar + org.yaml:snakeyaml:jar + software.amazon.ion:ion-java:jar + xerces:xercesImpl:jar + xmlenc:xmlenc:jar + xmlpull:xmlpull:jar + xpp3:xpp3_min:jar + + + + + + + ${basedir}/src/main/resources + + * + + 0777 + conf + unix + + + + + From 6a19b7770ee48a39b049cefc7da8408f01b6a7c8 Mon Sep 17 00:00:00 2001 From: xiaojie19852006 Date: Mon, 7 Feb 2022 14:33:18 +0800 Subject: [PATCH 3/6] add hive connection --- .../service/HiveConnection.java | 142 ++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/src/main/java/org/apache/linkis/metadatamanager/service/HiveConnection.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/src/main/java/org/apache/linkis/metadatamanager/service/HiveConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/src/main/java/org/apache/linkis/metadatamanager/service/HiveConnection.java new file mode 100644 index 0000000000..3ab0eee553 --- /dev/null +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/src/main/java/org/apache/linkis/metadatamanager/service/HiveConnection.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.metadatamanager.service; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.linkis.common.conf.CommonVars; + +import java.io.Closeable; +import java.io.IOException; +import java.net.URI; +import java.security.PrivilegedExceptionAction; +import java.util.Map; +import java.util.concurrent.locks.ReentrantLock; + +import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; + +public class HiveConnection implements Closeable { + + private Hive hiveClient; + + private IMetaStoreClient metaStoreClient; + + private static final CommonVars KERBEROS_DEFAULT_PRINCIPLE = + CommonVars.apply("wds.linkis.server.mdm.service.kerberos.principle", "hadoop/_HOST@EXAMPLE.COM"); + + private static final CommonVars DEFAULT_SERVICE_USER = + CommonVars.apply("wds.linkis.server.mdm.service.user", "hadoop"); + + private static final CommonVars KERBEROS_KRB5_CONF_PATH = + CommonVars.apply("wds.linkis.server.mdm.service.kerberos.krb5.path", ""); + + static{ + if(StringUtils.isNotBlank(KERBEROS_KRB5_CONF_PATH.getValue())){ + System.setProperty("java.security.krb5.conf", KERBEROS_KRB5_CONF_PATH.getValue()); + } + } + + public HiveConnection(String uris, String principle, String keytabFilePath, Map hadoopConf) throws Exception { + final HiveConf conf = new HiveConf(); + conf.setVar(HiveConf.ConfVars.METASTOREURIS, uris); + conf.setVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL, "true"); + conf.setVar(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL, KERBEROS_DEFAULT_PRINCIPLE.getValue()); + //Disable the cache in FileSystem + conf.setBoolean(String.format("fs.%s.impl.disable.cache", URI.create(conf.get(FS_DEFAULT_NAME_KEY, "")).getScheme()), true); + conf.set("hadoop.security.authentication", "kerberos"); + hadoopConf.forEach(conf::set); + principle = principle.substring(0, principle.indexOf("@")); + UserGroupInformation ugi = UserGroupInformationWrapper.loginUserFromKeytab(conf, + principle, keytabFilePath); + hiveClient = getHive(ugi, conf); + } + + public HiveConnection(String uris, Map hadoopConf) throws Exception{ + final HiveConf conf = new HiveConf(); + conf.setVar(HiveConf.ConfVars.METASTOREURIS, uris); + hadoopConf.forEach(conf::set); + //Disable the cache in FileSystem + conf.setBoolean(String.format("fs.%s.impl.disable.cache", URI.create(conf.get(FS_DEFAULT_NAME_KEY, "")).getScheme()), true); + //TODO choose an authentication strategy for hive, and then use createProxyUser + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(DEFAULT_SERVICE_USER.getValue()); + hiveClient = getHive(ugi, conf); + } + /** + * Get Hive client(Hive object) + * @return hive + */ + public Hive getClient(){ + return hiveClient; + } + + private Hive getHive(UserGroupInformation ugi, HiveConf conf) throws IOException, InterruptedException { + return ugi.doAs((PrivilegedExceptionAction) () -> { + Hive hive = Hive.get(conf); + metaStoreClient = hive.getMSC(); + //To remove thread Local vars + Hive.set(null); + return hive; + }); + } + @Override + public void close() throws IOException { + //Close meta store client + metaStoreClient.close(); + } + + /** + * Wrapper class of UserGroupInformation + */ + private static class UserGroupInformationWrapper{ + private static ReentrantLock globalLock = new ReentrantLock(); + + public static UserGroupInformation loginUserFromKeytab(final Configuration conf, String user, String path) throws Exception { + globalLock.lock(); + try{ + UserGroupInformation.setConfiguration(conf); + return UserGroupInformation.loginUserFromKeytabAndReturnUGI(user, path); + }finally{ + globalLock.unlock(); + } + } + public static UserGroupInformation createProxyUser(final Configuration conf, String user) throws Exception{ + globalLock.lock(); + try{ + UserGroupInformation.setLoginUser(null); + UserGroupInformation.setConfiguration(conf); + return UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); + }finally{ + globalLock.unlock(); + } + } + + public static UserGroupInformation getLoginUser() throws Exception{ + globalLock.lock(); + try{ + UserGroupInformation.setLoginUser(null); + return UserGroupInformation.getLoginUser(); + }finally{ + globalLock.unlock(); + } + } + } +} From be1a0bd91d33db8dc928f7dbad381aadd2fc8025 Mon Sep 17 00:00:00 2001 From: xiaojie19852006 Date: Mon, 7 Feb 2022 14:34:35 +0800 Subject: [PATCH 4/6] add hive params mapper --- .../service/HiveParamsMapper.java | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/src/main/java/org/apache/linkis/metadatamanager/service/HiveParamsMapper.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/src/main/java/org/apache/linkis/metadatamanager/service/HiveParamsMapper.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/src/main/java/org/apache/linkis/metadatamanager/service/HiveParamsMapper.java new file mode 100644 index 0000000000..c556dfb204 --- /dev/null +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/src/main/java/org/apache/linkis/metadatamanager/service/HiveParamsMapper.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.metadatamanager.service; + +import org.apache.linkis.common.conf.CommonVars; + +public class HiveParamsMapper { + + public static final CommonVars PARAM_HIVE_PRINCIPLE = + CommonVars.apply("wds.linkis.server.mdm.service.hive.principle", "principle"); + + public static final CommonVars PARAM_HIVE_URIS = + CommonVars.apply("wds.linkis.server.mdm.service.hive.uris", "uris"); + + public static final CommonVars PARAM_HIVE_KEYTAB = + CommonVars.apply("wds.linkis.server.mdm.service.hive.keytab", "keytab"); + + public static final CommonVars PARAM_HADOOP_CONF = + CommonVars.apply("wds.linkis.server.mdm.service.hadoop.conf", "hadoopConf"); +} From cf5ae7cc12399665be3102b469d27bb29df3249a Mon Sep 17 00:00:00 2001 From: xiaojie19852006 Date: Mon, 7 Feb 2022 14:35:39 +0800 Subject: [PATCH 5/6] add meta service for hive --- .../service/HiveMetaService.java | 217 ++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/src/main/java/org/apache/linkis/metadatamanager/service/HiveMetaService.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/src/main/java/org/apache/linkis/metadatamanager/service/HiveMetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/src/main/java/org/apache/linkis/metadatamanager/service/HiveMetaService.java new file mode 100644 index 0000000000..2d01e8a6bc --- /dev/null +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/src/main/java/org/apache/linkis/metadatamanager/service/HiveMetaService.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.metadatamanager.service; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.linkis.bml.client.BmlClient; +import org.apache.linkis.bml.client.BmlClientFactory; +import org.apache.linkis.bml.protocol.BmlDownloadResponse; +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.datasourcemanager.common.util.json.Json; +import org.apache.linkis.metadatamanager.common.domain.MetaColumnInfo; +import org.apache.linkis.metadatamanager.common.domain.MetaPartitionInfo; +import org.apache.linkis.metadatamanager.common.exception.MetaRuntimeException; +import org.apache.linkis.metadatamanager.common.service.AbstractMetaService; +import org.apache.linkis.metadatamanager.common.service.MetadataConnection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.io.Resource; +import org.springframework.core.io.support.PathMatchingResourcePatternResolver; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.*; + +@Component +public class HiveMetaService extends AbstractMetaService { + + private static final Logger LOG = LoggerFactory.getLogger(HiveMetaService.class); + private static final CommonVars TMP_FILE_STORE_LOCATION = + CommonVars.apply("wds.linkis.server.mdm.service.temp.location", "classpath:/tmp"); + + private BmlClient client; + + @PostConstruct + public void buildClient(){ + client = BmlClientFactory.createBmlClient(); + } + @Override + public MetadataConnection getConnection(String operator, Map params) throws Exception { + Resource resource = new PathMatchingResourcePatternResolver().getResource(TMP_FILE_STORE_LOCATION.getValue()); + String uris = String.valueOf(params.getOrDefault(HiveParamsMapper.PARAM_HIVE_URIS.getValue(), "")); + String principle = String.valueOf(params.getOrDefault(HiveParamsMapper.PARAM_HIVE_PRINCIPLE.getValue(), "")); + HiveConnection conn = null; + if(StringUtils.isNotBlank(principle)){ + LOG.info("Try to connect Hive MetaStore in kerberos with principle:[" + principle +"]"); + String keytabResourceId = String.valueOf(params.getOrDefault(HiveParamsMapper.PARAM_HIVE_KEYTAB.getValue(), "")); + if(StringUtils.isNotBlank(keytabResourceId)){ + LOG.info("Start to download resource id:[" + keytabResourceId +"]"); + String keytabFilePath = resource.getFile().getAbsolutePath() + "/" + UUID.randomUUID().toString().replace("-", ""); + if(!downloadResource(keytabResourceId, operator, keytabFilePath)){ + throw new MetaRuntimeException("Fail to download resource i:[" + keytabResourceId +"]", null); + } + conn = new HiveConnection(uris, principle, keytabFilePath, getExtraHadoopConf(params)); + }else{ + throw new MetaRuntimeException("Cannot find the keytab file in connect parameters", null); + } + }else{ + conn = new HiveConnection(uris, getExtraHadoopConf(params)); + } + return new MetadataConnection<>(conn, true); + } + + /** + * Download resource to path by id + * @param resourceId resource id + * @param user user + * @param absolutePath absolute path + * @return + * @throws IOException + */ + private boolean downloadResource(String resourceId, String user, String absolutePath) throws IOException { + BmlDownloadResponse downloadResponse = client.downloadResource(user, resourceId, absolutePath); + if(downloadResponse.isSuccess()){ + IOUtils.copy(downloadResponse.inputStream(), new FileOutputStream(absolutePath)); + return true; + } + return false; + } + + @Override + public List queryDatabases(HiveConnection connection) { + try { + return connection.getClient().getAllDatabases(); + } catch (HiveException e) { + throw new RuntimeException("Fail to get Hive databases(获取数据库列表失败)", e); + } + } + + @Override + public List queryTables(HiveConnection connection, String database) { + try { + return connection.getClient().getAllTables(database); + } catch (HiveException e) { + throw new RuntimeException("Fail to get Hive tables(获取表列表失败)", e); + } + } + + @Override + public MetaPartitionInfo queryPartitions(HiveConnection connection, String database, String table) { + List partitions; + Table rawTable; + try { + rawTable = connection.getClient().getTable(database, table); + partitions = connection.getClient().getPartitions(rawTable); + } catch (HiveException e) { + throw new RuntimeException("Fail to get Hive partitions(获取分区信息失败)", e); + } + MetaPartitionInfo info = new MetaPartitionInfo(); + List partitionKeys = rawTable.getPartitionKeys(); + List partKeys = new ArrayList<>(); + partitionKeys.forEach(e -> partKeys.add(e.getName())); + info.setPartKeys(partKeys); + //Static partitions + Map pMap = new HashMap<>(20); + MetaPartitionInfo.PartitionNode root = new MetaPartitionInfo.PartitionNode(); + info.setRoot(root); + long t = System.currentTimeMillis(); + for(Partition p : partitions){ + try { + List values = p.getValues(); + if(!partitionKeys.isEmpty()){ + String parentNameValue = ""; + for(int i = 0; i < values.size(); i++){ + FieldSchema fieldSchema = partitionKeys.get(i); + String name = fieldSchema.getName(); + String value = values.get(i); + String nameValue= name + "=" + value; + MetaPartitionInfo.PartitionNode node = new MetaPartitionInfo.PartitionNode(); + if(i > 0){ + MetaPartitionInfo.PartitionNode parent = pMap.get(parentNameValue); + parent.setName(name); + parent.getPartitions().putIfAbsent(value, node); + }else{ + root.setName(name); + root.getPartitions().putIfAbsent(value, node); + } + parentNameValue += "/" + nameValue; + pMap.putIfAbsent(parentNameValue, node); + } + } + }catch(Exception e){ + LOG.warn(e.getMessage(), e); + } + } + return info; + } + + @Override + public List queryColumns(HiveConnection connection, String database, String table) { + List columns = new ArrayList<>(); + Table tb; + try { + tb = connection.getClient().getTable(database, table); + } catch (HiveException e) { + throw new RuntimeException("Fail to get Hive columns(获得表字段信息失败)", e); + } + tb.getFields().forEach( field ->{ + MetaColumnInfo metaColumnInfo = new MetaColumnInfo(); + metaColumnInfo.setIndex(field.getFieldID()); + metaColumnInfo.setName(field.getFieldName()); + metaColumnInfo.setType(field.getFieldObjectInspector().getTypeName()); + columns.add(metaColumnInfo); + }); + return columns; + } + + @Override + public Map queryTableProps(HiveConnection connection, String database, String table) { + try { + Table rawTable = connection.getClient().getTable(database, table); + return new HashMap<>((Map)rawTable.getMetadata()); + }catch(Exception e){ + throw new RuntimeException("Fail to get Hive table properties(获取表参数信息失败)", e); + } + } + + @SuppressWarnings("unchecked") + private Map getExtraHadoopConf(Map connectParams){ + Map extraHadoopConf = new HashMap<>(); + Object extraHadoopConfObj = connectParams.get(HiveParamsMapper.PARAM_HADOOP_CONF.getValue()); + if (Objects.nonNull(extraHadoopConfObj)){ + try { + if (!(extraHadoopConfObj instanceof Map)) { + extraHadoopConf = Json.fromJson(String.valueOf(extraHadoopConfObj), Map.class, String.class, String.class); + }else{ + extraHadoopConf = (Map)extraHadoopConfObj; + } + }catch(Exception e){ + throw new MetaRuntimeException("Cannot parse the param:[" +HiveParamsMapper.PARAM_HADOOP_CONF.getValue() + "]", e); + } + } + return extraHadoopConf; + } + +} From 509bdbafc929babd13c78c4ff7e41b5ee3333254 Mon Sep 17 00:00:00 2001 From: xiaojie19852006 Date: Mon, 7 Feb 2022 15:53:11 +0800 Subject: [PATCH 6/6] add exclusion for hive-exec --- .../service/hive/pom.xml | 72 +++++++++++++++++-- 1 file changed, 68 insertions(+), 4 deletions(-) diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/pom.xml b/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/pom.xml index 19c1c34dcf..7fba922f52 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/pom.xml +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-manager/service/hive/pom.xml @@ -133,10 +133,6 @@ hive-exec ${hive.version} - - slf4j-log4j12 - org.slf4j - org.apache.ivy ivy @@ -173,6 +169,74 @@ hadoop-yarn-server-web-proxy org.apache.hadoop + + org.pentaho + pentaho-aggdesigner-algorithm + + + org.apache.hive + hive-ant + + + org.apache.hive + hive-serde + + + org.apache.hive + hive-shims + + + com.jolbox + bonecp + + + org.apache.derby + derby + + + org.apache.curator + curator-framework + + + org.apache.curator + apache-curator + + + org.codehaus.groovy + groovy-all + + + stax + stax-api + + + jline + jline + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-api + + + jsr311-api + javax.ws.rs + + + httpclient + org.apache.httpcomponents + + + commons-httpclient + commons-httpclient + + + httpcore + org.apache.httpcomponents +