From 2e7c6611cc92eddb3795d5575e1089498c865225 Mon Sep 17 00:00:00 2001
From: Alibaba-HZY <daowu.hzy@cainiao.com>
Date: Tue, 4 Apr 2023 18:25:57 +0800
Subject: [PATCH 01/10] [hive] Support hive-conf-dir and hadoop-conf-dir option
 in hive catalog  (#823)

---
 .../org/apache/paimon/hive/HiveCatalog.java   | 105 +++++++++++++++++-
 .../paimon/hive/HiveCatalogFactory.java       |  25 +++--
 .../hive/HiveCatalogFactoryOptions.java       |  36 ++++++
 .../paimon/hive/Hive23CatalogITCase.java      |  12 ++
 .../paimon/hive/Hive31CatalogITCase.java      |  12 ++
 .../paimon/hive/HiveCatalogITCaseBase.java    |  15 +++
 .../resources/hadoop-conf-dir/core-site.xml   |  21 ++++
 7 files changed, 212 insertions(+), 14 deletions(-)
 create mode 100644 paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactoryOptions.java
 create mode 100644 paimon-hive/paimon-hive-connector-common/src/test/resources/hadoop-conf-dir/core-site.xml

diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
index d167c02ca905..941ab93bc30d 100644
--- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
+++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
@@ -31,7 +31,6 @@
 import org.apache.paimon.schema.TableSchema;
 import org.apache.paimon.table.TableType;
 import org.apache.paimon.types.DataField;
-import org.apache.paimon.utils.StringUtils;
 
 import org.apache.flink.table.hive.LegacyHiveClasses;
 import org.apache.hadoop.conf.Configuration;
@@ -53,7 +52,12 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.io.FileNotFoundException;
 import java.lang.reflect.Method;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -68,6 +72,7 @@
 import static org.apache.paimon.options.CatalogOptions.LOCK_ENABLED;
 import static org.apache.paimon.options.CatalogOptions.TABLE_TYPE;
 import static org.apache.paimon.utils.Preconditions.checkState;
+import static org.apache.paimon.utils.StringUtils.isNullOrWhitespaceOnly;
 
 /** A catalog implementation for Hive. */
 public class HiveCatalog extends AbstractCatalog {
@@ -83,13 +88,15 @@ public class HiveCatalog extends AbstractCatalog {
     private static final String STORAGE_HANDLER_CLASS_NAME =
             "org.apache.paimon.hive.PaimonStorageHandler";
 
+    public static final String HIVE_SITE_FILE = "hive-site.xml";
+
     private final HiveConf hiveConf;
     private final String clientClassName;
     private final IMetaStoreClient client;
 
-    public HiveCatalog(FileIO fileIO, Configuration hadoopConfig, String clientClassName) {
+    public HiveCatalog(FileIO fileIO, HiveConf hiveConf, String clientClassName) {
         super(fileIO);
-        this.hiveConf = new HiveConf(hadoopConfig, HiveConf.class);
+        this.hiveConf = hiveConf;
         this.clientClassName = clientClassName;
         this.client = createClient(hiveConf, clientClassName);
     }
@@ -521,9 +528,97 @@ static IMetaStoreClient createClient(HiveConf hiveConf, String clientClassName)
         } catch (Exception e) {
             throw new RuntimeException(e);
         }
-        return StringUtils.isNullOrWhitespaceOnly(
-                        hiveConf.get(HiveConf.ConfVars.METASTOREURIS.varname))
+        return isNullOrWhitespaceOnly(hiveConf.get(HiveConf.ConfVars.METASTOREURIS.varname))
                 ? client
                 : HiveMetaStoreClient.newSynchronizedClient(client);
     }
+
+    public static HiveConf createHiveConf(
+            @Nullable String hiveConfDir, @Nullable String hadoopConfDir) {
+        // create HiveConf from hadoop configuration with hadoop conf directory configured.
+        Configuration hadoopConf = null;
+        if (!isNullOrWhitespaceOnly(hadoopConfDir)) {
+            hadoopConf = getHadoopConfiguration(hadoopConfDir);
+            if (hadoopConf == null) {
+                String possiableUsedConfFiles =
+                        "core-site.xml | hdfs-site.xml | yarn-site.xml | mapred-site.xml";
+                throw new RuntimeException(
+                        "Failed to load the hadoop conf from specified path:" + hadoopConfDir,
+                        new FileNotFoundException(
+                                "Please check the path none of the conf files ("
+                                        + possiableUsedConfFiles
+                                        + ") exist in the folder."));
+            }
+        }
+        if (hadoopConf == null) {
+            hadoopConf = new Configuration();
+        }
+        // ignore all the static conf file URLs that HiveConf may have set
+        HiveConf.setHiveSiteLocation(null);
+        HiveConf.setLoadMetastoreConfig(false);
+        HiveConf.setLoadHiveServer2Config(false);
+        HiveConf hiveConf = new HiveConf(hadoopConf, HiveConf.class);
+
+        LOG.info("Setting hive conf dir as {}", hiveConfDir);
+
+        if (hiveConfDir != null) {
+            org.apache.hadoop.fs.Path hiveSite =
+                    new org.apache.hadoop.fs.Path(hiveConfDir, HIVE_SITE_FILE);
+            if (!hiveSite.toUri().isAbsolute()) {
+                // treat relative URI as local file to be compatible with previous behavior
+                hiveSite = new org.apache.hadoop.fs.Path(new File(hiveSite.toString()).toURI());
+            }
+            hiveConf.addResource(hiveSite);
+        }
+        // else {
+        //    // user doesn't provide hive conf dir, we try to find it in classpath
+        //    URL hiveSite =
+        //            Thread.currentThread().getContextClassLoader().getResource(HIVE_SITE_FILE);
+        //    if (hiveSite != null) {
+        //        LOG.info("Found {} in classpath: {}", HIVE_SITE_FILE, hiveSite);
+        //        hiveConf.addResource(hiveSite);
+        //    }
+        // }
+        return hiveConf;
+    }
+
+    /**
+     * Returns a new Hadoop Configuration object using the path to the hadoop conf configured.
+     *
+     * @param hadoopConfDir Hadoop conf directory path.
+     * @return A Hadoop configuration instance.
+     */
+    public static Configuration getHadoopConfiguration(String hadoopConfDir) {
+        if (new File(hadoopConfDir).exists()) {
+            List<File> possiableConfFiles = new ArrayList<File>();
+            File coreSite = new File(hadoopConfDir, "core-site.xml");
+            if (coreSite.exists()) {
+                possiableConfFiles.add(coreSite);
+            }
+            File hdfsSite = new File(hadoopConfDir, "hdfs-site.xml");
+            if (hdfsSite.exists()) {
+                possiableConfFiles.add(hdfsSite);
+            }
+            File yarnSite = new File(hadoopConfDir, "yarn-site.xml");
+            if (yarnSite.exists()) {
+                possiableConfFiles.add(yarnSite);
+            }
+            // Add mapred-site.xml. We need to read configurations like compression codec.
+            File mapredSite = new File(hadoopConfDir, "mapred-site.xml");
+            if (mapredSite.exists()) {
+                possiableConfFiles.add(mapredSite);
+            }
+            if (possiableConfFiles.isEmpty()) {
+                return null;
+            } else {
+                Configuration hadoopConfiguration = new Configuration();
+                for (File confFile : possiableConfFiles) {
+                    hadoopConfiguration.addResource(
+                            new org.apache.hadoop.fs.Path(confFile.getAbsolutePath()));
+                }
+                return hadoopConfiguration;
+            }
+        }
+        return null;
+    }
 }
diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java
index 503739ebf354..4c050e998172 100644
--- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java
+++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java
@@ -28,14 +28,16 @@
 import org.apache.paimon.options.ConfigOptions;
 import org.apache.paimon.utils.Preconditions;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
 
+import static org.apache.paimon.hive.HiveCatalog.createHiveConf;
+import static org.apache.paimon.hive.HiveCatalogFactoryOptions.HADOOP_CONF_DIR;
+import static org.apache.paimon.hive.HiveCatalogFactoryOptions.HIVE_CONF_DIR;
+import static org.apache.paimon.hive.HiveCatalogFactoryOptions.IDENTIFIER;
+
 /** Factory to create {@link HiveCatalog}. */
 public class HiveCatalogFactory implements CatalogFactory {
 
-    private static final String IDENTIFIER = "hive";
-
     private static final ConfigOption<String> METASTORE_CLIENT_CLASS =
             ConfigOptions.key("metastore.client.class")
                     .stringType()
@@ -60,12 +62,17 @@ public Catalog create(FileIO fileIO, Path warehouse, CatalogContext context) {
                                 + IDENTIFIER
                                 + " catalog");
 
-        Configuration hadoopConfig = new Configuration();
-        context.options().toMap().forEach(hadoopConfig::set);
-        hadoopConfig.set(HiveConf.ConfVars.METASTOREURIS.varname, uri);
-        hadoopConfig.set(
-                HiveConf.ConfVars.METASTOREWAREHOUSE.varname, warehouse.toUri().toString());
+        String hiveConfDir = context.options().get(HIVE_CONF_DIR);
+        String hadoopConfDir = context.options().get(HADOOP_CONF_DIR);
+        HiveConf hiveConf = createHiveConf(hiveConfDir, hadoopConfDir);
+
+        // always using user-set parameters overwrite hive-site.xml parameters
+        context.options().toMap().forEach(hiveConf::set);
+        hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, uri);
+        hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, warehouse.toUri().toString());
+
+        String clientClassName = context.options().get(METASTORE_CLIENT_CLASS);
 
-        return new HiveCatalog(fileIO, hadoopConfig, context.options().get(METASTORE_CLIENT_CLASS));
+        return new HiveCatalog(fileIO, hiveConf, clientClassName);
     }
 }
diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactoryOptions.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactoryOptions.java
new file mode 100644
index 000000000000..643141a3ee1d
--- /dev/null
+++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactoryOptions.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.hive;
+
+import org.apache.paimon.options.ConfigOption;
+import org.apache.paimon.options.ConfigOptions;
+
+/** {@link ConfigOption}s for {@link HiveCatalog}. */
+public final class HiveCatalogFactoryOptions {
+
+    public static final String IDENTIFIER = "hive";
+
+    public static final ConfigOption<String> HIVE_CONF_DIR =
+            ConfigOptions.key("hive-conf-dir").stringType().noDefaultValue();
+
+    public static final ConfigOption<String> HADOOP_CONF_DIR =
+            ConfigOptions.key("hadoop-conf-dir").stringType().noDefaultValue();
+
+    private HiveCatalogFactoryOptions() {}
+}
diff --git a/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java
index b2d38da4bad9..f064ac312b6e 100644
--- a/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java
+++ b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java
@@ -72,6 +72,10 @@ public void testCustomMetastoreClient() throws Exception {
                                 "  'metastore' = 'hive',",
                                 "  'uri' = '',",
                                 "  'warehouse' = '" + path + "',",
+                                "  'hive-conf-dir' = '"
+                                        + hiveShell.getBaseDir().getRoot().getPath()
+                                        + HIVE_CONF
+                                        + "',",
                                 "  'metastore.client.class' = '"
                                         + TestHiveMetaStoreClient.class.getName()
                                         + "'",
@@ -96,6 +100,10 @@ public void testCreateExistTableInHive() throws Exception {
                         "  'metastore' = 'hive',",
                         "  'uri' = '',",
                         "  'warehouse' = '" + path + "',",
+                        "  'hive-conf-dir' = '"
+                                + hiveShell.getBaseDir().getRoot().getPath()
+                                + HIVE_CONF
+                                + "',",
                         "  'metastore.client.class' = '"
                                 + CreateFailHiveMetaStoreClient.class.getName()
                                 + "'",
@@ -127,6 +135,10 @@ public void testAlterTableFailedInHive() throws Exception {
                                 "  'metastore' = 'hive',",
                                 "  'uri' = '',",
                                 "  'warehouse' = '" + path + "',",
+                                "  'hive-conf-dir' = '"
+                                        + hiveShell.getBaseDir().getRoot().getPath()
+                                        + HIVE_CONF
+                                        + "',",
                                 "  'metastore.client.class' = '"
                                         + AlterFailHiveMetaStoreClient.class.getName()
                                         + "'",
diff --git a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java
index 84860dcdeb21..6ccaa871b84b 100644
--- a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java
+++ b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java
@@ -72,6 +72,10 @@ public void testCustomMetastoreClient() throws Exception {
                                 "  'metastore' = 'hive',",
                                 "  'uri' = '',",
                                 "  'warehouse' = '" + path + "',",
+                                "  'hive-conf-dir' = '"
+                                        + hiveShell.getBaseDir().getRoot().getPath()
+                                        + HIVE_CONF
+                                        + "',",
                                 "  'metastore.client.class' = '"
                                         + TestHiveMetaStoreClient.class.getName()
                                         + "'",
@@ -96,6 +100,10 @@ public void testCreateExistTableInHive() throws Exception {
                         "  'metastore' = 'hive',",
                         "  'uri' = '',",
                         "  'warehouse' = '" + path + "',",
+                        "  'hive-conf-dir' = '"
+                                + hiveShell.getBaseDir().getRoot().getPath()
+                                + HIVE_CONF
+                                + "',",
                         "  'metastore.client.class' = '"
                                 + CreateFailHiveMetaStoreClient.class.getName()
                                 + "'",
@@ -127,6 +135,10 @@ public void testAlterTableFailedInHive() throws Exception {
                                 "  'metastore' = 'hive',",
                                 "  'uri' = '',",
                                 "  'warehouse' = '" + path + "',",
+                                "  'hive-conf-dir' = '"
+                                        + hiveShell.getBaseDir().getRoot().getPath()
+                                        + HIVE_CONF
+                                        + "',",
                                 "  'metastore.client.class' = '"
                                         + AlterFailHiveMetaStoreClient.class.getName()
                                         + "'",
diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
index d50bb16e2db0..cdd56ef4a8f2 100644
--- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
+++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
@@ -67,12 +67,18 @@ public abstract class HiveCatalogITCaseBase {
     @HiveSQL(files = {})
     protected static HiveShell hiveShell;
 
+    public static String HIVE_CONF = "/hive-conf";
+
+    public static final String HADOOP_CONF_DIR =
+            Thread.currentThread().getContextClassLoader().getResource("hadoop-conf-dir").getPath();
+
     @Before
     public void before() throws Exception {
         hiveShell.execute("CREATE DATABASE IF NOT EXISTS test_db");
         hiveShell.execute("USE test_db");
         hiveShell.execute("CREATE TABLE hive_table ( a INT, b STRING )");
         hiveShell.execute("INSERT INTO hive_table VALUES (100, 'Hive'), (200, 'Table')");
+        hiveShell.executeQuery("SHOW TABLES");
 
         path = folder.newFolder().toURI().toString();
         EnvironmentSettings settings = EnvironmentSettings.newInstance().inBatchMode().build();
@@ -84,6 +90,11 @@ public void before() throws Exception {
                                 "  'type' = 'paimon',",
                                 "  'metastore' = 'hive',",
                                 "  'uri' = '',",
+                                "  'hive-conf-dir' = '"
+                                        + hiveShell.getBaseDir().getRoot().getPath()
+                                        + HIVE_CONF
+                                        + "',",
+                                "  'hadoop-conf-dir' = '" + HADOOP_CONF_DIR + "',",
                                 "  'warehouse' = '" + path + "',",
                                 "  'lock.enabled' = 'true'",
                                 ")"))
@@ -227,6 +238,10 @@ public void testCreateExternalTable() throws Exception {
                                 "  'metastore' = 'hive',",
                                 "  'uri' = '',",
                                 "  'warehouse' = '" + path + "',",
+                                "  'hive-conf-dir' = '"
+                                        + hiveShell.getBaseDir().getRoot().getPath()
+                                        + HIVE_CONF
+                                        + "',",
                                 "  'lock.enabled' = 'true',",
                                 "  'table.type' = 'EXTERNAL'",
                                 ")"))
diff --git a/paimon-hive/paimon-hive-connector-common/src/test/resources/hadoop-conf-dir/core-site.xml b/paimon-hive/paimon-hive-connector-common/src/test/resources/hadoop-conf-dir/core-site.xml
new file mode 100644
index 000000000000..768f207586d6
--- /dev/null
+++ b/paimon-hive/paimon-hive-connector-common/src/test/resources/hadoop-conf-dir/core-site.xml
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--Autogenerated by Cloudera Manager-->
+<configuration>
+  <property>
+    <name>fs.defaultFS</name>
+    <value>dummy-fs</value>
+  </property>
+  <property>
+    <name>fs.trash.interval</name>
+    <value>1</value>
+  </property>
+  <property>
+    <name>io.compression.codecs</name>
+    <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value>
+  </property>
+  <property>
+    <name>hadoop.security.authentication</name>
+    <value>none</value>
+  </property>
+</configuration>

From 8e02125484c3d8731d490d1c94aa9fb7f2bc55fa Mon Sep 17 00:00:00 2001
From: Alibaba-HZY <daowu.hzy@cainiao.com>
Date: Tue, 4 Apr 2023 19:45:55 +0800
Subject: [PATCH 02/10] [hive] Support hive-conf-dir and hadoop-conf-dir option
 in hive catalog  (#823)

---
 .../main/java/org/apache/paimon/hive/HiveCatalog.java  | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
index 941ab93bc30d..45539b5465a9 100644
--- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
+++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
@@ -570,15 +570,7 @@ public static HiveConf createHiveConf(
             }
             hiveConf.addResource(hiveSite);
         }
-        // else {
-        //    // user doesn't provide hive conf dir, we try to find it in classpath
-        //    URL hiveSite =
-        //            Thread.currentThread().getContextClassLoader().getResource(HIVE_SITE_FILE);
-        //    if (hiveSite != null) {
-        //        LOG.info("Found {} in classpath: {}", HIVE_SITE_FILE, hiveSite);
-        //        hiveConf.addResource(hiveSite);
-        //    }
-        // }
+
         return hiveConf;
     }
 

From 0d3ceeb3e54dd44825abe854cb1a7a04a3fb41db Mon Sep 17 00:00:00 2001
From: Alibaba-HZY <daowu.hzy@cainiao.com>
Date: Wed, 5 Apr 2023 13:37:32 +0800
Subject: [PATCH 03/10] [hive] Remove unless comment and add document for
 hive-conf-dif option (#823)

---
 docs/content/how-to/creating-catalogs.md           |  5 ++++-
 .../configuration/ConfigOptionsDocGenerator.java   |  4 +++-
 .../java/org/apache/paimon/hive/HiveCatalog.java   |  1 -
 .../org/apache/paimon/hive/HiveCatalogFactory.java |  4 ++--
 .../paimon/hive/HiveCatalogFactoryOptions.java     | 14 +++++++++++---
 5 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/docs/content/how-to/creating-catalogs.md b/docs/content/how-to/creating-catalogs.md
index 00ecbe6d75d5..b4d185bd2996 100644
--- a/docs/content/how-to/creating-catalogs.md
+++ b/docs/content/how-to/creating-catalogs.md
@@ -84,12 +84,15 @@ Paimon Hive catalog in Flink relies on Flink Hive connector bundled jar. You sho
 
 The following Flink SQL registers and uses a Paimon Hive catalog named `my_hive`. Metadata and table files are stored under `hdfs://path/to/warehouse`. In addition, metadata is also stored in Hive metastore.
 
+If your Hive requires security authentication such as Kerberos,LDAP,Ranger and so on. You can specify the hive-conf-dir parameter to the hive-site.xml file path
+
 ```sql
 CREATE CATALOG my_hive WITH (
     'type' = 'paimon',
     'metastore' = 'hive',
     'uri' = 'thrift://<hive-metastore-host-name>:<port>',
-    'warehouse' = 'hdfs://path/to/warehouse'
+    'warehouse' = 'hdfs://path/to/warehouse',
+    'hive-conf-dir' = 'hdfs://path/to/hive-conf',
 );
 
 USE CATALOG my_hive;
diff --git a/paimon-docs/src/main/java/org/apache/paimon/docs/configuration/ConfigOptionsDocGenerator.java b/paimon-docs/src/main/java/org/apache/paimon/docs/configuration/ConfigOptionsDocGenerator.java
index da1f97c1dcba..ab72438d2221 100644
--- a/paimon-docs/src/main/java/org/apache/paimon/docs/configuration/ConfigOptionsDocGenerator.java
+++ b/paimon-docs/src/main/java/org/apache/paimon/docs/configuration/ConfigOptionsDocGenerator.java
@@ -76,7 +76,9 @@ public class ConfigOptionsDocGenerator {
                 new OptionsClassLocation(
                         "paimon-flink/paimon-flink-common", "org.apache.paimon.flink"),
                 new OptionsClassLocation(
-                        "paimon-flink/paimon-flink-common", "org.apache.paimon.flink.kafka")
+                        "paimon-flink/paimon-flink-common", "org.apache.paimon.flink.kafka"),
+                new OptionsClassLocation(
+                        "paimon-hive/paimon-hive-catalog", "org.apache.paimon.hive")
             };
     static final String DEFAULT_PATH_PREFIX = "src/main/java";
 
diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
index 45539b5465a9..12ca3b4c6f73 100644
--- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
+++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
@@ -565,7 +565,6 @@ public static HiveConf createHiveConf(
             org.apache.hadoop.fs.Path hiveSite =
                     new org.apache.hadoop.fs.Path(hiveConfDir, HIVE_SITE_FILE);
             if (!hiveSite.toUri().isAbsolute()) {
-                // treat relative URI as local file to be compatible with previous behavior
                 hiveSite = new org.apache.hadoop.fs.Path(new File(hiveSite.toString()).toURI());
             }
             hiveConf.addResource(hiveSite);
diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java
index 4c050e998172..ec727f664986 100644
--- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java
+++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java
@@ -26,12 +26,12 @@
 import org.apache.paimon.options.CatalogOptions;
 import org.apache.paimon.options.ConfigOption;
 import org.apache.paimon.options.ConfigOptions;
+import org.apache.paimon.utils.HadoopUtils;
 import org.apache.paimon.utils.Preconditions;
 
 import org.apache.hadoop.hive.conf.HiveConf;
 
 import static org.apache.paimon.hive.HiveCatalog.createHiveConf;
-import static org.apache.paimon.hive.HiveCatalogFactoryOptions.HADOOP_CONF_DIR;
 import static org.apache.paimon.hive.HiveCatalogFactoryOptions.HIVE_CONF_DIR;
 import static org.apache.paimon.hive.HiveCatalogFactoryOptions.IDENTIFIER;
 
@@ -63,7 +63,7 @@ public Catalog create(FileIO fileIO, Path warehouse, CatalogContext context) {
                                 + " catalog");
 
         String hiveConfDir = context.options().get(HIVE_CONF_DIR);
-        String hadoopConfDir = context.options().get(HADOOP_CONF_DIR);
+        String hadoopConfDir = context.options().get(HadoopUtils.PATH_HADOOP_CONFIG);
         HiveConf hiveConf = createHiveConf(hiveConfDir, hadoopConfDir);
 
         // always using user-set parameters overwrite hive-site.xml parameters
diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactoryOptions.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactoryOptions.java
index 643141a3ee1d..8e25f7793ad0 100644
--- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactoryOptions.java
+++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactoryOptions.java
@@ -21,16 +21,24 @@
 import org.apache.paimon.options.ConfigOption;
 import org.apache.paimon.options.ConfigOptions;
 
-/** {@link ConfigOption}s for {@link HiveCatalog}. */
+/** Options for hive catalog. */
 public final class HiveCatalogFactoryOptions {
 
     public static final String IDENTIFIER = "hive";
 
     public static final ConfigOption<String> HIVE_CONF_DIR =
-            ConfigOptions.key("hive-conf-dir").stringType().noDefaultValue();
+            ConfigOptions.key("hive-conf-dir")
+                    .stringType()
+                    .noDefaultValue()
+                    .withDescription(
+                            "File directory of the hive-site.xml , used to create HiveMetastoreClient and security authentication, such as Kerberos, LDAP, Ranger and so on");
 
     public static final ConfigOption<String> HADOOP_CONF_DIR =
-            ConfigOptions.key("hadoop-conf-dir").stringType().noDefaultValue();
+            ConfigOptions.key("hadoop-conf-dir")
+                    .stringType()
+                    .noDefaultValue()
+                    .withDescription(
+                            "File directory of the core-site.xml、hdfs-site.xml、yarn-site.xml、mapred-site.xml. Currently, only local file system paths are supported. We recommend using the HADOOP_CONF_DIR environment variable to specify the Hadoop configuration. So consider using this parameter only if the environment variables don't meet your needs, such as when you want to set up the Hadoop configuration individually for each HiveCatalog");
 
     private HiveCatalogFactoryOptions() {}
 }

From 97227413cec1f57967bd5537961b6308acf2edb4 Mon Sep 17 00:00:00 2001
From: Alibaba-HZY <daowu.hzy@cainiao.com>
Date: Thu, 6 Apr 2023 10:59:27 +0800
Subject: [PATCH 04/10] [hive] add license (#823)

---
 .../test/resources/hadoop-conf-dir/core-site.xml  | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/paimon-hive/paimon-hive-connector-common/src/test/resources/hadoop-conf-dir/core-site.xml b/paimon-hive/paimon-hive-connector-common/src/test/resources/hadoop-conf-dir/core-site.xml
index 768f207586d6..db4d6376e8a2 100644
--- a/paimon-hive/paimon-hive-connector-common/src/test/resources/hadoop-conf-dir/core-site.xml
+++ b/paimon-hive/paimon-hive-connector-common/src/test/resources/hadoop-conf-dir/core-site.xml
@@ -1,5 +1,20 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
 
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
 <!--Autogenerated by Cloudera Manager-->
 <configuration>
   <property>

From deb255aaca346e1ac03e9112ca19857bc6b9d847 Mon Sep 17 00:00:00 2001
From: Alibaba-HZY <daowu.hzy@cainiao.com>
Date: Thu, 6 Apr 2023 12:57:34 +0800
Subject: [PATCH 05/10] [hive] rename HadoopUtils.PATH_HADOOP_CONFIG to
 hadoop-conf-dir to unify names. (#823)

---
 docs/content/filesystems/hdfs.md                              | 2 +-
 .../src/main/java/org/apache/paimon/utils/HadoopUtils.java    | 2 +-
 .../java/org/apache/paimon/fs/HadoopConfigLoadingTest.java    | 2 +-
 .../main/java/org/apache/paimon/hive/HiveCatalogFactory.java  | 4 ++--
 .../java/org/apache/paimon/hive/HiveCatalogITCaseBase.java    | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/content/filesystems/hdfs.md b/docs/content/filesystems/hdfs.md
index 7c4eddc390fa..7a45f0d910e1 100644
--- a/docs/content/filesystems/hdfs.md
+++ b/docs/content/filesystems/hdfs.md
@@ -40,7 +40,7 @@ You may not have to do anything, if you are in a hadoop environment. Otherwise p
 configure your HDFS:
 
 1. Set environment variable `HADOOP_HOME` or `HADOOP_CONF_DIR`.
-2. Configure `'fs.hdfs.hadoopconf'` in the paimon catalog.
+2. Configure `'hadoop-conf-dir'` in the paimon catalog.
 
 The first approach is recommended.
 
diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/HadoopUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/HadoopUtils.java
index bfeb288cc078..a478a08e21eb 100644
--- a/paimon-common/src/main/java/org/apache/paimon/utils/HadoopUtils.java
+++ b/paimon-common/src/main/java/org/apache/paimon/utils/HadoopUtils.java
@@ -39,7 +39,7 @@ public class HadoopUtils {
     public static final String HADOOP_CONF_ENV = "HADOOP_CONF_DIR";
 
     /** Path to Hadoop configuration. */
-    public static final String PATH_HADOOP_CONFIG = "fs.hdfs.hadoopconf";
+    public static final String PATH_HADOOP_CONFIG = "hadoop-conf-dir";
 
     public static Configuration getHadoopConfiguration(Options options) {
 
diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/HadoopConfigLoadingTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/HadoopConfigLoadingTest.java
index 2a998a2461d5..906f9786383e 100644
--- a/paimon-common/src/test/java/org/apache/paimon/fs/HadoopConfigLoadingTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/fs/HadoopConfigLoadingTest.java
@@ -160,7 +160,7 @@ public void loadOverlappingConfig() throws Exception {
         final String k5 = "key5";
 
         final String v1 = "from HADOOP_CONF_DIR";
-        final String v2 = "from Paimon config `fs.hdfs.hadoopconf`";
+        final String v2 = "from Paimon config `hadoop-conf-dir`";
         final String v4 = "from HADOOP_HOME/etc/hadoop";
         final String v5 = "from HADOOP_HOME/conf";
 
diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java
index ec727f664986..4c050e998172 100644
--- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java
+++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java
@@ -26,12 +26,12 @@
 import org.apache.paimon.options.CatalogOptions;
 import org.apache.paimon.options.ConfigOption;
 import org.apache.paimon.options.ConfigOptions;
-import org.apache.paimon.utils.HadoopUtils;
 import org.apache.paimon.utils.Preconditions;
 
 import org.apache.hadoop.hive.conf.HiveConf;
 
 import static org.apache.paimon.hive.HiveCatalog.createHiveConf;
+import static org.apache.paimon.hive.HiveCatalogFactoryOptions.HADOOP_CONF_DIR;
 import static org.apache.paimon.hive.HiveCatalogFactoryOptions.HIVE_CONF_DIR;
 import static org.apache.paimon.hive.HiveCatalogFactoryOptions.IDENTIFIER;
 
@@ -63,7 +63,7 @@ public Catalog create(FileIO fileIO, Path warehouse, CatalogContext context) {
                                 + " catalog");
 
         String hiveConfDir = context.options().get(HIVE_CONF_DIR);
-        String hadoopConfDir = context.options().get(HadoopUtils.PATH_HADOOP_CONFIG);
+        String hadoopConfDir = context.options().get(HADOOP_CONF_DIR);
         HiveConf hiveConf = createHiveConf(hiveConfDir, hadoopConfDir);
 
         // always using user-set parameters overwrite hive-site.xml parameters
diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
index cdd56ef4a8f2..fee29c4e4696 100644
--- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
+++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
@@ -67,7 +67,7 @@ public abstract class HiveCatalogITCaseBase {
     @HiveSQL(files = {})
     protected static HiveShell hiveShell;
 
-    public static String HIVE_CONF = "/hive-conf";
+    public static final String HIVE_CONF = "/hive-conf";
 
     public static final String HADOOP_CONF_DIR =
             Thread.currentThread().getContextClassLoader().getResource("hadoop-conf-dir").getPath();

From a5ff196ede7d9ca9cc01790384d0efc30fdac8cc Mon Sep 17 00:00:00 2001
From: Alibaba-HZY <daowu.hzy@cainiao.com>
Date: Thu, 6 Apr 2023 13:34:17 +0800
Subject: [PATCH 06/10] [hive] generate doc (#823)

---
 .../hive_catalog_factory_configuration.html   | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 docs/layouts/shortcodes/generated/hive_catalog_factory_configuration.html

diff --git a/docs/layouts/shortcodes/generated/hive_catalog_factory_configuration.html b/docs/layouts/shortcodes/generated/hive_catalog_factory_configuration.html
new file mode 100644
index 000000000000..bcc80350606f
--- /dev/null
+++ b/docs/layouts/shortcodes/generated/hive_catalog_factory_configuration.html
@@ -0,0 +1,24 @@
+<table class="configuration table table-bordered">
+    <thead>
+        <tr>
+            <th class="text-left" style="width: 20%">Key</th>
+            <th class="text-left" style="width: 15%">Default</th>
+            <th class="text-left" style="width: 10%">Type</th>
+            <th class="text-left" style="width: 55%">Description</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td><h5>hadoop-conf-dir</h5></td>
+            <td style="word-wrap: break-word;">(none)</td>
+            <td>String</td>
+            <td>File directory of the core-site.xml、hdfs-site.xml、yarn-site.xml、mapred-site.xml. Currently, only local file system paths are supported. We recommend using the HADOOP_CONF_DIR environment variable to specify the Hadoop configuration. So consider using this parameter only if the environment variables don't meet your needs, such as when you want to set up the Hadoop configuration individually for each HiveCatalog</td>
+        </tr>
+        <tr>
+            <td><h5>hive-conf-dir</h5></td>
+            <td style="word-wrap: break-word;">(none)</td>
+            <td>String</td>
+            <td>File directory of the hive-site.xml , used to create HiveMetastoreClient and security authentication, such as Kerberos, LDAP, Ranger and so on</td>
+        </tr>
+    </tbody>
+</table>

From 11015c8869b001a365f5774ca0ef184f78b566d8 Mon Sep 17 00:00:00 2001
From: Alibaba-HZY <daowu.hzy@cainiao.com>
Date: Thu, 6 Apr 2023 14:43:34 +0800
Subject: [PATCH 07/10] [hive] generate doc and hive-conf-dir support uri
 (#823)

---
 docs/content/how-to/creating-catalogs.md           |  3 +--
 ...ration.html => hive_catalog_configuration.html} |  0
 .../java/org/apache/paimon/hive/HiveCatalog.java   | 14 ++++++++++++++
 .../org/apache/paimon/hive/HiveCatalogFactory.java |  6 +++---
 ...FactoryOptions.java => HiveCatalogOptions.java} |  4 ++--
 5 files changed, 20 insertions(+), 7 deletions(-)
 rename docs/layouts/shortcodes/generated/{hive_catalog_factory_configuration.html => hive_catalog_configuration.html} (100%)
 rename paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/{HiveCatalogFactoryOptions.java => HiveCatalogOptions.java} (95%)

diff --git a/docs/content/how-to/creating-catalogs.md b/docs/content/how-to/creating-catalogs.md
index b4d185bd2996..9e0f290fe10e 100644
--- a/docs/content/how-to/creating-catalogs.md
+++ b/docs/content/how-to/creating-catalogs.md
@@ -84,7 +84,7 @@ Paimon Hive catalog in Flink relies on Flink Hive connector bundled jar. You sho
 
 The following Flink SQL registers and uses a Paimon Hive catalog named `my_hive`. Metadata and table files are stored under `hdfs://path/to/warehouse`. In addition, metadata is also stored in Hive metastore.
 
-If your Hive requires security authentication such as Kerberos,LDAP,Ranger and so on. You can specify the hive-conf-dir parameter to the hive-site.xml file path
+If your Hive requires security authentication such as Kerberos, LDAP, Ranger and so on. You can specify the hive-conf-dir parameter to the hive-site.xml file path.
 
 ```sql
 CREATE CATALOG my_hive WITH (
@@ -92,7 +92,6 @@ CREATE CATALOG my_hive WITH (
     'metastore' = 'hive',
     'uri' = 'thrift://<hive-metastore-host-name>:<port>',
     'warehouse' = 'hdfs://path/to/warehouse',
-    'hive-conf-dir' = 'hdfs://path/to/hive-conf',
 );
 
 USE CATALOG my_hive;
diff --git a/docs/layouts/shortcodes/generated/hive_catalog_factory_configuration.html b/docs/layouts/shortcodes/generated/hive_catalog_configuration.html
similarity index 100%
rename from docs/layouts/shortcodes/generated/hive_catalog_factory_configuration.html
rename to docs/layouts/shortcodes/generated/hive_catalog_configuration.html
diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
index 12ca3b4c6f73..382faaba569e 100644
--- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
+++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
@@ -56,6 +56,8 @@
 
 import java.io.File;
 import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -567,12 +569,24 @@ public static HiveConf createHiveConf(
             if (!hiveSite.toUri().isAbsolute()) {
                 hiveSite = new org.apache.hadoop.fs.Path(new File(hiveSite.toString()).toURI());
             }
+            try (InputStream inputStream = hiveSite.getFileSystem(hadoopConf).open(hiveSite)) {
+                hiveConf.addResource(inputStream, hiveSite.toString());
+                // trigger a read from the conf to avoid input stream is closed
+                isEmbeddedMetastore(hiveConf);
+            } catch (IOException e) {
+                throw new RuntimeException(
+                        "Failed to load hive-site.xml from specified path:" + hiveSite, e);
+            }
             hiveConf.addResource(hiveSite);
         }
 
         return hiveConf;
     }
 
+    public static boolean isEmbeddedMetastore(HiveConf hiveConf) {
+        return isNullOrWhitespaceOnly(hiveConf.getVar(HiveConf.ConfVars.METASTOREURIS));
+    }
+
     /**
      * Returns a new Hadoop Configuration object using the path to the hadoop conf configured.
      *
diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java
index 4c050e998172..a33463738e11 100644
--- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java
+++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java
@@ -31,9 +31,9 @@
 import org.apache.hadoop.hive.conf.HiveConf;
 
 import static org.apache.paimon.hive.HiveCatalog.createHiveConf;
-import static org.apache.paimon.hive.HiveCatalogFactoryOptions.HADOOP_CONF_DIR;
-import static org.apache.paimon.hive.HiveCatalogFactoryOptions.HIVE_CONF_DIR;
-import static org.apache.paimon.hive.HiveCatalogFactoryOptions.IDENTIFIER;
+import static org.apache.paimon.hive.HiveCatalogOptions.HADOOP_CONF_DIR;
+import static org.apache.paimon.hive.HiveCatalogOptions.HIVE_CONF_DIR;
+import static org.apache.paimon.hive.HiveCatalogOptions.IDENTIFIER;
 
 /** Factory to create {@link HiveCatalog}. */
 public class HiveCatalogFactory implements CatalogFactory {
diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactoryOptions.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java
similarity index 95%
rename from paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactoryOptions.java
rename to paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java
index 8e25f7793ad0..aeb4ef253627 100644
--- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactoryOptions.java
+++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java
@@ -22,7 +22,7 @@
 import org.apache.paimon.options.ConfigOptions;
 
 /** Options for hive catalog. */
-public final class HiveCatalogFactoryOptions {
+public final class HiveCatalogOptions {
 
     public static final String IDENTIFIER = "hive";
 
@@ -40,5 +40,5 @@ public final class HiveCatalogFactoryOptions {
                     .withDescription(
                             "File directory of the core-site.xml、hdfs-site.xml、yarn-site.xml、mapred-site.xml. Currently, only local file system paths are supported. We recommend using the HADOOP_CONF_DIR environment variable to specify the Hadoop configuration. So consider using this parameter only if the environment variables don't meet your needs, such as when you want to set up the Hadoop configuration individually for each HiveCatalog");
 
-    private HiveCatalogFactoryOptions() {}
+    private HiveCatalogOptions() {}
 }

From 73dd1b55b61f655c04e4ed18e673dc142fb98005 Mon Sep 17 00:00:00 2001
From: Alibaba-HZY <daowu.hzy@cainiao.com>
Date: Thu, 6 Apr 2023 15:29:31 +0800
Subject: [PATCH 08/10] [hive] remove , (#823)

---
 docs/content/how-to/creating-catalogs.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/content/how-to/creating-catalogs.md b/docs/content/how-to/creating-catalogs.md
index 9e0f290fe10e..2ead26372e67 100644
--- a/docs/content/how-to/creating-catalogs.md
+++ b/docs/content/how-to/creating-catalogs.md
@@ -91,7 +91,7 @@ CREATE CATALOG my_hive WITH (
     'type' = 'paimon',
     'metastore' = 'hive',
     'uri' = 'thrift://<hive-metastore-host-name>:<port>',
-    'warehouse' = 'hdfs://path/to/warehouse',
+    'warehouse' = 'hdfs://path/to/warehouse'
 );
 
 USE CATALOG my_hive;

From 63cea57bf03e4ca1fb6e697009896cfc81edb91c Mon Sep 17 00:00:00 2001
From: Alibaba-HZY <daowu.hzy@cainiao.com>
Date: Thu, 6 Apr 2023 16:28:03 +0800
Subject: [PATCH 09/10] [hive] Add hive catalog options reference (#823)

---
 docs/content/maintenance/configurations.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/content/maintenance/configurations.md b/docs/content/maintenance/configurations.md
index 721a8714a839..fdc5843867bb 100644
--- a/docs/content/maintenance/configurations.md
+++ b/docs/content/maintenance/configurations.md
@@ -38,6 +38,12 @@ Options for paimon catalog.
 
 {{< generated/catalog_configuration >}}
 
+### HiveCatalogOptions
+
+Options for Hive catalog.
+
+{{< generated/hive_catalog_configuration >}}
+
 ### FlinkConnectorOptions
 
 Flink connector options for paimon.

From 3da862bff4142cce6e3389bc4bb1a37a86f681e2 Mon Sep 17 00:00:00 2001
From: Alibaba-HZY <daowu.hzy@cainiao.com>
Date: Thu, 6 Apr 2023 21:01:06 +0800
Subject: [PATCH 10/10] [hive] add options test (#823)

---
 .../generated/hive_catalog_configuration.html |  2 +-
 .../paimon/hive/HiveCatalogOptions.java       |  2 +-
 .../paimon/hive/HiveCatalogITCaseBase.java    |  4 --
 .../paimon/hive/HiveCatalogOptionsTest.java   | 46 ++++++++++++++++++
 .../resources/hive-conf-dir/hive-site.xml     | 48 +++++++++++++++++++
 5 files changed, 96 insertions(+), 6 deletions(-)
 create mode 100644 paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogOptionsTest.java
 create mode 100644 paimon-hive/paimon-hive-connector-common/src/test/resources/hive-conf-dir/hive-site.xml

diff --git a/docs/layouts/shortcodes/generated/hive_catalog_configuration.html b/docs/layouts/shortcodes/generated/hive_catalog_configuration.html
index bcc80350606f..e7ec2f4687cb 100644
--- a/docs/layouts/shortcodes/generated/hive_catalog_configuration.html
+++ b/docs/layouts/shortcodes/generated/hive_catalog_configuration.html
@@ -12,7 +12,7 @@
             <td><h5>hadoop-conf-dir</h5></td>
             <td style="word-wrap: break-word;">(none)</td>
             <td>String</td>
-            <td>File directory of the core-site.xml、hdfs-site.xml、yarn-site.xml、mapred-site.xml. Currently, only local file system paths are supported. We recommend using the HADOOP_CONF_DIR environment variable to specify the Hadoop configuration. So consider using this parameter only if the environment variables don't meet your needs, such as when you want to set up the Hadoop configuration individually for each HiveCatalog</td>
+            <td>File directory of the core-site.xml、hdfs-site.xml、yarn-site.xml、mapred-site.xml. Currently, only local file system paths are supported.</td>
         </tr>
         <tr>
             <td><h5>hive-conf-dir</h5></td>
diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java
index aeb4ef253627..ceebdaa20f13 100644
--- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java
+++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java
@@ -38,7 +38,7 @@ public final class HiveCatalogOptions {
                     .stringType()
                     .noDefaultValue()
                     .withDescription(
-                            "File directory of the core-site.xml、hdfs-site.xml、yarn-site.xml、mapred-site.xml. Currently, only local file system paths are supported. We recommend using the HADOOP_CONF_DIR environment variable to specify the Hadoop configuration. So consider using this parameter only if the environment variables don't meet your needs, such as when you want to set up the Hadoop configuration individually for each HiveCatalog");
+                            "File directory of the core-site.xml、hdfs-site.xml、yarn-site.xml、mapred-site.xml. Currently, only local file system paths are supported.");
 
     private HiveCatalogOptions() {}
 }
diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
index fee29c4e4696..d9c1ab67f8c6 100644
--- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
+++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
@@ -69,9 +69,6 @@ public abstract class HiveCatalogITCaseBase {
 
     public static final String HIVE_CONF = "/hive-conf";
 
-    public static final String HADOOP_CONF_DIR =
-            Thread.currentThread().getContextClassLoader().getResource("hadoop-conf-dir").getPath();
-
     @Before
     public void before() throws Exception {
         hiveShell.execute("CREATE DATABASE IF NOT EXISTS test_db");
@@ -94,7 +91,6 @@ public void before() throws Exception {
                                         + hiveShell.getBaseDir().getRoot().getPath()
                                         + HIVE_CONF
                                         + "',",
-                                "  'hadoop-conf-dir' = '" + HADOOP_CONF_DIR + "',",
                                 "  'warehouse' = '" + path + "',",
                                 "  'lock.enabled' = 'true'",
                                 ")"))
diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogOptionsTest.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogOptionsTest.java
new file mode 100644
index 000000000000..f9cf0a96b30b
--- /dev/null
+++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogOptionsTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.hive;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Test for HiveCatalogOptions. */
+public class HiveCatalogOptionsTest {
+
+    public static final String HADOOP_CONF_DIR =
+            Thread.currentThread().getContextClassLoader().getResource("hadoop-conf-dir").getPath();
+
+    public static final String HIVE_CONF_DIR =
+            Thread.currentThread().getContextClassLoader().getResource("hive-conf-dir").getPath();
+
+    @Test
+    public void testHadoopConfDir() {
+        HiveConf hiveConf = HiveCatalog.createHiveConf(null, HADOOP_CONF_DIR);
+        assertThat(hiveConf.get("fs.defaultFS")).isEqualTo("dummy-fs");
+    }
+
+    @Test
+    public void testHiveConfDir() {
+        HiveConf hiveConf = HiveCatalog.createHiveConf(HIVE_CONF_DIR, null);
+        assertThat(hiveConf.get("hive.metastore.uris")).isEqualTo("dummy-hms");
+    }
+}
diff --git a/paimon-hive/paimon-hive-connector-common/src/test/resources/hive-conf-dir/hive-site.xml b/paimon-hive/paimon-hive-connector-common/src/test/resources/hive-conf-dir/hive-site.xml
new file mode 100644
index 000000000000..197217697adf
--- /dev/null
+++ b/paimon-hive/paimon-hive-connector-common/src/test/resources/hive-conf-dir/hive-site.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<configuration>
+
+	<property>
+		<name>hive.metastore.schema.verification</name>
+		<value>false</value>
+	</property>
+
+	<property>
+		<name>hive.metastore.client.capability.check</name>
+		<value>false</value>
+	</property>
+
+	<property>
+		<name>datanucleus.schema.autoCreateTables</name>
+		<value>true</value>
+	</property>
+
+	<property>
+		<name>datanucleus.schema.autoCreateAll</name>
+		<value>true</value>
+	</property>
+
+    <!-- production code doesn't allow embedded mode, set a dummy value here which is OK as long as the created HiveCatalog is not used -->
+	<property>
+		<name>hive.metastore.uris</name>
+		<value>dummy-hms</value>
+	</property>
+
+</configuration>