HIVE-26985: Create a trackable hive configuration object

beeline/src/java/org/apache/hive/beeline/Commands.java

@@ -773,7 +773,7 @@ public HiveConf getHiveConf(boolean call) {
   }
 
   public HiveConf getHiveConfHelper(boolean call) {
-    HiveConf conf = new HiveConf();
+    HiveConf conf = HiveConf.create();
     BufferedRows rows = getConfInternal(call);
     while (rows != null && rows.hasNext()) {
       addConf((Rows.Row) rows.next(), conf);

beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java

@@ -279,7 +279,7 @@ private void verifyCMD(String CMD, String keywords, OutputStream os, String[] op
   @BeforeClass
   public static void init(){
     // something changed scratch dir permissions, so test can't execute
-    HiveConf hiveConf = new HiveConf();
+    HiveConf hiveConf = HiveConf.create();
     String scratchDir = hiveConf.get(HiveConf.ConfVars.SCRATCHDIR.varname);
     File file = new File(scratchDir);
     if (file.exists()) {

cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java

@@ -740,7 +740,7 @@ public int run(String[] args) throws Exception {
       logInitDetailMessage = e.getMessage();
     }
 
-    CliSessionState ss = new CliSessionState(new HiveConf(SessionState.class));
+    CliSessionState ss = new CliSessionState(HiveConf.create(SessionState.class));
     ss.in = System.in;
     try {
       ss.out =

cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java

@@ -130,7 +130,7 @@ public void testThatCliDriverDoesNotStripComments() throws Exception {
     SessionStream err = new SessionStream(dataErr);
     System.setErr(err);
 
-    CliSessionState ss = new CliSessionState(new HiveConf());
+    CliSessionState ss = new CliSessionState(HiveConf.create());
     ss.out = out;
     ss.err = err;
 
@@ -226,7 +226,7 @@ public void testRun() throws Exception {
       File historyFile = new File(historyDirectory + File.separator + ".hivehistory");
       historyFile.delete();
     }
-    HiveConf configuration = new HiveConf();
+    HiveConf configuration = HiveConf.create();
     configuration.setBoolVar(ConfVars.HIVE_SESSION_HISTORY_ENABLED, true);
     PrintStream oldOut = System.out;
     ByteArrayOutputStream dataOut = new ByteArrayOutputStream();
@@ -260,7 +260,7 @@ public void testRun() throws Exception {
   @Test
   public void testQuit() throws Exception {
 
-    CliSessionState ss = new CliSessionState(new HiveConf());
+    CliSessionState ss = new CliSessionState(HiveConf.create());
     ss.err = new SessionStream(System.err);
     ss.out = new SessionStream(System.out);
 
@@ -290,7 +290,7 @@ public void testQuit() throws Exception {
 
   @Test
   public void testProcessSelectDatabase() throws Exception {
-    CliSessionState sessinState = new CliSessionState(new HiveConf());
+    CliSessionState sessinState = new CliSessionState(HiveConf.create());
     CliSessionState.start(sessinState);
     ByteArrayOutputStream data = new ByteArrayOutputStream();
     sessinState.err = new SessionStream(data);
@@ -325,7 +325,7 @@ public void testprocessInitFiles() throws Exception {
     FileUtils.write(homeFile, "-- init hive file for test ");
     setEnv("HIVE_HOME", homeFile.getParentFile().getParentFile().getAbsolutePath());
     setEnv("HIVE_CONF_DIR", homeFile.getParentFile().getAbsolutePath());
-    CliSessionState sessionState = new CliSessionState(new HiveConf());
+    CliSessionState sessionState = new CliSessionState(HiveConf.create());
 
     ByteArrayOutputStream data = new ByteArrayOutputStream();
 

cli/src/test/org/apache/hadoop/hive/cli/TestCliSessionState.java

@@ -34,7 +34,7 @@ public class TestCliSessionState {
    */
   @Test
   public void testgetDbName() throws Exception {
-    SessionState.start(new HiveConf());
+    SessionState.start(HiveConf.create());
     assertEquals(Warehouse.DEFAULT_DATABASE_NAME,
         SessionState.get().getCurrentDatabase());
   }

cli/src/test/org/apache/hadoop/hive/cli/TestOptionsProcessor.java

@@ -49,7 +49,7 @@ public void testOptionsProcessor() {
     assertEquals("D", processor.getHiveVariables().get("C"));
     assertEquals("Y", processor.getHiveVariables().get("X"));
 
-    CliSessionState sessionState = new CliSessionState(new HiveConf());
+    CliSessionState sessionState = new CliSessionState(HiveConf.create());
     // stage 2
     processor.process_stage2(sessionState);
     assertEquals("testDb", sessionState.database);
@@ -69,7 +69,7 @@ public void testFiles() {
     String[] args = {"-i", "f1", "-i", "f2","-f", "fileName",};
     assertTrue(processor.process_stage1(args));
 
-    CliSessionState sessionState = new CliSessionState(new HiveConf());
+    CliSessionState sessionState = new CliSessionState(HiveConf.create());
     processor.process_stage2(sessionState);
     assertEquals("fileName", sessionState.fileName);
     assertEquals(2, sessionState.initFiles.size());

common/src/java/org/apache/hadoop/hive/common/LogUtils.java

@@ -93,7 +93,7 @@ public static String initHiveExecLog4j()
 
   private static String initHiveLog4jCommon(ConfVars confVarName)
       throws LogInitializationException {
-    HiveConf conf = new HiveConf();
+    HiveConf conf = HiveConf.create();
     return initHiveLog4jCommon(conf, confVarName);
   }
 

common/src/java/org/apache/hadoop/hive/conf/HiveConf.java

@@ -477,6 +477,8 @@ public static final Set<String> getLlapDaemonConfVars() {
    * in the underlying Hadoop configuration.
    */
   public static enum ConfVars {
+    HIVE_CONF_PROPERTY_TRACKING("hive.conf.property.tracking", false,
+        "Whether to enable property tracking with TrackedHiveConf objects"),
     MSC_CACHE_ENABLED("hive.metastore.client.cache.v2.enabled", true,
             "This property enables a Caffeine Cache for Metastore client"),
     MSC_CACHE_MAX_SIZE("hive.metastore.client.cache.v2.maxSize", "1Gb", new SizeValidator(),
@@ -6331,29 +6333,76 @@ public ZooKeeperHiveHelper getZKConfig() {
       .trustStorePassword(trustStorePassword).build();
   }
 
-  public HiveConf() {
+  public static HiveConf create() {
+    HiveConf conf = new HiveConf();
+    if (conf.isPropertyTrackingEnabled()) {
+      return new TrackedHiveConf();
+    }
+    return conf;
+  }
+
+  public static HiveConf create(Class<?> cls) {
+    HiveConf conf = new HiveConf(cls);
+    if (conf.isPropertyTrackingEnabled()) {
+      return new TrackedHiveConf(cls);
+    }
+    return conf;
+  }
+
+  public static HiveConf create(Configuration other, Class<?> cls) {
+    if (other.getBoolean(ConfVars.HIVE_CONF_PROPERTY_TRACKING.varname, false)) {
+      return new TrackedHiveConf(other, cls);
+    }
+    return new HiveConf(other, cls);
+  }
+
+  public static HiveConf create(HiveConf other) {
+    if (other.isPropertyTrackingEnabled()) {
+      return new TrackedHiveConf(other);
+    }
+    return new HiveConf(other);
+  }
+
+  private boolean isPropertyTrackingEnabled() {
+    return getBoolVar(ConfVars.HIVE_CONF_PROPERTY_TRACKING);
+  }
+
+  /**
+   * @deprecated This method will become private eventually; Use {@link #create()} instead.
+   */
+  @Deprecated
+  protected HiveConf() {
     super();
     initialize(this.getClass());
   }
 
-  public HiveConf(Class<?> cls) {
+  /**
+   * @deprecated This method will become private eventually; Use {@link #create(cls)} instead.
+   */
+  @Deprecated
+  protected HiveConf(Class<?> cls) {
     super();
     initialize(cls);
   }
 
-  public HiveConf(Configuration other, Class<?> cls) {
+  /**
+   * @deprecated This method will become private eventually; Use {@link #create(other, cls)} instead.
+   */
+  @Deprecated
+  protected HiveConf(Configuration other, Class<?> cls) {
     super(other);
     initialize(cls);
   }
 
   /**
-   * Copy constructor
+   * @deprecated This method will become private eventually; Use {@link #create(other)} instead.
    */
-  public HiveConf(HiveConf other) {
+  @Deprecated
+  protected HiveConf(HiveConf other) {
     super(other);
     hiveJar = other.hiveJar;
     auxJars = other.auxJars;
-    origProp = (Properties)other.origProp.clone();
+    origProp = (Properties) other.origProp.clone();
     restrictList.addAll(other.restrictList);
     hiddenSet.addAll(other.hiddenSet);
     modWhiteListPattern = other.modWhiteListPattern;

common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java

@@ -77,7 +77,7 @@ public static StringBuilder dumpConfig(HiveConf conf) {
     sb.append("hiveServer2SiteUrl=").append(HiveConf.getHiveServer2SiteLocation()).append('\n');
     sb.append("hivemetastoreSiteUrl=").append(HiveConf.getMetastoreSiteLocation()).append('\n');
     dumpConfig(conf, sb);
-    return sb.append("END========\"new HiveConf()\"========\n");
+    return sb.append("END========\"HiveConf.create()\"========\n");
   }
 
   /**

common/src/java/org/apache/hadoop/hive/conf/TrackedHiveConf.java

@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * TrackedHiveConf is a HiveConf object where property changes are logged.
+ */
+public class TrackedHiveConf extends HiveConf {
+  private static final Logger LOG = LoggerFactory.getLogger(TrackedHiveConf.class);
+
+  protected TrackedHiveConf() {
+    super();
+  }
+
+  @Deprecated
+  protected TrackedHiveConf(Class<?> cls) {
+    super(cls);
+  }
+
+  @Deprecated
+  protected TrackedHiveConf(Configuration other, Class<?> cls) {
+    super(other, cls);
+  }
+
+  @Deprecated
+  protected TrackedHiveConf(HiveConf other) {
+    super(other);
+  }
+
+  @Override
+  public void set(String name, String value, String source) {
+    LOG.info("'{}' changed: '{}' -> '{}' (thread: {})", name, get(name), value, Thread.currentThread().getId());
+    LOG.info("Change stack", new RuntimeException("Fake exception, only for easy stack logging (set)"));
+    super.set(name, value, source);
+  }
+
+  @Override
+  public void unset(String name) {
+    LOG.info("'{}' unset, current value: '{}' (thread: {})", name, get(name), Thread.currentThread().getId());
+    LOG.info("Unset stack", new RuntimeException("Fake exception, only for easy stack logging (unset)"));
+    super.unset(name);
+  }
+
+  @Override
+  public void clear() {
+    LOG.info("Configuration is cleared (thread: {})", Thread.currentThread().getId());
+    LOG.info("Clear stack", new RuntimeException("Fake exception, only for easy stack logging (clear)"));
+    super.clear();
+  }
+}

common/src/java/org/apache/hive/http/HttpServer.java

@@ -186,7 +186,7 @@ public HttpServer build() throws IOException {
     }
 
     public Builder setConf(HiveConf origConf) {
-      this.conf = new HiveConf(origConf);
+      this.conf = HiveConf.create(origConf);
       origConf.stripHiddenConfigurations(conf);
       setContextAttribute(CONF_CONTEXT_ATTRIBUTE, conf);
       return this;

common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java

@@ -127,7 +127,7 @@ public void getParentRegardlessOfScheme_root() {
 
   @Test
   public void testGetJarFilesByPath() {
-    HiveConf conf = new HiveConf(this.getClass());
+    HiveConf conf = HiveConf.create(this.getClass());
     File tmpDir = Files.createTempDir();
     String jarFileName1 = tmpDir.getAbsolutePath() + File.separator + "a.jar";
     String jarFileName2 = tmpDir.getAbsolutePath() + File.separator + "b.jar";
@@ -220,7 +220,7 @@ private void verifyIfParentsContainPath(Path key, Set<Path> parents, boolean exp
   public void testCopyWithDistcp() throws IOException {
     Path copySrc = new Path("copySrc");
     Path copyDst = new Path("copyDst");
-    HiveConf conf = new HiveConf(TestFileUtils.class);
+    HiveConf conf = HiveConf.create(TestFileUtils.class);
 
     FileSystem mockFs = mock(FileSystem.class);
     when(mockFs.getUri()).thenReturn(URI.create("hdfs:///"));
@@ -241,7 +241,7 @@ public void testCopyWithDistcp() throws IOException {
   public void testCopyWithDistCpAs() throws IOException {
     Path copySrc = new Path("copySrc");
     Path copyDst = new Path("copyDst");
-    HiveConf conf = new HiveConf(TestFileUtils.class);
+    HiveConf conf = HiveConf.create(TestFileUtils.class);
 
     FileSystem fs = copySrc.getFileSystem(conf);
 
@@ -290,7 +290,7 @@ public void testMakeRelative() {
 
   @Test
   public void testListStatusIterator() throws Exception {
-    MockFileSystem fs = new MockFileSystem(new HiveConf(),
+    MockFileSystem fs = new MockFileSystem(HiveConf.create(),
         new MockFile("mock:/tmp/.staging", 500, new byte[0]),
         new MockFile("mock:/tmp/_dummy", 500, new byte[0]),
         new MockFile("mock:/tmp/dummy", 500, new byte[0]));

common/src/test/org/apache/hadoop/hive/conf/TestHiveAsyncLogging.java

@@ -36,7 +36,7 @@ public class TestHiveAsyncLogging {
   // this test requires disruptor jar in classpath
   @Test
   public void testAsyncLoggingInitialization() throws Exception {
-    HiveConf conf = new HiveConf();
+    HiveConf conf = HiveConf.create();
     conf.setBoolVar(ConfVars.HIVE_ASYNC_LOG_ENABLED, false);
     LogUtils.initHiveLog4jCommon(conf, ConfVars.HIVE_LOG4J_FILE);
     Log4jContextFactory log4jContextFactory = (Log4jContextFactory) LogManager.getFactory();

common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java

@@ -57,7 +57,7 @@ private void checkConfVar(ConfVars var, String expectedConfVarVal) throws Except
   }
 
   private void checkHiveConf(String name, String expectedHiveVal) throws Exception {
-    Assert.assertEquals(expectedHiveVal, new HiveConf().get(name));
+    Assert.assertEquals(expectedHiveVal, HiveConf.create().get(name));
   }
 
   @Test
@@ -137,7 +137,7 @@ public void testToSizeBytes() throws Exception {
 
   @Test
   public void testHiddenConfig() throws Exception {
-    HiveConf conf = new HiveConf();
+    HiveConf conf = HiveConf.create();
 
     // check that a change to the hidden list should fail
     try {
@@ -178,7 +178,7 @@ public void testHiddenConfig() throws Exception {
 
   @Test
   public void testEncodingDecoding() throws UnsupportedEncodingException {
-    HiveConf conf = new HiveConf();
+    HiveConf conf = HiveConf.create();
     String query = "select blah, '\u0001' from random_table";
     conf.setQueryString(query);
     Assert.assertEquals(URLEncoder.encode(query, "UTF-8"), conf.get(ConfVars.HIVEQUERYSTRING.varname));
@@ -229,7 +229,7 @@ public void testAdditionalConfigFiles() throws Exception{
     File f2 = new File(newFileName);
     FileUtils.writeStringToFile(f2, testLdapString);
 
-    HiveConf conf = new HiveConf();
+    HiveConf conf = HiveConf.create();
     String val = conf.getVar(ConfVars.HIVE_SERVER2_PLAIN_LDAP_DOMAIN);
     Assert.assertEquals("b.com", val);
     //restore and clean up

common/src/test/org/apache/hadoop/hive/conf/TestHiveConfRestrictList.java

@@ -37,7 +37,7 @@ public void setUp() throws Exception {
 
     System.setProperty(ConfVars.HIVE_CONF_RESTRICTED_LIST.varname,
         ConfVars.HIVETESTMODEPREFIX.varname);
-    conf = new HiveConf();
+    conf = HiveConf.create();
   }
 
   /**

common/src/test/org/apache/hadoop/hive/conf/TestHiveConfUtil.java

@@ -28,7 +28,7 @@
  */
 public class TestHiveConfUtil {
 
-  private HiveConf conf = new HiveConf();
+  private HiveConf conf = HiveConf.create();
 
   @Before
   public void init() {

common/src/test/org/apache/hadoop/hive/conf/TestHiveLogging.java

@@ -49,7 +49,7 @@ private void configLog(String hiveLog4jTest, String hiveExecLog4jTest)
 
     LogUtils.initHiveLog4j();
 
-    HiveConf conf = new HiveConf();
+    HiveConf conf = HiveConf.create();
     assertEquals(expectedLog4jTestPath, conf.getVar(ConfVars.HIVE_LOG4J_FILE));
     assertEquals(expectedLog4jExecPath, conf.getVar(ConfVars.HIVE_EXEC_LOG4J_FILE));
   }