[ALLUXIO-3216] Instanced config (#7349)

* Initial support for instanced configuration

* Use instanced configuration from AbstractFileSystem

* Fix checkstyle

* Cleanups

* Fix checkstyle

* Fix compile

* Prefer conf to config

* Address review comments

* Move InstancedConfiguration to alluxio.conf

build/checkstyle/alluxio_checks.xml

@@ -67,8 +67,6 @@
 
   <!-- All Java AST specific tests live under TreeWalker module. -->
   <module name="TreeWalker">
-    <!-- Checks that a class which has only private constructors is declared as final. -->
-    <module name="FinalClass"/>
     <module name="OuterTypeFilename"/>
     <module name="IllegalTokenText">
       <property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>

core/client/fs/src/main/java/alluxio/client/file/FileSystem.java

@@ -77,12 +77,7 @@ public static FileSystem get(FileSystemContext context) {
           String key = entry.getKey();
           String value = entry.getValue();
           Source source = Configuration.getSource(PropertyKey.fromString(key));
-          if (source == Source.SITE_PROPERTY) {
-            LOG.debug("{}={} ({}: {})",
-                key, value, source.name(), Configuration.getSitePropertiesFile());
-          } else {
-            LOG.debug("{}={} ({})", key, value, source.name());
-          }
+          LOG.debug("{}={} ({})", key, value, source);
         }
       }
       if (Configuration.getBoolean(PropertyKey.USER_LINEAGE_ENABLED)) {

core/client/hdfs/src/main/java/alluxio/hadoop/AbstractFileSystem.java

@@ -14,8 +14,10 @@
 import static java.util.stream.Collectors.toList;
 import static java.util.stream.Collectors.toMap;
 
+import alluxio.AlluxioConfiguration;
 import alluxio.AlluxioURI;
 import alluxio.Configuration;
+import alluxio.conf.InstancedConfiguration;
 import alluxio.PropertyKey;
 import alluxio.client.block.AlluxioBlockStore;
 import alluxio.client.block.BlockWorkerInfo;
@@ -495,7 +497,7 @@ void initializeInternal(URI uri, org.apache.hadoop.conf.Configuration conf) thro
     // Load Alluxio configuration if any and merge to the one in Alluxio file system. These
     // modifications to ClientContext are global, affecting all Alluxio clients in this JVM.
     // We assume here that all clients use the same configuration.
-    HadoopConfigurationUtils.mergeHadoopConfiguration(conf);
+    HadoopConfigurationUtils.mergeHadoopConfiguration(conf, Configuration.global());
     Configuration.set(PropertyKey.ZOOKEEPER_ENABLED, isZookeeperMode());
     // When using zookeeper we get the leader master address from the alluxio.zookeeper.address
     // configuration property, so the user doesn't need to specify the authority.
@@ -532,21 +534,11 @@ void initializeInternal(URI uri, org.apache.hadoop.conf.Configuration conf) thro
   private boolean connectDetailsMatch(URI uri, org.apache.hadoop.conf.Configuration conf) {
     // Create the master inquire client that we would have after merging the hadoop conf into
     // Alluxio Configuration.
-    MasterInquireClient.Factory.Config inquireClientConf =
-        MasterInquireClient.Factory.Config.defaults();
-    inquireClientConf.setZookeeperEnabled(conf.getBoolean(PropertyKey.ZOOKEEPER_ENABLED.getName(),
-        inquireClientConf.isZookeeperEnabled()));
-    inquireClientConf.setZookeeperAddress(
-        conf.get(PropertyKey.ZOOKEEPER_ADDRESS.getName(), inquireClientConf.getZookeeperAddress()));
-    inquireClientConf.setElectionPath(conf.get(PropertyKey.ZOOKEEPER_ELECTION_PATH.getName(),
-        inquireClientConf.getElectionPath()));
-    inquireClientConf.setLeaderPath(
-        conf.get(PropertyKey.ZOOKEEPER_ELECTION_PATH.getName(), inquireClientConf.getLeaderPath()));
-    inquireClientConf.setConnectHost(mUri.getHost());
-    inquireClientConf.setConnectPort(mUri.getPort());
-    MasterInquireClient configClient = MasterInquireClient.Factory.create(inquireClientConf);
-    MasterInquireClient contextClient = FileSystemContext.get().getMasterInquireClient();
-    return configClient.equals(contextClient);
+    AlluxioConfiguration alluxioConf = new InstancedConfiguration(Configuration.global());
+    HadoopConfigurationUtils.mergeHadoopConfiguration(conf, alluxioConf);
+    MasterInquireClient confClient = MasterInquireClient.Factory.create(alluxioConf);
+
+    return confClient.equals(FileSystemContext.get().getMasterInquireClient());
   }
 
   /**

core/client/hdfs/src/main/java/alluxio/hadoop/HadoopConfigurationUtils.java

@@ -11,6 +11,7 @@
 
 package alluxio.hadoop;
 
+import alluxio.AlluxioConfiguration;
 import alluxio.Configuration;
 import alluxio.PropertyKey;
 import alluxio.conf.Source;
@@ -57,8 +58,10 @@ public static void storeToHadoopConfiguration(org.apache.hadoop.conf.Configurati
    * Merges Hadoop {@link org.apache.hadoop.conf.Configuration} into the Alluxio configuration.
    *
    * @param source the {@link org.apache.hadoop.conf.Configuration} to merge
+   * @param alluxioConfiguration the Alluxio configuration to merge to
    */
-  public static void mergeHadoopConfiguration(org.apache.hadoop.conf.Configuration source) {
+  public static void mergeHadoopConfiguration(org.apache.hadoop.conf.Configuration source,
+      AlluxioConfiguration alluxioConfiguration) {
     // Load Alluxio configuration if any and merge to the one in Alluxio file system
     // Push Alluxio configuration to the Job configuration
     Properties alluxioConfProperties = new Properties();
@@ -72,7 +75,7 @@ public static void mergeHadoopConfiguration(org.apache.hadoop.conf.Configuration
     LOG.info("Loading Alluxio properties from Hadoop configuration: {}", alluxioConfProperties);
     // Merge the relevant Hadoop configuration into Alluxio's configuration.
     // TODO(jiri): support multiple client configurations (ALLUXIO-2034)
-    Configuration.merge(alluxioConfProperties, Source.RUNTIME);
-    Configuration.validate();
+    alluxioConfiguration.merge(alluxioConfProperties, Source.RUNTIME);
+    alluxioConfiguration.validate();
   }
 }

core/client/hdfs/src/test/java/alluxio/hadoop/HadoopConfigurationUtilsTest.java

@@ -42,7 +42,7 @@ public void after() {
   public void mergeEmptyHadoopConfiguration() {
     org.apache.hadoop.conf.Configuration hadoopConfig = new org.apache.hadoop.conf.Configuration();
     long beforeSize = Configuration.toMap().size();
-    HadoopConfigurationUtils.mergeHadoopConfiguration(hadoopConfig);
+    HadoopConfigurationUtils.mergeHadoopConfiguration(hadoopConfig, Configuration.global());
     long afterSize = Configuration.toMap().size();
     Assert.assertEquals(beforeSize, afterSize);
   }
@@ -59,7 +59,7 @@ public void mergeHadoopConfiguration() {
 
     // This hadoop config will not be loaded into Alluxio configuration.
     hadoopConfig.set("hadoop.config.parameter", "hadoop config value");
-    HadoopConfigurationUtils.mergeHadoopConfiguration(hadoopConfig);
+    HadoopConfigurationUtils.mergeHadoopConfiguration(hadoopConfig, Configuration.global());
     Assert.assertEquals(TEST_S3_ACCCES_KEY, Configuration.get(PropertyKey.S3A_ACCESS_KEY));
     Assert.assertEquals(TEST_S3_SECRET_KEY, Configuration.get(PropertyKey.S3A_SECRET_KEY));
     Assert.assertEquals(Source.RUNTIME, Configuration.getSource(PropertyKey.S3A_ACCESS_KEY));

core/common/src/main/java/alluxio/AlluxioConfiguration.java

@@ -0,0 +1,188 @@
+/*
+ * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
+ * (the "License"). You may not use this work except in compliance with the License, which is
+ * available at www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied, as more fully set forth in the License.
+ *
+ * See the NOTICE file distributed with this work for information regarding copyright ownership.
+ */
+
+package alluxio;
+
+import alluxio.conf.Source;
+import alluxio.wire.ConfigProperty;
+import alluxio.wire.Scope;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Alluxio configuration.
+ */
+public interface AlluxioConfiguration {
+  /**
+   * Gets the value for the given key in the {@link Properties}; if this key is not found, a
+   * RuntimeException is thrown.
+   *
+   * @param key the key to get the value for
+   * @return the value for the given key
+   */
+  String get(PropertyKey key);
+
+  /**
+   * Checks if the configuration contains value for the given key.
+   *
+   * @param key the key to check
+   * @return true if there is value for the key, false otherwise
+   */
+  boolean containsKey(PropertyKey key);
+
+  /**
+   * Gets the integer representation of the value for the given key.
+   *
+   * @param key the key to get the value for
+   * @return the value for the given key as an {@code int}
+   */
+  int getInt(PropertyKey key);
+
+  /**
+   * Gets the long representation of the value for the given key.
+   *
+   * @param key the key to get the value for
+   * @return the value for the given key as a {@code long}
+   */
+  long getLong(PropertyKey key);
+
+  /**
+   * Gets the double representation of the value for the given key.
+   *
+   * @param key the key to get the value for
+   * @return the value for the given key as a {@code double}
+   */
+  double getDouble(PropertyKey key);
+
+  /**
+   * Gets the float representation of the value for the given key.
+   *
+   * @param key the key to get the value for
+   * @return the value for the given key as a {@code float}
+   */
+  float getFloat(PropertyKey key);
+
+  /**
+   * Gets the boolean representation of the value for the given key.
+   *
+   * @param key the key to get the value for
+   * @return the value for the given key as a {@code boolean}
+   */
+  boolean getBoolean(PropertyKey key);
+
+  /**
+   * Gets the value for the given key as a list.
+   *
+   * @param key the key to get the value for
+   * @param delimiter the delimiter to split the values
+   * @return the list of values for the given key
+   */
+  List<String> getList(PropertyKey key, String delimiter);
+
+  /**
+   * Gets the value for the given key as an enum value.
+   *
+   * @param key the key to get the value for
+   * @param enumType the type of the enum
+   * @param <T> the type of the enum
+   * @return the value for the given key as an enum value
+   */
+  <T extends Enum<T>> T getEnum(PropertyKey key, Class<T> enumType);
+
+  /**
+   * Gets the bytes of the value for the given key.
+   *
+   * @param key the key to get the value for
+   * @return the bytes of the value for the given key
+   */
+  long getBytes(PropertyKey key);
+
+  /**
+   * Gets the time of key in millisecond unit.
+   *
+   * @param key the key to get the value for
+   * @return the time of key in millisecond unit
+   */
+  long getMs(PropertyKey key);
+
+  /**
+   * Gets the time of the key as a duration.
+   *
+   * @param key the key to get the value for
+   * @return the value of the key represented as a duration
+   */
+  Duration getDuration(PropertyKey key);
+
+  /**
+   * Gets the value for the given key as a class.
+   *
+   * @param key the key to get the value for
+   * @param <T> the type of the class
+   * @return the value for the given key as a class
+   */
+  <T> Class<T> getClass(PropertyKey key);
+
+  /**
+   * Gets a set of properties that share a given common prefix key as a map. E.g., if A.B=V1 and
+   * A.C=V2, calling this method with prefixKey=A returns a map of {B=V1, C=V2}, where B and C are
+   * also valid properties. If no property shares the prefix, an empty map is returned.
+   *
+   * @param prefixKey the prefix key
+   * @return a map from nested properties aggregated by the prefix
+   */
+  Map<String, String> getNestedProperties(PropertyKey prefixKey);
+
+  /**
+   * @return a view of the resolved properties represented by this configuration,
+   *         including all default properties
+   */
+  Map<String, String> toMap();
+
+  /**
+   * @return a map of the raw properties represented by this configuration,
+   *         including all default properties
+   */
+  Map<String, String> toRawMap();
+
+  /**
+   * @param key the property key
+   * @return the source for the given key
+   */
+  Source getSource(PropertyKey key);
+
+  /**
+   * Gets the raw configuration of a given scope.
+   *
+   * @param scope the property key scope
+   * @return a list of raw configurations inside the property scope
+   */
+  List<ConfigProperty> getConfiguration(Scope scope);
+
+  /**
+   * Merges the current configuration properties with new properties. If a property exists
+   * both in the new and current configuration, the one from the new configuration wins if
+   * its priority is higher or equal than the existing one.
+   *
+   * @param properties the source {@link Properties} to be merged
+   * @param source the source of the the properties (e.g., system property, default and etc)
+   */
+  void merge(Map<?, ?> properties, Source source);
+
+  /**
+   * Validates the configuration.
+   *
+   * @throws IllegalStateException if invalid configuration is encountered
+   */
+  void validate();
+}