HDDS-2444. Remove server side dependencies from ozonefs jar files

Revert "HDDS-2427. Exclude webapps from hadoop-ozone-filesystem-lib-current uber jar" This reverts commit 6a450fb.
apache · Nov 11, 2019 · e833021 · e833021
1 parent 6a450fb
commit e833021
Show file tree

Hide file tree

Showing 13 changed files with 140 additions and 135 deletions.
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
@@ -19,7 +19,6 @@
 
 import com.google.common.base.Strings;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.server.ServerUtils;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -345,18 +344,12 @@ public static String getOzoneDatanodeRatisDirectory(Configuration conf) {
             OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR);
 
     if (Strings.isNullOrEmpty(storageDir)) {
-      storageDir = getDefaultRatisDirectory(conf);
+      storageDir = ServerUtils.getDefaultRatisDirectory(conf);
     }
     return storageDir;
   }
 
-  public static String getDefaultRatisDirectory(Configuration conf) {
-    LOG.warn("Storage directory for Ratis is not configured. It is a good " +
-            "idea to map this to an SSD disk. Falling back to {}",
-        HddsConfigKeys.OZONE_METADATA_DIRS);
-    File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf);
-    return (new File(metaDirPath, "ratis")).getPath();
-  }
+
 
   /**
    * Get the path for datanode id file.

diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java
@@ -224,4 +224,12 @@ public static File getDBPath(Configuration conf, String key) {
         HddsConfigKeys.OZONE_METADATA_DIRS);
     return ServerUtils.getOzoneMetaDirPath(conf);
   }
+
+  public static String getDefaultRatisDirectory(Configuration conf) {
+    LOG.warn("Storage directory for Ratis is not configured. It is a good " +
+            "idea to map this to an SSD disk. Falling back to {}",
+        HddsConfigKeys.OZONE_METADATA_DIRS);
+    File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf);
+    return (new File(metaDirPath, "ratis")).getPath();
+  }
 }
diff --git a/hadoop-ozone/common/pom.xml b/hadoop-ozone/common/pom.xml
@@ -56,14 +56,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdds-common</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdds-server-framework</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdds-container-service</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdds-client</artifactId>

diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -26,7 +26,6 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.security.SecureRandom;
@@ -35,8 +34,6 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 
-import com.google.common.base.Strings;
-
 import org.apache.commons.compress.archivers.ArchiveEntry;
 import org.apache.commons.compress.archivers.ArchiveOutputStream;
 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
@@ -46,11 +43,8 @@
 import org.apache.commons.compress.utils.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.scm.HddsServerUtil;
-import org.apache.hadoop.hdds.server.ServerUtils;
 import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
 import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
@@ -197,16 +191,6 @@ public static int getOmRestPort(Configuration conf) {
     return port.orElse(OZONE_OM_HTTP_BIND_PORT_DEFAULT);
   }
 
-  /**
-   * Get the location where OM should store its metadata directories.
-   * Fall back to OZONE_METADATA_DIRS if not defined.
-   *
-   * @param conf - Config
-   * @return File path, after creating all the required Directories.
-   */
-  public static File getOmDbDir(Configuration conf) {
-    return ServerUtils.getDBPath(conf, OMConfigKeys.OZONE_OM_DB_DIRS);
-  }
 
   /**
    * Checks if the OM request is read only or not.
@@ -462,28 +446,6 @@ public static String getHttpsAddressForOMPeerNode(Configuration conf,
         addressPort.orElse(OZONE_OM_HTTPS_BIND_PORT_DEFAULT);
   }
 
-  /**
-   * Get the local directory where ratis logs will be stored.
-   */
-  public static String getOMRatisDirectory(Configuration conf) {
-    String storageDir = conf.get(OMConfigKeys.OZONE_OM_RATIS_STORAGE_DIR);
-
-    if (Strings.isNullOrEmpty(storageDir)) {
-      storageDir = HddsServerUtil.getDefaultRatisDirectory(conf);
-    }
-    return storageDir;
-  }
-
-  public static String getOMRatisSnapshotDirectory(Configuration conf) {
-    String snapshotDir = conf.get(OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_DIR);
-
-    if (Strings.isNullOrEmpty(snapshotDir)) {
-      snapshotDir = Paths.get(getOMRatisDirectory(conf),
-          "snapshot").toString();
-    }
-    return snapshotDir;
-  }
-
   public static File createOMDir(String dirPath) {
     File dirFile = new File(dirPath);
     if (!dirFile.exists() && !dirFile.mkdirs()) {

diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/TestOmUtils.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/TestOmUtils.java
@@ -21,14 +21,8 @@
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
 import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.test.PathUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.junit.Rule;
 import org.junit.Test;
-import org.junit.rules.ExpectedException;
 import org.junit.rules.TemporaryFolder;
 import org.junit.rules.Timeout;
 
@@ -40,9 +34,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
-import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
 
 /**
  * Unit tests for {@link OmUtils}.
@@ -55,54 +47,6 @@ public class TestOmUtils {
   @Rule
   public Timeout timeout = new Timeout(60_000);
 
-  @Rule
-  public ExpectedException thrown= ExpectedException.none();
-
-  /**
-   * Test {@link OmUtils#getOmDbDir}.
-   */
-  @Test
-  public void testGetOmDbDir() {
-    final File testDir = PathUtils.getTestDir(TestOmUtils.class);
-    final File dbDir = new File(testDir, "omDbDir");
-    final File metaDir = new File(testDir, "metaDir");   // should be ignored.
-    final Configuration conf = new OzoneConfiguration();
-    conf.set(OMConfigKeys.OZONE_OM_DB_DIRS, dbDir.getPath());
-    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.getPath());
-
-    try {
-      assertEquals(dbDir, OmUtils.getOmDbDir(conf));
-      assertTrue(dbDir.exists());          // should have been created.
-    } finally {
-      FileUtils.deleteQuietly(dbDir);
-    }
-  }
-
-  /**
-   * Test {@link OmUtils#getOmDbDir} with fallback to OZONE_METADATA_DIRS
-   * when OZONE_OM_DB_DIRS is undefined.
-   */
-  @Test
-  public void testGetOmDbDirWithFallback() {
-    final File testDir = PathUtils.getTestDir(TestOmUtils.class);
-    final File metaDir = new File(testDir, "metaDir");
-    final Configuration conf = new OzoneConfiguration();
-    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.getPath());
-
-    try {
-      assertEquals(metaDir, OmUtils.getOmDbDir(conf));
-      assertTrue(metaDir.exists());        // should have been created.
-    } finally {
-      FileUtils.deleteQuietly(metaDir);
-    }
-  }
-
-  @Test
-  public void testNoOmDbDirConfigured() {
-    thrown.expect(IllegalArgumentException.class);
-    OmUtils.getOmDbDir(new OzoneConfiguration());
-  }
-
   @Test
   public void testWriteCheckpointToOutputStream() throws Exception {
 

diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml
@@ -45,6 +45,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <artifactId>hadoop-hdds-docs</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdds-server-framework</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>org.bouncycastle</groupId>
       <artifactId>bcprov-jdk15on</artifactId>

diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java
@@ -17,13 +17,15 @@
  */
 package org.apache.hadoop.ozone.om;
 
+import java.io.File;
 import java.io.IOException;
 import java.util.Properties;
 import java.util.UUID;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
-import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.hdds.server.ServerUtils;
 import org.apache.hadoop.ozone.common.Storage;
 
 import static org.apache.hadoop.ozone.OzoneConsts.SCM_ID;
@@ -43,7 +45,7 @@ public class OMStorage extends Storage {
    * @throws IOException if any directories are inaccessible.
    */
   public OMStorage(OzoneConfiguration conf) throws IOException {
-    super(NodeType.OM, OmUtils.getOmDbDir(conf), STORAGE_DIR);
+    super(NodeType.OM, getOmDbDir(conf), STORAGE_DIR);
   }
 
   public void setScmId(String scmId) throws IOException {
@@ -104,4 +106,15 @@ protected Properties getNodeProperties() {
     }
     return omProperties;
   }
+
+  /**
+   * Get the location where OM should store its metadata directories.
+   * Fall back to OZONE_METADATA_DIRS if not defined.
+   *
+   * @param conf - Config
+   * @return File path, after creating all the required Directories.
+   */
+  public static File getOmDbDir(Configuration conf) {
+    return ServerUtils.getDBPath(conf, OMConfigKeys.OZONE_OM_DB_DIRS);
+  }
 }
diff --git a/...p-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/...p-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -40,7 +40,6 @@
 import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
 import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
 import org.apache.hadoop.hdds.utils.db.cache.TableCacheImpl;
-import org.apache.hadoop.ozone.OmUtils;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.common.BlockGroup;
 import org.apache.hadoop.ozone.om.codec.OmBucketInfoCodec;
@@ -243,7 +242,7 @@ public void start(OzoneConfiguration configuration) throws IOException {
     // We need to create the DB here, as when during restart, stop closes the
     // db, so we need to create the store object and initialize DB.
     if (store == null) {
-      File metaDir = OmUtils.getOmDbDir(configuration);
+      File metaDir = OMStorage.getOmDbDir(configuration);
 
       RocksDBConfiguration rocksDBConfiguration =
           configuration.getObject(RocksDBConfiguration.class);

diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -343,7 +343,7 @@ private OzoneManager(OzoneConfiguration conf) throws IOException,
 
     // Read configuration and set values.
     ozAdmins = conf.getTrimmedStringCollection(OZONE_ADMINISTRATORS);
-    omMetaDir = OmUtils.getOmDbDir(configuration);
+    omMetaDir = OMStorage.getOmDbDir(configuration);
     this.isAclEnabled = conf.getBoolean(OZONE_ACL_ENABLED,
         OZONE_ACL_ENABLED_DEFAULT);
     this.scmBlockSize = (long) conf.getStorageSize(OZONE_SCM_BLOCK_SIZE,
@@ -416,15 +416,16 @@ private OzoneManager(OzoneConfiguration conf) throws IOException,
 
     if (isRatisEnabled) {
       // Create Ratis storage dir
-      String omRatisDirectory = OmUtils.getOMRatisDirectory(configuration);
+      String omRatisDirectory =
+          OzoneManagerRatisServer.getOMRatisDirectory(configuration);
       if (omRatisDirectory == null || omRatisDirectory.isEmpty()) {
         throw new IllegalArgumentException(HddsConfigKeys.OZONE_METADATA_DIRS +
             " must be defined.");
       }
       OmUtils.createOMDir(omRatisDirectory);
       // Create Ratis snapshot dir
       omRatisSnapshotDir = OmUtils.createOMDir(
-          OmUtils.getOMRatisSnapshotDirectory(configuration));
+          OzoneManagerRatisServer.getOMRatisSnapshotDirectory(configuration));
 
       if (peerNodes != null && !peerNodes.isEmpty()) {
         this.omSnapshotProvider = new OzoneManagerSnapshotProvider(

diff --git a/...ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java b/...ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java
@@ -23,6 +23,7 @@
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -34,11 +35,12 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
+import com.google.common.base.Strings;
 import com.google.protobuf.InvalidProtocolBufferException;
 import com.google.protobuf.ServiceException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.StorageUnit;
-import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.hdds.server.ServerUtils;
 import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.om.ha.OMNodeDetails;
 import org.apache.hadoop.ozone.om.OzoneManager;
@@ -364,7 +366,7 @@ private RaftProperties newRaftProperties(Configuration conf) {
     }
 
     // Set Ratis storage directory
-    String storageDir = OmUtils.getOMRatisDirectory(conf);
+    String storageDir = OzoneManagerRatisServer.getOMRatisDirectory(conf);
     RaftServerConfigKeys.setStorageDirs(properties,
         Collections.singletonList(new File(storageDir)));
 
@@ -645,4 +647,26 @@ private UUID getRaftGroupIdFromOmServiceId(String omServiceId) {
   public long getStateMachineLastAppliedIndex() {
     return omStateMachine.getLastAppliedIndex();
   }
+
+  /**
+   * Get the local directory where ratis logs will be stored.
+   */
+  public static String getOMRatisDirectory(Configuration conf) {
+    String storageDir = conf.get(OMConfigKeys.OZONE_OM_RATIS_STORAGE_DIR);
+
+    if (Strings.isNullOrEmpty(storageDir)) {
+      storageDir = ServerUtils.getDefaultRatisDirectory(conf);
+    }
+    return storageDir;
+  }
+
+  public static String getOMRatisSnapshotDirectory(Configuration conf) {
+    String snapshotDir = conf.get(OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_DIR);
+
+    if (Strings.isNullOrEmpty(snapshotDir)) {
+      snapshotDir = Paths.get(getOMRatisDirectory(conf),
+          "snapshot").toString();
+    }
+    return snapshotDir;
+  }
 }