HADOOP-19081. Move some hadoop-common code to new hadoop-ftp module

hadoop-common-project/hadoop-common/pom.xml

@@ -289,10 +289,6 @@
       <artifactId>hadoop-minikdc</artifactId>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>com.jcraft</groupId>
-      <artifactId>jsch</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.apache.curator</groupId>
       <artifactId>curator-test</artifactId>
@@ -311,21 +307,6 @@
       <artifactId>jsr305</artifactId>
       <scope>compile</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.sshd</groupId>
-      <artifactId>sshd-core</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.sshd</groupId>
-      <artifactId>sshd-sftp</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.ftpserver</groupId>
-      <artifactId>ftpserver-core</artifactId>
-      <scope>test</scope>
-    </dependency>
 
     <dependency>
       <groupId>org.apache.zookeeper</groupId>

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java

@@ -23,7 +23,6 @@
 import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.ftp.FtpConfigKeys;
 import org.apache.hadoop.fs.local.LocalConfigKeys;
 import org.apache.hadoop.ha.SshFenceByTcpPort;
 import org.apache.hadoop.ha.ZKFailoverController;
@@ -40,7 +39,6 @@
  * {@link org.apache.hadoop.fs.AbstractFileSystem}
  * {@link org.apache.hadoop.fs.CommonConfigurationKeys}
  * {@link org.apache.hadoop.fs.CommonConfigurationKeysPublic}
- * {@link org.apache.hadoop.fs.ftp.FtpConfigKeys}
  * {@link org.apache.hadoop.fs.local.LocalConfigKeys}
  * {@link org.apache.hadoop.ha.SshFenceByTcpPort}
  * {@link org.apache.hadoop.http.HttpServer2}
@@ -65,7 +63,6 @@ public void initializeMemberVariables() {
         CommonConfigurationKeys.class,
         CommonConfigurationKeysPublic.class,
         LocalConfigKeys.class,
-        FtpConfigKeys.class,
         SshFenceByTcpPort.class,
         LdapGroupsMapping.class,
         ZKFailoverController.class,
@@ -85,11 +82,6 @@ public void initializeMemberVariables() {
     errorIfMissingXmlProps = false;
 
     // Lots of properties not in the above classes
-    xmlPropsToSkipCompare.add("fs.ftp.password.localhost");
-    xmlPropsToSkipCompare.add("fs.ftp.user.localhost");
-    xmlPropsToSkipCompare.add("fs.ftp.data.connection.mode");
-    xmlPropsToSkipCompare.add("fs.ftp.transfer.mode");
-    xmlPropsToSkipCompare.add("fs.ftp.timeout");
     xmlPropsToSkipCompare.add("hadoop.tmp.dir");
     xmlPropsToSkipCompare.add("nfs3.mountd.port");
     xmlPropsToSkipCompare.add("nfs3.server.port");
@@ -138,52 +130,13 @@ public void initializeMemberVariables() {
     xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.webhdfs.impl");
     xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.wasb.impl");
 
-    // Azure properties are in a different class
-    // - org.apache.hadoop.fs.azure.AzureNativeFileSystemStore
-    // - org.apache.hadoop.fs.azure.SASKeyGeneratorImpl
-    xmlPropsToSkipCompare.add("fs.azure.sas.expiry.period");
-    xmlPropsToSkipCompare.add("fs.azure.local.sas.key.mode");
-    xmlPropsToSkipCompare.add("fs.azure.secure.mode");
-    xmlPropsToSkipCompare.add("fs.azure.authorization");
-    xmlPropsToSkipCompare.add("fs.azure.authorization.caching.enable");
-    xmlPropsToSkipCompare.add("fs.azure.saskey.usecontainersaskeyforallaccess");
-    xmlPropsToSkipCompare.add("fs.azure.user.agent.prefix");
+    addAzureSettings();
 
     // Properties in enable callqueue overflow trigger failover for stateless servers.
     xmlPropsToSkipCompare.add("ipc.[port_number].callqueue.overflow.trigger.failover");
     xmlPropsToSkipCompare.add("ipc.callqueue.overflow.trigger.failover");
 
-    // FairCallQueue configs that includes dynamic ports in its keys
-    xmlPropsToSkipCompare.add("ipc.[port_number].backoff.enable");
-    xmlPropsToSkipCompare.add("ipc.backoff.enable");
-    xmlPropsToSkipCompare.add("ipc.[port_number].callqueue.impl");
-    xmlPropsToSkipCompare.add("ipc.callqueue.impl");
-    xmlPropsToSkipCompare.add("ipc.[port_number].scheduler.impl");
-    xmlPropsToSkipCompare.add("ipc.scheduler.impl");
-    xmlPropsToSkipCompare.add("ipc.[port_number].scheduler.priority.levels");
-    xmlPropsToSkipCompare.add("ipc.[port_number].callqueue.capacity.weights");
-    xmlPropsToSkipCompare.add(
-        "ipc.[port_number].faircallqueue.multiplexer.weights");
-    xmlPropsToSkipCompare.add("ipc.[port_number].identity-provider.impl");
-    xmlPropsToSkipCompare.add("ipc.identity-provider.impl");
-    xmlPropsToSkipCompare.add("ipc.[port_number].cost-provider.impl");
-    xmlPropsToSkipCompare.add("ipc.cost-provider.impl");
-    xmlPropsToSkipCompare.add("ipc.[port_number].decay-scheduler.period-ms");
-    xmlPropsToSkipCompare.add("ipc.[port_number].decay-scheduler.decay-factor");
-    xmlPropsToSkipCompare.add("ipc.[port_number].decay-scheduler.thresholds");
-    xmlPropsToSkipCompare.add(
-        "ipc.[port_number].decay-scheduler.backoff.responsetime.enable");
-    xmlPropsToSkipCompare.add(
-        "ipc.[port_number].decay-scheduler.backoff.responsetime.thresholds");
-    xmlPropsToSkipCompare.add(
-        "ipc.[port_number].decay-scheduler.metrics.top.user.count");
-    xmlPropsToSkipCompare.add(
-        "ipc.[port_number].decay-scheduler.service-users");
-    xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.lockshared");
-    xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.lockexclusive");
-    xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.handler");
-    xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.lockfree");
-    xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.response");
+    addFairCallQueueSettings();
 
     // Deprecated properties.  These should eventually be removed from the
     // class.
@@ -250,5 +203,70 @@ public void initializeMemberVariables() {
     xmlPropsToSkipCompare.add("io.seqfile.local.dir");
 
     xmlPropsToSkipCompare.add("hadoop.http.sni.host.check.enabled");
+
+    addFtpSettings();
+  }
+
+  private void addAzureSettings() {
+    // Azure properties are in a different class
+    // - org.apache.hadoop.fs.azure.AzureNativeFileSystemStore
+    // - org.apache.hadoop.fs.azure.SASKeyGeneratorImpl
+    xmlPropsToSkipCompare.add("fs.azure.sas.expiry.period");
+    xmlPropsToSkipCompare.add("fs.azure.local.sas.key.mode");
+    xmlPropsToSkipCompare.add("fs.azure.secure.mode");
+    xmlPropsToSkipCompare.add("fs.azure.authorization");
+    xmlPropsToSkipCompare.add("fs.azure.authorization.caching.enable");
+    xmlPropsToSkipCompare.add("fs.azure.saskey.usecontainersaskeyforallaccess");
+    xmlPropsToSkipCompare.add("fs.azure.user.agent.prefix");
+  }
+
+  private void addFairCallQueueSettings() {
+    // FairCallQueue configs that includes dynamic ports in its keys
+    xmlPropsToSkipCompare.add("ipc.[port_number].backoff.enable");
+    xmlPropsToSkipCompare.add("ipc.backoff.enable");
+    xmlPropsToSkipCompare.add("ipc.[port_number].callqueue.impl");
+    xmlPropsToSkipCompare.add("ipc.callqueue.impl");
+    xmlPropsToSkipCompare.add("ipc.[port_number].scheduler.impl");
+    xmlPropsToSkipCompare.add("ipc.scheduler.impl");
+    xmlPropsToSkipCompare.add("ipc.[port_number].scheduler.priority.levels");
+    xmlPropsToSkipCompare.add("ipc.[port_number].callqueue.capacity.weights");
+    xmlPropsToSkipCompare.add(
+            "ipc.[port_number].faircallqueue.multiplexer.weights");
+    xmlPropsToSkipCompare.add("ipc.[port_number].identity-provider.impl");
+    xmlPropsToSkipCompare.add("ipc.identity-provider.impl");
+    xmlPropsToSkipCompare.add("ipc.[port_number].cost-provider.impl");
+    xmlPropsToSkipCompare.add("ipc.cost-provider.impl");
+    xmlPropsToSkipCompare.add("ipc.[port_number].decay-scheduler.period-ms");
+    xmlPropsToSkipCompare.add("ipc.[port_number].decay-scheduler.decay-factor");
+    xmlPropsToSkipCompare.add("ipc.[port_number].decay-scheduler.thresholds");
+    xmlPropsToSkipCompare.add(
+            "ipc.[port_number].decay-scheduler.backoff.responsetime.enable");
+    xmlPropsToSkipCompare.add(
+            "ipc.[port_number].decay-scheduler.backoff.responsetime.thresholds");
+    xmlPropsToSkipCompare.add(
+            "ipc.[port_number].decay-scheduler.metrics.top.user.count");
+    xmlPropsToSkipCompare.add(
+            "ipc.[port_number].decay-scheduler.service-users");
+    xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.lockshared");
+    xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.lockexclusive");
+    xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.handler");
+    xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.lockfree");
+    xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.response");
+  }
+
+  private void addFtpSettings() {
+    // skip all FTP props (some were already skipped above)
+    xmlPropsToSkipCompare.add("fs.AbstractFileSystem.ftp.impl");
+    xmlPropsToSkipCompare.add("fs.ftp.impl");
+    xmlPropsToSkipCompare.add("fs.ftp.host");
+    xmlPropsToSkipCompare.add("fs.ftp.host.port");
+    xmlPropsToSkipCompare.add("fs.ftp.data.connection.mode");
+    xmlPropsToSkipCompare.add("fs.ftp.transfer.mode");
+    xmlPropsToSkipCompare.add("fs.ftp.timeout");
+    xmlPropsToSkipCompare.add("ftp.stream-buffer-size");
+    xmlPropsToSkipCompare.add("ftp.bytes-per-checksum");
+    xmlPropsToSkipCompare.add("ftp.client-write-packet-size");
+    xmlPropsToSkipCompare.add("ftp.blocksize");
+    xmlPropsToSkipCompare.add("ftp.replication");
   }
 }

hadoop-tools/hadoop-ftp/dev-support/findbugsExcludeFile.xml

@@ -0,0 +1,19 @@
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<FindBugsFilter>
+
+</FindBugsFilter>

hadoop-tools/hadoop-ftp/pom.xml

@@ -0,0 +1,130 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+                      https://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hadoop</groupId>
+    <artifactId>hadoop-project-dist</artifactId>
+    <version>3.5.0-SNAPSHOT</version>
+    <relativePath>../../hadoop-project-dist</relativePath>
+  </parent>
+  <artifactId>hadoop-ftp</artifactId>
+  <version>3.5.0-SNAPSHOT</version>
+  <description>Apache Hadoop FTP/SFTP support</description>
+  <name>Apache Hadoop FTP/SFTP support</name>
+  <packaging>jar</packaging>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-annotations</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>commons-net</groupId>
+      <artifactId>commons-net</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.jcraft</groupId>
+      <artifactId>jsch</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.assertj</groupId>
+      <artifactId>assertj-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-reload4j</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>ch.qos.reload4j</groupId>
+      <artifactId>reload4j</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.sshd</groupId>
+      <artifactId>sshd-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.sshd</groupId>
+      <artifactId>sshd-sftp</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.ftpserver</groupId>
+      <artifactId>ftpserver-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <systemPropertyVariables>
+            <runningWithNative>${runningWithNative}</runningWithNative>
+          </systemPropertyVariables>
+          <properties>
+            <property>
+              <name>listener</name>
+              <value>org.apache.hadoop.test.TimedOutTestsListener</value>
+            </property>
+          </properties>
+        </configuration>
+      </plugin>
+
+      <plugin>
+        <groupId>org.apache.rat</groupId>
+        <artifactId>apache-rat-plugin</artifactId>
+        <configuration>
+          <excludes>
+          </excludes>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+</project>
+