[FLINK-4895] Drop Hadoop1 support and remove related build infrastruc…

…ture This closes apache#2850
rmetzger · Nov 29, 2016 · f6c8307 · f6c8307
1 parent 910f733
commit f6c8307
Show file tree

Hide file tree

Showing 24 changed files with 63 additions and 637 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -35,11 +35,6 @@ matrix:
     - jdk: "oraclejdk7"
       env: PROFILE="-Dhadoop.version=2.3.0 -Pflink-fast-tests-b,include-kinesis"
 
-    - jdk: "openjdk7" 
-      env: PROFILE="-Dhadoop.profile=1 -Pflink-fast-tests-a,include-kinesis"
-    - jdk: "openjdk7" 
-      env: PROFILE="-Dhadoop.profile=1 -Pflink-fast-tests-b,include-kinesis"
-
 
 git:
   depth: 100

diff --git a/docs/_config.yml b/docs/_config.yml
@@ -27,7 +27,6 @@
 # we change the version for the complete docs when forking of a release branch
 # etc.
 version: "1.2-SNAPSHOT"
-version_hadoop1: "1.2-hadoop1-SNAPSHOT"
 version_short: "1.2" # Used for the top navbar w/o snapshot suffix
 is_snapshot_version: true
 

diff --git a/docs/setup/building.md b/docs/setup/building.md
@@ -76,31 +76,11 @@ mvn clean install
 
 ## Hadoop Versions
 
-{% info %} Most users do not need to do this manually. The [download page]({{ site.download_url }})  contains binary packages for common Hadoop versions.
+{% info %} Most users do not need to do this manually. The [download page]({{ site.download_url }}) contains binary packages for common Hadoop versions.
 
 Flink has dependencies to HDFS and YARN which are both dependencies from [Apache Hadoop](http://hadoop.apache.org). There exist many different versions of Hadoop (from both the upstream project and the different Hadoop distributions). If you are using a wrong combination of versions, exceptions can occur.
 
-There are two main versions of Hadoop that we need to differentiate:
-- **Hadoop 1**, with all versions starting with zero or one, like *0.20*, *0.23* or *1.2.1*.
-- **Hadoop 2**, with all versions starting with 2, like *2.6.0*.
-
-The main differentiation between Hadoop 1 and Hadoop 2 is the availability of [Hadoop YARN](https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html), Hadoop's cluster resource manager.
-
-**By default, Flink is using the Hadoop 2 dependencies**.
-
-### Hadoop 1
-
-To build Flink for Hadoop 1, issue the following command:
-
-~~~bash
-mvn clean install -DskipTests -Dhadoop.profile=1
-~~~
-
-The `-Dhadoop.profile=1` flag instructs Maven to build Flink for Hadoop 1. Note that the features included in Flink change when using a different Hadoop profile. In particular, there is no support for YARN and HBase in Hadoop 1 builds.
-
-### Hadoop 2.x
-
-Hadoop 2.X versions are only supported from version 2.3.0 upwards.
+Hadoop is only supported from version 2.3.0 upwards.
 You can also specify a specific Hadoop version to build against:
 
 ~~~bash
@@ -176,12 +156,3 @@ in the compiler configuration of the `pom.xml` file of the module causing the er
 
 {% top %}
 
-## Internals
-
-The builds with Maven are controlled by [properties](http://maven.apache.org/pom.html#Properties) and [build profiles](http://maven.apache.org/guides/introduction/introduction-to-profiles.html). There are two profiles, one for `hadoop1` and one for `hadoop2`. When the `hadoop2` profile is enabled (default), the system will also build the YARN client.
-
-To enable the `hadoop1` profile, set `-Dhadoop.profile=1` when building. Depending on the profile, there are two Hadoop versions, set via properties. For `hadoop1`, we use 1.2.1 by default, for `hadoop2` it is 2.3.0.
-
-You can change these versions with the `hadoop-two.version` (or `hadoop-one.version`) property. For example `-Dhadoop-two.version=2.4.0`.
-
-{% top %}
diff --git a/flink-batch-connectors/flink-hadoop-compatibility/pom.xml b/flink-batch-connectors/flink-hadoop-compatibility/pom.xml
@@ -62,7 +62,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>${shading-artifact.name}</artifactId>
+			<artifactId>flink-shaded-hadoop2</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 

diff --git a/.../src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/example/WordCount.java b/.../src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/example/WordCount.java
@@ -77,7 +77,6 @@ public static void main(String[] args) throws Exception {
 		HadoopOutputFormat<Text, IntWritable> hadoopOutputFormat = new HadoopOutputFormat<Text, IntWritable>(new TextOutputFormat<Text, IntWritable>(), job);
 		hadoopOutputFormat.getConfiguration().set("mapreduce.output.textoutputformat.separator", " ");
 		hadoopOutputFormat.getConfiguration().set("mapred.textoutputformat.separator", " "); // set the value for both, since this test
-		// is being executed with both types (hadoop1 and hadoop2 profile)
 		TextOutputFormat.setOutputPath(job, new Path(outputPath));
 
 		// Output & Execute

diff --git a/flink-batch-connectors/flink-hbase/pom.xml b/flink-batch-connectors/flink-hbase/pom.xml
@@ -34,8 +34,7 @@ under the License.
 	<packaging>jar</packaging>
 
 	<properties>
-		<hbase.hadoop1.version>0.98.22-hadoop1</hbase.hadoop1.version>
-		<hbase.hadoop2.version>1.2.3</hbase.hadoop2.version>
+		<hbase.version>1.2.3</hbase.version>
 	</properties>
 
 	<build>
@@ -85,7 +84,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>${shading-artifact.name}</artifactId>
+			<artifactId>flink-shaded-hadoop2</artifactId>
 			<version>${project.version}</version>
 			<scope>provided</scope>
 		</dependency>
@@ -198,100 +197,51 @@ under the License.
 			</exclusions>
 		</dependency>
 
-	</dependencies>
-
-	<profiles>
-		<profile>
-			<id>hadoop-1</id>
-			<activation>
-				<property>
-					<!-- Please do not remove the 'hadoop1' comment. See ./tools/generate_specific_pom.sh -->
-					<!--hadoop1--><name>hadoop.profile</name><value>1</value>
-				</property>
-			</activation>
-			<properties>
-				<hbase.version>${hbase.hadoop1.version}</hbase.version>
-				<!--
-				Required test dependencies are only available for Hadoop-2.
-				Disabling tests for Hadoop-1 profile.
-				-->
-				<maven.test.skip>true</maven.test.skip>
-			</properties>
-
-		</profile>
-
-		<profile>
-			<id>hadoop-2</id>
-			<repositories>
-				<repository>
-					<id>hadoop-2-repo2</id>
-					<url>https://repo.maven.apache.org/maven2</url>
-					<releases>
-						<enabled>true</enabled>
-					</releases>
-					<snapshots>
-						<enabled>false</enabled>
-					</snapshots>
-				</repository>
-			</repositories>
-			<activation>
-				<property>
-					<!-- Please do not remove the 'hadoop2' comment. See ./tools/generate_specific_pom.sh -->
-					<!--hadoop2--><name>!hadoop.profile</name>
-				</property>
-			</activation>
-			<properties>
-				<hbase.version>${hbase.hadoop2.version}</hbase.version>
-			</properties>
-
-			<dependencies>
-				<!-- Test dependencies are only available for Hadoop-2. -->
-				<dependency>
-					<groupId>org.apache.hbase</groupId>
-					<artifactId>hbase-server</artifactId>
-					<version>${hbase.version}</version>
-					<classifier>tests</classifier>
-					<scope>test</scope>
-				</dependency>
-
-				<dependency>
-					<groupId>org.apache.hadoop</groupId>
-					<artifactId>hadoop-minicluster</artifactId>
-					<version>${hadoop.version}</version>
-					<scope>test</scope>
-				</dependency>
+		<!-- Test dependencies are only available for Hadoop-2. -->
+		<dependency>
+			<groupId>org.apache.hbase</groupId>
+			<artifactId>hbase-server</artifactId>
+			<version>${hbase.version}</version>
+			<classifier>tests</classifier>
+			<scope>test</scope>
+		</dependency>
 
-				<dependency>
-					<groupId>org.apache.hbase</groupId>
-					<artifactId>hbase-hadoop-compat</artifactId>
-					<version>${hbase.version}</version>
-					<scope>test</scope>
-					<type>test-jar</type>
-				</dependency>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-minicluster</artifactId>
+			<version>${hadoop.version}</version>
+			<scope>test</scope>
+		</dependency>
 
-				<dependency>
-					<groupId>org.apache.hadoop</groupId>
-					<artifactId>hadoop-hdfs</artifactId>
-					<version>${hadoop.version}</version>
-					<type>test-jar</type>
-					<scope>test</scope>
-				</dependency>
+		<dependency>
+			<groupId>org.apache.hbase</groupId>
+			<artifactId>hbase-hadoop-compat</artifactId>
+			<version>${hbase.version}</version>
+			<scope>test</scope>
+			<type>test-jar</type>
+		</dependency>
 
-				<dependency>
-					<groupId>org.apache.hbase</groupId>
-					<artifactId>hbase-hadoop2-compat</artifactId>
-					<version>${hbase.version}</version>
-					<scope>test</scope>
-					<type>test-jar</type>
-				</dependency>
-			</dependencies>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-hdfs</artifactId>
+			<version>${hadoop.version}</version>
+			<type>test-jar</type>
+			<scope>test</scope>
+		</dependency>
 
-		</profile>
+		<dependency>
+			<groupId>org.apache.hbase</groupId>
+			<artifactId>hbase-hadoop2-compat</artifactId>
+			<version>${hbase.version}</version>
+			<scope>test</scope>
+			<type>test-jar</type>
+		</dependency>
+	</dependencies>
 
+	<profiles>
 		<profile>
 			<id>cdh5.1.3</id>
 			<properties>
-				<hadoop.profile>2</hadoop.profile>
 				<hbase.version>0.98.1-cdh5.1.3</hbase.version>
 				<hadoop.version>2.3.0-cdh5.1.3</hadoop.version>
 				<!-- Cloudera use different versions for hadoop core and commons-->

diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
@@ -131,28 +131,16 @@ under the License.
 			<artifactId>flink-statebackend-rocksdb_2.10</artifactId>
 			<version>${project.version}</version>
 		</dependency>
+
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-yarn_2.10</artifactId>
+			<version>${project.version}</version>
+		</dependency>
 
 	</dependencies>
 
 	<profiles>
-		<!-- See main pom.xml for explanation of Hadoop profiles -->
-		<profile>
-			<id>include-yarn</id>
-			<activation>
-				<property>
-					<!-- Please do not remove the 'hadoop2' comment. See ./tools/generate_specific_pom.sh -->
-					<!--hadoop2--><name>!hadoop.profile</name>
-				</property>
-			</activation>
-			<dependencies>
-				<dependency>
-					<groupId>org.apache.flink</groupId>
-					<artifactId>flink-yarn_2.10</artifactId>
-					<version>${project.version}</version>
-				</dependency>
-			</dependencies>
-		</profile>
-
 		<profile>
 			<!-- Creates/Removes the 'build-target' symlink in the root directory (only Unix systems) -->
 			<id>symlink-build-target</id>

diff --git a/flink-fs-tests/pom.xml b/flink-fs-tests/pom.xml
@@ -38,7 +38,7 @@ under the License.
 	<dependencies>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>${shading-artifact.name}</artifactId>
+			<artifactId>flink-shaded-hadoop2</artifactId>
 			<version>${project.version}</version>
 			<scope>test</scope>
 		</dependency>

diff --git a/flink-java/pom.xml b/flink-java/pom.xml
@@ -43,7 +43,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>${shading-artifact.name}</artifactId>
+			<artifactId>flink-shaded-hadoop2</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 

diff --git a/flink-mesos/pom.xml b/flink-mesos/pom.xml
@@ -50,7 +50,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>${shading-artifact.name}</artifactId>
+			<artifactId>flink-shaded-hadoop2</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 

diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
@@ -166,7 +166,6 @@ under the License.
 									Everything else will be packaged into the fat-jar
 									-->
 									<exclude>org.apache.flink:flink-annotations</exclude>
-									<exclude>org.apache.flink:flink-shaded-hadoop1</exclude>
 									<exclude>org.apache.flink:flink-shaded-hadoop2</exclude>
 									<exclude>org.apache.flink:flink-shaded-curator-recipes</exclude>
 									<exclude>org.apache.flink:flink-core</exclude>

diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
@@ -170,7 +170,6 @@ under the License.
 									Everything else will be packaged into the fat-jar
 									-->
 									<exclude>org.apache.flink:flink-annotations</exclude>
-									<exclude>org.apache.flink:flink-shaded-hadoop1</exclude>
 									<exclude>org.apache.flink:flink-shaded-hadoop2</exclude>
 									<exclude>org.apache.flink:flink-shaded-curator-recipes</exclude>
 									<exclude>org.apache.flink:flink-core</exclude>

diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
@@ -52,7 +52,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>${shading-artifact.name}</artifactId>
+			<artifactId>flink-shaded-hadoop2</artifactId>
 			<version>${project.version}</version>
 		</dependency>