Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Updated to latest Hadoop minor releases.

  • Loading branch information...
commit 06a8a8cdead267d964ae0ec1e2440267c50d73bb 1 parent a505766
@tomwhite authored
Showing with 105 additions and 62 deletions.
  1. +2 −2 README
  2. +103 −60 hadoop-meta/pom.xml
View
4 README
@@ -9,7 +9,7 @@ for the first edition at http://github.com/tomwhite/hadoop-book/tree/1e, and
for the second edition at http://github.com/tomwhite/hadoop-book/tree/2e.
This version of the code has been tested with:
- * Hadoop 1.0.2/0.22.0/0.23.0
+ * Hadoop 1.0.4/0.22.0/0.23.0/2.0.2-alpha
* Avro 1.5.4
* Pig 0.9.1
* Hive 0.8.0
@@ -25,7 +25,7 @@ You also need to install Maven.
Then you can build the code with:
-% mvn package -DskipTests -Dhadoop.version=1.0.2
+% mvn package -DskipTests -Dhadoop.version=1.0.4
(Note that only the Hadoop version numbers specified in hadoop-meta/pom.xml
can be used here, not any version of Hadoop.)
View
163 hadoop-meta/pom.xml
@@ -28,15 +28,15 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
<profiles>
<profile>
- <id>apache-20</id>
+ <id>apache-1</id>
<activation>
<property>
<name>hadoop.version</name>
- <value>0.20.2</value>
+ <value>1.0.4</value>
</property>
</activation>
<properties>
- <hadoop.version>0.20.2</hadoop.version>
+ <hadoop.version>1.0.4</hadoop.version>
</properties>
<dependencies>
<dependency>
@@ -55,10 +55,20 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
<artifactId>log4j</artifactId>
</dependency>
<dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ <version>1.8</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ <version>1.5.2</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.mrunit</groupId>
<artifactId>mrunit</artifactId>
<version>0.8.1-incubating</version>
- <classifier>hadoop020</classifier>
+ <classifier>hadoop100</classifier>
<scope>test</scope>
</dependency>
</dependencies>
@@ -69,17 +79,11 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<excludes>
- <exclude>MaxTemperatureWithMultipleInputs.java</exclude>
- <exclude>PartitionByStationUsingMultipleOutputs.java</exclude>
- <exclude>PartitionByStationYearUsingMultipleOutputs.java</exclude>
- <exclude>JoinRecordWithStationName.java</exclude>
<exclude>LookupRecordByTemperature.java</exclude>
<exclude>LookupRecordsByTemperature.java</exclude>
<exclude>MaxTemperatureByStationNameUsingDistributedCacheFileApi.java</exclude>
- <exclude>MissingTemperatureFields.java</exclude>
<exclude>NewMissingTemperatureFields.java</exclude>
<exclude>SortByTemperatureToMapFile.java</exclude>
- <exclude>SortByTemperatureUsingTotalOrderPartitioner.java</exclude>
</excludes>
</configuration>
</plugin>
@@ -87,21 +91,27 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
</build>
</profile>
<profile>
- <id>apache-1.0.2</id>
+ <id>cdh3</id>
<activation>
<property>
<name>hadoop.version</name>
- <value>1.0.2</value>
+ <value>0.20.2-cdh3u5</value>
</property>
</activation>
<properties>
- <hadoop.version>1.0.2</hadoop.version>
+ <hadoop.version>0.20.2-cdh3u5</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -114,14 +124,9 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
<artifactId>log4j</artifactId>
</dependency>
<dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-core</artifactId>
- <version>1.8</version>
- </dependency>
- <dependency>
<groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- <version>1.5.2</version>
+ <artifactId>jackson-core-asl</artifactId>
+ <version>1.7.3</version> <!-- To match that in Avro -->
</dependency>
<dependency>
<groupId>org.apache.mrunit</groupId>
@@ -143,6 +148,7 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
<exclude>MaxTemperatureByStationNameUsingDistributedCacheFileApi.java</exclude>
<exclude>NewMissingTemperatureFields.java</exclude>
<exclude>SortByTemperatureToMapFile.java</exclude>
+ <exclude>SortByTemperatureUsingTotalOrderPartitioner.java</exclude>
</excludes>
</configuration>
</plugin>
@@ -150,48 +156,55 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
</build>
</profile>
<profile>
- <id>cdh3</id>
+ <id>apache-22</id>
<activation>
<property>
<name>hadoop.version</name>
- <value>0.20.2-cdh3u3</value>
+ <value>0.22.0</value>
</property>
</activation>
<properties>
- <hadoop.version>0.20.2-cdh3u3</hadoop.version>
+ <hadoop.version>0.22.0</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
+ <artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-test</artifactId>
+ <artifactId>hadoop-common-test</artifactId>
<version>${hadoop.version}</version>
+ <scope>test</scope>
</dependency>
- <!-- Required to run -->
<dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
</dependency>
<dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- <version>1.7.3</version> <!-- To match that in Avro -->
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-test</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapred</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapred-test</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.mrunit</groupId>
<artifactId>mrunit</artifactId>
<version>0.8.1-incubating</version>
- <classifier>hadoop100</classifier>
+ <classifier>hadoop023</classifier>
<scope>test</scope>
</dependency>
</dependencies>
@@ -202,28 +215,29 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<excludes>
- <exclude>LookupRecordByTemperature.java</exclude>
- <exclude>LookupRecordsByTemperature.java</exclude>
- <exclude>MaxTemperatureByStationNameUsingDistributedCacheFileApi.java</exclude>
- <exclude>NewMissingTemperatureFields.java</exclude>
- <exclude>SortByTemperatureToMapFile.java</exclude>
- <exclude>SortByTemperatureUsingTotalOrderPartitioner.java</exclude>
+ <exclude>MissingTemperatureFields.java</exclude> <!-- Task.Counter missing -->
</excludes>
+ <testExcludes>
+ <exclude>CoherencyModelTest.java</exclude>
+ <exclude>ShowFileStatusTest.java</exclude>
+ <exclude>v1/MaxTemperatureReducerTest.java</exclude> <!-- Class cast problem in MRUnit -->
+ <exclude>v3/MaxTemperatureDriverMiniTest.java</exclude> <!-- Failing -->
+ </testExcludes>
</configuration>
</plugin>
</plugins>
</build>
</profile>
<profile>
- <id>apache-22</id>
+ <id>apache-23</id>
<activation>
<property>
<name>hadoop.version</name>
- <value>0.22.0</value>
+ <value>0.23.4</value>
</property>
</activation>
<properties>
- <hadoop.version>0.22.0</hadoop.version>
+ <hadoop.version>0.23.4</hadoop.version>
</properties>
<dependencies>
<dependency>
@@ -233,8 +247,9 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common-test</artifactId>
+ <artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
+ <type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
@@ -244,19 +259,53 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs-test</artifactId>
+ <artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
+ <type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapred</artifactId>
+ <artifactId>hadoop-mapreduce-client-app</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapred-test</artifactId>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-hs</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <version>${hadoop.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-tests</artifactId>
<version>${hadoop.version}</version>
+ <type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
@@ -273,13 +322,7 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
- <excludes>
- <exclude>MissingTemperatureFields.java</exclude> <!-- Task.Counter missing -->
- </excludes>
<testExcludes>
- <exclude>CoherencyModelTest.java</exclude>
- <exclude>ShowFileStatusTest.java</exclude>
- <exclude>v1/MaxTemperatureReducerTest.java</exclude> <!-- Class cast problem in MRUnit -->
<exclude>v3/MaxTemperatureDriverMiniTest.java</exclude> <!-- Failing -->
</testExcludes>
</configuration>
@@ -288,15 +331,15 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
</build>
</profile>
<profile>
- <id>apache-23</id>
+ <id>apache-2</id>
<activation>
<property>
<name>hadoop.version</name>
- <value>0.23.1</value>
+ <value>2.0.2-alpha</value>
</property>
</activation>
<properties>
- <hadoop.version>0.23.1</hadoop.version>
+ <hadoop.version>2.0.2-alpha</hadoop.version>
</properties>
<dependencies>
<dependency>
Please sign in to comment.
Something went wrong with that request. Please try again.