Skip to content

Commit

Permalink
Allow different Hadoop versions to be specified.
Browse files Browse the repository at this point in the history 
See #14 and
#15
  • Loading branch information
@tomwhite
tomwhite committed Jan 9, 2014
1 parent c148d8c commit fb237f6
Show file tree
Hide file tree
Showing 4 changed files with 121 additions and 37 deletions.
17 changes: 14 additions & 3 deletions README
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,21 @@ You also need to install Maven.

Then you can build the code with:

% mvn package -DskipTests -Dhadoop.version=1.2.1
% mvn package -DskipTests

(Note that only the Hadoop version numbers specified in hadoop-meta/pom.xml
can be used here, not any version of Hadoop.)
By default Hadoop 1.2.1 is used. This can be changed by specifying the hadoop.version
property, e.g.

% mvn package -DskipTests -Dhadoop.version=1.2.0

There are also profiles for different Hadoop major versions,
specified in hadoop-meta/pom.xml. For example, to use Hadoop 2:

% mvn package -DskipTests -P apache-2

Again, you can specify hadoop.version to use a particular Hadoop 2 version:

% mvn package -DskipTests -P apache-2 -Dhadoop.version=2.1.1-beta

You should then be able to run the examples from the book.

Expand Down
Empty file modified ch05/output/_SUCCESS
100644 → 100755
View file
Empty file.
Empty file modified ch05/output/part-r-00000
100644 → 100755
View file
Empty file.
141 changes: 107 additions & 34 deletions hadoop-meta/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,11 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
</build>

<profiles>

<profile>
<id>apache-1</id>
<activation>
<property>
<name>hadoop.version</name>
<value>1.2.1</value>
</property>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<hadoop.version>1.2.1</hadoop.version>
Expand Down Expand Up @@ -90,14 +88,9 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
</plugins>
</build>
</profile>

<profile>
<id>cdh3</id>
<activation>
<property>
<name>hadoop.version</name>
<value>0.20.2-cdh3u5</value>
</property>
</activation>
<properties>
<hadoop.version>0.20.2-cdh3u5</hadoop.version>
</properties>
Expand Down Expand Up @@ -155,14 +148,9 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
</plugins>
</build>
</profile>

<profile>
<id>apache-22</id>
<activation>
<property>
<name>hadoop.version</name>
<value>0.22.0</value>
</property>
</activation>
<properties>
<hadoop.version>0.22.0</hadoop.version>
</properties>
Expand Down Expand Up @@ -228,14 +216,9 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
</plugins>
</build>
</profile>

<profile>
<id>apache-23</id>
<activation>
<property>
<name>hadoop.version</name>
<value>0.23.10</value>
</property>
</activation>
<properties>
<hadoop.version>0.23.10</hadoop.version>
</properties>
Expand Down Expand Up @@ -330,14 +313,9 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
</plugins>
</build>
</profile>

<profile>
<id>apache-2</id>
<activation>
<property>
<name>hadoop.version</name>
<value>2.2.0</value>
</property>
</activation>
<properties>
<hadoop.version>2.2.0</hadoop.version>
</properties>
Expand Down Expand Up @@ -432,14 +410,109 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
</plugins>
</build>
</profile>

<profile>
<id>cdh4</id>
<properties>
<hadoop.version>2.0.0-cdh4.5.0</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-app</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-common</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-hs</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
<version>${hadoop.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-tests</artifactId>
<version>${hadoop.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.mrunit</groupId>
<artifactId>mrunit</artifactId>
<version>0.8.1-incubating</version>
<classifier>hadoop023</classifier>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<excludes>
<exclude>MaxTemperatureByStationNameUsingDistributedCacheFileApi.java</exclude> <!-- missing API -->
</excludes>
<testExcludes>
<exclude>v3/MaxTemperatureDriverMiniTest.java</exclude> <!-- Failing -->
</testExcludes>
</configuration>
</plugin>
</plugins>
</build>
</profile>

<profile>
<id>apache-trunk</id>
<activation>
<property>
<name>hadoop.version</name>
<value>3.0.0-SNAPSHOT</value>
</property>
</activation>
<properties>
<hadoop.version>3.0.0-SNAPSHOT</hadoop.version>
</properties>
Expand Down

0 comments on commit fb237f6

Please sign in to comment.