Permalink
Browse files

Support Avro 1.5.4.

  • Loading branch information...
1 parent 898df28 commit 5ff156674a1c50bddb85cc0abcd820d10f46d1e7 @tomwhite committed Jan 12, 2012
View
2 README
@@ -10,7 +10,7 @@ for the second edition at http://github.com/tomwhite/hadoop-book/tree/2e.
This version of the code has been tested with:
* Hadoop 1.0.0/0.22.0/0.23.0
- * Avro 1.4.1
+ * Avro 1.5.4
* Pig 0.9.0
* Hive 0.8.0
* HBase 0.90.4
View
6 book/pom.xml
@@ -14,7 +14,7 @@
<!-- Component versions are defined here -->
<!-- Note that Hadoop dependencies are defined in hadoop-meta -->
- <avro.version>1.6.1</avro.version>
+ <avro.version>1.5.4</avro.version>
<pig.version>0.9.0</pig.version>
<hive.version>0.8.0</hive.version>
<hbase.version>0.90.4</hbase.version>
@@ -133,21 +133,25 @@
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.8.1</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
<version>1.1</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.8.5</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>junit-addons</groupId>
<artifactId>junit-addons</artifactId>
<version>1.4</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
View
23 ch04-avro/pom.xml
@@ -44,14 +44,35 @@
<include>StringPair.avsc</include>
<include>WeatherRecord.avsc</include>
</includes>
- <stringType>String</stringType>
+ <stringType>String</stringType> <!-- Avro 1.6.0 onwards -->
<sourceDirectory>src/main/resources</sourceDirectory>
<outputDirectory>${project.build.directory}/generated-sources/java
</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.2.1</version>
+ <executions>
+ <execution>
+ <id>job</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assembly/job.xml</descriptor>
+ </descriptors>
+ <finalName>avro-examples</finalName>
+ <outputDirectory>${project.build.directory}/../..</outputDirectory>
+ <appendAssemblyId>false</appendAssemblyId>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
View
27 ch04-avro/src/main/assembly/job.xml
@@ -0,0 +1,27 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>job</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <useTransitiveFiltering>true</useTransitiveFiltering>
+ <unpack>false</unpack>
+ <scope>runtime</scope>
+ <outputDirectory>lib</outputDirectory>
+ <useProjectArtifact>false</useProjectArtifact>
+ <excludes>
+ <exclude>org.apache.hadoop:*</exclude>
+ </excludes>
+ </dependencySet>
+ </dependencySets>
+ <fileSets>
+ <fileSet>
+ <directory>target/classes</directory>
+ <outputDirectory>/</outputDirectory>
+ </fileSet>
+ </fileSets>
+</assembly>
View
14 ch04-avro/src/main/java/AvroSpecificMaxTemperature.java
@@ -30,9 +30,9 @@ public void map(Utf8 line,
throws IOException {
parser.parse(line.toString());
if (parser.isValidTemperature()) {
- record.setYear(parser.getYearInt());
- record.setTemperature(parser.getAirTemperature());
- record.setStationId(parser.getStationId());
+ record.year = parser.getYearInt();
+ record.temperature = parser.getAirTemperature();
+ record.stationId = new Utf8(parser.getStationId());
collector.collect(
new Pair<Integer, WeatherRecord>(parser.getYearInt(), record));
}
@@ -48,17 +48,17 @@ public void reduce(Integer key, Iterable<WeatherRecord> values,
throws IOException {
WeatherRecord max = null;
for (WeatherRecord value : values) {
- if (max == null || value.getTemperature() > max.getTemperature()) {
+ if (max == null || value.temperature > max.temperature) {
max = newWeatherRecord(value);
}
}
collector.collect(max);
}
private WeatherRecord newWeatherRecord(WeatherRecord value) {
WeatherRecord record = new WeatherRecord();
- record.setYear(value.getYear());
- record.setTemperature(value.getTemperature());
- record.setStationId(value.getStationId());
+ record.year = value.year;
+ record.temperature = value.temperature;
+ record.stationId = value.stationId;
return record;
}
}
View
15 ch04-avro/src/test/java/AvroTest.java
@@ -69,6 +69,7 @@ public void testInt() throws IOException {
}
@Test
+ @Ignore("Requires Avro 1.6.0 or later")
public void testGenericString() throws IOException {
Schema schema = new Schema.Parser().parse("{\"type\": \"string\", \"avro.java.string\": \"String\"}");
@@ -131,21 +132,23 @@ public void testPairSpecific() throws IOException {
// vv AvroSpecificStringPair
/*[*/StringPair datum = new StringPair();
- datum.setLeft("L");
- datum.setRight("R");/*]*/
+ datum.left = "L";
+ datum.right = "R";/*]*/
ByteArrayOutputStream out = new ByteArrayOutputStream();
- /*[*/DatumWriter<StringPair> writer = new SpecificDatumWriter<StringPair>(StringPair.class);/*]*/
+ /*[*/DatumWriter<StringPair> writer =
+ new SpecificDatumWriter<StringPair>(StringPair.class);/*]*/
Encoder encoder = EncoderFactory.get().binaryEncoder(out, null);
writer.write(datum, encoder);
encoder.flush();
out.close();
- /*[*/DatumReader<StringPair> reader = new SpecificDatumReader<StringPair>(StringPair.class);/*]*/
+ /*[*/DatumReader<StringPair> reader =
+ new SpecificDatumReader<StringPair>(StringPair.class);/*]*/
Decoder decoder = DecoderFactory.get().binaryDecoder(out.toByteArray(), null);
StringPair result = reader.read(null, decoder);
- /*[*/assertThat((String) result.getLeft(), is("L"));
- assertThat((String) result.getRight(), is("R"));/*]*/
+ assertThat(result./*[*/left/*]*/.toString(), is("L"));
+ assertThat(result./*[*/right/*]*/.toString(), is("R"));
// ^^ AvroSpecificStringPair
}
View
4 hadoop-meta/pom.xml
@@ -205,11 +205,11 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
<activation>
<property>
<name>hadoop.version</name>
- <value>0.20.2-cdh3u2</value>
+ <value>0.20.2-cdh3u1</value>
</property>
</activation>
<properties>
- <hadoop.version>0.20.2-cdh3u2</hadoop.version>
+ <hadoop.version>0.20.2-cdh3u1</hadoop.version>
</properties>
<dependencies>
<dependency>

0 comments on commit 5ff1566

Please sign in to comment.