Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

CDH4 Fixes

Made three updates
1. pom.xml to require more resent stuff like pig 9 and CDH 4
2. Combinatorial.java: Still testing it but it looks like there was a
bug in the outputSchema method
3. EBCI : I had to remove @Override to build.  I forgot the reason but
my compiler didn't want this tag from methods from interfaces.
  • Loading branch information...
commit eb67e919106105be1372f0b92e9dcd57f1aeead8 1 parent f888d93
@tmalaska tmalaska authored
View
12 pom.xml
@@ -17,14 +17,22 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
- <version>0.20.2-cdh3u1</version>
+ <version>0.23.1-mr1-cdh4.0.0b2</version>
<scope>provided</scope>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>0.23.1-mr1-cdh4.0.0b2</version>
+ <scope>provided</scope>
+ </dependency>
+
+
+ <dependency>
<groupId>org.apache.pig</groupId>
<artifactId>pig</artifactId>
- <version>0.8.1</version>
+ <version>0.9.2</version>
<scope>provided</scope>
</dependency>
View
36 src/main/java/com/cloudera/science/pig/Combinatorial.java
@@ -106,13 +106,48 @@ public Schema outputSchema(Schema input) {
throw new IllegalArgumentException("Expected a bag; input has > 1 field");
}
try {
+
+
if (input.getField(0).type != DataType.BAG) {
throw new IllegalArgumentException("Expected a bag; found: " +
DataType.findTypeName(input.getField(0).type));
}
+ if (input.getField(0).schema.getField(0).type != DataType.TUPLE) {
+ throw new IllegalArgumentException("Expected a tuple in a bag; found: " +
+ DataType.findTypeName(input.getField(0).type));
+ }
if (input.getField(0).schema.size() != 1) {
throw new IllegalArgumentException("The bag must contain a single field");
}
+
+ Schema bagSchema = input.getField(0).schema;
+ Schema tupleSchema = bagSchema.getField(0).schema;
+
+ byte fieldType = tupleSchema.getField(0).type;
+
+ if (!isComparable(fieldType)) {
+ throw new IllegalArgumentException("The bag's Tulple's field must be a comparable type");
+ }
+
+ //What does this code want to do ??
+ FieldSchema inputField = tupleSchema.getField(0);
+ String inputName = inputField.alias;
+ List<FieldSchema> fields = Lists.newArrayList();
+ for (int i = 0; i < arity; i++) {
+ fields.add(new FieldSchema(inputName + i, inputField.type));
+ }
+ Schema newTupleSchema = new Schema(fields);
+
+ FieldSchema tupleFieldSchema = new FieldSchema(inputName + "tuple", newTupleSchema,
+ DataType.TUPLE);
+
+ Schema newBagSchema = new Schema(tupleFieldSchema);
+ //bagSchema.setTwoLevelAccessRequired(true);
+ Schema.FieldSchema bagFieldSchema = new Schema.FieldSchema(inputName + "bag",
+ newBagSchema, DataType.BAG);
+ return new Schema(bagFieldSchema);
+
+ /*
byte bagType = input.getField(0).schema.getField(0).type;
if (bagType == DataType.TUPLE) {
bagType = input.getField(0).schema.getField(0).schema.getField(0).type;
@@ -137,6 +172,7 @@ public Schema outputSchema(Schema input) {
Schema.FieldSchema bagFieldSchema = new Schema.FieldSchema(inputName + "bag",
bagSchema, DataType.BAG);
return new Schema(bagFieldSchema);
+ */
} catch (Exception e) {
throw new RuntimeException(e);
}
View
4 src/main/java/com/cloudera/science/pig/EBCI.java
@@ -45,7 +45,7 @@ public PiFunction(double p, GammaDistribution g1, GammaDistribution g2) {
this.g2 = g2;
}
- @Override
+
public double value(double lambda) throws FunctionEvaluationException {
return p * g1.density(lambda) + (1.0 - p) * g2.density(lambda);
}
@@ -62,7 +62,6 @@ public PiFunctionIntegral(PiFunction pi, double target) {
this.integrator = new SimpsonIntegrator();
}
- @Override
public double value(double lambda) throws FunctionEvaluationException {
try {
if (lambda == 0.0) {
@@ -77,7 +76,6 @@ public double value(double lambda) throws FunctionEvaluationException {
return Double.POSITIVE_INFINITY;
}
- @Override
public UnivariateRealFunction derivative() {
return pi;
}
Please sign in to comment.
Something went wrong with that request. Please try again.