Skip to content
Permalink
Browse files
more unit tests
  • Loading branch information
jmalkin committed Jun 23, 2017
1 parent 9cde068 commit 9a6d09e33f71f50a2270db8dd80f445709525642
Showing 3 changed files with 131 additions and 6 deletions.
@@ -31,8 +31,6 @@
* @author Jon Malkin
*/
public class GetVarOptSamples extends EvalFunc<DataBag> {
// defined for test consistency

private static final ArrayOfTuplesSerDe SERDE = new ArrayOfTuplesSerDe();

@Override
@@ -51,16 +49,20 @@ public DataBag exec(final Tuple inputTuple) throws IOException {
@Override
public Schema outputSchema(final Schema input) {
try {
if (input == null || input.size() == 0
|| input.getField(0).type != DataType.BYTEARRAY) {
throw new IllegalArgumentException("Input to GetVarOptSamples must be a DataByteArray: "
+ (input == null ? "null" : input.toString()));
}

final Schema weightedSampleSchema = new Schema();
weightedSampleSchema.add(new Schema.FieldSchema(WEIGHT_ALIAS, DataType.DOUBLE));
weightedSampleSchema.add(new Schema.FieldSchema(RECORD_ALIAS, DataType.TUPLE));

return new Schema(new Schema.FieldSchema(getSchemaName(this
.getClass().getName().toLowerCase(), input), weightedSampleSchema, DataType.BAG));
} catch (final FrontendException e) {
// fall through
return null;
}

return null;
}
}
@@ -0,0 +1,123 @@
package com.yahoo.sketches.pig.sampling;

import static com.yahoo.sketches.pig.sampling.VarOptCommonImpl.RECORD_ALIAS;
import static com.yahoo.sketches.pig.sampling.VarOptCommonImpl.WEIGHT_ALIAS;
import static com.yahoo.sketches.pig.sampling.VarOptSamplingTest.EPS;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;

import java.io.IOException;
import java.util.List;

import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.testng.annotations.Test;

import com.yahoo.sketches.sampling.VarOptItemsSketch;

public class GetVarOptSamplesTest {
private static final ArrayOfTuplesSerDe serDe_ = new ArrayOfTuplesSerDe();

@Test
public void checkDegenerateInput() {
final GetVarOptSamples udf = new GetVarOptSamples();

try {
assertNull(udf.exec(null));
assertNull(udf.exec(TupleFactory.getInstance().newTuple(0)));
assertNull(udf.exec(TupleFactory.getInstance().newTuple((Object) null)));
} catch (final IOException e) {
fail("Unexpected IOException");
}
}

@Test
public void checkExec() {
final int k = 10;
final int n = 25; // exact mode
final GetVarOptSamples udf = new GetVarOptSamples();

try {
final VarOptItemsSketch<Tuple> vis = VarOptItemsSketch.newInstance(k);
double cumWt = 0.0;
for (int i = 1; i <= n; ++i) {
final Tuple t = TupleFactory.getInstance().newTuple(2);
final double wt = 1.0 * i;
t.set(0, wt);
t.set(1, i);
vis.update(t, wt);
cumWt += wt;
}

final DataByteArray dba = new DataByteArray(vis.toByteArray(serDe_));
final Tuple inputTuple = TupleFactory.getInstance().newTuple(dba);
final DataBag result = udf.exec(inputTuple);

double cumResultWt = 0.0;
for (Tuple sample : result) {
cumResultWt += (double) sample.get(0);
final Tuple record = (Tuple) sample.get(1);
final int id = (int) record.get(1);
assertTrue(id >= 1 && id <= n);
}
assertEquals(cumResultWt, cumWt, EPS);
} catch (final IOException e) {
fail("Unexpected IOException");
}
}

@Test
public void validOutputSchemaTest() throws IOException {
final GetVarOptSamples udf = new GetVarOptSamples();

final Schema serializedSketch = new Schema();
serializedSketch.add(new Schema.FieldSchema("field1", DataType.BYTEARRAY));
//final Schema inputSchema = new Schema();
//inputSchema.add(new Schema.FieldSchema("record", serializedSketch, DataType.TUPLE));

//final Schema output = udf.outputSchema(inputSchema);
final Schema output = udf.outputSchema(serializedSketch);
assertEquals(output.size(), 1);
assertEquals(output.getField(0).type, DataType.BAG);

final List<Schema.FieldSchema> outputFields = output.getField(0).schema.getFields();
assertEquals(outputFields.size(), 2);

// check high-level structure
assertEquals(outputFields.get(0).alias, WEIGHT_ALIAS);
assertEquals(outputFields.get(0).type, DataType.DOUBLE);
assertEquals(outputFields.get(1).alias, RECORD_ALIAS);
assertEquals(outputFields.get(1).type, DataType.TUPLE);
}

@Test
public void badOutputSchemaTest() throws IOException {
final GetVarOptSamples udf = new GetVarOptSamples();

try {
udf.outputSchema(null);
} catch (final IllegalArgumentException e) {
// expected
}

try {
udf.outputSchema(new Schema());
} catch (final IllegalArgumentException e) {
// expected
}

try {
final Schema wrongSchema = new Schema();
wrongSchema.add(new Schema.FieldSchema("field", DataType.BOOLEAN));
udf.outputSchema(wrongSchema);
} catch (final IllegalArgumentException e) {
// expected
}
}
}
@@ -23,7 +23,7 @@
import com.yahoo.sketches.sampling.VarOptItemsSketch;

public class VarOptSamplingTest {
private static final double EPS = 1e-10;
static final double EPS = 1e-10;
private static final ArrayOfTuplesSerDe serDe_ = new ArrayOfTuplesSerDe();

@Test

0 comments on commit 9a6d09e

Please sign in to comment.