Skip to content
Permalink
Browse files
Merge pull request #54 from DataSketches/quantiles-sketch-to-string
to string UDFs
  • Loading branch information
leerho committed Feb 12, 2019
2 parents 9c208ad + 9b09539 commit 0fab110c101d59b98a23f88bb597469530bb58ba
Showing 4 changed files with 205 additions and 0 deletions.
@@ -0,0 +1,41 @@
/*
* Copyright 2019, Verizon Media.
* Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms.
*/

package com.yahoo.sketches.pig.quantiles;

import java.io.IOException;

import org.apache.pig.EvalFunc;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.Tuple;

import com.yahoo.memory.Memory;
import com.yahoo.sketches.quantiles.DoublesSketch;

/**
* This UDF is to get a human-readable summary of a given sketch.
*/
public class DoublesSketchToString extends EvalFunc<String> {

@Override
public String exec(final Tuple input) throws IOException {
if (input == null) {
return null;
}
if (input.size() != 1) {
throw new IllegalArgumentException("expected one input");
}

if (!(input.get(0) instanceof DataByteArray)) {
throw new IllegalArgumentException("expected a DataByteArray as a sketch, got "
+ input.get(0).getClass().getSimpleName());
}
final DataByteArray dba = (DataByteArray) input.get(0);
final DoublesSketch sketch = DoublesSketch.wrap(Memory.wrap(dba.get()));

return sketch.toString();
}

}
@@ -0,0 +1,44 @@
/*
* Copyright 2019, Verizon Media.
* Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms.
*/

package com.yahoo.sketches.pig.quantiles;

import java.io.IOException;
import java.util.Comparator;

import org.apache.pig.EvalFunc;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.Tuple;

import com.yahoo.memory.Memory;
import com.yahoo.sketches.ArrayOfStringsSerDe;
import com.yahoo.sketches.quantiles.ItemsSketch;

/**
* This UDF is to get a human-readable summary of a given sketch.
*/
public class StringsSketchToString extends EvalFunc<String> {

@Override
public String exec(final Tuple input) throws IOException {
if (input == null) {
return null;
}
if (input.size() != 1) {
throw new IllegalArgumentException("expected one input");
}

if (!(input.get(0) instanceof DataByteArray)) {
throw new IllegalArgumentException("expected a DataByteArray as a sketch, got "
+ input.get(0).getClass().getSimpleName());
}
final DataByteArray dba = (DataByteArray) input.get(0);
final ItemsSketch<String> sketch =
ItemsSketch.getInstance(Memory.wrap(dba.get()), Comparator.naturalOrder(),
new ArrayOfStringsSerDe());
return sketch.toString();
}

}
@@ -0,0 +1,57 @@
/*
* Copyright 2019, Verizon Media.
* Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms.
*/

package com.yahoo.sketches.pig.quantiles;

import java.util.Arrays;

import org.apache.pig.EvalFunc;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.TupleFactory;

import org.testng.annotations.Test;

import com.yahoo.sketches.quantiles.DoublesSketch;

import org.testng.Assert;

public class DoublesSketchToStringTest {
private static final TupleFactory TUPLE_FACTORY = TupleFactory.getInstance();

@Test
public void nullInputTuple() throws Exception {
final EvalFunc<String> func = new DoublesSketchToString();
final String result = func.exec(null);
Assert.assertNull(result);
}

@Test(expectedExceptions = IllegalArgumentException.class)
public void emptyInputTuple() throws Exception {
final EvalFunc<String> func = new DoublesSketchToString();
final String result = func.exec(TUPLE_FACTORY.newTuple());
Assert.assertNull(result);
}

@Test(expectedExceptions = IllegalArgumentException.class)
public void tooManyInputs() throws Exception {
final EvalFunc<String> func = new DoublesSketchToString();
func.exec(TUPLE_FACTORY.newTuple(2));
}

@Test(expectedExceptions = IllegalArgumentException.class)
public void wrongTypeForSketch() throws Exception {
final EvalFunc<String> func = new DoublesSketchToString();
func.exec(TUPLE_FACTORY.newTuple(Arrays.asList(1.0)));
}

@Test
public void normalCase() throws Exception {
final EvalFunc<String> func = new DoublesSketchToString();
final DoublesSketch sketch = DoublesSketch.builder().build();
final String result = func.exec(TUPLE_FACTORY.newTuple(Arrays.asList(new DataByteArray(sketch.toByteArray()))));
Assert.assertNotNull(result);
}

}
@@ -0,0 +1,63 @@
/*
* Copyright 2019, Verizon Media.
* Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms.
*/

package com.yahoo.sketches.pig.quantiles;

import java.util.Arrays;
import java.util.Comparator;

import org.apache.pig.EvalFunc;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.TupleFactory;

import org.testng.annotations.Test;

import com.yahoo.sketches.ArrayOfItemsSerDe;
import com.yahoo.sketches.ArrayOfStringsSerDe;
import com.yahoo.sketches.quantiles.ItemsSketch;

import org.testng.Assert;

public class StringsSketchToStringTest {
private static final TupleFactory TUPLE_FACTORY = TupleFactory.getInstance();

private static final Comparator<String> COMPARATOR = Comparator.naturalOrder();
private static final ArrayOfItemsSerDe<String> SER_DE = new ArrayOfStringsSerDe();

@Test
public void nullInputTuple() throws Exception {
final EvalFunc<String> func = new StringsSketchToString();
final String result = func.exec(null);
Assert.assertNull(result);
}

@Test(expectedExceptions = IllegalArgumentException.class)
public void emptyInputTuple() throws Exception {
final EvalFunc<String> func = new StringsSketchToString();
final String result = func.exec(TUPLE_FACTORY.newTuple());
Assert.assertNull(result);
}

@Test(expectedExceptions = IllegalArgumentException.class)
public void tooManyInputs() throws Exception {
final EvalFunc<String> func = new StringsSketchToString();
func.exec(TUPLE_FACTORY.newTuple(2));
}

@Test(expectedExceptions = IllegalArgumentException.class)
public void wrongTypeForSketch() throws Exception {
final EvalFunc<String> func = new StringsSketchToString();
func.exec(TUPLE_FACTORY.newTuple(Arrays.asList(1.0)));
}

@Test
public void normalCase() throws Exception {
final EvalFunc<String> func = new DoublesSketchToString();
ItemsSketch<String> sketch = ItemsSketch.getInstance(COMPARATOR);
final String result = func.exec(TUPLE_FACTORY.newTuple(new DataByteArray(sketch.toByteArray(SER_DE))));
Assert.assertNotNull(result);
}

}

0 comments on commit 0fab110

Please sign in to comment.