Skip to content
Permalink
Browse files
Merge pull request #37 from DataSketches/core-0.9.0
updated to use the latest sketches-core 0.9.0
  • Loading branch information
leerho committed Mar 29, 2017
2 parents e451146 + fd9c4b0 commit 086e68bf085a17f121a8c334568c53d04a6e7af6
Showing 9 changed files with 32 additions and 30 deletions.
@@ -145,13 +145,13 @@
<dependency>
<groupId>com.yahoo.datasketches</groupId>
<artifactId>sketches-core</artifactId>
<version>0.8.4</version>
<version>0.9.0</version>
</dependency>

<dependency>
<groupId>com.yahoo.datasketches</groupId>
<artifactId>memory</artifactId>
<version>0.8.4</version>
<version>0.9.0</version>
</dependency>

<!-- Pig -->
@@ -322,7 +322,7 @@
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version> 0.7.9</version>
<version>0.7.9</version>
<executions>
<!-- Prepares the property pointing to the JaCoCo runtime agent which
is passed as VM argument when Maven the Surefire plugin is executed. -->
@@ -132,11 +132,11 @@ public Tuple exec(final Tuple inputTuple) throws IOException {
}
final DoublesSketch resultSketch = union.getResultAndReset();
if (resultSketch != null) {
return tupleFactory_.newTuple(new DataByteArray(resultSketch.toByteArray()));
return tupleFactory_.newTuple(new DataByteArray(resultSketch.toByteArray(true)));
}
}
// return empty sketch
return tupleFactory_.newTuple(new DataByteArray(unionBuilder_.build().getResult().toByteArray()));
return tupleFactory_.newTuple(new DataByteArray(unionBuilder_.build().getResult().toByteArray(true)));
}

@Override
@@ -191,11 +191,11 @@ public Tuple getValue() {
if (accumUnion_ != null) {
final DoublesSketch resultSketch = accumUnion_.getResultAndReset();
if (resultSketch != null) {
return tupleFactory_.newTuple(new DataByteArray(resultSketch.toByteArray()));
return tupleFactory_.newTuple(new DataByteArray(resultSketch.toByteArray(true)));
}
}
// return empty sketch
return tupleFactory_.newTuple(new DataByteArray(unionBuilder_.build().getResult().toByteArray()));
return tupleFactory_.newTuple(new DataByteArray(unionBuilder_.build().getResult().toByteArray(true)));
}

/**
@@ -332,11 +332,11 @@ public Tuple exec(final Tuple inputTuple) throws IOException { //throws is in AP
}
final DoublesSketch resultSketch = union.getResultAndReset();
if (resultSketch != null) {
return tupleFactory_.newTuple(new DataByteArray(resultSketch.toByteArray()));
return tupleFactory_.newTuple(new DataByteArray(resultSketch.toByteArray(true)));
}
}
// return empty sketch
return tupleFactory_.newTuple(new DataByteArray(unionBuilder_.build().getResult().toByteArray()));
return tupleFactory_.newTuple(new DataByteArray(unionBuilder_.build().getResult().toByteArray(true)));
}
} // end IntermediateFinal

@@ -121,11 +121,11 @@ public Tuple exec(final Tuple inputTuple) throws IOException {
updateUnion(bag, union);
final DoublesSketch resultSketch = union.getResultAndReset();
if (resultSketch != null) {
return tupleFactory_.newTuple(new DataByteArray(resultSketch.toByteArray()));
return tupleFactory_.newTuple(new DataByteArray(resultSketch.toByteArray(true)));
}
}
// return empty sketch
return tupleFactory_.newTuple(new DataByteArray(unionBuilder_.build().getResult().toByteArray()));
return tupleFactory_.newTuple(new DataByteArray(unionBuilder_.build().getResult().toByteArray(true)));
}

@Override
@@ -176,11 +176,11 @@ public Tuple getValue() {
if (accumUnion_ != null) {
final DoublesSketch resultSketch = accumUnion_.getResultAndReset();
if (resultSketch != null) {
return tupleFactory_.newTuple(new DataByteArray(resultSketch.toByteArray()));
return tupleFactory_.newTuple(new DataByteArray(resultSketch.toByteArray(true)));
}
}
// return empty sketch
return tupleFactory_.newTuple(new DataByteArray(unionBuilder_.build().getResult().toByteArray()));
return tupleFactory_.newTuple(new DataByteArray(unionBuilder_.build().getResult().toByteArray(true)));
}

/**
@@ -336,11 +336,11 @@ public Tuple exec(final Tuple inputTuple) throws IOException {
}
final DoublesSketch resultSketch = union.getResultAndReset();
if (resultSketch != null) {
return tupleFactory_.newTuple(new DataByteArray(resultSketch.toByteArray()));
return tupleFactory_.newTuple(new DataByteArray(resultSketch.toByteArray(true)));
}
}
// return empty sketch
return tupleFactory_.newTuple(new DataByteArray(unionBuilder_.build().getResult().toByteArray()));
return tupleFactory_.newTuple(new DataByteArray(unionBuilder_.build().getResult().toByteArray(true)));
}
} // end IntermediateFinal

@@ -12,8 +12,7 @@
import org.apache.pig.data.Tuple;

import com.yahoo.memory.NativeMemory;
import com.yahoo.sketches.quantiles.DoublesSketch;
import com.yahoo.sketches.quantiles.DoublesSketchBuilder;
import com.yahoo.sketches.quantiles.UpdateDoublesSketch;
import com.yahoo.sketches.tuple.DoubleSummary;
import com.yahoo.sketches.tuple.Sketch;
import com.yahoo.sketches.tuple.SketchIterator;
@@ -45,7 +44,7 @@ public Double exec(final Tuple input) throws IOException {
throw new IllegalArgumentException("percentile must be between 0 and 100");
}

final DoublesSketch qs = new DoublesSketchBuilder().build(QUANTILES_SKETCH_SIZE);
final UpdateDoublesSketch qs = UpdateDoublesSketch.builder().build(QUANTILES_SKETCH_SIZE);
final SketchIterator<DoubleSummary> it = sketch.iterator();
while (it.next()) {
qs.update(it.getSummary().getValue());
@@ -18,7 +18,7 @@

import com.yahoo.memory.NativeMemory;
import com.yahoo.sketches.quantiles.DoublesSketch;

import com.yahoo.sketches.quantiles.UpdateDoublesSketch;

public class DataToDoublesSketchTest {
private static final TupleFactory tupleFactory = TupleFactory.getInstance();
@@ -151,7 +151,7 @@ public void algebraicIntermediateFinalNormalCase() throws Exception {
}

{ // this is to simulate an output from a prior call of IntermediateFinal
DoublesSketch qs = DoublesSketch.builder().build();
UpdateDoublesSketch qs = UpdateDoublesSketch.builder().build();
qs.update(2.0);
bag.add(tupleFactory.newTuple(new DataByteArray(qs.toByteArray())));
}
@@ -5,6 +5,7 @@
package com.yahoo.sketches.pig.quantiles;

import com.yahoo.sketches.quantiles.DoublesSketch;
import com.yahoo.sketches.quantiles.UpdateDoublesSketch;

import java.util.Arrays;

@@ -33,7 +34,7 @@ public void emptySketch() throws Exception {
@Test
public void normalCase() throws Exception {
EvalFunc<Tuple> func = new GetPmfFromDoublesSketch();
DoublesSketch sketch = DoublesSketch.builder().build();
UpdateDoublesSketch sketch = DoublesSketch.builder().build();
for (int i = 1; i <= 10; i++) sketch.update(i);
Tuple resultTuple = func.exec(tupleFactory.newTuple(Arrays.asList(new DataByteArray(sketch.toByteArray()), 2.0, 7.0)));
Assert.assertNotNull(resultTuple);
@@ -5,6 +5,7 @@
package com.yahoo.sketches.pig.quantiles;

import com.yahoo.sketches.quantiles.DoublesSketch;
import com.yahoo.sketches.quantiles.UpdateDoublesSketch;

import java.util.Arrays;

@@ -29,7 +30,7 @@ public void emptySketch() throws Exception {
@Test
public void normalCase() throws Exception {
EvalFunc<Double> func = new GetQuantileFromDoublesSketch();
DoublesSketch sketch = DoublesSketch.builder().build();
UpdateDoublesSketch sketch = DoublesSketch.builder().build();
sketch.update(1.0);
Double result = func.exec(tupleFactory.newTuple(Arrays.asList(new DataByteArray(sketch.toByteArray()), 0.5)));
Assert.assertEquals(result, 1.0);
@@ -5,6 +5,7 @@
package com.yahoo.sketches.pig.quantiles;

import com.yahoo.sketches.quantiles.DoublesSketch;
import com.yahoo.sketches.quantiles.UpdateDoublesSketch;

import java.util.Arrays;

@@ -58,7 +59,7 @@ public void wrongTypeAmongFractions() throws Exception {
@Test
public void oneFraction() throws Exception {
EvalFunc<Tuple> func = new GetQuantilesFromDoublesSketch();
DoublesSketch sketch = DoublesSketch.builder().build();
UpdateDoublesSketch sketch = DoublesSketch.builder().build();
for (int i = 1; i <= 10; i++) sketch.update(i);
Tuple resultTuple = func.exec(tupleFactory.newTuple(Arrays.asList(new DataByteArray(sketch.toByteArray()), 0.5)));
Assert.assertNotNull(resultTuple);
@@ -69,7 +70,7 @@ public void oneFraction() throws Exception {
@Test
public void severalFractions() throws Exception {
EvalFunc<Tuple> func = new GetQuantilesFromDoublesSketch();
DoublesSketch sketch = DoublesSketch.builder().build();
UpdateDoublesSketch sketch = DoublesSketch.builder().build();
for (int i = 1; i <= 10; i++) sketch.update(i);
Tuple resultTuple = func.exec(tupleFactory.newTuple(Arrays.asList(new DataByteArray(sketch.toByteArray()), 0.0, 0.5, 1.0)));
Assert.assertNotNull(resultTuple);
@@ -82,7 +83,7 @@ public void severalFractions() throws Exception {
@Test
public void numberOfEvenlySpacedIntervals() throws Exception {
EvalFunc<Tuple> func = new GetQuantilesFromDoublesSketch();
DoublesSketch sketch = DoublesSketch.builder().build();
UpdateDoublesSketch sketch = DoublesSketch.builder().build();
for (int i = 1; i <= 10; i++) sketch.update(i);
Tuple resultTuple = func.exec(tupleFactory.newTuple(Arrays.asList(new DataByteArray(sketch.toByteArray()), 3)));
Assert.assertNotNull(resultTuple);
@@ -18,7 +18,7 @@

import com.yahoo.memory.NativeMemory;
import com.yahoo.sketches.quantiles.DoublesSketch;

import com.yahoo.sketches.quantiles.UpdateDoublesSketch;

public class UnionDoublesSketchTest {
private static final TupleFactory tupleFactory = TupleFactory.getInstance();
@@ -52,7 +52,7 @@ public void execEmptyBag() throws Exception {
public void execNormalCase() throws Exception {
EvalFunc<Tuple> func = new UnionDoublesSketch();
DataBag bag = bagFactory.newDefaultBag();
DoublesSketch inputSketch = DoublesSketch.builder().build();
UpdateDoublesSketch inputSketch = DoublesSketch.builder().build();
inputSketch.update(1.0);
bag.add(tupleFactory.newTuple(new DataByteArray(inputSketch.toByteArray())));
Tuple resultTuple = func.exec(tupleFactory.newTuple(bag));
@@ -90,7 +90,7 @@ public void accumulator() throws Exception {

// normal case
DataBag bag = bagFactory.newDefaultBag();
DoublesSketch inputSketch = DoublesSketch.builder().build();
UpdateDoublesSketch inputSketch = DoublesSketch.builder().build();
inputSketch.update(1.0);
bag.add(tupleFactory.newTuple(new DataByteArray(inputSketch.toByteArray())));
func.accumulate(tupleFactory.newTuple(bag));
@@ -142,14 +142,14 @@ public void algebraicIntermediateFinalNormalCase() throws Exception {

{ // this is to simulate an output from Initial
DataBag innerBag = bagFactory.newDefaultBag();
DoublesSketch qs = DoublesSketch.builder().build();
UpdateDoublesSketch qs = DoublesSketch.builder().build();
qs.update(1.0);
innerBag.add(tupleFactory.newTuple(new DataByteArray(qs.toByteArray())));
bag.add(tupleFactory.newTuple(innerBag));
}

{ // this is to simulate an output from a prior call of IntermediateFinal
DoublesSketch qs = DoublesSketch.builder().build();
UpdateDoublesSketch qs = DoublesSketch.builder().build();
qs.update(2.0);
bag.add(tupleFactory.newTuple(new DataByteArray(qs.toByteArray())));
}

0 comments on commit 086e68b

Please sign in to comment.