Skip to content
Permalink
Browse files
Merge pull request #39 from DataSketches/new_memory
move to new memory package
  • Loading branch information
jmalkin committed Jun 20, 2017
2 parents fa0581a + 60a7811 commit 3b4a815ddb4504a5d15079db29ba7f1b4d1e15a8
Showing 41 changed files with 152 additions and 149 deletions.
10 pom.xml
@@ -145,13 +145,13 @@
<dependency>
<groupId>com.yahoo.datasketches</groupId>
<artifactId>sketches-core</artifactId>
<version>0.9.0</version>
<version>0.10.0</version>
</dependency>

<dependency>
<groupId>com.yahoo.datasketches</groupId>
<artifactId>memory</artifactId>
<version>0.9.0</version>
<version>0.10.1</version>
</dependency>

<!-- Pig -->
@@ -167,7 +167,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.7.3</version>
<version>2.8.1</version>
<optional>true</optional>
<exclusions>
<exclusion>
@@ -180,7 +180,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>2.7.3</version>
<version>2.8.0</version>
<optional>true</optional>
<exclusions>
<exclusion>
@@ -201,7 +201,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-common</artifactId>
<version>2.7.3</version>
<version>2.8.0</version>
<optional>true</optional>
</dependency>

@@ -17,7 +17,7 @@
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;

import com.yahoo.memory.NativeMemory;
import com.yahoo.memory.Memory;
import com.yahoo.sketches.ArrayOfStringsSerDe;
import com.yahoo.sketches.frequencies.ErrorType;
import com.yahoo.sketches.frequencies.ItemsSketch;
@@ -53,7 +53,7 @@ public DataBag exec(final Tuple input) throws IOException {

final DataByteArray dba = (DataByteArray) input.get(0);
final ItemsSketch<String> sketch =
ItemsSketch.getInstance(new NativeMemory(dba.get()), new ArrayOfStringsSerDe());
ItemsSketch.getInstance(Memory.wrap(dba.get()), new ArrayOfStringsSerDe());
final ItemsSketch.Row<String>[] result = sketch.getFrequentItems(errorType);

final DataBag bag = BagFactory.getInstance().newDefaultBag();
@@ -10,7 +10,7 @@
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;

import com.yahoo.memory.NativeMemory;
import com.yahoo.memory.Memory;
import com.yahoo.sketches.ArrayOfItemsSerDe;
import com.yahoo.sketches.frequencies.ItemsSketch;

@@ -28,7 +28,7 @@ static <T> Tuple serializeSketchToTuple(
static <T> ItemsSketch<T> deserializeSketchFromTuple(
final Tuple tuple, final ArrayOfItemsSerDe<T> serDe) throws ExecException {
final byte[] bytes = ((DataByteArray) tuple.get(0)).get();
return ItemsSketch.getInstance(new NativeMemory(bytes), serDe);
return ItemsSketch.getInstance(Memory.wrap(bytes), serDe);
}

}
@@ -18,7 +18,7 @@
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;

import com.yahoo.memory.NativeMemory;
import com.yahoo.memory.Memory;
import com.yahoo.sketches.quantiles.DoublesSketch;
import com.yahoo.sketches.quantiles.DoublesUnion;
import com.yahoo.sketches.quantiles.DoublesUnionBuilder;
@@ -324,7 +324,7 @@ public Tuple exec(final Tuple inputTuple) throws IOException { //throws is in AP
// due to system bagged outputs from multiple mapper Intermediate functions.
// Each dataTuple.DBA:sketch will merged into the union.
final DataByteArray dba = (DataByteArray) f0;
union.update(new NativeMemory(dba.get()));
union.update(Memory.wrap(dba.get()));
} else {
throw new IllegalArgumentException("dataTuple.Field0: Is not a DataByteArray: "
+ f0.getClass().getName());
@@ -19,7 +19,7 @@
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;

import com.yahoo.memory.NativeMemory;
import com.yahoo.memory.Memory;
import com.yahoo.sketches.ArrayOfItemsSerDe;
import com.yahoo.sketches.quantiles.ItemsSketch;
import com.yahoo.sketches.quantiles.ItemsUnion;
@@ -314,7 +314,7 @@ public Tuple exec(final Tuple inputTuple) throws IOException { //throws is in AP
// due to system bagged outputs from multiple mapper Intermediate functions.
// Each dataTuple.DBA:sketch will merged into the union.
final DataByteArray dba = (DataByteArray) f0;
union.update(ItemsSketch.getInstance(new NativeMemory(dba.get()), comparator_, serDe_));
union.update(ItemsSketch.getInstance(Memory.wrap(dba.get()), comparator_, serDe_));
} else {
throw new IllegalArgumentException("dataTuple.Field0: Is not a DataByteArray: "
+ f0.getClass().getName());
@@ -11,7 +11,7 @@
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.Tuple;

import com.yahoo.memory.NativeMemory;
import com.yahoo.memory.Memory;
import com.yahoo.sketches.quantiles.DoublesSketch;

/**
@@ -32,7 +32,7 @@ public Integer exec(final Tuple input) throws IOException {
+ input.get(0).getClass().getSimpleName());
}
final DataByteArray dba = (DataByteArray) input.get(0);
final DoublesSketch sketch = DoublesSketch.heapify(new NativeMemory(dba.get()));
final DoublesSketch sketch = DoublesSketch.heapify(Memory.wrap(dba.get()));

return sketch.getK();
}
@@ -12,7 +12,7 @@
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.Tuple;

import com.yahoo.memory.NativeMemory;
import com.yahoo.memory.Memory;
import com.yahoo.sketches.ArrayOfStringsSerDe;
import com.yahoo.sketches.quantiles.ItemsSketch;

@@ -35,7 +35,7 @@ public Integer exec(final Tuple input) throws IOException {
}
final DataByteArray dba = (DataByteArray) input.get(0);
final ItemsSketch<String> sketch =
ItemsSketch.getInstance(new NativeMemory(dba.get()), Comparator.naturalOrder(),
ItemsSketch.getInstance(Memory.wrap(dba.get()), Comparator.naturalOrder(),
new ArrayOfStringsSerDe());

return sketch.getK();
@@ -11,7 +11,7 @@
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.Tuple;

import com.yahoo.memory.NativeMemory;
import com.yahoo.memory.Memory;
import com.yahoo.sketches.quantiles.DoublesSketch;

/**
@@ -36,7 +36,7 @@ public Tuple exec(final Tuple input) throws IOException {
+ input.get(0).getClass().getSimpleName());
}
final DataByteArray dba = (DataByteArray) input.get(0);
final DoublesSketch sketch = DoublesSketch.heapify(new NativeMemory(dba.get()));
final DoublesSketch sketch = DoublesSketch.heapify(Memory.wrap(dba.get()));

final double[] splitPoints = new double[input.size() - 1];
for (int i = 1; i < input.size(); i++) {
@@ -12,7 +12,7 @@
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.Tuple;

import com.yahoo.memory.NativeMemory;
import com.yahoo.memory.Memory;
import com.yahoo.sketches.ArrayOfStringsSerDe;
import com.yahoo.sketches.quantiles.ItemsSketch;

@@ -38,7 +38,7 @@ public Tuple exec(final Tuple input) throws IOException {
}
final DataByteArray dba = (DataByteArray) input.get(0);
final ItemsSketch<String> sketch =
ItemsSketch.getInstance(new NativeMemory(dba.get()), Comparator.naturalOrder(),
ItemsSketch.getInstance(Memory.wrap(dba.get()), Comparator.naturalOrder(),
new ArrayOfStringsSerDe());

final String[] splitPoints = new String[input.size() - 1];
@@ -11,7 +11,7 @@
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.Tuple;

import com.yahoo.memory.NativeMemory;
import com.yahoo.memory.Memory;
import com.yahoo.sketches.quantiles.DoublesSketch;

/**
@@ -34,7 +34,7 @@ public Double exec(final Tuple input) throws IOException {
+ input.get(0).getClass().getSimpleName());
}
final DataByteArray dba = (DataByteArray) input.get(0);
final DoublesSketch sketch = DoublesSketch.heapify(new NativeMemory(dba.get()));
final DoublesSketch sketch = DoublesSketch.heapify(Memory.wrap(dba.get()));

if (!(input.get(1) instanceof Double)) {
throw new IllegalArgumentException("expected a double value as a fraction, got "
@@ -12,7 +12,7 @@
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.Tuple;

import com.yahoo.memory.NativeMemory;
import com.yahoo.memory.Memory;
import com.yahoo.sketches.ArrayOfStringsSerDe;
import com.yahoo.sketches.quantiles.ItemsSketch;

@@ -37,7 +37,7 @@ public String exec(final Tuple input) throws IOException {
}
final DataByteArray dba = (DataByteArray) input.get(0);
final ItemsSketch<String> sketch = ItemsSketch.getInstance(
new NativeMemory(dba.get()), Comparator.naturalOrder(), new ArrayOfStringsSerDe());
Memory.wrap(dba.get()), Comparator.naturalOrder(), new ArrayOfStringsSerDe());

if (!(input.get(1) instanceof Double)) {
throw new IllegalArgumentException(
@@ -13,7 +13,7 @@
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;

import com.yahoo.memory.NativeMemory;
import com.yahoo.memory.Memory;
import com.yahoo.sketches.quantiles.DoublesSketch;

/**
@@ -36,7 +36,7 @@ public Tuple exec(final Tuple input) throws IOException {
+ input.get(0).getClass().getSimpleName());
}
final DataByteArray dba = (DataByteArray) input.get(0);
final DoublesSketch sketch = DoublesSketch.heapify(new NativeMemory(dba.get()));
final DoublesSketch sketch = DoublesSketch.heapify(Memory.wrap(dba.get()));

if (input.size() == 2) {
final Object arg = input.get(1);
@@ -14,7 +14,7 @@
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;

import com.yahoo.memory.NativeMemory;
import com.yahoo.memory.Memory;
import com.yahoo.sketches.ArrayOfStringsSerDe;
import com.yahoo.sketches.quantiles.ItemsSketch;

@@ -39,7 +39,7 @@ public Tuple exec(final Tuple input) throws IOException {
}
final DataByteArray dba = (DataByteArray) input.get(0);
final ItemsSketch<String> sketch = ItemsSketch.getInstance(
new NativeMemory(dba.get()), Comparator.naturalOrder(), new ArrayOfStringsSerDe());
Memory.wrap(dba.get()), Comparator.naturalOrder(), new ArrayOfStringsSerDe());
if (sketch.isEmpty()) { return null; }

if (input.size() == 2) {
@@ -19,7 +19,7 @@
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;

import com.yahoo.memory.NativeMemory;
import com.yahoo.memory.Memory;
import com.yahoo.sketches.quantiles.DoublesSketch;
import com.yahoo.sketches.quantiles.DoublesUnion;
import com.yahoo.sketches.quantiles.DoublesUnionBuilder;
@@ -225,7 +225,7 @@ private static void updateUnion(final DataBag bag, final DoublesUnion union) thr
if (f0 instanceof DataByteArray) {
final DataByteArray dba = (DataByteArray) f0;
if (dba.size() > 0) {
union.update(new NativeMemory(dba.get()));
union.update(Memory.wrap(dba.get()));
}
} else {
throw new IllegalArgumentException("Field type was not DataType.BYTEARRAY: " + innerTuple.getType(0));
@@ -328,7 +328,7 @@ public Tuple exec(final Tuple inputTuple) throws IOException {
// It is due to system bagged outputs from multiple mapper Intermediate functions.
// Each dataTuple.DBA:sketch will merged into the union.
final DataByteArray dba = (DataByteArray) f0;
union.update(new NativeMemory(dba.get()));
union.update(Memory.wrap(dba.get()));
} else {
throw new IllegalArgumentException("dataTuple.Field0: Is not a DataByteArray: "
+ f0.getClass().getName());
@@ -20,7 +20,7 @@
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;

import com.yahoo.memory.NativeMemory;
import com.yahoo.memory.Memory;
import com.yahoo.sketches.ArrayOfItemsSerDe;
import com.yahoo.sketches.quantiles.ItemsSketch;
import com.yahoo.sketches.quantiles.ItemsUnion;
@@ -209,7 +209,7 @@ private static <T> void updateUnion(final DataBag bag, final ItemsUnion<T> union
if (f0 instanceof DataByteArray) {
final DataByteArray dba = (DataByteArray) f0;
if (dba.size() > 0) {
union.update(ItemsSketch.getInstance(new NativeMemory(dba.get()), comparator, serDe));
union.update(ItemsSketch.getInstance(Memory.wrap(dba.get()), comparator, serDe));
}
} else {
throw new IllegalArgumentException("Field type was not DataType.BYTEARRAY: " + innerTuple.getType(0));
@@ -301,7 +301,7 @@ public Tuple exec(final Tuple inputTuple) throws IOException {
// It is due to system bagged outputs from multiple mapper Intermediate functions.
// Each dataTuple.DBA:sketch will merged into the union.
final DataByteArray dba = (DataByteArray) f0;
union.update(ItemsSketch.getInstance(new NativeMemory(dba.get()), comparator_, serDe_));
union.update(ItemsSketch.getInstance(Memory.wrap(dba.get()), comparator_, serDe_));
} else {
throw new IllegalArgumentException("dataTuple.Field0: Is not a DataByteArray: "
+ f0.getClass().getName());
@@ -82,7 +82,7 @@ public void accumulate(final Tuple inputTuple) throws IOException {
final DataBag samples = (DataBag) inputTuple.get(0);

if (reservoir_ == null) {
reservoir_ = ReservoirItemsSketch.getInstance(targetK_);
reservoir_ = ReservoirItemsSketch.newInstance(targetK_);
}

for (Tuple t : samples) {
@@ -96,7 +96,7 @@ public Tuple getValue() {
return null;
}

final List<Tuple> data = reservoir_.getRawSamplesAsList();
final List<Tuple> data = SamplingPigUtil.getRawSamplesAsList(reservoir_);
final DataBag sampleBag = BagFactory.getInstance().newDefaultBag(data);

return createResultTuple(reservoir_.getN(), reservoir_.getK(), sampleBag);
@@ -199,7 +199,7 @@ public Tuple exec(final Tuple inputTuple) throws IOException {
if (records.size() <= targetK_) {
outputBag = records;
} else {
reservoir = ReservoirItemsSketch.getInstance(targetK_);
reservoir = ReservoirItemsSketch.newInstance(targetK_);
for (Tuple t : records) {
reservoir.update(t);
}
@@ -244,7 +244,7 @@ public Tuple exec(final Tuple inputTuple) throws IOException {
return null;
}

final ReservoirItemsUnion<Tuple> union = ReservoirItemsUnion.getInstance(targetK_);
final ReservoirItemsUnion<Tuple> union = ReservoirItemsUnion.newInstance(targetK_);

final DataBag outerBag = (DataBag) inputTuple.get(0);
for (Tuple reservoir : outerBag) {
@@ -262,7 +262,7 @@ public Tuple exec(final Tuple inputTuple) throws IOException {
}

final ReservoirItemsSketch<Tuple> result = union.getResult();
final ArrayList<Tuple> data = result.getRawSamplesAsList();
final ArrayList<Tuple> data = SamplingPigUtil.getRawSamplesAsList(result);
final DataBag sampleBag = BagFactory.getInstance().newDefaultBag(data);

final Tuple output = TupleFactory.getInstance().newTuple(3);
@@ -60,7 +60,7 @@ public void accumulate(final Tuple inputTuple) throws IOException {
final DataBag reservoirs = (DataBag) inputTuple.get(0);

if (union_ == null) {
union_ = ReservoirItemsUnion.getInstance(maxK_);
union_ = ReservoirItemsUnion.newInstance(maxK_);
}

try {
@@ -20,7 +20,6 @@
import org.apache.pig.impl.logicalLayer.schema.Schema;

import com.yahoo.memory.Memory;
import com.yahoo.memory.NativeMemory;
import com.yahoo.sketches.theta.AnotB;
import com.yahoo.sketches.theta.CompactSketch;
import com.yahoo.sketches.theta.SetOperation;
@@ -112,14 +111,14 @@ public Tuple exec(final Tuple inputTuple) throws IOException {
Sketch sketchA = null;
if (objA != null) {
final DataByteArray dbaA = (DataByteArray)objA;
final Memory srcMem = new NativeMemory(dbaA.get());
final Memory srcMem = Memory.wrap(dbaA.get());
sketchA = Sketch.wrap(srcMem, seed_);
}
final Object objB = extractFieldAtIndex(inputTuple, 1);
Sketch sketchB = null;
if (objB != null) {
final DataByteArray dbaB = (DataByteArray)objB;
final Memory srcMem = new NativeMemory(dbaB.get());
final Memory srcMem = Memory.wrap(dbaB.get());
sketchB = Sketch.wrap(srcMem, seed_);
}

0 comments on commit 3b4a815

Please sign in to comment.