Skip to content
Permalink
Browse files
Interim 16
  • Loading branch information
leerho committed Apr 14, 2022
1 parent a871741 commit 8f01cbd33187030f26bf5f7ae035892b088ec940
Showing 21 changed files with 703 additions and 342 deletions.
@@ -0,0 +1,112 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.datasketches.kll;

import static org.apache.datasketches.kll.KllPreambleUtil.DATA_START_ADR;
import static org.apache.datasketches.kll.KllPreambleUtil.DATA_START_ADR_SINGLE_ITEM;
import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryEmptyFlag;
import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryN;
import static org.apache.datasketches.kll.KllPreambleUtil.getMemorySingleItemFlag;
import static org.apache.datasketches.kll.KllSketch.Error.EMPTY_NO_DATA;
import static org.apache.datasketches.kll.KllSketch.Error.SINGLE_ITEM_IMPROPER_CALL;
import static org.apache.datasketches.kll.KllSketch.Error.kllSketchThrow;

import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.memory.WritableMemory;

class KllDirectCompactDoublesSketch extends KllDirectDoublesSketch {
final boolean empty;
final boolean singleItem;

KllDirectCompactDoublesSketch(final Memory srcMem, final KllMemoryValidate memVal) {
super((WritableMemory) srcMem, null, memVal);
this.empty = getMemoryEmptyFlag(srcMem);
this.singleItem = getMemorySingleItemFlag(srcMem);
}

@Override
public long getN() {
if (empty) { return 0; }
if (singleItem ) { return 1; }
return getMemoryN(wmem);
}

@Override
public byte[] toByteArray() {
final int bytes = (int) wmem.getCapacity();
final byte[] byteArr = new byte[bytes];
wmem.getByteArray(0, byteArr, 0, bytes);
return byteArr;
}

@Override //returns expanded array including empty space at bottom
double[] getDoubleItemsArray() {
final int k = getK();
if (empty) { return new double[k]; }
final double[] itemsArr;
if (singleItem) {
itemsArr = new double[k];
itemsArr[k - 1] = wmem.getDouble(DATA_START_ADR_SINGLE_ITEM);
return itemsArr;
}
final int numLevels = getNumLevels();
final int levelsBytes = numLevels * Integer.BYTES; //compact format!
final int allItems = levelsArr[numLevels];
itemsArr = new double[allItems];
final int shift = levelsArr[0];
final int offset = DATA_START_ADR + levelsBytes + 2 * Double.BYTES;
wmem.getDoubleArray(offset, itemsArr, shift, allItems - shift);
return itemsArr;
}

@Override
double getDoubleItemsArrayAt(final int index) {
if (empty) { kllSketchThrow(EMPTY_NO_DATA); }
if (singleItem) { kllSketchThrow(SINGLE_ITEM_IMPROPER_CALL); }
final int offset =
DATA_START_ADR + (getLevelsArray().length - 1) * Integer.BYTES + (index + 2) * Double.BYTES;
return wmem.getDouble(offset);
}

@Override
double getDoubleSingleItem() {
if (!singleItem) { kllSketchThrow(SINGLE_ITEM_IMPROPER_CALL); }
return wmem.getDouble(DATA_START_ADR_SINGLE_ITEM);
}

@Override
double getMaxDoubleValue() {
if (empty) { return Double.NaN; }
if (singleItem) { return getDoubleSingleItem(); }
final int offset =
DATA_START_ADR + (getLevelsArray().length - 1) * Integer.BYTES + Double.BYTES;
return wmem.getDouble(offset);
}

@Override
double getMinDoubleValue() {
if (empty) { return Double.NaN; }
if (singleItem) { return getDoubleSingleItem(); }
final int offset =
DATA_START_ADR + (getLevelsArray().length - 1) * Integer.BYTES;
return wmem.getDouble(offset);
}

}
@@ -0,0 +1,112 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.datasketches.kll;

import static org.apache.datasketches.kll.KllPreambleUtil.DATA_START_ADR;
import static org.apache.datasketches.kll.KllPreambleUtil.DATA_START_ADR_SINGLE_ITEM;
import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryEmptyFlag;
import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryN;
import static org.apache.datasketches.kll.KllPreambleUtil.getMemorySingleItemFlag;
import static org.apache.datasketches.kll.KllSketch.Error.EMPTY_NO_DATA;
import static org.apache.datasketches.kll.KllSketch.Error.SINGLE_ITEM_IMPROPER_CALL;
import static org.apache.datasketches.kll.KllSketch.Error.kllSketchThrow;

import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.memory.WritableMemory;

class KllDirectCompactFloatsSketch extends KllDirectFloatsSketch {
final boolean empty;
final boolean singleItem;

KllDirectCompactFloatsSketch(final Memory srcMem, final KllMemoryValidate memVal) {
super((WritableMemory) srcMem, null, memVal);
this.empty = getMemoryEmptyFlag(srcMem);
this.singleItem = getMemorySingleItemFlag(srcMem);
}

@Override
public long getN() {
if (empty) { return 0; }
if (singleItem ) { return 1; }
return getMemoryN(wmem);
}

@Override
public byte[] toByteArray() {
final int bytes = (int) wmem.getCapacity();
final byte[] byteArr = new byte[bytes];
wmem.getByteArray(0, byteArr, 0, bytes);
return byteArr;
}

@Override //returns expanded array including empty space at bottom
float[] getFloatItemsArray() {
final int k = getK();
if (empty) { return new float[k]; }
final float[] itemsArr;
if (singleItem) {
itemsArr = new float[k];
itemsArr[k - 1] = wmem.getFloat(DATA_START_ADR_SINGLE_ITEM);
return itemsArr;
}
final int numLevels = getNumLevels();
final int levelsBytes = numLevels * Integer.BYTES; //compact format!
final int allItems = levelsArr[numLevels];
itemsArr = new float[allItems];
final int shift = levelsArr[0];
final int offset = DATA_START_ADR + levelsBytes + 2 * Float.BYTES;
wmem.getFloatArray(offset, itemsArr, shift, allItems - shift);
return itemsArr;
}

@Override
float getFloatItemsArrayAt(final int index) {
if (empty) { kllSketchThrow(EMPTY_NO_DATA); }
if (singleItem) { kllSketchThrow(SINGLE_ITEM_IMPROPER_CALL); }
final int offset =
DATA_START_ADR + (getLevelsArray().length - 1) * Integer.BYTES + (index + 2) * Float.BYTES;
return wmem.getFloat(offset);
}

@Override
float getFloatSingleItem() {
if (!singleItem) { kllSketchThrow(SINGLE_ITEM_IMPROPER_CALL); }
return wmem.getFloat(DATA_START_ADR_SINGLE_ITEM);
}

@Override
float getMaxFloatValue() {
if (empty) { return Float.NaN; }
if (singleItem) { return getFloatSingleItem(); }
final int offset =
DATA_START_ADR + (getLevelsArray().length - 1) * Integer.BYTES + Float.BYTES;
return wmem.getFloat(offset);
}

@Override
float getMinFloatValue() {
if (empty) { return Float.NaN; }
if (singleItem) { return getFloatSingleItem(); }
final int offset =
DATA_START_ADR + (getLevelsArray().length - 1) * Integer.BYTES;
return wmem.getFloat(offset);
}

}
@@ -41,6 +41,7 @@
import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryNumLevels;
import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryPreInts;
import static org.apache.datasketches.kll.KllPreambleUtil.setMemorySerVer;
import static org.apache.datasketches.kll.KllSketch.Error.MUST_NOT_CALL;
import static org.apache.datasketches.kll.KllSketch.Error.TGT_IS_READ_ONLY;
import static org.apache.datasketches.kll.KllSketch.Error.kllSketchThrow;

@@ -56,7 +57,7 @@
*
* @author Lee Rhodes, Kevin Lang
*/
final class KllDirectDoublesSketch extends KllDoublesSketch {
class KllDirectDoublesSketch extends KllDoublesSketch {

/**
* The constructor with Memory that can be off-heap.
@@ -121,11 +122,12 @@ public byte[] toUpdatableByteArray() {
return byteArr;
}

@Override
@Override //returns entire array including empty space at bottom
double[] getDoubleItemsArray() {
final int items = getItemsArrLengthItems();
final int items = levelsArr[getNumLevels()];
final double[] itemsArr = new double[items];
final int offset = DATA_START_ADR + getLevelsArray().length * Integer.BYTES + 2 * Double.BYTES;
final int levelsBytes = levelsArr.length * Integer.BYTES;
final int offset = DATA_START_ADR + levelsBytes + 2 * Double.BYTES;
wmem.getDoubleArray(offset, itemsArr, 0, items);
return itemsArr;
}
@@ -137,9 +139,11 @@ public byte[] toUpdatableByteArray() {
return wmem.getDouble(offset);
}

int getItemsArrLengthItems() {
return getLevelsArray()[getNumLevels()];
}
@Override
double getDoubleSingleItem() { kllSketchThrow(MUST_NOT_CALL); return Double.NaN; }

@Override
float getFloatSingleItem() { kllSketchThrow(MUST_NOT_CALL); return Float.NaN; }

@Override
int getM() {
@@ -163,11 +167,6 @@ int getMinK() {
return getMemoryMinK(wmem);
}

@Override
int getNumLevels() {
return getMemoryNumLevels(wmem);
}

@Override
void incN() {
if (readOnly) { kllSketchThrow(TGT_IS_READ_ONLY); }
@@ -40,6 +40,7 @@
import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryNumLevels;
import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryPreInts;
import static org.apache.datasketches.kll.KllPreambleUtil.setMemorySerVer;
import static org.apache.datasketches.kll.KllSketch.Error.MUST_NOT_CALL;
import static org.apache.datasketches.kll.KllSketch.Error.TGT_IS_READ_ONLY;
import static org.apache.datasketches.kll.KllSketch.Error.kllSketchThrow;

@@ -122,24 +123,27 @@ public byte[] toUpdatableByteArray() {
}

@Override
double getDoubleSingleItem() { kllSketchThrow(MUST_NOT_CALL); return Double.NaN; }

@Override //returns entire array including empty space at bottom
float[] getFloatItemsArray() {
final int items = getItemsArrLengthItems();
final int items = levelsArr[getNumLevels()];
final float[] itemsArr = new float[items];
final int offset = DATA_START_ADR + getLevelsArray().length * Integer.BYTES + 2 * Float.BYTES;
final int levelsBytes = levelsArr.length * Integer.BYTES;
final int offset = DATA_START_ADR + levelsBytes + 2 * Float.BYTES;
wmem.getFloatArray(offset, itemsArr, 0, items);
return itemsArr;
}

@Override
float getFloatItemsArrayAt(final int index) {
final int offset =
DATA_START_ADR + getLevelsArray().length * Integer.BYTES + 2 * Float.BYTES + index * Float.BYTES;
DATA_START_ADR + getLevelsArray().length * Integer.BYTES + (index + 2) * Float.BYTES;
return wmem.getFloat(offset);
}

int getItemsArrLengthItems() {
return getLevelsArray()[getNumLevels()];
}
@Override
float getFloatSingleItem() { kllSketchThrow(MUST_NOT_CALL); return Float.NaN; }

@Override
int getM() {
@@ -163,11 +167,6 @@ int getMinK() {
return getMemoryMinK(wmem);
}

@Override
int getNumLevels() {
return getMemoryNumLevels(wmem);
}

@Override
void incN() {
if (readOnly) { kllSketchThrow(TGT_IS_READ_ONLY); }
@@ -198,7 +197,7 @@ void setFloatItemsArray(final float[] floatItems) {
void setFloatItemsArrayAt(final int index, final float value) {
if (readOnly) { kllSketchThrow(TGT_IS_READ_ONLY); }
final int offset =
DATA_START_ADR + getLevelsArray().length * Integer.BYTES + 2 * Float.BYTES + index * Float.BYTES;
DATA_START_ADR + getLevelsArray().length * Integer.BYTES + (index + 2) * Float.BYTES;
wmem.putFloat(offset, value);
}

@@ -28,7 +28,6 @@
import java.util.Random;

import org.apache.datasketches.SketchesArgumentException;
import org.apache.datasketches.memory.WritableMemory;

/**
* Static methods to support KllDoublesSketch
@@ -139,8 +138,12 @@ static void mergeDoubleImpl(final KllSketch mine, final KllSketch other) {
final int myMinK = mine.getMinK();

//update this sketch with level0 items from the other sketch
for (int i = otherLevelsArr[0]; i < otherLevelsArr[1]; i++) {
KllDoublesHelper.updateDouble(mine, otherDoubleItemsArr[i]);
if (other.isCompactSingleItem()) {
updateDouble(mine, other.getFloatSingleItem());
} else {
for (int i = otherLevelsArr[0]; i < otherLevelsArr[1]; i++) {
KllDoublesHelper.updateDouble(mine, otherDoubleItemsArr[i]);
}
}
// after the level 0 update, we capture the state of levels and items arrays
final int myCurNumLevels = mine.getNumLevels();
@@ -212,22 +215,16 @@ static void mergeDoubleImpl(final KllSketch mine, final KllSketch other) {
mine.setMinK(min(myMinK, other.getMinK()));
}

//Update numLevels, levelsArray, items
mine.setNumLevels(myNewNumLevels);
mine.setLevelsArray(myNewLevelsArr);
mine.setDoubleItemsArray(myNewDoubleItemsArr);

//Update min, max values
final double otherMin = other.getMinDoubleValue();
final double otherMax = other.getMaxDoubleValue();
mine.setMinDoubleValue(resolveDoubleMinValue(myMin, otherMin));
mine.setMaxDoubleValue(resolveDoubleMaxValue(myMax, otherMax));
//System.out.println(mine.toString(true, true));

//Update numLevels, levelsArray, items
mine.setNumLevels(myNewNumLevels);
//final WritableMemory mymem = mine.getWritableMemory();
//System.out.println(KllPreambleUtil.memoryToString(mymem, true));
System.out.println(mine.toString(true, true));
mine.setLevelsArray(myNewLevelsArr);
System.out.println(mine.toString(true, true));
mine.setDoubleItemsArray(myNewDoubleItemsArr);
System.out.println(mine.toString(true, true));
assert KllHelper.sumTheSampleWeights(mine.getNumLevels(), mine.getLevelsArray()) == mine.getN();
}

0 comments on commit 8f01cbd

Please sign in to comment.