Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions src/main/java/com/yahoo/sketches/cpc/CompressedState.java
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,12 @@
*/
final class CompressedState {
private static final String LS = System.getProperty("line.separator");
private boolean csvIsValid = false;
private boolean windowIsValid = false;
final int lgK;
final short seedHash;
int fiCol = 0;
boolean mergeFlag = false; //compliment of HIP Flag
boolean csvIsValid = false;
boolean windowIsValid = false;

long numCoupons = 0;

double kxp;
Expand All @@ -60,7 +59,7 @@ final class CompressedState {
int[] cwStream = null; //may be longer than required
int cwLengthInts = 0;

int cpcRequiredBytes = 0;
//int cpcRequiredBytes = 0;

private CompressedState(final int lgK, final short seedHash) {
this.lgK = lgK;
Expand Down
11 changes: 10 additions & 1 deletion src/main/java/com/yahoo/sketches/theta/CompactSketch.java
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,14 @@ public abstract class CompactSketch extends Sketch {

//Sketch

@Override
public CompactSketch compact() { return this; }

@Override
public CompactSketch compact(final boolean dstOrdered, final WritableMemory dstMem) {
return this;
}

@Override
public Family getFamily() {
return Family.COMPACT;
Expand Down Expand Up @@ -112,6 +120,7 @@ static final long[] compactCachePart(final long[] srcCache, final int lgArrLongs
return cacheOut;
}

//compactCache and dstMem must be valid
static final Memory loadCompactMemory(final long[] compactCache, final short seedHash,
final int curCount, final long thetaLong, final WritableMemory dstMem,
final byte flags, final int preLongs) {
Expand Down Expand Up @@ -144,7 +153,7 @@ static final Memory loadCompactMemory(final long[] compactCache, final short see
if (preLongs > 2) {
insertThetaLong(dstMem, thetaLong);
}
if ((compactCache != null) && (curCount > 0)) {
if (curCount > 0) {
dstMem.putLongArray(preLongs << 3, compactCache, 0, curCount);
}
return dstMem;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,8 @@ private void advanceEpoch() {
ConcurrentPropagationService.resetExecutorService(Thread.currentThread().getId());
//noinspection NonAtomicOperationOnVolatileField
// this increment of a volatile field is done within the scope of the propagation
// synchronization and hence is done by a single thread
// synchronization and hence is done by a single thread.
// Ignore a FindBugs warning
epoch_++;
endPropagation(null, true);
initBgPropagationService();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,7 @@ private void advanceEpoch() {
//noinspection NonAtomicOperationOnVolatileField
// this increment of a volatile field is done within the scope of the propagation
// synchronization and hence is done by a single thread
// Ignore a FindBugs warning
epoch_++;
endPropagation(null, true);
initBgPropagationService();
Expand Down
33 changes: 33 additions & 0 deletions src/main/java/com/yahoo/sketches/theta/Sketch.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import static com.yahoo.sketches.theta.PreambleUtil.SER_VER_BYTE;

import com.yahoo.memory.Memory;
import com.yahoo.memory.WritableMemory;
import com.yahoo.sketches.BinomialBoundsN;
import com.yahoo.sketches.Family;
import com.yahoo.sketches.SketchesArgumentException;
Expand Down Expand Up @@ -142,6 +143,38 @@ else if (serVer == 2) {

//Sketch interface, defined here with Javadocs

/**
* Converts this sketch to an ordered CompactSketch on the Java heap.
*
* <p>If this sketch is already in compact form this operation returns <i>this</i>.
*
* @return this sketch as an ordered CompactSketch on the Java heap.
*/
public abstract CompactSketch compact();

/**
* Convert this sketch to a CompactSketch in the chosen form.
*
* <p>If this sketch is already in compact form this operation returns <i>this</i>.
*
* <p>Otherwise, this compacting process converts the hash table form of an UpdateSketch to
* a simple list of the valid hash values from the hash table. Any hash values equal to or
* greater than theta will be discarded. The number of valid values remaining in the
* Compact Sketch depends on a number of factors, but may be larger or smaller than
* <i>Nominal Entries</i> (or <i>k</i>). It will never exceed 2<i>k</i>. If it is critical
* to always limit the size to no more than <i>k</i>, then <i>rebuild()</i> should be called
* on the UpdateSketch prior to this.
*
* @param dstOrdered
* <a href="{@docRoot}/resources/dictionary.html#dstOrdered">See Destination Ordered</a>
*
* @param dstMem
* <a href="{@docRoot}/resources/dictionary.html#dstMem">See Destination Memory</a>.
*
* @return this sketch as a CompactSketch in the chosen form
*/
public abstract CompactSketch compact(final boolean dstOrdered, final WritableMemory dstMem);

/**
* Gets the number of hash values less than the given theta.
* @param theta the given theta as a double between zero and one.
Expand Down
9 changes: 9 additions & 0 deletions src/main/java/com/yahoo/sketches/theta/UnionImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,15 @@
* @author Kevin Lang
*/
final class UnionImpl extends Union {
/**
* Although the gadget object is initially an UpdateSketch, in the context of a Union it is used
* as a specialized buffer that happens to leverage much of the machinery of an UpdateSketch.
* However, in this context some of the key invariants of the sketch algorithm are intentionally
* violated as an optimization. As a result this object can not be considered as an UpdateSketch
* and should never be exported as an UpdateSketch. It's internal state is not necessarily
* finalized and may contain garbage. Also its internal concept of "nominal entries" or "k" can
* be meaningless. It is private for very good reasons.
*/
private final UpdateSketch gadget_;
private final short seedHash_; //eliminates having to compute the seedHash on every update.
private long unionThetaLong_; //when on-heap, this is the only copy
Expand Down
69 changes: 24 additions & 45 deletions src/main/java/com/yahoo/sketches/theta/UpdateSketch.java
Original file line number Diff line number Diff line change
Expand Up @@ -115,50 +115,11 @@ public static UpdateSketch heapify(final Memory srcMem, final long seed) {
//Sketch interface

@Override
public boolean isCompact() {
return false;
public CompactSketch compact() {
return compact(true, null);
}

@Override
public boolean isOrdered() {
return false;
}

//UpdateSketch interface

/**
* Returns a new builder
*
* @return a new builder
*/
public static final UpdateSketchBuilder builder() {
return new UpdateSketchBuilder();
}

/**
* Resets this sketch back to a virgin empty state.
*/
public abstract void reset();

/**
* Convert this UpdateSketch to a CompactSketch in the chosen form.
*
* <p>This compacting process converts the hash table form of an UpdateSketch to
* a simple list of the valid hash values from the hash table. Any hash values equal to or
* greater than theta will be discarded. The number of valid values remaining in the
* Compact Sketch depends on a number of factors, but may be larger or smaller than
* <i>Nominal Entries</i> (or <i>k</i>). It will never exceed 2<i>k</i>. If it is critical
* to always limit the size to no more than <i>k</i>, then <i>rebuild()</i> should be called
* on the UpdateSketch prior to this.
*
* @param dstOrdered
* <a href="{@docRoot}/resources/dictionary.html#dstOrdered">See Destination Ordered</a>
*
* @param dstMem
* <a href="{@docRoot}/resources/dictionary.html#dstMem">See Destination Memory</a>.
*
* @return this sketch as a CompactSketch in the chosen form
*/
public CompactSketch compact(final boolean dstOrdered, final WritableMemory dstMem) {
CompactSketch sketchOut = null;
final int sw = (dstOrdered ? 2 : 0) | ((dstMem != null) ? 1 : 0);
Expand All @@ -184,14 +145,32 @@ public CompactSketch compact(final boolean dstOrdered, final WritableMemory dstM
return sketchOut;
}

@Override
public boolean isCompact() {
return false;
}

@Override
public boolean isOrdered() {
return false;
}

//UpdateSketch interface

/**
* Converts this UpdateSketch to an ordered CompactSketch on the Java heap.
* @return this sketch as an ordered CompactSketch on the Java heap.
* Returns a new builder
*
* @return a new builder
*/
public CompactSketch compact() {
return compact(true, null);
public static final UpdateSketchBuilder builder() {
return new UpdateSketchBuilder();
}

/**
* Resets this sketch back to a virgin empty state.
*/
public abstract void reset();

/**
* Rebuilds the hash table to remove dirty values or to reduce the size
* to nominal entries.
Expand Down
2 changes: 2 additions & 0 deletions src/test/java/com/yahoo/sketches/theta/CompactSketchTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,8 @@ public void checkDirectCompactSingleItemSketch() {
sk.update(1);
csk = sk.compact(true, WritableMemory.allocate(16));
assertEquals(csk.getCurrentBytes(true), 16);
assertTrue(csk == csk.compact());
assertTrue(csk == csk.compact(true, null));
}

@Test
Expand Down