Skip to content

Commit

Permalink
Merge a0888e9 into 0ba8f4c
Browse files Browse the repository at this point in the history
  • Loading branch information
leerho committed Aug 26, 2019
2 parents 0ba8f4c + a0888e9 commit f006324
Show file tree
Hide file tree
Showing 38 changed files with 1,714 additions and 1,116 deletions.
3 changes: 2 additions & 1 deletion src/main/java/com/yahoo/sketches/Util.java
Expand Up @@ -311,7 +311,8 @@ public static final String characterPad(final String s, final int fieldLength, f
public static final short checkSeedHashes(final short seedHashA, final short seedHashB) {
if (seedHashA != seedHashB) {
throw new SketchesArgumentException(
"Incompatible Seed Hashes. " + seedHashA + ", " + seedHashB);
"Incompatible Seed Hashes. " + Integer.toHexString(seedHashA & 0XFFFF)
+ ", " + Integer.toHexString(seedHashB & 0XFFFF));
}
return seedHashA;
}
Expand Down
Expand Up @@ -42,7 +42,7 @@
*/
final class DirectCompactOrderedSketch extends DirectCompactSketch {

private DirectCompactOrderedSketch(final Memory mem) {
DirectCompactOrderedSketch(final Memory mem) {
super(mem);
}

Expand All @@ -60,29 +60,6 @@ static DirectCompactOrderedSketch wrapInstance(final Memory srcMem, final long s
return new DirectCompactOrderedSketch(srcMem);
}

/**
* Converts the given UpdateSketch to this compact form.
* @param sketch the given UpdateSketch
* @param dstMem the given destination Memory. This clears it before use.
* @return a DirectCompactOrderedSketch.
*/
static DirectCompactOrderedSketch compact(final UpdateSketch sketch, final WritableMemory dstMem) {
final int curCount = sketch.getRetainedEntries(true);
long thetaLong = sketch.getThetaLong();
boolean empty = sketch.isEmpty();
thetaLong = thetaOnCompact(empty, curCount, thetaLong);
empty = emptyOnCompact(curCount, thetaLong);
final int preLongs = computeCompactPreLongs(thetaLong, empty, curCount);
final short seedHash = sketch.getSeedHash();
final long[] cache = sketch.getCache();
final int requiredFlags = READ_ONLY_FLAG_MASK | COMPACT_FLAG_MASK | ORDERED_FLAG_MASK;
final byte flags = (byte) (requiredFlags | (empty ? EMPTY_FLAG_MASK : 0));
final boolean ordered = true;
final long[] compactCache = CompactSketch.compactCache(cache, curCount, thetaLong, ordered);
loadCompactMemory(compactCache, seedHash, curCount, thetaLong, dstMem, flags, preLongs);
return new DirectCompactOrderedSketch(dstMem);
}

/**
* Constructs this sketch from correct, valid components.
* @param cache in compact, ordered form
Expand Down
15 changes: 3 additions & 12 deletions src/main/java/com/yahoo/sketches/theta/DirectCompactSketch.java
Expand Up @@ -39,14 +39,10 @@ abstract class DirectCompactSketch extends CompactSketch {
}

//Sketch

//overidden by EmptyCompactSketch and SingleItemSketch
@Override
public int getCurrentBytes(final boolean compact) { //compact is ignored here
final int preLongs = getCurrentPreambleLongs(true);
final boolean empty = PreambleUtil.isEmpty(mem_);
if (preLongs == 1) {
return (empty) ? 8 : 16; //empty or singleItem
}
//preLongs > 1
final int curCount = extractCurCount(mem_);
return (preLongs + curCount) << 3;
Expand All @@ -57,16 +53,11 @@ public HashIterator iterator() {
return new MemoryHashIterator(mem_, getRetainedEntries(), getThetaLong());
}

//overidden by EmptyCompactSketch and SingleItemSketch
@Override
public int getRetainedEntries(final boolean valid) { //compact is always valid
final int preLongs = getCurrentPreambleLongs(true);
final boolean empty = PreambleUtil.isEmpty(mem_);
if (preLongs == 1) {
return (empty) ? 0 : 1;
}
//preLongs > 1
final int curCount = extractCurCount(mem_);
return curCount;
return extractCurCount(mem_);
}

@Override
Expand Down
Expand Up @@ -41,7 +41,7 @@
*/
final class DirectCompactUnorderedSketch extends DirectCompactSketch {

private DirectCompactUnorderedSketch(final Memory mem) {
DirectCompactUnorderedSketch(final Memory mem) {
super(mem);
}

Expand All @@ -59,30 +59,6 @@ static DirectCompactUnorderedSketch wrapInstance(final Memory srcMem, final long
return new DirectCompactUnorderedSketch(srcMem);
}

/**
* Constructs given an UpdateSketch.
* @param sketch the given UpdateSketch
* @param dstMem the given destination Memory. This clears it before use.
* @return a DirectCompactUnorderedSketch
*/
static DirectCompactUnorderedSketch compact(final UpdateSketch sketch,
final WritableMemory dstMem) {
final int curCount = sketch.getRetainedEntries(true);
long thetaLong = sketch.getThetaLong();
boolean empty = sketch.isEmpty();
thetaLong = thetaOnCompact(empty, curCount, thetaLong);
empty = emptyOnCompact(curCount, thetaLong);
final int preLongs = computeCompactPreLongs(thetaLong, empty, curCount);
final short seedHash = sketch.getSeedHash();
final long[] cache = sketch.getCache();
final int requiredFlags = READ_ONLY_FLAG_MASK | COMPACT_FLAG_MASK;
final byte flags = (byte) (requiredFlags | (empty ? EMPTY_FLAG_MASK : 0));
final boolean ordered = false;
final long[] compactCache = CompactSketch.compactCache(cache, curCount, thetaLong, ordered);
loadCompactMemory(compactCache, seedHash, curCount, thetaLong, dstMem, flags, preLongs);
return new DirectCompactUnorderedSketch(dstMem);
}

/**
* Constructs this sketch from correct, valid components.
* @param cache in compact, ordered form
Expand All @@ -104,8 +80,6 @@ static DirectCompactUnorderedSketch compact(final long[] cache, final boolean em
return new DirectCompactUnorderedSketch(dstMem);
}

//restricted methods

@Override
public boolean isOrdered() {
return false;
Expand Down
Expand Up @@ -20,9 +20,7 @@
package com.yahoo.sketches.theta;

import static com.yahoo.sketches.Util.REBUILD_THRESHOLD;
import static com.yahoo.sketches.theta.PreambleUtil.EMPTY_FLAG_MASK;
import static com.yahoo.sketches.theta.PreambleUtil.FAMILY_BYTE;
import static com.yahoo.sketches.theta.PreambleUtil.FLAGS_BYTE;
import static com.yahoo.sketches.theta.PreambleUtil.LG_ARR_LONGS_BYTE;
import static com.yahoo.sketches.theta.PreambleUtil.LG_NOM_LONGS_BYTE;
import static com.yahoo.sketches.theta.PreambleUtil.LG_RESIZE_FACTOR_BIT;
Expand Down Expand Up @@ -148,7 +146,7 @@ public boolean isDirect() {

@Override
public boolean isEmpty() {
return (mem_.getByte(FLAGS_BYTE) & EMPTY_FLAG_MASK) > 0;
return PreambleUtil.isEmpty(mem_);
}

@Override
Expand Down
133 changes: 133 additions & 0 deletions src/main/java/com/yahoo/sketches/theta/EmptyCompactSketch.java
@@ -0,0 +1,133 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package com.yahoo.sketches.theta;

import com.yahoo.memory.Memory;
import com.yahoo.sketches.SketchesArgumentException;

/**
* Singleton empty CompactSketch.
*
* @author Lee Rhodes
*/
final class EmptyCompactSketch extends CompactSketch {

//For backward compatibility, a candidate long must have Flags= compact, read-only,
// COMPACT-Family=3, SerVer=3, PreLongs=1, and be exactly 8 bytes long. The seedHash is ignored.
// NOTE: The empty and ordered flags may or may not be set
private static final long EMPTY_SKETCH_MASK = 0X00_00_EB_FF_FF_FF_FF_FFL;
private static final long EMPTY_SKETCH_TEST = 0X00_00_0A_00_00_03_03_01L;
//When returning a byte array the empty and ordered bits are also set
static final byte[] EMPTY_COMPACT_SKETCH_ARR = { 1, 3, 3, 0, 0, 0x1E, 0, 0 };
private static final EmptyCompactSketch EMPTY_COMPACT_SKETCH = new EmptyCompactSketch();

private EmptyCompactSketch() {}

static EmptyCompactSketch getInstance() {
return EMPTY_COMPACT_SKETCH;
}

static EmptyCompactSketch getInstance(final Memory srcMem) {
final long pre0 = srcMem.getLong(0);
if (testCandidatePre0(pre0)) {
return EMPTY_COMPACT_SKETCH;
}
final long maskedPre0 = pre0 & EMPTY_SKETCH_MASK;
throw new SketchesArgumentException("Input Memory does not match required Preamble. "
+ "Memory Pre0: " + maskedPre0 + ", required Pre0: " + EMPTY_SKETCH_MASK);
}

//static

static boolean testCandidatePre0(final long candidate) {
return (candidate & EMPTY_SKETCH_MASK) == EMPTY_SKETCH_TEST;
}

@Override
public int getCurrentBytes(final boolean compact) {
return 8;
}

@Override
public HashIterator iterator() {
return new HeapHashIterator(new long[0], 0, Long.MAX_VALUE);
}

@Override
public int getRetainedEntries(final boolean valid) {
return 0;
}

@Override
public long getThetaLong() {
return Long.MAX_VALUE;
}

@Override
public boolean hasMemory() {
return false;
}

@Override
public boolean isDirect() {
return false;
}

@Override
public boolean isEmpty() {
return true;
}

@Override
public boolean isOrdered() {
return true;
}

/**
* Returns 8 bytes representing a CompactSketch that the following flags set:
* ordered, compact, empty, readOnly. The SerVer is 3, the Family is COMPACT(3),
* and the PreLongs = 1. The seedHash is zero.
*/
@Override
public byte[] toByteArray() {
return EMPTY_COMPACT_SKETCH_ARR;
}

@Override
long[] getCache() {
return new long[0];
}

@Override
int getCurrentPreambleLongs(final boolean compact) {
return 1;
}

@Override
Memory getMemory() {
return null;
}

@Override
short getSeedHash() {
return 0;
}

}

0 comments on commit f006324

Please sign in to comment.