Skip to content
Permalink
Browse files
HBASE-25465 Use javac --release option for supporting cross version c…
…ompilation (#4164)

Signed-off-by: Andrew Purtell <apurtell@apache.org>
  • Loading branch information
Apache9 committed Mar 12, 2022
1 parent f4866a4 commit c2485215dee14ae9888af235dda9826b83507456
Show file tree
Hide file tree
Showing 14 changed files with 187 additions and 389 deletions.
@@ -23,19 +23,20 @@
import java.util.List;
import java.util.Objects;
import java.util.PriorityQueue;

import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.yetus.audience.InterfaceAudience;

import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;

import org.apache.hadoop.hbase.shaded.protobuf.generated.FilterProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.BytesBytesPair;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.UnsafeAvailChecker;

/**
* This is optimized version of a standard FuzzyRowFilter Filters data based on fuzzy row key.
@@ -57,7 +58,7 @@
*/
@InterfaceAudience.Public
public class FuzzyRowFilter extends FilterBase {
private static final boolean UNSAFE_UNALIGNED = UnsafeAvailChecker.unaligned();
private static final boolean UNSAFE_UNALIGNED = HBasePlatformDependent.unaligned();
private List<Pair<byte[], byte[]>> fuzzyKeysData;
private boolean done = false;

@@ -167,6 +167,10 @@
<groupId>org.apache.hbase.thirdparty</groupId>
<artifactId>hbase-shaded-netty</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hbase.thirdparty</groupId>
<artifactId>hbase-unsafe</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
@@ -30,10 +30,10 @@
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.nio.SingleByteBuff;
import org.apache.hadoop.hbase.util.ReflectionUtils;
import org.apache.hadoop.hbase.util.UnsafeAccess;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sun.nio.ch.DirectBuffer;

import org.apache.hbase.thirdparty.com.google.common.collect.Sets;

@@ -335,11 +335,8 @@ public ByteBuff allocate(int size) {
public void clean() {
while (!buffers.isEmpty()) {
ByteBuffer b = buffers.poll();
if (b instanceof DirectBuffer) {
DirectBuffer db = (DirectBuffer) b;
if (db.cleaner() != null) {
db.cleaner().clean();
}
if (b.isDirect()) {
UnsafeAccess.freeDirectBuffer(b);
}
}
this.usedBufCount.set(0);
@@ -23,25 +23,22 @@
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;

import org.apache.hadoop.hbase.io.ByteBuffAllocator.Recycler;
import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent;
import org.apache.hadoop.hbase.util.ByteBufferUtils;
import org.apache.hadoop.hbase.util.ObjectIntPair;
import org.apache.hadoop.hbase.util.UnsafeAccess;
import org.apache.hadoop.hbase.util.UnsafeAvailChecker;
import org.apache.yetus.audience.InterfaceAudience;

import sun.nio.ch.DirectBuffer;

/**
* An implementation of ByteBuff where a single BB backs the BBI. This just acts as a wrapper over a
* normal BB - offheap or onheap
*/
@InterfaceAudience.Private
public class SingleByteBuff extends ByteBuff {

private static final boolean UNSAFE_AVAIL = UnsafeAvailChecker.isAvailable();
private static final boolean UNSAFE_UNALIGNED = UnsafeAvailChecker.unaligned();
private static final boolean UNSAFE_AVAIL = HBasePlatformDependent.isUnsafeAvailable();
private static final boolean UNSAFE_UNALIGNED = HBasePlatformDependent.unaligned();

// Underlying BB
private final ByteBuffer buf;
@@ -65,7 +62,7 @@ public SingleByteBuff(Recycler recycler, ByteBuffer buf) {
this.unsafeOffset = UnsafeAccess.BYTE_ARRAY_BASE_OFFSET + buf.arrayOffset();
this.unsafeRef = buf.array();
} else {
this.unsafeOffset = ((DirectBuffer) buf).address();
this.unsafeOffset = UnsafeAccess.directBufferAddress(buf);
}
}

@@ -30,24 +30,23 @@
import org.apache.hadoop.hbase.io.ByteBufferWriter;
import org.apache.hadoop.hbase.io.util.StreamUtils;
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.WritableUtils;
import org.apache.yetus.audience.InterfaceAudience;
import sun.nio.ch.DirectBuffer;

/**
* Utility functions for working with byte buffers, such as reading/writing
* variable-length long numbers.
*/
@SuppressWarnings("restriction")
@InterfaceAudience.Private
public final class ByteBufferUtils {
// "Compressed integer" serialization helper constants.
public final static int VALUE_MASK = 0x7f;
public final static int NEXT_BIT_SHIFT = 7;
public final static int NEXT_BIT_MASK = 1 << 7;
final static boolean UNSAFE_AVAIL = UnsafeAvailChecker.isAvailable();
public final static boolean UNSAFE_UNALIGNED = UnsafeAvailChecker.unaligned();
final static boolean UNSAFE_AVAIL = HBasePlatformDependent.isUnsafeAvailable();
public final static boolean UNSAFE_UNALIGNED = HBasePlatformDependent.unaligned();

private ByteBufferUtils() {
}
@@ -78,11 +77,10 @@ static class ComparerHolder {

static Comparer getBestComparer() {
try {
Class<?> theClass = Class.forName(UNSAFE_COMPARER_NAME);
Class<? extends Comparer> theClass =
Class.forName(UNSAFE_COMPARER_NAME).asSubclass(Comparer.class);

@SuppressWarnings("unchecked")
Comparer comparer = (Comparer) theClass.getConstructor().newInstance();
return comparer;
return theClass.getConstructor().newInstance();
} catch (Throwable t) { // ensure we really catch *everything*
return PureJavaComparer.INSTANCE;
}
@@ -137,7 +135,7 @@ public int compareTo(byte[] buf1, int o1, int l1, ByteBuffer buf2, int o2, int l
long offset2Adj;
Object refObj2 = null;
if (buf2.isDirect()) {
offset2Adj = o2 + ((DirectBuffer)buf2).address();
offset2Adj = o2 + UnsafeAccess.directBufferAddress(buf2);
} else {
offset2Adj = o2 + buf2.arrayOffset() + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET;
refObj2 = buf2.array();
@@ -151,13 +149,13 @@ public int compareTo(ByteBuffer buf1, int o1, int l1, ByteBuffer buf2, int o2, i
long offset1Adj, offset2Adj;
Object refObj1 = null, refObj2 = null;
if (buf1.isDirect()) {
offset1Adj = o1 + ((DirectBuffer) buf1).address();
offset1Adj = o1 + UnsafeAccess.directBufferAddress(buf1);
} else {
offset1Adj = o1 + buf1.arrayOffset() + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET;
refObj1 = buf1.array();
}
if (buf2.isDirect()) {
offset2Adj = o2 + ((DirectBuffer) buf2).address();
offset2Adj = o2 + UnsafeAccess.directBufferAddress(buf2);
} else {
offset2Adj = o2 + buf2.arrayOffset() + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET;
refObj2 = buf2.array();
@@ -175,12 +173,11 @@ static class ConverterHolder {

static Converter getBestConverter() {
try {
Class<?> theClass = Class.forName(UNSAFE_CONVERTER_NAME);
Class<? extends Converter> theClass =
Class.forName(UNSAFE_CONVERTER_NAME).asSubclass(Converter.class);

// yes, UnsafeComparer does implement Comparer<byte[]>
@SuppressWarnings("unchecked")
Converter converter = (Converter) theClass.getConstructor().newInstance();
return converter;
return theClass.getConstructor().newInstance();
} catch (Throwable t) { // ensure we really catch *everything*
return PureJavaConverter.INSTANCE;
}
@@ -922,8 +919,8 @@ static int compareToUnsafe(Object obj1, long o1, int l1, Object obj2, long o2, i
* 64-bit.
*/
for (i = 0; i < strideLimit; i += stride) {
long lw = UnsafeAccess.theUnsafe.getLong(obj1, o1 + (long) i);
long rw = UnsafeAccess.theUnsafe.getLong(obj2, o2 + (long) i);
long lw = HBasePlatformDependent.getLong(obj1, o1 + (long) i);
long rw = HBasePlatformDependent.getLong(obj2, o2 + (long) i);
if (lw != rw) {
if (!UnsafeAccess.LITTLE_ENDIAN) {
return ((lw + Long.MIN_VALUE) < (rw + Long.MIN_VALUE)) ? -1 : 1;
@@ -944,8 +941,8 @@ static int compareToUnsafe(Object obj1, long o1, int l1, Object obj2, long o2, i

// The epilogue to cover the last (minLength % stride) elements.
for (; i < minLength; i++) {
int il = (UnsafeAccess.theUnsafe.getByte(obj1, o1 + i) & 0xFF);
int ir = (UnsafeAccess.theUnsafe.getByte(obj2, o2 + i) & 0xFF);
int il = (HBasePlatformDependent.getByte(obj1, o1 + i) & 0xFF);
int ir = (HBasePlatformDependent.getByte(obj2, o2 + i) & 0xFF);
if (il != ir) {
return il - ir;
}
@@ -41,13 +41,13 @@

import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.io.WritableUtils;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sun.misc.Unsafe;

import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils;

@@ -56,7 +56,6 @@
* comparisons, hash code generation, manufacturing keys for HashMaps or
* HashSets, and can be used as key in maps or trees.
*/
@SuppressWarnings("restriction")
@InterfaceAudience.Public
@edu.umd.cs.findbugs.annotations.SuppressWarnings(
value="EQ_CHECK_FOR_OPERAND_NOT_COMPATIBLE_WITH_THIS",
@@ -126,7 +125,7 @@ public class Bytes implements Comparable<Bytes> {
// SizeOf which uses java.lang.instrument says 24 bytes. (3 longs?)
public static final int ESTIMATED_HEAP_TAX = 16;

static final boolean UNSAFE_UNALIGNED = UnsafeAvailChecker.unaligned();
static final boolean UNSAFE_UNALIGNED = HBasePlatformDependent.unaligned();

/**
* Returns length of the byte array, returning 0 if the array is null.
@@ -1359,21 +1358,17 @@ int putShort(byte[] bytes, int offset, short val) {

protected static final class UnsafeConverter extends Converter {

static final Unsafe theUnsafe;

public UnsafeConverter() {}

static {
if (UNSAFE_UNALIGNED) {
theUnsafe = UnsafeAccess.theUnsafe;
} else {
if (!UNSAFE_UNALIGNED) {
// It doesn't matter what we throw;
// it's swallowed in getBestComparer().
throw new Error();
}

// sanity check - this should never fail
if (theUnsafe.arrayIndexScale(byte[].class) != 1) {
if (HBasePlatformDependent.arrayIndexScale(byte[].class) != 1) {
throw new AssertionError();
}
}
@@ -1412,7 +1407,7 @@ int putShort(byte[] bytes, int offset, short val) {

/**
* Provides a lexicographical comparer implementation; either a Java
* implementation or a faster implementation based on {@link Unsafe}.
* implementation or a faster implementation based on {@code Unsafe}.
*
* <p>Uses reflection to gracefully fall back to the Java implementation if
* {@code Unsafe} isn't available.
@@ -1469,18 +1464,15 @@ public int compareTo(byte[] buffer1, int offset1, int length1,
enum UnsafeComparer implements Comparer<byte[]> {
INSTANCE;

static final Unsafe theUnsafe;
static {
if (UNSAFE_UNALIGNED) {
theUnsafe = UnsafeAccess.theUnsafe;
} else {
if (!UNSAFE_UNALIGNED) {
// It doesn't matter what we throw;
// it's swallowed in getBestComparer().
throw new Error();
}

// sanity check - this should never fail
if (theUnsafe.arrayIndexScale(byte[].class) != 1) {
if (HBasePlatformDependent.arrayIndexScale(byte[].class) != 1) {
throw new AssertionError();
}
}
@@ -1518,8 +1510,8 @@ public int compareTo(byte[] buffer1, int offset1, int length1,
* than 4 bytes even on 32-bit. On the other hand, it is substantially faster on 64-bit.
*/
for (i = 0; i < strideLimit; i += stride) {
long lw = theUnsafe.getLong(buffer1, offset1Adj + i);
long rw = theUnsafe.getLong(buffer2, offset2Adj + i);
long lw = HBasePlatformDependent.getLong(buffer1, offset1Adj + i);
long rw = HBasePlatformDependent.getLong(buffer2, offset2Adj + i);
if (lw != rw) {
if(!UnsafeAccess.LITTLE_ENDIAN) {
return ((lw + Long.MIN_VALUE) < (rw + Long.MIN_VALUE)) ? -1 : 1;
@@ -24,7 +24,7 @@
import java.lang.reflect.Modifier;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;

import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -186,13 +186,19 @@ private static final class HeaderSize {
private byte a;
}

private static final int ARRAY_OBJECT_INDEX_SCALE =
HBasePlatformDependent.arrayIndexScale(Object[].class);

private static final int ARRAY_BYTE_INDEX_SCALE =
HBasePlatformDependent.arrayIndexScale(byte[].class);

public UnsafeLayout() {
}

@Override
int headerSize() {
try {
return (int) UnsafeAccess.theUnsafe.objectFieldOffset(
return (int) HBasePlatformDependent.objectFieldOffset(
HeaderSize.class.getDeclaredField("a"));
} catch (NoSuchFieldException | SecurityException e) {
LOG.error(e.toString(), e);
@@ -202,29 +208,28 @@ int headerSize() {

@Override
int arrayHeaderSize() {
return UnsafeAccess.theUnsafe.arrayBaseOffset(byte[].class);
return HBasePlatformDependent.arrayBaseOffset(byte[].class);
}

@Override
@SuppressWarnings("static-access")
int oopSize() {
// Unsafe.addressSize() returns 8, even with CompressedOops. This is how many bytes each
// element is allocated in an Object[].
return UnsafeAccess.theUnsafe.ARRAY_OBJECT_INDEX_SCALE;
return ARRAY_OBJECT_INDEX_SCALE;
}

@Override
@SuppressWarnings("static-access")
long sizeOfByteArray(int len) {
return align(ARRAY + len * UnsafeAccess.theUnsafe.ARRAY_BYTE_INDEX_SCALE);
return align(ARRAY + len * ARRAY_BYTE_INDEX_SCALE);
}
}

private static MemoryLayout getMemoryLayout() {
// Have a safeguard in case Unsafe estimate is wrong. This is static context, there is
// no configuration, so we look at System property.
String enabled = System.getProperty("hbase.memorylayout.use.unsafe");
if (UnsafeAvailChecker.isAvailable() && (enabled == null || Boolean.parseBoolean(enabled))) {
if (HBasePlatformDependent.isUnsafeAvailable() &&
(enabled == null || Boolean.parseBoolean(enabled))) {
LOG.debug("Using Unsafe to estimate memory layout");
return new UnsafeLayout();
}

0 comments on commit c248521

Please sign in to comment.