Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ARROW-9: Replace straggler references to Drill #2

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 18 additions & 18 deletions java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable {
private final boolean isEmpty;
private volatile int length;
private final HistoricalLog historicalLog = BaseAllocator.DEBUG ?
new HistoricalLog(BaseAllocator.DEBUG_LOG_LENGTH, "DrillBuf[%d]", id) : null;
new HistoricalLog(BaseAllocator.DEBUG_LOG_LENGTH, "ArrowBuf[%d]", id) : null;

public ArrowBuf(
final AtomicInteger refCnt,
Expand Down Expand Up @@ -155,18 +155,18 @@ private void ensure(int width) {
}

/**
* Create a new DrillBuf that is associated with an alternative allocator for the purposes of memory ownership and
* accounting. This has no impact on the reference counting for the current DrillBuf except in the situation where the
* Create a new ArrowBuf that is associated with an alternative allocator for the purposes of memory ownership and
* accounting. This has no impact on the reference counting for the current ArrowBuf except in the situation where the
* passed in Allocator is the same as the current buffer.
*
* This operation has no impact on the reference count of this DrillBuf. The newly created DrillBuf with either have a
* This operation has no impact on the reference count of this ArrowBuf. The newly created ArrowBuf with either have a
* reference count of 1 (in the case that this is the first time this memory is being associated with the new
* allocator) or the current value of the reference count + 1 for the other AllocationManager/BufferLedger combination
* in the case that the provided allocator already had an association to this underlying memory.
*
* @param target
* The target allocator to create an association with.
* @return A new DrillBuf which shares the same underlying memory as this DrillBuf.
* @return A new ArrowBuf which shares the same underlying memory as this ArrowBuf.
*/
public ArrowBuf retain(BufferAllocator target) {

Expand All @@ -178,17 +178,17 @@ public ArrowBuf retain(BufferAllocator target) {
historicalLog.recordEvent("retain(%s)", target.getName());
}
final BufferLedger otherLedger = this.ledger.getLedgerForAllocator(target);
return otherLedger.newDrillBuf(offset, length, null);
return otherLedger.newArrowBuf(offset, length, null);
}

/**
* Transfer the memory accounting ownership of this DrillBuf to another allocator. This will generate a new DrillBuf
* that carries an association with the underlying memory of this DrillBuf. If this DrillBuf is connected to the
* Transfer the memory accounting ownership of this ArrowBuf to another allocator. This will generate a new ArrowBuf
* that carries an association with the underlying memory of this ArrowBuf. If this ArrowBuf is connected to the
* owning BufferLedger of this memory, that memory ownership/accounting will be transferred to the taret allocator. If
* this DrillBuf does not currently own the memory underlying it (and is only associated with it), this does not
* transfer any ownership to the newly created DrillBuf.
* this ArrowBuf does not currently own the memory underlying it (and is only associated with it), this does not
* transfer any ownership to the newly created ArrowBuf.
*
* This operation has no impact on the reference count of this DrillBuf. The newly created DrillBuf with either have a
* This operation has no impact on the reference count of this ArrowBuf. The newly created ArrowBuf with either have a
* reference count of 1 (in the case that this is the first time this memory is being associated with the new
* allocator) or the current value of the reference count for the other AllocationManager/BufferLedger combination in
* the case that the provided allocator already had an association to this underlying memory.
Expand All @@ -203,7 +203,7 @@ public ArrowBuf retain(BufferAllocator target) {
* @param target
* The allocator to transfer ownership to.
* @return A new transfer result with the impact of the transfer (whether it was overlimit) as well as the newly
* created DrillBuf.
* created ArrowBuf.
*/
public TransferResult transferOwnership(BufferAllocator target) {

Expand All @@ -212,7 +212,7 @@ public TransferResult transferOwnership(BufferAllocator target) {
}

final BufferLedger otherLedger = this.ledger.getLedgerForAllocator(target);
final ArrowBuf newBuf = otherLedger.newDrillBuf(offset, length, null);
final ArrowBuf newBuf = otherLedger.newArrowBuf(offset, length, null);
final boolean allocationFit = this.ledger.transferBalance(otherLedger);
return new TransferResult(allocationFit, newBuf);
}
Expand Down Expand Up @@ -267,7 +267,7 @@ public boolean release(int decrement) {

if (refCnt < 0) {
throw new IllegalStateException(
String.format("DrillBuf[%d] refCnt has gone negative. Buffer Info: %s", id, toVerboseString()));
String.format("ArrowBuf[%d] refCnt has gone negative. Buffer Info: %s", id, toVerboseString()));
}

return refCnt == 0;
Expand Down Expand Up @@ -370,7 +370,7 @@ public ArrowBuf slice(int index, int length) {
* Re the behavior of reference counting, see http://netty.io/wiki/reference-counted-objects.html#wiki-h3-5, which
* explains that derived buffers share their reference count with their parent
*/
final ArrowBuf newBuf = ledger.newDrillBuf(offset + index, length);
final ArrowBuf newBuf = ledger.newArrowBuf(offset + index, length);
newBuf.writerIndex(length);
return newBuf;
}
Expand Down Expand Up @@ -437,7 +437,7 @@ public long memoryAddress() {

@Override
public String toString() {
return String.format("DrillBuf[%d], udle: [%d %d..%d]", id, udle.id, offset, offset + capacity());
return String.format("ArrowBuf[%d], udle: [%d %d..%d]", id, udle.id, offset, offset + capacity());
}

@Override
Expand Down Expand Up @@ -782,7 +782,7 @@ public void close() {
}

/**
* Returns the possible memory consumed by this DrillBuf in the worse case scenario. (not shared, connected to larger
* Returns the possible memory consumed by this ArrowBuf in the worse case scenario. (not shared, connected to larger
* underlying buffer of allocated memory)
*
* @return Size in bytes.
Expand Down Expand Up @@ -833,7 +833,7 @@ public String toHexString(final int start, final int length) {
}

/**
* Get the integer id assigned to this DrillBuf for debugging purposes.
* Get the integer id assigned to this ArrowBuf for debugging purposes.
*
* @return integer id
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import org.apache.arrow.memory.BufferAllocator;

/**
* Allows us to decorate DrillBuf to make it expandable so that we can use them in the context of the Netty framework
* Allows us to decorate ArrowBuf to make it expandable so that we can use them in the context of the Netty framework
* (thus supporting RPC level memory accounting).
*/
public class ExpandableByteBuf extends MutableWrappedByteBuf {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
import com.codahale.metrics.MetricRegistry;

/**
* The base allocator that we use for all of Drill's memory management. Returns UnsafeDirectLittleEndian buffers.
* The base allocator that we use for all of Arrow's memory management. Returns UnsafeDirectLittleEndian buffers.
*/
public class PooledByteBufAllocatorL {
private static final org.slf4j.Logger memoryLogger = org.slf4j.LoggerFactory.getLogger("drill.allocator");
Expand Down Expand Up @@ -184,7 +184,7 @@ private UnsafeDirectLittleEndian newDirectBufferL(int initialCapacity, int maxCa

private UnsupportedOperationException fail() {
return new UnsupportedOperationException(
"Drill requries that the JVM used supports access sun.misc.Unsafe. This platform didn't provide that functionality.");
"Arrow requries that the JVM used supports access sun.misc.Unsafe. This platform didn't provide that functionality.");
}

public UnsafeDirectLittleEndian directBuffer(int initialCapacity, int maxCapacity) {
Expand All @@ -197,7 +197,7 @@ public UnsafeDirectLittleEndian directBuffer(int initialCapacity, int maxCapacit

@Override
public ByteBuf heapBuffer(int initialCapacity, int maxCapacity) {
throw new UnsupportedOperationException("Drill doesn't support using heap buffers.");
throw new UnsupportedOperationException("Arrow doesn't support using heap buffers.");
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import java.util.concurrent.atomic.AtomicLong;

/**
* The underlying class we use for little-endian access to memory. Is used underneath DrillBufs to abstract away the
* The underlying class we use for little-endian access to memory. Is used underneath ArrowBufs to abstract away the
* Netty classes and underlying Netty memory management.
*/
public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
Expand Down Expand Up @@ -55,7 +55,7 @@ public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
private UnsafeDirectLittleEndian(AbstractByteBuf buf, boolean fake, AtomicLong bufferCount, AtomicLong bufferSize) {
super(buf);
if (!NATIVE_ORDER || buf.order() != ByteOrder.BIG_ENDIAN) {
throw new IllegalStateException("Drill only runs on LittleEndian systems.");
throw new IllegalStateException("Arrow only runs on LittleEndian systems.");
}

this.bufferCount = bufferCount;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
* This class is also responsible for managing when memory is allocated and returned to the Netty-based
* PooledByteBufAllocatorL.
*
* The only reason that this isn't package private is we're forced to put DrillBuf in Netty's package which need access
* The only reason that this isn't package private is we're forced to put ArrowBuf in Netty's package which need access
* to these objects or methods.
*
* Threading: AllocationManager manages thread-safety internally. Operations within the context of a single BufferLedger
Expand Down Expand Up @@ -185,8 +185,8 @@ public void release() {

/**
* The reference manager that binds an allocator manager to a particular BaseAllocator. Also responsible for creating
* a set of DrillBufs that share a common fate and set of reference counts.
* As with AllocationManager, the only reason this is public is due to DrillBuf being in io.netty.buffer package.
* a set of ArrowBufs that share a common fate and set of reference counts.
* As with AllocationManager, the only reason this is public is due to ArrowBuf being in io.netty.buffer package.
*/
public class BufferLedger {

Expand Down Expand Up @@ -322,7 +322,7 @@ public int decrement(int decrement) {
/**
* Returns the ledger associated with a particular BufferAllocator. If the BufferAllocator doesn't currently have a
* ledger associated with this AllocationManager, a new one is created. This is placed on BufferLedger rather than
* AllocationManager directly because DrillBufs don't have access to AllocationManager and they are the ones
* AllocationManager directly because ArrowBufs don't have access to AllocationManager and they are the ones
* responsible for exposing the ability to associate multiple allocators with a particular piece of underlying
* memory. Note that this will increment the reference count of this ledger by one to ensure the ledger isn't
* destroyed before use.
Expand All @@ -335,32 +335,32 @@ public BufferLedger getLedgerForAllocator(BufferAllocator allocator) {
}

/**
* Create a new DrillBuf associated with this AllocationManager and memory. Does not impact reference count.
* Create a new ArrowBuf associated with this AllocationManager and memory. Does not impact reference count.
* Typically used for slicing.
* @param offset
* The offset in bytes to start this new DrillBuf.
* The offset in bytes to start this new ArrowBuf.
* @param length
* The length in bytes that this DrillBuf will provide access to.
* @return A new DrillBuf that shares references with all DrillBufs associated with this BufferLedger
* The length in bytes that this ArrowBuf will provide access to.
* @return A new ArrowBuf that shares references with all ArrowBufs associated with this BufferLedger
*/
public ArrowBuf newDrillBuf(int offset, int length) {
public ArrowBuf newArrowBuf(int offset, int length) {
allocator.assertOpen();
return newDrillBuf(offset, length, null);
return newArrowBuf(offset, length, null);
}

/**
* Create a new DrillBuf associated with this AllocationManager and memory.
* Create a new ArrowBuf associated with this AllocationManager and memory.
* @param offset
* The offset in bytes to start this new DrillBuf.
* The offset in bytes to start this new ArrowBuf.
* @param length
* The length in bytes that this DrillBuf will provide access to.
* The length in bytes that this ArrowBuf will provide access to.
* @param manager
* An optional BufferManager argument that can be used to manage expansion of this DrillBuf
* An optional BufferManager argument that can be used to manage expansion of this ArrowBuf
* @param retain
* Whether or not the newly created buffer should get an additional reference count added to it.
* @return A new DrillBuf that shares references with all DrillBufs associated with this BufferLedger
* @return A new ArrowBuf that shares references with all ArrowBufs associated with this BufferLedger
*/
public ArrowBuf newDrillBuf(int offset, int length, BufferManager manager) {
public ArrowBuf newArrowBuf(int offset, int length, BufferManager manager) {
allocator.assertOpen();

final ArrowBuf buf = new ArrowBuf(
Expand All @@ -375,7 +375,7 @@ public ArrowBuf newDrillBuf(int offset, int length, BufferManager manager) {

if (BaseAllocator.DEBUG) {
historicalLog.recordEvent(
"DrillBuf(BufferLedger, BufferAllocator[%s], UnsafeDirectLittleEndian[identityHashCode == "
"ArrowBuf(BufferLedger, BufferAllocator[%s], UnsafeDirectLittleEndian[identityHashCode == "
+ "%d](%s)) => ledger hc == %d",
allocator.name, System.identityHashCode(buf), buf.toString(),
System.identityHashCode(this));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,19 +23,19 @@
import io.netty.buffer.ExpandableByteBuf;

/**
* An implementation of ByteBufAllocator that wraps a Drill BufferAllocator. This allows the RPC layer to be accounted
* and managed using Drill's BufferAllocator infrastructure. The only thin different from a typical BufferAllocator is
* An implementation of ByteBufAllocator that wraps a Arrow BufferAllocator. This allows the RPC layer to be accounted
* and managed using Arrow's BufferAllocator infrastructure. The only thin different from a typical BufferAllocator is
* the signature and the fact that this Allocator returns ExpandableByteBufs which enable otherwise non-expandable
* DrillBufs to be expandable.
* ArrowBufs to be expandable.
*/
public class DrillByteBufAllocator implements ByteBufAllocator {
public class ArrowByteBufAllocator implements ByteBufAllocator {

private static final int DEFAULT_BUFFER_SIZE = 4096;
private static final int DEFAULT_MAX_COMPOSITE_COMPONENTS = 16;

private final BufferAllocator allocator;

public DrillByteBufAllocator(BufferAllocator allocator) {
public ArrowByteBufAllocator(BufferAllocator allocator) {
this.allocator = allocator;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ protected BaseAllocator(
this.parentAllocator = parentAllocator;
this.name = name;

this.thisAsByteBufAllocator = new DrillByteBufAllocator(this);
this.thisAsByteBufAllocator = new ArrowByteBufAllocator(this);

if (DEBUG) {
childAllocators = new IdentityHashMap<>();
Expand Down Expand Up @@ -236,7 +236,7 @@ private ArrowBuf bufferWithoutReservation(final int size, BufferManager bufferMa

final AllocationManager manager = new AllocationManager(this, size);
final BufferLedger ledger = manager.associate(this); // +1 ref cnt (required)
final ArrowBuf buffer = ledger.newDrillBuf(0, size, bufferManager);
final ArrowBuf buffer = ledger.newArrowBuf(0, size, bufferManager);

// make sure that our allocation is equal to what we expected.
Preconditions.checkArgument(buffer.capacity() == size,
Expand Down Expand Up @@ -314,9 +314,9 @@ public ArrowBuf allocateBuffer() {
Preconditions.checkState(!closed, "Attempt to allocate after closed");
Preconditions.checkState(!used, "Attempt to allocate more than once");

final ArrowBuf drillBuf = allocate(nBytes);
final ArrowBuf arrowBuf = allocate(nBytes);
used = true;
return drillBuf;
return arrowBuf;
}

public int getSize() {
Expand Down Expand Up @@ -397,13 +397,13 @@ private ArrowBuf allocate(int nBytes) {
* as well, so we need to return the same number back to avoid double-counting them.
*/
try {
final ArrowBuf drillBuf = BaseAllocator.this.bufferWithoutReservation(nBytes, null);
final ArrowBuf arrowBuf = BaseAllocator.this.bufferWithoutReservation(nBytes, null);

if (DEBUG) {
historicalLog.recordEvent("allocate() => %s", String.format("DrillBuf[%d]", drillBuf.getId()));
historicalLog.recordEvent("allocate() => %s", String.format("ArrowBuf[%d]", arrowBuf.getId()));
}
success = true;
return drillBuf;
return arrowBuf;
} finally {
if (!success) {
releaseBytes(nBytes);
Expand Down Expand Up @@ -565,7 +565,7 @@ void verifyAllocator() {
* Verifies the accounting state of the allocator. Only works for DEBUG.
*
* <p>
* This overload is used for recursive calls, allowing for checking that DrillBufs are unique across all allocators
* This overload is used for recursive calls, allowing for checking that ArrowBufs are unique across all allocators
* that are checked.
* </p>
*
Expand Down Expand Up @@ -594,7 +594,7 @@ private void verifyAllocator(final IdentityHashMap<UnsafeDirectLittleEndian, Bas
* Verify my relationships with my descendants.
*
* The sum of direct child allocators' owned memory must be <= my allocated memory; my allocated memory also
* includes DrillBuf's directly allocated by me.
* includes ArrowBuf's directly allocated by me.
*/
long childTotal = 0;
for (final BaseAllocator childAllocator : childSet) {
Expand Down Expand Up @@ -623,11 +623,11 @@ private void verifyAllocator(final IdentityHashMap<UnsafeDirectLittleEndian, Bas

final UnsafeDirectLittleEndian udle = ledger.getUnderlying();
/*
* Even when shared, DrillBufs are rewrapped, so we should never see the same instance twice.
* Even when shared, ArrowBufs are rewrapped, so we should never see the same instance twice.
*/
final BaseAllocator otherOwner = buffersSeen.get(udle);
if (otherOwner != null) {
throw new IllegalStateException("This allocator's drillBuf already owned by another allocator");
throw new IllegalStateException("This allocator's ArrowBuf already owned by another allocator");
}
buffersSeen.put(udle, this);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public interface BufferAllocator extends AutoCloseable {
*
* @param size
* The size in bytes.
* @return a new DrillBuf, or null if the request can't be satisfied
* @return a new ArrowBuf, or null if the request can't be satisfied
* @throws OutOfMemoryException
* if buffer cannot be allocated
*/
Expand All @@ -44,7 +44,7 @@ public interface BufferAllocator extends AutoCloseable {
* The size in bytes.
* @param manager
* A buffer manager to manage reallocation.
* @return a new DrillBuf, or null if the request can't be satisfied
* @return a new ArrowBuf, or null if the request can't be satisfied
* @throws OutOfMemoryException
* if buffer cannot be allocated
*/
Expand Down
Loading