Skip to content

Commit

Permalink
Minor follow-up optimizations (#148)
Browse files Browse the repository at this point in the history 
* Make JIT to fold more constants in Tsavorite

* Use BitOperations.Log2 in SectorAlignedBufferPool.Position

* Use BitOperations.Log2 in LimitedFixedBufferPool.Position

* Simplify MallocFixedPageSize further by removing ReturnPhysicalAddress

---------
  • Loading branch information
@PaulusParssinen
PaulusParssinen committed Mar 27, 2024
1 parent d45629a commit 922c975
Show file tree
Hide file tree
Showing 5 changed files with 52 additions and 120 deletions.
21 changes: 5 additions & 16 deletions libs/common/Memory/LimitedFixedBufferPool.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System;
using System.Diagnostics;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Threading;
using Microsoft.Extensions.Logging;
Expand Down Expand Up @@ -150,29 +151,17 @@ public void Print()
[MethodImpl(MethodImplOptions.AggressiveInlining)]
int Position(int v)
{
if (v < minAllocationSize || !IsPowerOfTwo(v))
if (v < minAllocationSize || !BitOperations.IsPow2(v))
return -1;

v /= minAllocationSize;

if (v == 1) return 0;
v--;

int r = 0; // r will be lg(v)
while (true) // unroll for more speed...
{
v >>= 1;
if (v == 0) break;
r++;
}
if (r + 1 >= numLevels)
int level = BitOperations.Log2((uint)v - 1) + 1;
if (level >= numLevels)
return -1;
return r + 1;
}

static bool IsPowerOfTwo(long x)
{
return (x > 0) && ((x & (x - 1)) == 0);
return level;
}
}
}
10 changes: 3 additions & 7 deletions libs/storage/Tsavorite/cs/src/core/Allocator/GenericAllocator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,9 @@ internal sealed unsafe class GenericAllocator<Key, Value> : AllocatorBase<Key, V
private readonly int ObjectBlockSize = 100 * (1 << 20);
// Tail offsets per segment, in object log
public readonly long[] segmentOffsets;
// Record sizes
private readonly SerializerSettings<Key, Value> SerializerSettings;
private readonly bool keyBlittable = Utility.IsBlittable<Key>();
private readonly bool valueBlittable = Utility.IsBlittable<Value>();


// We do not support variable-length keys in GenericAllocator
// Record sizes. We do not support variable-length keys in GenericAllocator
internal static int KeySize => Unsafe.SizeOf<Key>();
internal static int ValueSize => Unsafe.SizeOf<Value>();
internal static int RecordSize => Unsafe.SizeOf<Record<Key, Value>>();
Expand All @@ -75,7 +71,7 @@ internal sealed unsafe class GenericAllocator<Key, Value> : AllocatorBase<Key, V

SerializerSettings = serializerSettings ?? new SerializerSettings<Key, Value>();

if ((!keyBlittable) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.keySerializer == null)))
if ((!Utility.IsBlittable<Key>()) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.keySerializer == null)))
{
#if DEBUG
if (typeof(Key) != typeof(byte[]) && typeof(Key) != typeof(string))
Expand All @@ -84,7 +80,7 @@ internal sealed unsafe class GenericAllocator<Key, Value> : AllocatorBase<Key, V
SerializerSettings.keySerializer = ObjectSerializer.Get<Key>();
}

if ((!valueBlittable) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.valueSerializer == null)))
if ((!Utility.IsBlittable<Value>()) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.valueSerializer == null)))
{
#if DEBUG
if (typeof(Value) != typeof(byte[]) && typeof(Value) != typeof(string))
Expand Down
121 changes: 38 additions & 83 deletions libs/storage/Tsavorite/cs/src/core/Allocator/MallocFixedPageSize.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,13 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;

namespace Tsavorite.core
{
internal class CountdownWrapper
internal sealed class CountdownWrapper
{
// Separate event for sync code and tcs for async code: Do not block on async code.
private readonly CountdownEvent syncEvent;
Expand Down Expand Up @@ -60,12 +59,10 @@ internal void Decrement()
/// Memory allocator for objects
/// </summary>
/// <typeparam name="T"></typeparam>
public class MallocFixedPageSize<T> : IDisposable
public sealed class MallocFixedPageSize<T> : IDisposable
{
private const bool ForceUnpinnedAllocation = false;

// We will never get an index of 0 from (Bulk)Allocate
internal const long kInvalidAllocationIndex = 0;
private const long InvalidAllocationIndex = 0;

private const int PageSizeBits = 16;
private const int PageSize = 1 << PageSizeBits;
Expand All @@ -76,21 +73,17 @@ public class MallocFixedPageSize<T> : IDisposable
private readonly T[][] values = new T[LevelSize][];
private readonly IntPtr[] pointers = new IntPtr[LevelSize];

private readonly int RecordSize;

private volatile int writeCacheLevel;

private volatile int count;

internal bool IsPinned => isPinned;
private readonly bool isPinned;

private const bool ReturnPhysicalAddress = false;
internal static int RecordSize => Unsafe.SizeOf<T>();
internal static bool IsBlittable => Utility.IsBlittable<T>();

private int checkpointCallbackCount;
private SemaphoreSlim checkpointSemaphore;

private ConcurrentQueue<long> freeList;
private readonly ConcurrentQueue<long> freeList;

readonly ILogger logger;

Expand All @@ -99,7 +92,7 @@ private enum AllocationMode { None, Single, Bulk };
private AllocationMode allocationMode;
#endif

const int sector_size = 512;
const int SectorSize = 512;

private int initialAllocation = 0;

Expand All @@ -113,17 +106,11 @@ public unsafe MallocFixedPageSize(ILogger logger = null)
{
this.logger = logger;
freeList = new ConcurrentQueue<long>();
if (ForceUnpinnedAllocation)
isPinned = false;
else
isPinned = Utility.IsBlittable<T>();
Debug.Assert(isPinned || !ReturnPhysicalAddress, "ReturnPhysicalAddress requires pinning");

values[0] = GC.AllocateArray<T>(PageSize + sector_size, isPinned);
if (isPinned)
values[0] = GC.AllocateArray<T>(PageSize + SectorSize, pinned: IsBlittable);
if (IsBlittable)
{
pointers[0] = (IntPtr)(((long)Unsafe.AsPointer(ref values[0][0]) + (sector_size - 1)) & ~(sector_size - 1));
RecordSize = Marshal.SizeOf(values[0][0]);
pointers[0] = (IntPtr)(((long)Unsafe.AsPointer(ref values[0][0]) + (SectorSize - 1)) & ~(SectorSize - 1));
}

#if !(CALLOC)
Expand All @@ -136,24 +123,22 @@ public unsafe MallocFixedPageSize(ILogger logger = null)
// Allocate one block so we never return a null pointer; this allocation is never freed.
// Use BulkAllocate so the caller can still do either BulkAllocate or single Allocate().
BulkAllocate();
initialAllocation = kAllocateChunkSize;
initialAllocation = AllocateChunkSize;
#if DEBUG
// Clear this for the next allocation.
this.allocationMode = AllocationMode.None;
allocationMode = AllocationMode.None;
#endif
}

/// <summary>
/// Get physical address -- for Pinned only
/// Get physical address -- for blittable objects only
/// </summary>
/// <param name="logicalAddress">The logicalAddress of the allocation. For BulkAllocate, this may be an address within the chunk size, to reference that particular record.</param>
/// <returns></returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public long GetPhysicalAddress(long logicalAddress)
{
Debug.Assert(isPinned, "GetPhysicalAddress requires pinning");
if (ReturnPhysicalAddress)
return logicalAddress;
Debug.Assert(IsBlittable, "GetPhysicalAddress requires the values to be blittable");
return (long)pointers[logicalAddress >> PageSizeBits] + (logicalAddress & PageSizeMask) * RecordSize;
}

Expand All @@ -165,10 +150,8 @@ public long GetPhysicalAddress(long logicalAddress)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public unsafe ref T Get(long index)
{
if (ReturnPhysicalAddress)
throw new TsavoriteException("Physical pointer returned by allocator: de-reference pointer to get records instead of calling Get");
Debug.Assert(index != kInvalidAllocationIndex, "Invalid allocation index");
if (isPinned)
Debug.Assert(index != InvalidAllocationIndex, "Invalid allocation index");
if (IsBlittable)
return ref Unsafe.AsRef<T>((byte*)(pointers[index >> PageSizeBits]) + (index & PageSizeMask) * RecordSize);
else
return ref values[index >> PageSizeBits][index & PageSizeMask];
Expand All @@ -182,10 +165,8 @@ public long GetPhysicalAddress(long logicalAddress)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public unsafe void Set(long index, ref T value)
{
if (ReturnPhysicalAddress)
throw new TsavoriteException("Physical pointer returned by allocator: de-reference pointer to set records instead of calling Set (otherwise, set ForceUnpinnedAllocation to true)");
Debug.Assert(index != kInvalidAllocationIndex, "Invalid allocation index");
if (isPinned)
Debug.Assert(index != InvalidAllocationIndex, "Invalid allocation index");
if (IsBlittable)
Unsafe.AsRef<T>((byte*)(pointers[index >> PageSizeBits]) + (index & PageSizeMask) * RecordSize) = value;
else
values[index >> PageSizeBits][index & PageSizeMask] = value;
Expand All @@ -198,14 +179,12 @@ public unsafe void Set(long index, ref T value)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Free(long pointer)
{
if (!ReturnPhysicalAddress)
Get(pointer) = default;
freeList.Enqueue(pointer);
}

internal int FreeListCount => freeList.Count; // For test

internal const int kAllocateChunkSize = 16; // internal for test
internal const int AllocateChunkSize = 16; // internal for test

/// <summary>
/// Allocate a block of size RecordSize * kAllocateChunkSize.
Expand All @@ -217,10 +196,10 @@ public void Free(long pointer)
public unsafe long BulkAllocate()
{
#if DEBUG
Debug.Assert(this.allocationMode != AllocationMode.Single, "Cannot mix Single and Bulk allocation modes");
this.allocationMode = AllocationMode.Bulk;
Debug.Assert(allocationMode != AllocationMode.Single, "Cannot mix Single and Bulk allocation modes");
allocationMode = AllocationMode.Bulk;
#endif
return InternalAllocate(kAllocateChunkSize);
return InternalAllocate(AllocateChunkSize);
}

/// <summary>
Expand All @@ -231,8 +210,8 @@ public unsafe long BulkAllocate()
public unsafe long Allocate()
{
#if DEBUG
Debug.Assert(this.allocationMode != AllocationMode.Bulk, "Cannot mix Single and Bulk allocation modes");
this.allocationMode = AllocationMode.Single;
Debug.Assert(allocationMode != AllocationMode.Bulk, "Cannot mix Single and Bulk allocation modes");
allocationMode = AllocationMode.Single;
#endif
return InternalAllocate(1);
}
Expand All @@ -253,9 +232,9 @@ private unsafe long InternalAllocate(int blockSize)
// If index 0, then allocate space for next level.
if (index == 0)
{
var tmp = GC.AllocateArray<T>(PageSize + sector_size, isPinned);
if (isPinned)
pointers[1] = (IntPtr)(((long)Unsafe.AsPointer(ref tmp[0]) + (sector_size - 1)) & ~(sector_size - 1));
var tmp = GC.AllocateArray<T>(PageSize + SectorSize, pinned: IsBlittable);
if (IsBlittable)
pointers[1] = (IntPtr)(((long)Unsafe.AsPointer(ref tmp[0]) + (SectorSize - 1)) & ~(SectorSize - 1));

#if !(CALLOC)
Array.Clear(tmp, 0, PageSize);
Expand All @@ -265,10 +244,7 @@ private unsafe long InternalAllocate(int blockSize)
}

// Return location.
if (ReturnPhysicalAddress)
return ((long)pointers[0]) + index * RecordSize;
else
return index;
return index;
}

// See if write cache contains corresponding array.
Expand All @@ -281,16 +257,10 @@ private unsafe long InternalAllocate(int blockSize)
if (cache == baseAddr)
{
// Return location.
if (ReturnPhysicalAddress)
return ((long)pointers[baseAddr]) + (long)offset * RecordSize;
else
return index;
return index;
}
}

// Write cache did not work, so get level information from index.
// int level = GetLevelFromIndex(index);

// Spin-wait until level has an allocated array.
var spinner = new SpinWait();
while (true)
Expand All @@ -313,9 +283,9 @@ private unsafe long InternalAllocate(int blockSize)
// Allocate for next page
int newBaseAddr = baseAddr + 1;

var tmp = GC.AllocateArray<T>(PageSize + sector_size, isPinned);
if (isPinned)
pointers[newBaseAddr] = (IntPtr)(((long)Unsafe.AsPointer(ref tmp[0]) + (sector_size - 1)) & ~(sector_size - 1));
var tmp = GC.AllocateArray<T>(PageSize + SectorSize, pinned: IsBlittable);
if (IsBlittable)
pointers[newBaseAddr] = (IntPtr)(((long)Unsafe.AsPointer(ref tmp[0]) + (SectorSize - 1)) & ~(SectorSize - 1));

#if !(CALLOC)
Array.Clear(tmp, 0, PageSize);
Expand All @@ -326,19 +296,13 @@ private unsafe long InternalAllocate(int blockSize)
}

// Return location.
if (ReturnPhysicalAddress)
return ((long)pointers[baseAddr]) + (long)offset * RecordSize;
else
return index;
return index;
}

/// <summary>
/// Dispose
/// </summary>
public void Dispose()
{
count = 0;
}
public void Dispose() => count = 0;


#region Checkpoint
Expand All @@ -347,10 +311,7 @@ public void Dispose()
/// Is checkpoint complete
/// </summary>
/// <returns></returns>
public bool IsCheckpointCompleted()
{
return checkpointCallbackCount == 0;
}
public bool IsCheckpointCompleted() => checkpointCallbackCount == 0;

/// <summary>
/// Is checkpoint completed
Expand Down Expand Up @@ -416,7 +377,7 @@ internal unsafe void BeginCheckpoint(IDevice device, ulong offset, out ulong num

Buffer.MemoryCopy((void*)pointers[i], result.mem.aligned_pointer, writeSize, writeSize);
int j = 0;
if (i == 0) j += kAllocateChunkSize * RecordSize;
if (i == 0) j += AllocateChunkSize * RecordSize;
for (; j < writeSize; j += sizeof(HashBucket))
{
skipReadCache((HashBucket*)(result.mem.aligned_pointer + j));
Expand Down Expand Up @@ -450,19 +411,13 @@ private unsafe void AsyncFlushCallback(uint errorCode, uint numBytes, object con
/// Max valid address
/// </summary>
/// <returns></returns>
public int GetMaxValidAddress()
{
return count;
}
public int GetMaxValidAddress() => count;

/// <summary>
/// Get page size
/// </summary>
/// <returns></returns>
public int GetPageSize()
{
return PageSize;
}
public int GetPageSize() => PageSize;
#endregion

#region Recover
Expand Down

0 comments on commit 922c975

Please sign in to comment.