Skip to content

Commit

Permalink
Help compiler with cctor checks in AOT mode
Browse files Browse the repository at this point in the history 
- hoist static holder contents to locals when compiler can't hoist init checks out of loops
- reorder static holder contents' accesses when compiler fails to CSE multiple static init checks
- remove module initializer because it is no longer relevant for JIT and inconsistently regresses numbers for AOT
  • Loading branch information
@neon-sunset
neon-sunset committed Jun 5, 2023
1 parent dbd1fa1 commit 1db6be9
Show file tree
Hide file tree
Showing 5 changed files with 60 additions and 51 deletions.
27 changes: 19 additions & 8 deletions src/FastCache.Cached/CacheManager.cs
View file
Expand Up @@ -211,9 +211,13 @@ internal static void ReportEvictions(uint count)

private static void ImmediateFullEviction<K, V>() where K : notnull
{
CacheStaticHolder<K, V>.EvictionJob.RescheduleConsideringExpiration();
var (evictionJob, quickList) = (
CacheStaticHolder<K, V>.EvictionJob,
CacheStaticHolder<K, V>.QuickList);

if (CacheStaticHolder<K, V>.QuickList.Evict(resize: true))
evictionJob.RescheduleConsideringExpiration();

if (quickList.Evict(resize: true))
{
return;
}
Expand All @@ -237,9 +241,11 @@ internal static void ReportEvictions(uint count)

private static async Task StaggeredFullEviction<K, V>() where K : notnull
{
var evictionJob = CacheStaticHolder<K, V>.EvictionJob;
var (quickList, evictionJob) = (
CacheStaticHolder<K, V>.QuickList,
CacheStaticHolder<K, V>.EvictionJob);

if (CacheStaticHolder<K, V>.QuickList.Evict())
if (quickList.Evict())
{
// When a lot of items are being added to cache, it triggers GC and its callbacks
// which may decrease throughput by accessing the same memory locations
Expand Down Expand Up @@ -279,11 +285,15 @@ internal static void ReportEvictions(uint count)

private static uint EvictFromCacheStore<K, V>() where K : notnull
{
var evictedCount = CacheStaticHolder<K, V>.Store.Count > Constants.ParallelEvictionThreshold
var (store, quicklist) = (
CacheStaticHolder<K, V>.Store,
CacheStaticHolder<K, V>.QuickList);

var evictedCount = store.Count > Constants.ParallelEvictionThreshold
? EvictFromCacheStoreParallel<K, V>()
: EvictFromCacheStoreSingleThreaded<K, V>();

CacheStaticHolder<K, V>.QuickList.PullFromCacheStore();
quicklist.PullFromCacheStore();

return evictedCount;
}
Expand All @@ -309,6 +319,7 @@ internal static void ReportEvictions(uint count)
private static uint EvictFromCacheStoreParallel<K, V>() where K : notnull
{
var now = TimeUtils.Now;
var store = CacheStaticHolder<K, V>.Store;
uint totalRemoved = 0;

void CheckAndRemove(KeyValuePair<K, CachedInner<V>> kvp)
Expand All @@ -318,12 +329,12 @@ void CheckAndRemove(KeyValuePair<K, CachedInner<V>> kvp)

if (now > timestamp)
{
CacheStaticHolder<K, V>.Store.TryRemove(key, out _);
store.TryRemove(key, out _);
count++;
}
}

CacheStaticHolder<K, V>.Store
store
.AsParallel()
.AsUnordered()
.ForAll(CheckAndRemove);
Expand Down
26 changes: 0 additions & 26 deletions src/FastCache.Cached/CacheStaticHolder.cs
View file
@@ -1,33 +1,7 @@
using FastCache.Collections;
using FastCache.Extensions;
using FastCache.Helpers;
using FastCache.Services;
using NonBlocking;

namespace FastCache;

#if NET5_0_OR_GREATER
#pragma warning disable CA2255 // Static initialization to ensure cctor checks are removed in JITted code
internal static class CacheStaticHolder
{
[ModuleInitializer]
public static void Initialize()
{
RuntimeHelpers.RunClassConstructor(typeof(Constants).TypeHandle);
RuntimeHelpers.RunClassConstructor(typeof(CacheStaticHolder<,>).TypeHandle);
RuntimeHelpers.RunClassConstructor(typeof(CacheManager).TypeHandle);

RuntimeHelpers.RunClassConstructor(typeof(Cached<,>).TypeHandle);
RuntimeHelpers.RunClassConstructor(typeof(Cached).TypeHandle);
RuntimeHelpers.RunClassConstructor(typeof(CachedExtensions).TypeHandle);
RuntimeHelpers.RunClassConstructor(typeof(CachedRange<>).TypeHandle);

RuntimeHelpers.RunClassConstructor(typeof(TimeUtils).TypeHandle);
}
}
#pragma warning restore CA2255
#endif

internal static class CacheStaticHolder<K, V> where K : notnull
{
internal static readonly ConcurrentDictionary<K, CachedInner<V>> Store = new();
Expand Down
49 changes: 35 additions & 14 deletions src/FastCache.Cached/Collections/CachedRange.Impl.cs
View file
Expand Up @@ -83,6 +83,9 @@ private static void SaveMultithreaded<K>(ReadOnlyMemory<K> keys, ReadOnlyMemory<
where K : notnull
where TList : IList<(K, V)>
{
var (store, quickList) = (
CacheStaticHolder<K, V>.Store,
CacheStaticHolder<K, V>.QuickList);
var timestamp = GetAndReportTimestamp<K>(expiration);

var sliceLength = range.Count / parallelism;
Expand All @@ -104,34 +107,39 @@ private static void SaveMultithreaded<K>(ReadOnlyMemory<K> keys, ReadOnlyMemory<
{
var (key, value) = range[i];

CacheStaticHolder<K, V>.Store[key] = new(value, timestamp);
CacheStaticHolder<K, V>.QuickList.OverwritingAdd(key, timestamp);
store[key] = new(value, timestamp);
quickList.OverwritingAdd(key, timestamp);
}
}

private static void SaveEnumerableSinglethreaded<K>(IEnumerable<(K, V)> range, TimeSpan expiration) where K : notnull
{
var (store, quickList) = (
CacheStaticHolder<K, V>.Store,
CacheStaticHolder<K, V>.QuickList);
var timestamp = GetAndReportTimestamp<K>(expiration);

foreach (var (key, value) in range)
{
CacheStaticHolder<K, V>.Store[key] = new(value, timestamp);
store[key] = new(value, timestamp);
}

CacheStaticHolder<K, V>.QuickList.PullFromCacheStore();
quickList.PullFromCacheStore();
}

private static void SaveEnumerableMultithreaded<K>(IEnumerable<(K, V)> range, TimeSpan expiration) where K : notnull
{
var (store, quickList) = (
CacheStaticHolder<K, V>.Store,
CacheStaticHolder<K, V>.QuickList);
var timestamp = GetAndReportTimestamp<K>(expiration);

range
.AsParallel()
.AsUnordered()
.ForAll(
kvp => CacheStaticHolder<K, V>.Store[kvp.Item1] = new(kvp.Item2, timestamp));
.ForAll(kvp => store[kvp.Item1] = new(kvp.Item2, timestamp));

CacheStaticHolder<K, V>.QuickList.PullFromCacheStore();
quickList.PullFromCacheStore();
}

private static void RemoveMultithreaded<K>(ReadOnlyMemory<K> keys, int parallelism) where K : notnull
Expand All @@ -157,44 +165,56 @@ private static void SaveMultithreaded<K>(ReadOnlyMemory<K> keys, ReadOnlyMemory<

private static void RemoveSlice<K>(ReadOnlySpan<K> slice) where K : notnull
{
var store = CacheStaticHolder<K, V>.Store;
foreach (var key in slice)
{
_ = CacheStaticHolder<K, V>.Store.TryRemove(key, out _);
store.TryRemove(key, out _);
}
}

private static void SaveSlice<K>(ReadOnlySpan<(K key, V value)> slice, long timestamp) where K : notnull
{
var (store, quickList) = (
CacheStaticHolder<K, V>.Store,
CacheStaticHolder<K, V>.QuickList);

foreach (var (key, value) in slice)
{
CacheStaticHolder<K, V>.Store[key] = new(value, timestamp);
store[key] = new(value, timestamp);
}

var quickListLimit = GetQuickListInsertLength<K>(slice.Length);
for (var i = 0; i < quickListLimit; i++)
{
CacheStaticHolder<K, V>.QuickList.OverwritingAdd(slice[i].key, timestamp);
quickList.OverwritingAdd(slice[i].key, timestamp);
}
}

private static void SaveSlice<K>(ReadOnlySpan<K> keys, ReadOnlySpan<V> values, long timestamp) where K : notnull
{
var (store, quickList) = (
CacheStaticHolder<K, V>.Store,
CacheStaticHolder<K, V>.QuickList);

for (var i = 0; i < keys.Length; i++)
{
CacheStaticHolder<K, V>.Store[keys[i]] = new(values[i], timestamp);
store[keys[i]] = new(values[i], timestamp);
}

var quickListLimit = GetQuickListInsertLength<K>(keys.Length);
for (var i = 0; i < quickListLimit; i++)
{
CacheStaticHolder<K, V>.QuickList.OverwritingAdd(keys[i], timestamp);
quickList.OverwritingAdd(keys[i], timestamp);
}
}

private static void SaveListSlice<K, TList>(ListSlice<(K, V), TList> slice, long timestamp)
where K : notnull
where TList : IList<(K Key, V Value)>
{
var (store, quickList) = (
CacheStaticHolder<K, V>.Store,
CacheStaticHolder<K, V>.QuickList);
var (list, start, end) = slice;

// Surprisingly, on plain lists the performance is within 1.1x of array/span impl.
Expand All @@ -203,14 +223,14 @@ private static void SaveMultithreaded<K>(ReadOnlyMemory<K> keys, ReadOnlyMemory<
{
var (key, value) = list[i];

CacheStaticHolder<K, V>.Store[key] = new(value, timestamp);
store[key] = new(value, timestamp);
}

var quickListLimit = GetQuickListInsertLength<K>(end - start) + start;

for (var i = start; i < quickListLimit; i++)
{
CacheStaticHolder<K, V>.QuickList.OverwritingAdd(list[i].Key, timestamp);
quickList.OverwritingAdd(list[i].Key, timestamp);
}
}

Expand All @@ -222,6 +242,7 @@ private static void SaveMultithreaded<K>(ReadOnlyMemory<K> keys, ReadOnlyMemory<
return timestamp;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int GetQuickListInsertLength<K>(int sliceLength) where K : notnull
{
return (int)Math.Min(CacheStaticHolder<K, V>.QuickList.FreeSpace, (uint)sliceLength);
Expand Down
6 changes: 4 additions & 2 deletions src/FastCache.Cached/Collections/CachedRange.cs
View file
Expand Up @@ -142,15 +142,17 @@ public static void Save<K>(IEnumerable<(K, V)> range, TimeSpan expiration)
#if NET6_0_OR_GREATER
else if (keys.TryGetNonEnumeratedCount(out var count) && GetParallelism((uint)count) > 1)
{
var store = CacheStaticHolder<K, V>.Store;
keys.AsParallel()
.ForAll(static key => CacheStaticHolder<K, V>.Store.TryRemove(key, out _));
.ForAll(key => store.TryRemove(key, out _));
}
#endif
else
{
var store = CacheStaticHolder<K, V>.Store;
foreach (var key in keys)
{
CacheStaticHolder<K, V>.Store.TryRemove(key, out _);
store.TryRemove(key, out _);
}
}
}
Expand Down
3 changes: 2 additions & 1 deletion src/FastCache.Cached/EvictionQuickList.cs
View file
Expand Up @@ -247,10 +247,11 @@ internal uint Trim(uint count)

uint countAfterTrim = currentCount - toRemoveCount;

var store = CacheStaticHolder<K, V>.Store;
uint removed = 0;
for (uint i = countAfterTrim; i < countAfterTrim + toRemoveCount; i++)
{
if (CacheStaticHolder<K, V>.Store.TryRemove(_active[i].Key, out var _))
if (store.TryRemove(_active[i].Key, out var _))
{
removed++;
}
Expand Down

0 comments on commit 1db6be9

Please sign in to comment.