Skip to content

Analyze hit rate with sequential scan #40

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jun 17, 2020
Merged

Analyze hit rate with sequential scan #40

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
291ea20
scantest
Jun 17, 2020
5a30e4e
update readme
Jun 17, 2020
ce01577
update
Jun 17, 2020
71410bc
fix charts
Jun 17, 2020
6ce5e6a
fix dupe %
Jun 17, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions BitFaster.Caching.HitRateAnalysis/AnalysisResult.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,14 @@ public class AnalysisResult

public int Samples { get; set; }

public bool IsScan { get; set; }

public double CacheSizePercent { get; set; }

public double HitRatio { get; set; }

public TimeSpan Duration { get; set; }

public static void WriteToFile(string path, IEnumerable<AnalysisResult> results)
{
using (var writer = new StreamWriter(path))
Expand Down
64 changes: 63 additions & 1 deletion BitFaster.Caching.HitRateAnalysis/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using BitFaster.Caching.HitRateAnalysis;
using BitFaster.Caching.Lru;
using MathNet.Numerics;
Expand Down Expand Up @@ -30,7 +31,7 @@ static void Main(string[] args)
double[] sValues = { 0.5, 0.86 };

// % of total number of items
double[] cacheSizes = { 0.025, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4 };
double[] cacheSizes = { 0.0125, 0.025, 0.05, 0.075, 0.1, 0.125, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4 };

List<Analysis> analysis = new List<Analysis>();

Expand All @@ -52,8 +53,10 @@ static void Main(string[] args)
for (int i = 0; i < sValues.Length; i++)
{
Console.WriteLine($"Generating Zipfan distribution with {sampleCount} samples, s = {sValues[i]}, N = {n}");
var sw = Stopwatch.StartNew();
zipdfDistribution[i] = new int[sampleCount];
Zipf.Samples(zipdfDistribution[i], sValues[i], n);
Console.WriteLine($"Took {sw.Elapsed}.");
}

List<AnalysisResult> results = new List<AnalysisResult>();
Expand All @@ -68,13 +71,44 @@ static void Main(string[] args)
var concurrentLru = new ConcurrentLru<int, int>(1, cacheSize, EqualityComparer<int>.Default);
var classicLru = new ClassicLru<int, int>(1, cacheSize, EqualityComparer<int>.Default);

var concurrentLruScan = new ConcurrentLru<int, int>(1, cacheSize, EqualityComparer<int>.Default);
var classicLruScan = new ClassicLru<int, int>(1, cacheSize, EqualityComparer<int>.Default);

var d = a.s == 0.5 ? 0 : 1;

var lruSw = Stopwatch.StartNew();
for (int i = 0; i < sampleCount; i++)
{
concurrentLru.GetOrAdd(zipdfDistribution[d][i], func);
}
lruSw.Stop();
Console.WriteLine($"concurrentLru size={cacheSize} took {lruSw.Elapsed}.");

var clruSw = Stopwatch.StartNew();
for (int i = 0; i < sampleCount; i++)
{
classicLru.GetOrAdd(zipdfDistribution[d][i], func);
}
clruSw.Stop();
Console.WriteLine($"classic lru size={cacheSize} took {clruSw.Elapsed}.");

var lruSwScan = Stopwatch.StartNew();
for (int i = 0; i < sampleCount; i++)
{
concurrentLruScan.GetOrAdd(zipdfDistribution[d][i], func);
concurrentLruScan.GetOrAdd(i % n, func);
}
lruSwScan.Stop();
Console.WriteLine($"concurrentLruScan lru size={cacheSize} took {lruSwScan.Elapsed}.");

var clruSwScan = Stopwatch.StartNew();
for (int i = 0; i < sampleCount; i++)
{
classicLruScan.GetOrAdd(zipdfDistribution[d][i], func);
classicLruScan.GetOrAdd(i % n, func);
}
clruSwScan.Stop();
Console.WriteLine($"classicLruScan lru size={cacheSize} took {clruSwScan.Elapsed}.");

results.Add(new AnalysisResult
{
Expand All @@ -83,7 +117,9 @@ static void Main(string[] args)
s = a.s,
CacheSizePercent = a.CacheSizePercent * 100.0,
Samples = a.Samples,
IsScan = false,
HitRatio = classicLru.HitRatio * 100.0,
Duration = clruSw.Elapsed,
});

results.Add(new AnalysisResult
Expand All @@ -93,7 +129,33 @@ static void Main(string[] args)
s = a.s,
CacheSizePercent = a.CacheSizePercent * 100.0,
Samples = a.Samples,
IsScan = false,
HitRatio = concurrentLru.HitRatio * 100.0,
Duration = lruSw.Elapsed,
});

results.Add(new AnalysisResult
{
Cache = "ClassicLru",
N = a.N,
s = a.s,
CacheSizePercent = a.CacheSizePercent * 100.0,
Samples = a.Samples,
IsScan = true,
HitRatio = classicLruScan.HitRatio * 100.0,
Duration = clruSwScan.Elapsed,
});

results.Add(new AnalysisResult
{
Cache = "ConcurrentLru",
N = a.N,
s = a.s,
CacheSizePercent = a.CacheSizePercent * 100.0,
Samples = a.Samples,
IsScan = true,
HitRatio = concurrentLruScan.HitRatio * 100.0,
Duration = lruSwScan.Elapsed,
});
}

Expand Down
134 changes: 97 additions & 37 deletions BitFaster.Caching.HitRateAnalysis/results.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,37 +1,97 @@
Cache,N,s,Samples,CacheSizePercent,HitRatio
ClassicLru,50000,0.5,1000000,2.5,5.8545
ConcurrentLru,50000,0.5,1000000,2.5,8.8714
ClassicLru,50000,0.5,1000000,5,10.4898
ConcurrentLru,50000,0.5,1000000,5,13.6905
ClassicLru,50000,0.5,1000000,10,18.3537
ConcurrentLru,50000,0.5,1000000,10,21.301000000000002
ClassicLru,50000,0.5,1000000,15,25.2029
ConcurrentLru,50000,0.5,1000000,15,27.750200000000003
ClassicLru,50000,0.5,1000000,20,31.443900000000003
ConcurrentLru,50000,0.5,1000000,20,33.4923
ClassicLru,50000,0.5,1000000,25,37.1243
ConcurrentLru,50000,0.5,1000000,25,38.8337
ClassicLru,50000,0.5,1000000,30,42.4998
ConcurrentLru,50000,0.5,1000000,30,43.7457
ClassicLru,50000,0.5,1000000,35,47.5692
ConcurrentLru,50000,0.5,1000000,35,48.4488
ClassicLru,50000,0.5,1000000,40,52.3858
ConcurrentLru,50000,0.5,1000000,40,52.9031
ClassicLru,50000,0.86,1000000,2.5,35.9167
ConcurrentLru,50000,0.86,1000000,2.5,40.2709
ClassicLru,50000,0.86,1000000,5,44.2585
ConcurrentLru,50000,0.86,1000000,5,47.7731
ClassicLru,50000,0.86,1000000,10,53.8678
ConcurrentLru,50000,0.86,1000000,10,56.2986
ClassicLru,50000,0.86,1000000,15,60.2531
ConcurrentLru,50000,0.86,1000000,15,61.9663
ClassicLru,50000,0.86,1000000,20,65.1358
ConcurrentLru,50000,0.86,1000000,20,66.3116
ClassicLru,50000,0.86,1000000,25,69.1442
ConcurrentLru,50000,0.86,1000000,25,69.9031
ClassicLru,50000,0.86,1000000,30,72.5617
ConcurrentLru,50000,0.86,1000000,30,72.95270000000001
ClassicLru,50000,0.86,1000000,35,75.54610000000001
ConcurrentLru,50000,0.86,1000000,35,75.6668
ClassicLru,50000,0.86,1000000,40,78.20899999999999
ConcurrentLru,50000,0.86,1000000,40,78.0539
Cache,N,s,Samples,IsScan,CacheSizePercent,HitRatio,Duration
ClassicLru,50000,0.5,1000000,False,1.25,3.1815,00:00:00.2219936
ConcurrentLru,50000,0.5,1000000,False,1.25,5.8393,00:00:00.2553361
ClassicLru,50000,0.5,1000000,True,1.25,1.46085,00:00:00.3538231
ConcurrentLru,50000,0.5,1000000,True,1.25,2.8294,00:00:00.4064362
ClassicLru,50000,0.5,1000000,False,2.5,5.8342,00:00:00.1863446
ConcurrentLru,50000,0.5,1000000,False,2.5,8.8863,00:00:00.1956831
ClassicLru,50000,0.5,1000000,True,2.5,2.83075,00:00:00.3777144
ConcurrentLru,50000,0.5,1000000,True,2.5,4.4782,00:00:00.3984226
ClassicLru,50000,0.5,1000000,False,5,10.4677,00:00:00.1988212
ConcurrentLru,50000,0.5,1000000,False,5,13.6737,00:00:00.1985374
ClassicLru,50000,0.5,1000000,True,5,5.35485,00:00:00.4012853
ConcurrentLru,50000,0.5,1000000,True,5,7.24645,00:00:00.4069157
ClassicLru,50000,0.5,1000000,False,7.5,14.5759,00:00:00.2001636
ConcurrentLru,50000,0.5,1000000,False,7.5,17.6997,00:00:00.1955304
ClassicLru,50000,0.5,1000000,True,7.5,7.7721,00:00:00.4027166
ConcurrentLru,50000,0.5,1000000,True,7.5,9.6753,00:00:00.4025873
ClassicLru,50000,0.5,1000000,False,10,18.348300000000002,00:00:00.2080376
ConcurrentLru,50000,0.5,1000000,False,10,21.299599999999998,00:00:00.1955030
ClassicLru,50000,0.5,1000000,True,10,10.07585,00:00:00.4116798
ConcurrentLru,50000,0.5,1000000,True,10,11.9522,00:00:00.4029283
ClassicLru,50000,0.5,1000000,False,1.25,3.1815,00:00:00.1864771
ConcurrentLru,50000,0.5,1000000,False,1.25,5.8393,00:00:00.1990631
ClassicLru,50000,0.5,1000000,True,1.25,1.46085,00:00:00.3600207
ConcurrentLru,50000,0.5,1000000,True,1.25,2.8294,00:00:00.4222347
ClassicLru,50000,0.5,1000000,False,15,25.181700000000003,00:00:00.2282900
ConcurrentLru,50000,0.5,1000000,False,15,27.7221,00:00:00.2087622
ClassicLru,50000,0.5,1000000,True,15,14.496400000000001,00:00:00.4531845
ConcurrentLru,50000,0.5,1000000,True,15,16.217699999999997,00:00:00.4159303
ClassicLru,50000,0.5,1000000,False,20,31.4061,00:00:00.2372111
ConcurrentLru,50000,0.5,1000000,False,20,33.5112,00:00:00.1965954
ClassicLru,50000,0.5,1000000,True,20,18.72495,00:00:00.4799955
ConcurrentLru,50000,0.5,1000000,True,20,20.244699999999998,00:00:00.4276315
ClassicLru,50000,0.5,1000000,False,25,37.160399999999996,00:00:00.2471366
ConcurrentLru,50000,0.5,1000000,False,25,38.822,00:00:00.2056509
ClassicLru,50000,0.5,1000000,True,25,22.76605,00:00:00.5113088
ConcurrentLru,50000,0.5,1000000,True,25,24.1382,00:00:00.4348177
ClassicLru,50000,0.5,1000000,False,30,42.5495,00:00:00.2536968
ConcurrentLru,50000,0.5,1000000,False,30,43.802600000000005,00:00:00.2038566
ClassicLru,50000,0.5,1000000,True,30,26.716450000000002,00:00:00.5487216
ConcurrentLru,50000,0.5,1000000,True,30,27.87425,00:00:00.4446285
ClassicLru,50000,0.5,1000000,False,35,47.5886,00:00:00.2674640
ConcurrentLru,50000,0.5,1000000,False,35,48.4817,00:00:00.2008129
ClassicLru,50000,0.5,1000000,True,35,30.5348,00:00:00.5652294
ConcurrentLru,50000,0.5,1000000,True,35,31.53125,00:00:00.4446676
ClassicLru,50000,0.5,1000000,False,40,52.39509999999999,00:00:00.2679638
ConcurrentLru,50000,0.5,1000000,False,40,52.912099999999995,00:00:00.1968226
ClassicLru,50000,0.5,1000000,True,40,34.319300000000005,00:00:00.5922026
ConcurrentLru,50000,0.5,1000000,True,40,35.1464,00:00:00.4419703
ClassicLru,50000,0.86,1000000,False,1.25,28.7188,00:00:00.1590029
ConcurrentLru,50000,0.86,1000000,False,1.25,33.7497,00:00:00.1745940
ClassicLru,50000,0.86,1000000,True,1.25,11.44975,00:00:00.3356978
ConcurrentLru,50000,0.86,1000000,True,1.25,16.35335,00:00:00.3678454
ClassicLru,50000,0.86,1000000,False,2.5,35.9255,00:00:00.1521401
ConcurrentLru,50000,0.86,1000000,False,2.5,40.2911,00:00:00.1459784
ClassicLru,50000,0.86,1000000,True,2.5,14.897750000000002,00:00:00.3460469
ConcurrentLru,50000,0.86,1000000,True,2.5,19.612199999999998,00:00:00.3438083
ClassicLru,50000,0.86,1000000,False,5,44.2184,00:00:00.1503402
ConcurrentLru,50000,0.86,1000000,False,5,47.7282,00:00:00.1351654
ClassicLru,50000,0.86,1000000,True,5,19.223100000000002,00:00:00.3556054
ConcurrentLru,50000,0.86,1000000,True,5,23.50135,00:00:00.3422126
ClassicLru,50000,0.86,1000000,False,7.5,49.681999999999995,00:00:00.1527913
ConcurrentLru,50000,0.86,1000000,False,7.5,52.61130000000001,00:00:00.1313441
ClassicLru,50000,0.86,1000000,True,7.5,22.28575,00:00:00.3620021
ConcurrentLru,50000,0.86,1000000,True,7.5,26.25925,00:00:00.3426238
ClassicLru,50000,0.86,1000000,False,10,53.8481,00:00:00.1599107
ConcurrentLru,50000,0.86,1000000,False,10,56.3201,00:00:00.1267778
ClassicLru,50000,0.86,1000000,True,10,24.804750000000002,00:00:00.3666843
ConcurrentLru,50000,0.86,1000000,True,10,28.510999999999996,00:00:00.3390340
ClassicLru,50000,0.86,1000000,False,1.25,28.7188,00:00:00.1596049
ConcurrentLru,50000,0.86,1000000,False,1.25,33.7497,00:00:00.1566299
ClassicLru,50000,0.86,1000000,True,1.25,11.44975,00:00:00.3432512
ConcurrentLru,50000,0.86,1000000,True,1.25,16.35335,00:00:00.3590832
ClassicLru,50000,0.86,1000000,False,15,60.2324,00:00:00.1584608
ConcurrentLru,50000,0.86,1000000,False,15,61.978100000000005,00:00:00.1257132
ClassicLru,50000,0.86,1000000,True,15,28.9737,00:00:00.3945479
ConcurrentLru,50000,0.86,1000000,True,15,32.244499999999995,00:00:00.3474522
ClassicLru,50000,0.86,1000000,False,20,65.12790000000001,00:00:00.1625112
ConcurrentLru,50000,0.86,1000000,False,20,66.2993,00:00:00.1190971
ClassicLru,50000,0.86,1000000,True,20,32.4771,00:00:00.4151890
ConcurrentLru,50000,0.86,1000000,True,20,35.393550000000005,00:00:00.3661938
ClassicLru,50000,0.86,1000000,False,25,69.0926,00:00:00.1765459
ConcurrentLru,50000,0.86,1000000,False,25,69.8668,00:00:00.1220536
ClassicLru,50000,0.86,1000000,True,25,35.5605,00:00:00.4374009
ConcurrentLru,50000,0.86,1000000,True,25,38.201049999999995,00:00:00.3574605
ClassicLru,50000,0.86,1000000,False,30,72.4841,00:00:00.1744888
ConcurrentLru,50000,0.86,1000000,False,30,72.91159999999999,00:00:00.1207647
ClassicLru,50000,0.86,1000000,True,30,38.38225,00:00:00.4761851
ConcurrentLru,50000,0.86,1000000,True,30,40.8101,00:00:00.3642631
ClassicLru,50000,0.86,1000000,False,35,75.457,00:00:00.1756699
ConcurrentLru,50000,0.86,1000000,False,35,75.5849,00:00:00.1246051
ClassicLru,50000,0.86,1000000,True,35,40.98865,00:00:00.5014744
ConcurrentLru,50000,0.86,1000000,True,35,43.285050000000005,00:00:00.3813605
ClassicLru,50000,0.86,1000000,False,40,78.0983,00:00:00.1715080
ConcurrentLru,50000,0.86,1000000,False,40,77.9631,00:00:00.1156444
ClassicLru,50000,0.86,1000000,True,40,43.426550000000006,00:00:00.4955222
ConcurrentLru,50000,0.86,1000000,True,40,45.6681,00:00:00.3754359
30 changes: 28 additions & 2 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,10 +105,36 @@ When the cache is small, below 15% of the total key space, ConcurrentLru outperf
<table>
<tr>
<td>
<img src="https://user-images.githubusercontent.com/12851828/84707621-e2a62480-af13-11ea-91e7-726911bce162.png" width="400"/>
<img src="https://user-images.githubusercontent.com/12851828/84844130-a00d4680-affe-11ea-8f7a-e3c66180d8b9.png" width="250"/>
</td>
<td>
<img src="https://user-images.githubusercontent.com/12851828/84707663-f81b4e80-af13-11ea-96d4-1ba71444d333.png" width="400"/>
<img src="https://user-images.githubusercontent.com/12851828/84844172-b6b39d80-affe-11ea-9a29-cbdae6020246.png" width="250"/>
</td>
</tr>
</table>

This is the same test, but interleaving a sequential scan of every key. In this case, ConcurrentLru performs better across the board.

<table>
<tr>
<td>
<img src="https://user-images.githubusercontent.com/12851828/84841922-a4366580-aff8-11ea-93dd-568d60cd82d9.png" width="250"/>
</td>
<td>
<img src="https://user-images.githubusercontent.com/12851828/84842237-730a6500-aff9-11ea-9a46-40141adff920.png" width="250"/>
</td>
</tr>
</table>

These charts summarize the percentage increase in hit rate ConcurrentLru vs LRU. Increase during sequential scan can be > 90%, depending on cache size relative to data set.

<table>
<tr>
<td>
<img src="https://user-images.githubusercontent.com/12851828/84843966-283f1c00-affe-11ea-99c9-20aa01f307f0.png" width="250"/>
</td>
<td>
<img src="https://user-images.githubusercontent.com/12851828/84844003-3d1baf80-affe-11ea-9266-e83efe2e8c35.png" width="250"/>
</td>
</tr>
</table>
Expand Down