Skip to content

Commit ff293eb

Browse files
committed
FNV-1a-128 high-speed implementation using LX4Cnh
1 parent 116c440 commit ff293eb

16 files changed

Lines changed: 575 additions & 5 deletions

File tree

algorithms/LodgeX4CorrNoHigh/Readme.md

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,27 @@ Copyright (c) 2021 Denis Kuzmin <x-3F@outlook.com> github/3F
88

99
Part of https://twitter.com/github3F/status/1403748080760111106
1010

11-
* Part of https://twitter.com/github3F/status/1403748080760111106
11+
### MLnoCS vs LX4Cnh
12+
13+
Algorithm | Maximum bits | One multiplication
14+
----------|--------------|-------------------
15+
LX4Cnh | 128 × 128 | less than ~ **4.3 ns** == 0.0000000043 sec
16+
MLnoCS | 128 × 16 (*<sup>1</sup>32) | less than ~ **0.31 ns** == 0.00000000031 sec
17+
LX4Cnh optimized\*<sup>2</sup> | 128 × 128 | \*<sup>2</sup> less than ~ **0.86 ns** == 0.00000000086 sec
18+
19+
* \*<sup>1</sup> - theoretically up to 128 x 32 with some correction.
20+
* \*<sup>2</sup> - The actual calculation using LX4Cnh can be a bit optimized such for FNV1a-128 implementation (find it in my repo):
21+
22+
[![](fnvOptimization.png)](#)
23+
24+
(**1 ns** == 0.000000001 sec)
1225

1326
## .NET implementation
1427

1528
*LodgeX4CorrNoHigh* class provides several ways of setting and getting numbers by using uint, ulong, or bytes array. Just play with available [Unit-Tests](tests) and [Speed-Tests](https://github.com/3F/sandbox/tree/master/csharp/numbers/BigNum).
1629

30+
[![](benchmark.png)](https://twitter.com/github3F/status/1410358979033813000)
31+
1732
### Examples
1833

1934
For example, using ulong (UInt64)
@@ -37,8 +52,6 @@ ulong high = LodgeX4CorrNoHigh.Multiply
3752

3853
To reduce the amount of unnecessary stack manipulations (ldloca.s/ldarg.. etc), meet an *embeddable* version.
3954

40-
One 128x128 multiplication requires less than ~ **0.02 ns** == 0.00000000002 sec
41-
4255
```csharp
4356
// 0xC1F4271980F30FED81EF70CCBC6EF2EF
4457
// × 0xDEF03F0142D0ACD21749BEF1EA30FF94
@@ -47,10 +60,9 @@ uint a = 0xC1F42719, b = 0x80F30FED, c = 0x81EF70CC, d = 0xBC6EF2EF;
4760
uint ma = 0xDEF03F01, mb = 0x42D0ACD2, mc = 0x1749BEF1, md = 0xEA30FF94;
4861
//-
4962
ulong high, low;
50-
unchecked{/*(c) Denis Kuzmin <x-3F@outlook.com> github/3F */ulong A=(ulong)b*mb;ulong B=A&0xFFFF_FFFF;ulong C=((A>>32)+B+(a*ma))&0xFFFF_FFFF;ulong D=(a>b)?a-b:b-a;ulong E=(ma>mb)?ma-mb:mb-ma;if(D!=0&&E!=0){ulong F=D*E;if((!(a>b)&&(ma>mb))||((a>b)&&!(ma>mb))){C+=F&0xFFFF_FFFF;}else{C-=F&0xFFFF_FFFF;}}ulong G=(C<<32)+B;A=(ulong)c*mc;ulong H=(ulong)d*md;B=(H>>32)+(H&0xFFF_FFFF_FFFF_FFFF)+(A&0xFFF_FFFF_FFFF_FFFF)+((A&0xFFF_FFFF)<<32);C=((((A>>28)+(A>>60)+(H>>60))<<28)>>16)+(B>>48);ulong I=B&0xFFFF_FFFF_FFFF;D=(c>d)?c-d:d-c;E=(mc>md)?mc-md:md-mc;if(D!=0&&E!=0){ulong F=D*E;ulong J=(F>>48);ulong K=F&0xFFFF_FFFF_FFFF;B=I;if((!(c>d)&&(mc>md))||((c>d)&&!(mc>md))){I+=K;C+=J;if(B>(I&0xFFFF_FFFF_FFFF))++C;}else{I-=K;C-=J;if(B<(I&0xFFFF_FFFF_FFFF))--C;}}ulong L=((I&0xFFFF_FFFF)<<32)+(H&0xFFFF_FFFF);C=G+L+((C<<16)+((I>>32)&0xFFFF));G=((ulong)a<<32)+b;I=((ulong)c<<32)+d;A=((ulong)ma<<32)+mb;H=((ulong)mc<<32)+md;D=(G>I)?G-I:I-G;E=(A>H)?A-H:H-A;if(D!=0&&E!=0){ulong F=D*E;if((!(G>I)&&(A>H))||((G>I)&&!(A>H))){C+=F;}else{C-=F;}}low=L;high=C;}
63+
unchecked{/*LX4Cnh (c) Denis Kuzmin <x-3F@outlook.com> github/3F */ulong A=(ulong)b*mb;ulong B=A&0xFFFF_FFFF;ulong C=((A>>32)+B+(a*ma))&0xFFFF_FFFF;ulong D=(a>b)?a-b:b-a;ulong E=(ma>mb)?ma-mb:mb-ma;if(D!=0&&E!=0){ulong F=D*E;if(((a<b)&&(ma>mb))||((a>b)&&(ma<mb))){C+=F&0xFFFF_FFFF;}else{C-=F&0xFFFF_FFFF;}}ulong G=(C<<32)+B;A=(ulong)c*mc;ulong H=(ulong)d*md;B=(H>>32)+(H&0xFFF_FFFF_FFFF_FFFF)+(A&0xFFF_FFFF_FFFF_FFFF)+((A&0xFFF_FFFF)<<32);C=(((A>>28)+(A>>60)+(H>>60))<<28);ulong I=B;D=(c>d)?c-d:d-c;E=(mc>md)?mc-md:md-mc;if(D!=0&&E!=0){ulong F=D*E;if(((c<d)&&(mc>md))||((c>d)&&(mc<md))){I+=F;if(B>I)C+=0x100000000;}else{I-=F;if(B<I)C-=0x100000000;}}ulong J=((I&0xFFFF_FFFF)<<32)+(H&0xFFFF_FFFF);C=G+J+C+(I>>32);G=((ulong)a<<32)+b;I=((ulong)c<<32)+d;A=((ulong)ma<<32)+mb;H=((ulong)mc<<32)+md;D=(G>I)?G-I:I-G;E=(A>H)?A-H:H-A;if(D!=0&&E!=0){ulong F=D*E;if(((G<I)&&(A>H))||((G>I)&&(A<H))){C+=F;}else{C-=F;}}low=J;high=C;}
5164

5265
// high low
5366
// ________________|_______________
5467
// = 0x9633C106748CB7D96650F9EA76F0832C
5568
```
56-
(**1 ns** == 0.000000001 sec)
4.55 KB
Loading

algorithms/MLnoCS/Readme.md

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,67 @@ Copyright (c) 2021 Denis Kuzmin <x-3F@outlook.com> github/3F
1111
[ [](https://3F.github.io/Donation/) ]
1212

1313
Part of https://twitter.com/github3F/status/1403748080760111106
14+
15+
## MLnoCS vs LX4Cnh
16+
17+
Algorithm | Maximum bits | One multiplication
18+
----------|--------------|-------------------
19+
LX4Cnh | 128 × 128 | less than ~ **4.3 ns** == 0.0000000043 sec
20+
MLnoCS | 128 × 16 (*<sup>1</sup>32) | less than ~ **0.31 ns** == 0.00000000031 sec
21+
LX4Cnh optimized\*<sup>2</sup> | 128 × 128 | \*<sup>2</sup> less than ~ **0.86 ns** == 0.00000000086 sec
22+
23+
* \*<sup>1</sup> - theoretically up to 128 x 32 with some correction.
24+
* \*<sup>2</sup> - The actual calculation using LX4Cnh can be a bit optimized such for FNV1a-128 implementation (find it in my repo).
25+
26+
(**1 ns** == 0.000000001 sec)
27+
28+
## .NET implementation
29+
30+
*MulLowNoCorrShifts16* class provides several ways of setting and getting numbers by using uint, ulong, or bytes array. Just play with available [Unit-Tests](tests) and [Speed-Tests](https://github.com/3F/sandbox/tree/master/csharp/numbers/BigNum).
31+
32+
[![](benchmark.png)](https://twitter.com/github3F/status/1410358979033813000)
33+
34+
### Examples
35+
36+
For example, using ulong (UInt64)
37+
38+
```csharp
39+
// 0x4BD4823ECC5D03EB19E07DB8FFD5DABE
40+
// × 0x6D5C
41+
42+
ulong high = MulLowNoCorrShifts16.Multiply
43+
(
44+
0x4BD4823ECC5D03EB, 0x19E07DB8FFD5DABE,
45+
0x6D5C,
46+
out ulong low
47+
);
48+
// high low
49+
// ________________|_______________
50+
// = 0xBBD38B950C188281E234F369FEFD8248
51+
```
52+
53+
### Embeddable superfast version
54+
55+
To reduce the amount of unnecessary stack manipulations (ldloca.s/ldarg.. etc), meet an *embeddable* version.
56+
57+
One 128×16 multiplication using *MLnoCS* requires less than ~ 0.31 ns == 0.00000000031 sec
58+
59+
```csharp
60+
// 0xC1F4271980F30FED81EF70CCBC6EF2EF
61+
// × 0x1036
62+
63+
uint a = 0xC1F42719, b = 0x80F30FED, c = 0x81EF70CC, d = 0xBC6EF2EF;
64+
ushort prime = 0x1036;
65+
//-
66+
uint ra, rb, rc, rd;
67+
unchecked{/* MLnoCS (c) Denis Kuzmin <x-3F@outlook.com> github/3F */ulong e=a,f=b,g=c,h=d,l,o,k;e*=prime;f*=prime;g*=prime;h*=prime;l=(g&0xFFFF_FFFF)+(h>>32);o=(f&0xFFFF_FFFF)+(g>>32);k=(e&0xFFFF_FFFF)+(f>>32);ra=(uint)k;rb=(uint)o;rc=(uint)l;rd=(uint)h;}
68+
// high A | high B | low C | low D
69+
// ______HIGH______|______LOW______
70+
// = 0x2BF1D770644434385F8E96F6AE962E6A
71+
```
72+
(**1 ns** == 0.000000001 sec)
73+
74+
See also *LodgeX4CorrNoHigh* for 128 × 128,
75+
76+
* https://twitter.com/github3F/status/1410669589734768643
77+
* https://twitter.com/github3F/status/1410358979033813000

csharp/FNV-1a/FNV-1a.sln

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
2+
Microsoft Visual Studio Solution File, Format Version 12.00
3+
# Visual Studio Version 16
4+
VisualStudioVersion = 16.0.31402.337
5+
MinimumVisualStudioVersion = 10.0.40219.1
6+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FNV-1a", "src\FNV-1a.csproj", "{5A2DDFC2-9402-44FB-BFF6-5071F15615B2}"
7+
EndProject
8+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests", "tests\UnitTest\UnitTest.csproj", "{A93ACA73-DA16-461D-BF06-9941671C7924}"
9+
EndProject
10+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Benchmark", "tests\Benchmark\Benchmark.csproj", "{39EEFDAE-C950-497A-A435-A05175D62800}"
11+
EndProject
12+
Global
13+
GlobalSection(SolutionConfigurationPlatforms) = preSolution
14+
Debug|Any CPU = Debug|Any CPU
15+
Release|Any CPU = Release|Any CPU
16+
EndGlobalSection
17+
GlobalSection(ProjectConfigurationPlatforms) = postSolution
18+
{5A2DDFC2-9402-44FB-BFF6-5071F15615B2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
19+
{5A2DDFC2-9402-44FB-BFF6-5071F15615B2}.Debug|Any CPU.Build.0 = Debug|Any CPU
20+
{5A2DDFC2-9402-44FB-BFF6-5071F15615B2}.Release|Any CPU.ActiveCfg = Release|Any CPU
21+
{5A2DDFC2-9402-44FB-BFF6-5071F15615B2}.Release|Any CPU.Build.0 = Release|Any CPU
22+
{A93ACA73-DA16-461D-BF06-9941671C7924}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
23+
{A93ACA73-DA16-461D-BF06-9941671C7924}.Debug|Any CPU.Build.0 = Debug|Any CPU
24+
{A93ACA73-DA16-461D-BF06-9941671C7924}.Release|Any CPU.ActiveCfg = Release|Any CPU
25+
{A93ACA73-DA16-461D-BF06-9941671C7924}.Release|Any CPU.Build.0 = Release|Any CPU
26+
{39EEFDAE-C950-497A-A435-A05175D62800}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
27+
{39EEFDAE-C950-497A-A435-A05175D62800}.Debug|Any CPU.Build.0 = Debug|Any CPU
28+
{39EEFDAE-C950-497A-A435-A05175D62800}.Release|Any CPU.ActiveCfg = Release|Any CPU
29+
{39EEFDAE-C950-497A-A435-A05175D62800}.Release|Any CPU.Build.0 = Release|Any CPU
30+
EndGlobalSection
31+
GlobalSection(SolutionProperties) = preSolution
32+
HideSolutionNode = FALSE
33+
EndGlobalSection
34+
GlobalSection(ExtensibilityGlobals) = postSolution
35+
SolutionGuid = {577340DB-CBCA-4482-8E93-9FFB42035889}
36+
EndGlobalSection
37+
EndGlobal

csharp/FNV-1a/LICENSE

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
The MIT License (MIT)
2+
3+
Copyright (c) 2021 Denis Kuzmin <x-3F@outlook.com> github/3F
4+
Copyright (c) sandbox contributors https://github.com/3F/sandbox/graphs/contributors
5+
6+
Permission is hereby granted, free of charge, to any person obtaining a copy
7+
of this software and associated documentation files (the "Software"), to deal
8+
in the Software without restriction, including without limitation the rights
9+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
copies of the Software, and to permit persons to whom the Software is
11+
furnished to do so, subject to the following conditions:
12+
13+
The above copyright notice and this permission notice shall be included in
14+
all copies or substantial portions of the Software.
15+
16+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22+
THE SOFTWARE.

csharp/FNV-1a/Readme.md

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
*FNV-1a* high-speed implementations
2+
3+
```
4+
Copyright (c) 2021 Denis Kuzmin <x-3F@outlook.com> github/3F
5+
```
6+
7+
[ [](https://3F.github.io/Donation/) ]
8+
9+
Part of https://twitter.com/github3F/status/1403748080760111106
10+
11+
## speed
12+
13+
### FNV-1a-128 using LX4Cnh
14+
15+
120 Unicode (UTF-16) characters as follow:
16+
17+
`*LodgeX4CorrNoHigh* (LX4Cnh) algorithm of the high-speed multiplications of **128-bit** numbers (full range, 128 × 128).`
18+
19+
[![](benchmark.120Utf16.table.png)](#)
20+
21+
> 128-bit hash value: 0x8e719ac9080952dec9c90a46279bfcc9
22+
23+
[![](benchmark.120Utf16.png)](#)
13.2 KB
Loading
3.63 KB
Loading

csharp/FNV-1a/src/FNV-1a.csproj

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFrameworks>net472;netcoreapp2.1</TargetFrameworks>
5+
<RootNamespace>net.r_eg.sandbox.Hash</RootNamespace>
6+
</PropertyGroup>
7+
8+
<PropertyGroup>
9+
<CheckForOverflowUnderflow>false</CheckForOverflowUnderflow>
10+
</PropertyGroup>
11+
12+
<ItemGroup>
13+
<Compile Include="..\..\..\algorithms\LodgeX4CorrNoHigh\src\LodgeX4CorrNoHigh.cs" />
14+
</ItemGroup>
15+
16+
</Project>

csharp/FNV-1a/src/FNV1a.cs

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
/*
2+
* The MIT License (MIT)
3+
*
4+
* Copyright (c) 2021 Denis Kuzmin <x-3F@outlook.com> github/3F
5+
* Copyright (c) sandbox contributors https://github.com/3F/sandbox/graphs/contributors
6+
*
7+
* Permission is hereby granted, free of charge, to any person obtaining a copy
8+
* of this software and associated documentation files (the "Software"), to deal
9+
* in the Software without restriction, including without limitation the rights
10+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11+
* copies of the Software, and to permit persons to whom the Software is
12+
* furnished to do so, subject to the following conditions:
13+
*
14+
* The above copyright notice and this permission notice shall be included in
15+
* all copies or substantial portions of the Software.
16+
*
17+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23+
* THE SOFTWARE.
24+
*/
25+
26+
using net.r_eg.sandbox.algorithms;
27+
28+
namespace net.r_eg.sandbox.Hash
29+
{
30+
public static class FNV1a
31+
{
32+
private const ulong OFS64 = 0xcbf29ce484222325;
33+
private const ulong PRIME64 = 0x00000100000001B3;
34+
35+
private const uint P128_B = 0x01000000, P128_D = 0x0000013B, P128_BD = 0xFFFEC5;
36+
37+
public static ulong GetHash128LX4Cnh(string input, out ulong low)
38+
{
39+
ulong a = 0x6c62272e, b = 0x07bb0142, c = 0x62b82175, d = 0x6295c58d;
40+
41+
ulong f = 0, fLm = 0;
42+
unchecked
43+
{
44+
for(int i = 0; i < input.Length; ++i)
45+
{
46+
d ^= input[i];
47+
48+
// Below is an optimized implementation (limited) of the LX4Cnh algorithm specially for Fnv1a128
49+
// (c) Denis Kuzmin <x-3F@outlook.com> github/3F
50+
51+
f = b * P128_B;
52+
53+
ulong v = (uint)f;
54+
55+
f = (f >> 32) + v;
56+
57+
if(a > b)
58+
{
59+
f += (uint)((a - b) * P128_B);
60+
}
61+
else if(a < b)
62+
{
63+
f -= (uint)((b - a) * P128_B);
64+
}
65+
66+
ulong fHigh = (f << 32) + (uint)v;
67+
ulong r2 = d * P128_D;
68+
69+
v = (r2 >> 32) + (r2 & 0xFFF_FFFF_FFFF_FFFF);
70+
71+
f = (r2 & 0xF000_0000_0000_0000) >> 32;
72+
73+
if(c > d)
74+
{
75+
fLm = v;
76+
v += (c - d) * P128_D;
77+
if(fLm > v) f += 0x100000000;
78+
}
79+
else if(c < d)
80+
{
81+
fLm = v;
82+
v -= (d - c) * P128_D;
83+
if(fLm < v) f -= 0x100000000;
84+
}
85+
86+
fLm = (((ulong)(uint)v) << 32) + (uint)r2;
87+
88+
f = fHigh + fLm + f + (v >> 32);
89+
90+
fHigh = (a << 32) + b; //fa
91+
v = (c << 32) + d; //fb
92+
93+
if(fHigh < v)
94+
{
95+
f += (v - fHigh) * P128_BD;
96+
}
97+
else if(fHigh > v)
98+
{
99+
f -= (fHigh - v) * P128_BD;
100+
}
101+
102+
a = f >> 32;
103+
b = (uint)f;
104+
c = fLm >> 32;
105+
d = (uint)fLm;
106+
}
107+
}
108+
109+
low = fLm;
110+
return f;
111+
}
112+
113+
public static ulong GetHash64(string input)
114+
{
115+
ulong hash = OFS64;
116+
117+
unchecked
118+
{
119+
for(int i = 0; i < input.Length; ++i)
120+
{
121+
hash = (hash ^ input[i]) * PRIME64;
122+
}
123+
}
124+
125+
return hash;
126+
}
127+
128+
public static ulong GetHash128Call(string input, out ulong low)
129+
{
130+
uint a = 0x6c62272e, b = 0x07bb0142, c = 0x62b82175, d = 0x6295c58d;
131+
132+
ulong f = 0; low = 0;
133+
unchecked
134+
{
135+
for(int i = 0; i < input.Length; ++i)
136+
{
137+
d ^= input[i];
138+
139+
f = LodgeX4CorrNoHigh.Multiply(a, b, c, d, 0, P128_B, 0, P128_D, out low);
140+
141+
a = (uint)(f >> 32);
142+
b = (uint)f;
143+
c = (uint)(low >> 32);
144+
d = (uint)low;
145+
}
146+
}
147+
148+
return f;
149+
}
150+
}
151+
}

0 commit comments

Comments
 (0)