Skip to content

Commit

Permalink
FNV-1a-128 high-speed implementation using LX4Cnh
Browse files Browse the repository at this point in the history
  • Loading branch information
3F committed Jul 17, 2021
1 parent 116c440 commit ff293eb
Show file tree
Hide file tree
Showing 16 changed files with 575 additions and 5 deletions.
22 changes: 17 additions & 5 deletions algorithms/LodgeX4CorrNoHigh/Readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,27 @@ Copyright (c) 2021 Denis Kuzmin <x-3F@outlook.com> github/3F

Part of https://twitter.com/github3F/status/1403748080760111106

* Part of https://twitter.com/github3F/status/1403748080760111106
### MLnoCS vs LX4Cnh

Algorithm | Maximum bits | One multiplication
----------|--------------|-------------------
LX4Cnh | 128 × 128 | less than ~ **4.3 ns** == 0.0000000043 sec
MLnoCS | 128 × 16 (*<sup>1</sup>32) | less than ~ **0.31 ns** == 0.00000000031 sec
LX4Cnh optimized\*<sup>2</sup> | 128 × 128 | \*<sup>2</sup> less than ~ **0.86 ns** == 0.00000000086 sec

* \*<sup>1</sup> - theoretically up to 128 x 32 with some correction.
* \*<sup>2</sup> - The actual calculation using LX4Cnh can be a bit optimized such for FNV1a-128 implementation (find it in my repo):

[![](fnvOptimization.png)](#)

(**1 ns** == 0.000000001 sec)

## .NET implementation

*LodgeX4CorrNoHigh* class provides several ways of setting and getting numbers by using uint, ulong, or bytes array. Just play with available [Unit-Tests](tests) and [Speed-Tests](https://github.com/3F/sandbox/tree/master/csharp/numbers/BigNum).

[![](benchmark.png)](https://twitter.com/github3F/status/1410358979033813000)

### Examples

For example, using ulong (UInt64)
Expand All @@ -37,8 +52,6 @@ ulong high = LodgeX4CorrNoHigh.Multiply

To reduce the amount of unnecessary stack manipulations (ldloca.s/ldarg.. etc), meet an *embeddable* version.

One 128x128 multiplication requires less than ~ **0.02 ns** == 0.00000000002 sec

```csharp
// 0xC1F4271980F30FED81EF70CCBC6EF2EF
// × 0xDEF03F0142D0ACD21749BEF1EA30FF94
Expand All @@ -47,10 +60,9 @@ uint a = 0xC1F42719, b = 0x80F30FED, c = 0x81EF70CC, d = 0xBC6EF2EF;
uint ma = 0xDEF03F01, mb = 0x42D0ACD2, mc = 0x1749BEF1, md = 0xEA30FF94;
//-
ulong high, low;
unchecked{/*(c) Denis Kuzmin <x-3F@outlook.com> github/3F */ulong A=(ulong)b*mb;ulong B=A&0xFFFF_FFFF;ulong C=((A>>32)+B+(a*ma))&0xFFFF_FFFF;ulong D=(a>b)?a-b:b-a;ulong E=(ma>mb)?ma-mb:mb-ma;if(D!=0&&E!=0){ulong F=D*E;if((!(a>b)&&(ma>mb))||((a>b)&&!(ma>mb))){C+=F&0xFFFF_FFFF;}else{C-=F&0xFFFF_FFFF;}}ulong G=(C<<32)+B;A=(ulong)c*mc;ulong H=(ulong)d*md;B=(H>>32)+(H&0xFFF_FFFF_FFFF_FFFF)+(A&0xFFF_FFFF_FFFF_FFFF)+((A&0xFFF_FFFF)<<32);C=((((A>>28)+(A>>60)+(H>>60))<<28)>>16)+(B>>48);ulong I=B&0xFFFF_FFFF_FFFF;D=(c>d)?c-d:d-c;E=(mc>md)?mc-md:md-mc;if(D!=0&&E!=0){ulong F=D*E;ulong J=(F>>48);ulong K=F&0xFFFF_FFFF_FFFF;B=I;if((!(c>d)&&(mc>md))||((c>d)&&!(mc>md))){I+=K;C+=J;if(B>(I&0xFFFF_FFFF_FFFF))++C;}else{I-=K;C-=J;if(B<(I&0xFFFF_FFFF_FFFF))--C;}}ulong L=((I&0xFFFF_FFFF)<<32)+(H&0xFFFF_FFFF);C=G+L+((C<<16)+((I>>32)&0xFFFF));G=((ulong)a<<32)+b;I=((ulong)c<<32)+d;A=((ulong)ma<<32)+mb;H=((ulong)mc<<32)+md;D=(G>I)?G-I:I-G;E=(A>H)?A-H:H-A;if(D!=0&&E!=0){ulong F=D*E;if((!(G>I)&&(A>H))||((G>I)&&!(A>H))){C+=F;}else{C-=F;}}low=L;high=C;}
unchecked{/*LX4Cnh (c) Denis Kuzmin <x-3F@outlook.com> github/3F */ulong A=(ulong)b*mb;ulong B=A&0xFFFF_FFFF;ulong C=((A>>32)+B+(a*ma))&0xFFFF_FFFF;ulong D=(a>b)?a-b:b-a;ulong E=(ma>mb)?ma-mb:mb-ma;if(D!=0&&E!=0){ulong F=D*E;if(((a<b)&&(ma>mb))||((a>b)&&(ma<mb))){C+=F&0xFFFF_FFFF;}else{C-=F&0xFFFF_FFFF;}}ulong G=(C<<32)+B;A=(ulong)c*mc;ulong H=(ulong)d*md;B=(H>>32)+(H&0xFFF_FFFF_FFFF_FFFF)+(A&0xFFF_FFFF_FFFF_FFFF)+((A&0xFFF_FFFF)<<32);C=(((A>>28)+(A>>60)+(H>>60))<<28);ulong I=B;D=(c>d)?c-d:d-c;E=(mc>md)?mc-md:md-mc;if(D!=0&&E!=0){ulong F=D*E;if(((c<d)&&(mc>md))||((c>d)&&(mc<md))){I+=F;if(B>I)C+=0x100000000;}else{I-=F;if(B<I)C-=0x100000000;}}ulong J=((I&0xFFFF_FFFF)<<32)+(H&0xFFFF_FFFF);C=G+J+C+(I>>32);G=((ulong)a<<32)+b;I=((ulong)c<<32)+d;A=((ulong)ma<<32)+mb;H=((ulong)mc<<32)+md;D=(G>I)?G-I:I-G;E=(A>H)?A-H:H-A;if(D!=0&&E!=0){ulong F=D*E;if(((G<I)&&(A>H))||((G>I)&&(A<H))){C+=F;}else{C-=F;}}low=J;high=C;}

// high low
// ________________|_______________
// = 0x9633C106748CB7D96650F9EA76F0832C
```
(**1 ns** == 0.000000001 sec)
Binary file added algorithms/LodgeX4CorrNoHigh/fnvOptimization.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
64 changes: 64 additions & 0 deletions algorithms/MLnoCS/Readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,67 @@ Copyright (c) 2021 Denis Kuzmin <x-3F@outlook.com> github/3F
[ [](https://3F.github.io/Donation/) ]

Part of https://twitter.com/github3F/status/1403748080760111106

## MLnoCS vs LX4Cnh

Algorithm | Maximum bits | One multiplication
----------|--------------|-------------------
LX4Cnh | 128 × 128 | less than ~ **4.3 ns** == 0.0000000043 sec
MLnoCS | 128 × 16 (*<sup>1</sup>32) | less than ~ **0.31 ns** == 0.00000000031 sec
LX4Cnh optimized\*<sup>2</sup> | 128 × 128 | \*<sup>2</sup> less than ~ **0.86 ns** == 0.00000000086 sec

* \*<sup>1</sup> - theoretically up to 128 x 32 with some correction.
* \*<sup>2</sup> - The actual calculation using LX4Cnh can be a bit optimized such for FNV1a-128 implementation (find it in my repo).

(**1 ns** == 0.000000001 sec)

## .NET implementation

*MulLowNoCorrShifts16* class provides several ways of setting and getting numbers by using uint, ulong, or bytes array. Just play with available [Unit-Tests](tests) and [Speed-Tests](https://github.com/3F/sandbox/tree/master/csharp/numbers/BigNum).

[![](benchmark.png)](https://twitter.com/github3F/status/1410358979033813000)

### Examples

For example, using ulong (UInt64)

```csharp
// 0x4BD4823ECC5D03EB19E07DB8FFD5DABE
// × 0x6D5C
ulong high = MulLowNoCorrShifts16.Multiply
(
0x4BD4823ECC5D03EB, 0x19E07DB8FFD5DABE,
0x6D5C,
out ulong low
);
// high low
// ________________|_______________
// = 0xBBD38B950C188281E234F369FEFD8248
```

### Embeddable superfast version

To reduce the amount of unnecessary stack manipulations (ldloca.s/ldarg.. etc), meet an *embeddable* version.

One 128×16 multiplication using *MLnoCS* requires less than ~ 0.31 ns == 0.00000000031 sec

```csharp
// 0xC1F4271980F30FED81EF70CCBC6EF2EF
// × 0x1036
uint a = 0xC1F42719, b = 0x80F30FED, c = 0x81EF70CC, d = 0xBC6EF2EF;
ushort prime = 0x1036;
//-
uint ra, rb, rc, rd;
unchecked{/* MLnoCS (c) Denis Kuzmin <x-3F@outlook.com> github/3F */ulong e=a,f=b,g=c,h=d,l,o,k;e*=prime;f*=prime;g*=prime;h*=prime;l=(g&0xFFFF_FFFF)+(h>>32);o=(f&0xFFFF_FFFF)+(g>>32);k=(e&0xFFFF_FFFF)+(f>>32);ra=(uint)k;rb=(uint)o;rc=(uint)l;rd=(uint)h;}
// high A | high B | low C | low D
// ______HIGH______|______LOW______
// = 0x2BF1D770644434385F8E96F6AE962E6A
```
(**1 ns** == 0.000000001 sec)

See also *LodgeX4CorrNoHigh* for 128 × 128,

* https://twitter.com/github3F/status/1410669589734768643
* https://twitter.com/github3F/status/1410358979033813000
37 changes: 37 additions & 0 deletions csharp/FNV-1a/FNV-1a.sln
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.31402.337
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FNV-1a", "src\FNV-1a.csproj", "{5A2DDFC2-9402-44FB-BFF6-5071F15615B2}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests", "tests\UnitTest\UnitTest.csproj", "{A93ACA73-DA16-461D-BF06-9941671C7924}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Benchmark", "tests\Benchmark\Benchmark.csproj", "{39EEFDAE-C950-497A-A435-A05175D62800}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{5A2DDFC2-9402-44FB-BFF6-5071F15615B2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5A2DDFC2-9402-44FB-BFF6-5071F15615B2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5A2DDFC2-9402-44FB-BFF6-5071F15615B2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5A2DDFC2-9402-44FB-BFF6-5071F15615B2}.Release|Any CPU.Build.0 = Release|Any CPU
{A93ACA73-DA16-461D-BF06-9941671C7924}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A93ACA73-DA16-461D-BF06-9941671C7924}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A93ACA73-DA16-461D-BF06-9941671C7924}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A93ACA73-DA16-461D-BF06-9941671C7924}.Release|Any CPU.Build.0 = Release|Any CPU
{39EEFDAE-C950-497A-A435-A05175D62800}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{39EEFDAE-C950-497A-A435-A05175D62800}.Debug|Any CPU.Build.0 = Debug|Any CPU
{39EEFDAE-C950-497A-A435-A05175D62800}.Release|Any CPU.ActiveCfg = Release|Any CPU
{39EEFDAE-C950-497A-A435-A05175D62800}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {577340DB-CBCA-4482-8E93-9FFB42035889}
EndGlobalSection
EndGlobal
22 changes: 22 additions & 0 deletions csharp/FNV-1a/LICENSE
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
The MIT License (MIT)

Copyright (c) 2021 Denis Kuzmin <x-3F@outlook.com> github/3F
Copyright (c) sandbox contributors https://github.com/3F/sandbox/graphs/contributors

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
23 changes: 23 additions & 0 deletions csharp/FNV-1a/Readme.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
*FNV-1a* high-speed implementations

```
Copyright (c) 2021 Denis Kuzmin <x-3F@outlook.com> github/3F
```

[ [](https://3F.github.io/Donation/) ]

Part of https://twitter.com/github3F/status/1403748080760111106

## speed

### FNV-1a-128 using LX4Cnh

120 Unicode (UTF-16) characters as follow:

`*LodgeX4CorrNoHigh* (LX4Cnh) algorithm of the high-speed multiplications of **128-bit** numbers (full range, 128 × 128).`

[![](benchmark.120Utf16.table.png)](#)

> 128-bit hash value: 0x8e719ac9080952dec9c90a46279bfcc9
[![](benchmark.120Utf16.png)](#)
Binary file added csharp/FNV-1a/benchmark.120Utf16.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added csharp/FNV-1a/benchmark.120Utf16.table.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
16 changes: 16 additions & 0 deletions csharp/FNV-1a/src/FNV-1a.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFrameworks>net472;netcoreapp2.1</TargetFrameworks>
<RootNamespace>net.r_eg.sandbox.Hash</RootNamespace>
</PropertyGroup>

<PropertyGroup>
<CheckForOverflowUnderflow>false</CheckForOverflowUnderflow>
</PropertyGroup>

<ItemGroup>
<Compile Include="..\..\..\algorithms\LodgeX4CorrNoHigh\src\LodgeX4CorrNoHigh.cs" />
</ItemGroup>

</Project>
151 changes: 151 additions & 0 deletions csharp/FNV-1a/src/FNV1a.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
/*
* The MIT License (MIT)
*
* Copyright (c) 2021 Denis Kuzmin <x-3F@outlook.com> github/3F
* Copyright (c) sandbox contributors https://github.com/3F/sandbox/graphs/contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/

using net.r_eg.sandbox.algorithms;

namespace net.r_eg.sandbox.Hash
{
public static class FNV1a
{
private const ulong OFS64 = 0xcbf29ce484222325;
private const ulong PRIME64 = 0x00000100000001B3;

private const uint P128_B = 0x01000000, P128_D = 0x0000013B, P128_BD = 0xFFFEC5;

public static ulong GetHash128LX4Cnh(string input, out ulong low)
{
ulong a = 0x6c62272e, b = 0x07bb0142, c = 0x62b82175, d = 0x6295c58d;

ulong f = 0, fLm = 0;
unchecked
{
for(int i = 0; i < input.Length; ++i)
{
d ^= input[i];

// Below is an optimized implementation (limited) of the LX4Cnh algorithm specially for Fnv1a128
// (c) Denis Kuzmin <x-3F@outlook.com> github/3F

f = b * P128_B;

ulong v = (uint)f;

f = (f >> 32) + v;

if(a > b)
{
f += (uint)((a - b) * P128_B);
}
else if(a < b)
{
f -= (uint)((b - a) * P128_B);
}

ulong fHigh = (f << 32) + (uint)v;
ulong r2 = d * P128_D;

v = (r2 >> 32) + (r2 & 0xFFF_FFFF_FFFF_FFFF);

f = (r2 & 0xF000_0000_0000_0000) >> 32;

if(c > d)
{
fLm = v;
v += (c - d) * P128_D;
if(fLm > v) f += 0x100000000;
}
else if(c < d)
{
fLm = v;
v -= (d - c) * P128_D;
if(fLm < v) f -= 0x100000000;
}

fLm = (((ulong)(uint)v) << 32) + (uint)r2;

f = fHigh + fLm + f + (v >> 32);

fHigh = (a << 32) + b; //fa
v = (c << 32) + d; //fb

if(fHigh < v)
{
f += (v - fHigh) * P128_BD;
}
else if(fHigh > v)
{
f -= (fHigh - v) * P128_BD;
}

a = f >> 32;
b = (uint)f;
c = fLm >> 32;
d = (uint)fLm;
}
}

low = fLm;
return f;
}

public static ulong GetHash64(string input)
{
ulong hash = OFS64;

unchecked
{
for(int i = 0; i < input.Length; ++i)
{
hash = (hash ^ input[i]) * PRIME64;
}
}

return hash;
}

public static ulong GetHash128Call(string input, out ulong low)
{
uint a = 0x6c62272e, b = 0x07bb0142, c = 0x62b82175, d = 0x6295c58d;

ulong f = 0; low = 0;
unchecked
{
for(int i = 0; i < input.Length; ++i)
{
d ^= input[i];

f = LodgeX4CorrNoHigh.Multiply(a, b, c, d, 0, P128_B, 0, P128_D, out low);

a = (uint)(f >> 32);
b = (uint)f;
c = (uint)(low >> 32);
d = (uint)low;
}
}

return f;
}
}
}
20 changes: 20 additions & 0 deletions csharp/FNV-1a/tests/Benchmark/Benchmark.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFrameworks>net472;net5.0</TargetFrameworks>
<PlatformTarget>AnyCPU</PlatformTarget>
<IsPackable>false</IsPackable>
<RootNamespace>net.r_eg.sandbox.Hash.Tests</RootNamespace>
<NoWarn>1701;1702;IDE0059</NoWarn>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="BenchmarkDotNet" Version="0.13.0" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\..\src\FNV-1a.csproj" />
</ItemGroup>

</Project>
Loading

0 comments on commit ff293eb

Please sign in to comment.