Skip to content

2 new implementations, to process data in parts. largerecs and smallrecs #165

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
May 9, 2024
Merged

2 new implementations, to process data in parts. largerecs and smallrecs #165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b20bfa9
merge optimizations into v.largerec
georges-hatem May 9, 2024
29c1ebc
new implementation that processes data in small parts
georges-hatem May 9, 2024
990d8d6
in parts, but large records to avoid overflow
georges-hatem May 9, 2024
c78e4fc
Merge branch 'gcarreno:main' into main
georges-hatem May 9, 2024
b8ae339
readme improvements
georges-hatem May 9, 2024
c108b4a
Merge branch 'main' of https://github.com/georges-hatem/1brc-ObjectPa…
georges-hatem May 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions entries/ghatem-fpc/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,12 @@
- -t flag to specify the thread-count (default reads the thread-count available on the CPU)

currently there are 2 versions that can be compiled / run:
- `OneBRC.lpr -> ghatem `: compact record, optimal for the 1B row / 41k stations, will fail on the other tests due to overflow
- `OneBRC-largerec.lpr -> ghatem-largerec `: same as OneBRC, but the StationData's "count" is UInt32 instead of 16. Passes all the tests
- `OneBRC.lpr -> ghatem `: compact record, optimal for the 1B row / 41k stations, will fail on the other tests due to overflow
- `OneBRC-largerec.lpr -> ghatem-largerec `: same as OneBRC, but the StationData's "count" is UInt32 instead of 16. Passes all the tests

- `OneBRC-parts.lpr -> ghatem-parts `: compact record, processes in parts as attempt stability, optimal for the 1B row / 41k stations, will fail on the other tests due to overflow
- `OneBRC-parts-largerec.lpr -> ghatem-partslarge`: same as OneBRC-parts, but the StationData's "count" is UInt32 instead of 16. Passes all the tests

## Hardware + Environment
host:
- Dell XPS 15 (9560, 2017)
Expand Down
41 changes: 28 additions & 13 deletions entries/ghatem-fpc/src/OneBRC-largerec.lpr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -459,8 +459,27 @@ procedure TOneBRC.ProcessData (aThreadNb: TThreadCount; aStartIdx: Int64; aEndId
vLineStart := i;

while i < aEndIdx do begin
while FData[i] <> #10 do begin
Inc (I);

// can still skip some chars
if FData[i] > ';' then begin
Inc (I, 5);
end;

// unroll a few seems to be improving?
if FData[i] <> #10 then begin
Inc (i);
if FData[i] <> #10 then begin
Inc (i);
if FData[i] <> #10 then begin
Inc (I);
if FData[i] <> #10 then begin
Inc (i);
while FData[i] <> #10 do begin
Inc (I);
end;
end;
end;
end;
end;

// new line parsed, process its contents
Expand Down Expand Up @@ -557,13 +576,11 @@ function MyFormatInt (const aIn: SmallInt): AnsiString; inline;
//---------------------------------------------------

procedure TOneBRC.GenerateOutput;
var vMean: Integer;
vStream: TStringStream;
I, N: Int64;
var vStream: TStringStream;
I, N: Int32;
vData: PStationData;
vHash: Cardinal;
vStations: TStringList;
iStationName: AnsiString;
vIdx: THashSize;
vRes: Boolean;
begin
Expand All @@ -573,10 +590,11 @@ procedure TOneBRC.GenerateOutput;
vStations.UseLocale := False;
try
vStations.BeginUpdate;
for iStationName in FDictionary.FStationNames do begin
if iStationName <> '' then
vStations.Add(iStationName);
for i := 0 to cNumStations - 1 do begin
if FDictionary.FStationNames[i] <> '' then
vStations.Add (FDictionary.FStationNames[i]);
end;

vStations.EndUpdate;
vStations.CustomSort (@Compare);

Expand All @@ -593,11 +611,9 @@ procedure TOneBRC.GenerateOutput;
FDictionary.InternalFind (vHash, vRes, vIdx);
vData := @FDictionary.FThreadData[0][FDictionary.FIndexes[vIdx]];

vMean := RoundExInteger(vData^.Sum/vData^.Count/10);

vStream.WriteString(
vStations[i] + '=' + MyFormatInt(vData^.Min)
+ '/' + MyFormatInt(vMean)
+ '/' + MyFormatInt(RoundExInteger(vData^.Sum/vData^.Count/10))
+ '/' + MyFormatInt(vData^.Max) + ', '
);
Inc(I);
Expand All @@ -622,7 +638,6 @@ procedure TOneBRC.GenerateOutput;
procedure TBRCThread.Execute;
begin
FProc (FThreadNb, FStart, FEnd);
Terminate;
end;

constructor TBRCThread.Create(aProc: TThreadProc; aThreadNb: TThreadCount; aStart: Int64; aEnd: Int64);
Expand Down
166 changes: 166 additions & 0 deletions entries/ghatem-fpc/src/OneBRC-parts-largerec.lpi
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
<?xml version="1.0" encoding="UTF-8"?>
<CONFIG>
<ProjectOptions>
<Version Value="12"/>
<PathDelim Value="\"/>
<General>
<Flags>
<MainUnitHasCreateFormStatements Value="False"/>
<MainUnitHasTitleStatement Value="False"/>
<MainUnitHasScaledStatement Value="False"/>
<CompatibilityMode Value="True"/>
</Flags>
<SessionStorage Value="InProjectDir"/>
<Title Value="1 BRC"/>
<UseAppBundle Value="False"/>
<ResourceType Value="res"/>
</General>
<BuildModes Count="4">
<Item1 Name="Default" Default="True"/>
<Item2 Name="Debug">
<CompilerOptions>
<Version Value="11"/>
<PathDelim Value="\"/>
<Target>
<Filename Value="..\..\..\bin\ghatem-partslarge"/>
</Target>
<SearchPaths>
<IncludeFiles Value="$(ProjOutDir)"/>
<UnitOutputDirectory Value="..\..\..\bin\lib\$(TargetCPU)-$(TargetOS)"/>
</SearchPaths>
<Parsing>
<SyntaxOptions>
<IncludeAssertionCode Value="True"/>
</SyntaxOptions>
</Parsing>
<CodeGeneration>
<Checks>
<IOChecks Value="True"/>
<RangeChecks Value="True"/>
<OverflowChecks Value="True"/>
<StackChecks Value="True"/>
</Checks>
<VerifyObjMethodCallValidity Value="True"/>
</CodeGeneration>
<Linking>
<Debugging>
<DebugInfoType Value="dsDwarf3"/>
<TrashVariables Value="True"/>
<UseValgrind Value="True"/>
<UseExternalDbgSyms Value="True"/>
</Debugging>
</Linking>
<Other>
<CustomOptions Value="-dDEBUG"/>
</Other>
</CompilerOptions>
</Item2>
<Item3 Name="Release">
<CompilerOptions>
<Version Value="11"/>
<PathDelim Value="\"/>
<Target>
<Filename Value="..\..\..\bin\ghatem-partslarge"/>
</Target>
<SearchPaths>
<IncludeFiles Value="$(ProjOutDir)"/>
<UnitOutputDirectory Value="..\..\..\bin\lib\$(TargetCPU)-$(TargetOS)"/>
</SearchPaths>
<CodeGeneration>
<SmartLinkUnit Value="True"/>
<Optimizations>
<OptimizationLevel Value="3"/>
</Optimizations>
</CodeGeneration>
<Linking>
<Debugging>
<GenerateDebugInfo Value="False"/>
<RunWithoutDebug Value="True"/>
</Debugging>
<LinkSmart Value="True"/>
</Linking>
<Other>
<CustomOptions Value="-dRELEASE"/>
</Other>
</CompilerOptions>
</Item3>
<Item4 Name="Valgrind">
<CompilerOptions>
<Version Value="11"/>
<PathDelim Value="\"/>
<Target>
<Filename Value="..\..\..\bin\ghatem-partslarge"/>
</Target>
<SearchPaths>
<IncludeFiles Value="$(ProjOutDir)"/>
<UnitOutputDirectory Value="..\..\..\bin\lib\$(TargetCPU)-$(TargetOS)"/>
</SearchPaths>
<Linking>
<Debugging>
<DebugInfoType Value="dsDwarf3"/>
<TrashVariables Value="True"/>
<UseValgrind Value="True"/>
<UseExternalDbgSyms Value="True"/>
</Debugging>
</Linking>
<Other>
<CustomOptions Value="-dDEBUG"/>
</Other>
</CompilerOptions>
</Item4>
</BuildModes>
<PublishOptions>
<Version Value="2"/>
<UseFileFilters Value="True"/>
</PublishOptions>
<RunParams>
<FormatVersion Value="2"/>
</RunParams>
<RequiredPackages Count="2">
<Item1>
<PackageName Value="mormot2"/>
</Item1>
<Item2>
<PackageName Value="LCLBase"/>
</Item2>
</RequiredPackages>
<Units Count="1">
<Unit0>
<Filename Value="OneBRC-parts-largerec.lpr"/>
<IsPartOfProject Value="True"/>
</Unit0>
</Units>
</ProjectOptions>
<CompilerOptions>
<Version Value="11"/>
<PathDelim Value="\"/>
<Target>
<Filename Value="..\..\..\bin\ghatem-partslarge"/>
</Target>
<SearchPaths>
<IncludeFiles Value="$(ProjOutDir)"/>
<UnitOutputDirectory Value="..\..\..\bin\lib\$(TargetCPU)-$(TargetOS)"/>
</SearchPaths>
<Linking>
<Debugging>
<DebugInfoType Value="dsDwarf3"/>
</Debugging>
</Linking>
<Other>
<CustomOptions Value="-dDEBUG"/>
</Other>
</CompilerOptions>
<Debugging>
<Exceptions Count="3">
<Item1>
<Name Value="EAbort"/>
</Item1>
<Item2>
<Name Value="ECodetoolError"/>
</Item2>
<Item3>
<Name Value="EFOpenError"/>
</Item3>
</Exceptions>
</Debugging>
</CONFIG>
Loading