diff --git a/entries/bfire/src/MultiThreadUnit.pas b/entries/bfire/src/MultiThreadUnit.pas index 73e7d21..8a3ae89 100644 --- a/entries/bfire/src/MultiThreadUnit.pas +++ b/entries/bfire/src/MultiThreadUnit.pas @@ -8,17 +8,18 @@ interface System.SyncObjs, System.Classes, System.SysUtils, - System.StrUtils, - System.Generics.Collections; + System.StrUtils; type // size entry_array to hold longest (by byte count) name // Dolores Hidalgo Cuna de la Independencia Nacional = 98 bytes // with temperature as large as: -999.9 (maybe just -99.9 ?) // so, add a bit, and use... - // entry_array: array [0 .. 127] of Byte; // will hold place name and temp + // entry_array: array [0 .. 107] of Byte; // will hold place name and temp + // note shortest seems to be: Xyz;0.0 -> 8 bytes TRawRecord = record - RawData: array [0 .. 127] of Byte; // bytes for each entry line + RawData: array [0 .. 107] of Byte; + // bytes for each entry line // 105 works end; THashRecord = record // each array item is compiled data for a place @@ -35,73 +36,95 @@ THashRecord = record // each array item is compiled data for a place // see http://compoasso.free.fr/primelistweb/page/prime/liste_online_en.php Prime: Integer = 18701; - ReadBufferSize: Integer = 32768; // 16384 OK; 32768 seems optimum + ReadBufferSize: Integer = 32768; // 16384 is OK, 32768 seems optimum-ish + ReadBufferMargin: Integer = 128; // was 128 + // reduce requested number of bytes by this much when reading - // DataBufferSize (combined) max is about 100000000 for 10 GB free - DataBufferSize1: Integer = 1000001; - DataBufferSize2: Integer = 1000001; - DataBufferSize3: Integer = 1000001; - DataBufferSize4: Integer = 1000001; - DataBufferCushion: Integer = 10000; + StackCount: Integer = 5; // how many groups to split up the alphabet - // split points for stacks - // ABCDE FGHIJK LMNOPQR STUVWXYZ (and extended) - SplitPoint1: Byte = 68; // letter "D" - SplitPoint2: Byte = 76; // letter "L" - SplitPoint3: Byte = 82; // letter "R" + // near max on my PC memory, roughly 10 GB for 5 buffers at 3.5 million items each + + DataBufferSize1: Integer = 3500001; + DataBufferSize2: Integer = 3500001; + DataBufferSize3: Integer = 3500001; + DataBufferSize4: Integer = 3500001; + DataBufferSize5: Integer = 3500001; + DataBufferCushion: Integer = 10; // beyound this point, sleep happens + + // split points for stacks, using first three characters + // so, ignoring Unicode, Aaa through Zzz + // decimal 65/97/97 through 90/122/122 + // define choice as sum of three byte values minus 258 + // that makes range 1 to 76 + SplitPoint1: Integer = 21; + SplitPoint2: Integer = 29; + SplitPoint3: Integer = 35; + SplitPoint4: Integer = 44; var inputFilename: String; outputFilename: String; - iBytesRead: Integer; Challenge: TFileStream; - ReadBuffer: PByte; // read file into this buffer + + UseStdOut: Boolean; // True unless output file is defined // set up for 41351 places (actually seems to be 41343 entries) - // split into four roughly equal stack to ho to separate hash tables + // split into five roughly equal groups for separate hash tables + HashRecordMap1: array [0 .. 18700] of THashRecord; // for Prime := 18701 HashRecordMap2: array [0 .. 18700] of THashRecord; // for Prime := 18701 HashRecordMap3: array [0 .. 18700] of THashRecord; // for Prime := 18701 HashRecordMap4: array [0 .. 18700] of THashRecord; // for Prime := 18701 + HashRecordMap5: array [0 .. 18700] of THashRecord; // for Prime := 18701 - // DataBufferSize: Integer = 1000001; - // I think this fits in about 4 GB - DataStack1: array [0 .. 1000000] of TRawRecord; - DataStack2: array [0 .. 1000000] of TRawRecord; - DataStack3: array [0 .. 1000000] of TRawRecord; - DataStack4: array [0 .. 1000000] of TRawRecord; + DataStack1: array [0 .. 3500000] of TRawRecord; + DataStack2: array [0 .. 3500000] of TRawRecord; + DataStack3: array [0 .. 3500000] of TRawRecord; + DataStack4: array [0 .. 3500000] of TRawRecord; + DataStack5: array [0 .. 3500000] of TRawRecord; DataStackCount1: Integer; DataStackCount2: Integer; DataStackCount3: Integer; DataStackCount4: Integer; + DataStackCount5: Integer; DataStackLock1: TCriticalSection; DataStackLock2: TCriticalSection; DataStackLock3: TCriticalSection; DataStackLock4: TCriticalSection; + DataStackLock5: TCriticalSection; + + HashRecordLock1: TCriticalSection; + HashRecordLock2: TCriticalSection; + HashRecordLock3: TCriticalSection; + HashRecordLock4: TCriticalSection; + HashRecordLock5: TCriticalSection; + + ReadLock: TCriticalSection; // controls reading file StationsForSort: TStringList; LineCount: Int64; - ReadFile_Done: Boolean; - ParseDataQ_Done1: Boolean; - ParseDataQ_Done2: Boolean; - ParseDataQ_Done3: Boolean; - ParseDataQ_Done4: Boolean; + ReadFile_Done1: Boolean; + ReadFile_Done2: Boolean; + ParseData_Done: Boolean; StackMax1: Integer; // sample stack count for peak value (approx.) StackMax2: Integer; StackMax3: Integer; StackMax4: Integer; + StackMax5: Integer; procedure FileToArrays(inputFilename: String; UseStdOut: Boolean); // read -procedure LaunchReadingThread(inputFilename: String); +procedure LaunchReadingThread1; +procedure LaunchReadingThread2; procedure LaunchTabulateThread1; procedure LaunchTabulateThread2; procedure LaunchTabulateThread3; procedure LaunchTabulateThread4; +procedure LaunchTabulateThread5; procedure SortArrays; procedure ArrayToFile(outFile: String; UseStdOut: Boolean); @@ -192,7 +215,6 @@ procedure SortArrays; inline; StationsForSort.Sorted := False; // sort later // load strings from HashRecordMaps with the corresponding indexes - for i := 0 to 18700 do // for Prime := 18701 begin if HashRecordMap1[i].DataCount > 0 then // add @@ -214,6 +236,11 @@ procedure SortArrays; inline; begin StationsForSort.AddObject(HashRecordMap4[i].DataName, @HashRecordMap4[i]); end; + + if HashRecordMap5[i].DataCount > 0 then // add + begin + StationsForSort.AddObject(HashRecordMap5[i].DataName, @HashRecordMap5[i]); + end; end; StationsForSort.Sort; @@ -271,7 +298,7 @@ procedure ArrayToFile(outFile: String; UseStdOut: Boolean); end; bufferStr := '}' + Chr(10); // linefeed appears at end of baseline file - if UseStdOut then // send to STDOUT, where it gets mangled + if UseStdOut then // send to STDOUT, where it gets mangled in Windows begin write(bufferStr); end @@ -298,29 +325,38 @@ procedure FileToArrays(inputFilename: String; UseStdOut: Boolean); i: Integer; begin - ReadBuffer := System.AllocMem(ReadBufferSize); - DataStackCount1 := 0; DataStackCount2 := 0; DataStackCount3 := 0; DataStackCount4 := 0; + DataStackCount5 := 0; // initialize a few things - ReadFile_Done := False; - ParseDataQ_Done1 := False; - ParseDataQ_Done2 := False; - ParseDataQ_Done3 := False; - ParseDataQ_Done4 := False; + ReadFile_Done1 := False; + ReadFile_Done2 := False; + ParseData_Done := False; DataStackLock1 := TCriticalSection.Create; DataStackLock2 := TCriticalSection.Create; DataStackLock3 := TCriticalSection.Create; DataStackLock4 := TCriticalSection.Create; + DataStackLock5 := TCriticalSection.Create; + + HashRecordLock1 := TCriticalSection.Create; + HashRecordLock2 := TCriticalSection.Create; + HashRecordLock3 := TCriticalSection.Create; + HashRecordLock4 := TCriticalSection.Create; + HashRecordLock5 := TCriticalSection.Create; + + ReadLock := TCriticalSection.Create; StackMax1 := 0; StackMax2 := 0; StackMax3 := 0; StackMax4 := 0; + StackMax5 := 0; + + LineCount := 0; // pre-fill arrays for i := 0 to Prime - 1 do @@ -329,6 +365,7 @@ procedure FileToArrays(inputFilename: String; UseStdOut: Boolean); HashRecordMap2[i].DataCount := 0; HashRecordMap3[i].DataCount := 0; HashRecordMap4[i].DataCount := 0; + HashRecordMap5[i].DataCount := 0; end; // setup TStringList for sorting @@ -348,7 +385,14 @@ procedure FileToArrays(inputFilename: String; UseStdOut: Boolean); LaunchTabulateThread2; LaunchTabulateThread3; LaunchTabulateThread4; - LaunchReadingThread(inputFilename); + LaunchTabulateThread5; + + Challenge := TFileStream.Create(inputFilename, fmOpenRead, fmShareDenyNone); + // Challenge := TBufferedFileStream.Create(inputFilename, fmOpenRead, fmShareDenyNone); + + LaunchReadingThread1; + LaunchReadingThread2; + end else begin @@ -357,196 +401,638 @@ procedure FileToArrays(inputFilename: String; UseStdOut: Boolean); end; -procedure LaunchReadingThread(inputFilename: String); +procedure LaunchReadingThread1; var - ReadingThread: TThread; - DataStackItem: TRawRecord; + ReadingThread1: TThread; begin - ReadingThread := TThread.CreateAnonymousThread( + ReadingThread1 := TThread.CreateAnonymousThread( procedure var + ReadItem1: array [0 .. 107] of Byte; BufferIndex: Integer; // index into read buffer DataIndex: Integer; // index into raw data array - Choice: Byte; - TimeOut: Integer; + Choice: Integer; + BytesRead: Integer; + SafetyCount: Integer; + PReadBuffer1: PByte; + Posted: Boolean; // flag for successfully sending to queue begin + PReadBuffer1 := System.AllocMem(ReadBufferSize); + BytesRead := 1; // some value > 0 + while BytesRead > 0 do + begin + ReadLock.Acquire; + BytesRead := Challenge.ReadData(PReadBuffer1, + ReadBufferSize - ReadBufferMargin); - try - Challenge := TFileStream.Create(inputFilename, fmOpenRead); - Challenge.Seek(0, soFromBeginning); - LineCount := 0; - DataIndex := 0; + if BytesRead < 8 then + // problem, or just end of file? ================ + begin + if Not(UseStdOut) then + begin + WriteLn('EOF 1: ' + IntToStr(BytesRead)); + end; + ReadLock.Release; + Break; + end; - iBytesRead := Challenge.Read(ReadBuffer^, ReadBufferSize); - while iBytesRead > 0 do + if Not((PReadBuffer1 + BytesRead - 1)^ = 10) then + // did not get LF at end begin - BufferIndex := 0; - while BufferIndex < iBytesRead do + // read a few more bytes until we get a line feed + for SafetyCount := 1 to ReadBufferMargin do + begin + if Challenge.ReadData((PReadBuffer1 + BytesRead)^, 1) <= 0 then + // EOF? + begin + Break; + end + else + begin + inc(BytesRead); + if ((PReadBuffer1 + BytesRead - 1)^ = 10) then // done + begin + Break; + end; + end; + end; + if SafetyCount = ReadBufferMargin then // problem begin - if ((ReadBuffer + BufferIndex)^ = 10) then // line feed + if Not(UseStdOut) then begin - // done collecting bytes, tack on carriage return as data end flag - DataStackItem.RawData[DataIndex] := 13; + WriteLn('Need bigger ReadBufferMargin'); + end; + end; + end; + + ReadLock.Release; + + DataIndex := 0; + + BufferIndex := 0; + while BufferIndex < BytesRead do + begin + if ((PReadBuffer1 + BufferIndex)^ = 10) then // line feed + begin + // done collecting bytes, tack on null as data end flag + ReadItem1[DataIndex] := 0; - // send to the appropriate stack - // ABCD EFGHIJKL MNOPQR STUVWXYZ (and extended) - // SplitPoint1: Byte = 68; // letter "D" - // SplitPoint2: Byte = 76; // letter "L" - // SplitPoint3: Byte = 82; // letter "R" + Posted := False; - Choice := DataStackItem.RawData[0]; + // try for an even distribution to the five stacks + // define choice as sum of two byte values minus 161 + // Choice := ReadItem1[0] + ReadItem1[1] - 161; + // define choice as sum of three byte values minus 258 + Choice := ReadItem1[0] + ReadItem1[1] + ReadItem1[2] - 258; - if Choice <= SplitPoint2 then // A thru K + if Choice <= SplitPoint2 then + begin + if Choice <= SplitPoint1 then + begin + Choice := 0; + end + else + begin + Choice := 1; + end; + end + else + begin + if Choice <= SplitPoint3 then begin - if Choice <= SplitPoint1 then // A thru E + Choice := 2; + end + else + begin + if Choice <= SplitPoint4 then + begin + Choice := 3; + end + else + begin + Choice := 4; + end; + end; + end; + + case Choice of + 0: begin while True do // break on success begin - DataStackLock1.Acquire; - if (DataStackCount1 < DataBufferSize1 - 1) then - // there is room + if (DataStackCount1 < DataBufferSize1 - DataBufferCushion) + then begin - DataStack1[DataStackCount1] := DataStackItem; - inc(DataStackCount1); - DataStackLock1.Release; - Break; + // lock and attempt to store the data + while Not Posted do + begin + DataStackLock1.Acquire; + if (DataStackCount1 < DataBufferSize1 - 2) then + begin + Move(ReadItem1[0], DataStack1[DataStackCount1].RawData + [0], SizeOf(ReadItem1)); + inc(DataStackCount1); + DataStackLock1.Release; + Posted := True; + end + else + begin + DataStackLock1.Release; + Sleep(1); // stack is full + end; + end; + If Posted then + Break; end - else // need to wait, hope for room later + else begin - DataStackLock1.Release; - while (DataStackCount1 > DataBufferSize1 - - DataBufferCushion) do + Sleep(1); // stack is full + end; + end; + end; + 1: + begin + while True do // break on success + begin + if (DataStackCount2 < DataBufferSize2 - DataBufferCushion) + then + begin + // lock and attempt to store the data + while Not Posted do begin - // nothing real -// TimeOut := DataStackCount1 mod 16001; -// TimeOut := DataStackCount1 +1; + DataStackLock2.Acquire; + if (DataStackCount2 < DataBufferSize2 - 2) then + begin + Move(ReadItem1[0], DataStack2[DataStackCount2].RawData + [0], SizeOf(ReadItem1)); + inc(DataStackCount2); + DataStackLock2.Release; + Posted := True; + end + else + begin + DataStackLock2.Release; + Sleep(1); // stack is full + end; end; + If Posted then + Break; + end + else + begin + Sleep(1); // stack is full end; end; - end - else // F thru K + end; + 2: + begin + while True do // break on success + begin + if (DataStackCount3 < DataBufferSize3 - DataBufferCushion) + then + begin + // lock and attempt to store the data + while Not Posted do + begin + DataStackLock3.Acquire; + if (DataStackCount3 < DataBufferSize3 - 2) then + begin + Move(ReadItem1[0], DataStack3[DataStackCount3].RawData + [0], SizeOf(ReadItem1)); + inc(DataStackCount3); + DataStackLock3.Release; + Posted := True; + end + else + begin + DataStackLock3.Release; + Sleep(1); // stack is full + end; + end; + If Posted then + Break; + end + else + begin + Sleep(1); // stack is full + end; + end; + end; + 3: begin while True do // break on success begin - DataStackLock2.Acquire; - if (DataStackCount2 < DataBufferSize2 - 1) then - // there is room + if (DataStackCount4 < DataBufferSize4 - DataBufferCushion) + then begin - DataStack2[DataStackCount2] := DataStackItem; - inc(DataStackCount2); - DataStackLock2.Release; - Break; + // lock and attempt to store the data + while Not Posted do + begin + DataStackLock4.Acquire; + if (DataStackCount4 < DataBufferSize4 - 2) then + begin + Move(ReadItem1[0], DataStack4[DataStackCount4].RawData + [0], SizeOf(ReadItem1)); + inc(DataStackCount4); + DataStackLock4.Release; + Posted := True; + end + else + begin + DataStackLock4.Release; + Sleep(1); // stack is full + end; + end; + If Posted then + Break; end - else // need to wait, hope for room later + else + begin + Sleep(1); // stack is full + end; + end; + end; + 4: + begin + while True do // break on success + begin + if (DataStackCount5 < DataBufferSize5 - DataBufferCushion) + then begin - DataStackLock2.Release; - while (DataStackCount2 > DataBufferSize2 - - DataBufferCushion) do + // lock and attempt to store the data + while Not Posted do begin - // nothing real -// TimeOut := DataStackCount2 mod 16001; -// TimeOut := DataStackCount2 +1; + DataStackLock5.Acquire; + if (DataStackCount5 < DataBufferSize5 - 2) then + begin + Move(ReadItem1[0], DataStack5[DataStackCount5].RawData + [0], SizeOf(ReadItem1)); + inc(DataStackCount5); + DataStackLock5.Release; + Posted := True; + end + else + begin + DataStackLock5.Release; + Sleep(1); // stack is full + end; end; + If Posted then + Break; + end + else + begin + Sleep(1); // stack is full end; end; end; + end; + + // done processing, reset for next line + inc(LineCount); + DataIndex := 0; + end + else // accumulate bytes + begin + ReadItem1[DataIndex] := (PReadBuffer1 + BufferIndex)^; + inc(DataIndex); + end; + inc(BufferIndex); + + end; // of: while i < iBytesRead do + + end; // of while iBytesRead > 0 do --- implies end of file + + ReadFile_Done1 := True; + + end); + ReadingThread1.Start; +end; + +procedure LaunchReadingThread2; +var + ReadingThread2: TThread; + +begin + ReadingThread2 := TThread.CreateAnonymousThread( + procedure + var + ReadItem2: array [0 .. 107] of Byte; + BufferIndex: Integer; // index into read buffer + DataIndex: Integer; // index into raw data array + Choice: Integer; + BytesRead: Integer; + SafetyCount: Integer; + PReadBuffer2: PByte; + Posted: Boolean; // flag for successfully sending to queue + + begin + PReadBuffer2 := System.AllocMem(ReadBufferSize); + BytesRead := 1; // some value > 0 + while BytesRead > 0 do + begin + ReadLock.Acquire; + BytesRead := Challenge.ReadData(PReadBuffer2, + ReadBufferSize - ReadBufferMargin); + + if BytesRead < 8 then + // problem, or just end of file? ================ + begin + if Not(UseStdOut) then + begin + WriteLn('EOF 2: ' + IntToStr(BytesRead)); + end; + ReadLock.Release; + Break; + end; + + if Not((PReadBuffer2 + BytesRead - 1)^ = 10) then + // did not get LF at end + begin + // read a few more bytes until we get a line feed + for SafetyCount := 1 to ReadBufferMargin do + begin + if Challenge.ReadData((PReadBuffer2 + BytesRead)^, 1) <= 0 then + // EOF? + begin + Break; + end + else + begin + inc(BytesRead); + if ((PReadBuffer2 + BytesRead - 1)^ = 10) then // done + begin + Break; + end; + end; + end; + if SafetyCount = ReadBufferMargin then // problem + begin + if Not(UseStdOut) then + begin + WriteLn('Need bigger ReadBufferMargin'); + end; + end; + end; + + ReadLock.Release; + + DataIndex := 0; + + BufferIndex := 0; + while BufferIndex < BytesRead do + begin + if ((PReadBuffer2 + BufferIndex)^ = 10) then // line feed + begin + // done collecting bytes, tack on null as data end flag + // DataStackItem.RawData[DataIndex] := 0; + ReadItem2[DataIndex] := 0; + + Posted := False; + + // try for an even distribution to the five stacks + // define choice as sum of two byte values minus 161 + // Choice := ReadItem1[0] + ReadItem1[1] - 161; + // define choice as sum of three byte values minus 258 + Choice := ReadItem2[0] + ReadItem2[1] + ReadItem2[2] - 258; + + if Choice <= SplitPoint2 then + begin + if Choice <= SplitPoint1 then + begin + Choice := 0; end - else // K and beyond + else + begin + Choice := 1; + end; + end + else + begin + if Choice <= SplitPoint3 then begin - if Choice <= SplitPoint3 then // L thru R + Choice := 2; + end + else + begin + if Choice <= SplitPoint4 then + begin + Choice := 3; + end + else + begin + Choice := 4; + end; + end; + end; + case Choice of + 0: begin while True do // break on success begin - DataStackLock3.Acquire; - if (DataStackCount3 < DataBufferSize3 - 1) then - // there is room + if (DataStackCount1 < DataBufferSize1 - DataBufferCushion) + then begin - DataStack3[DataStackCount3] := DataStackItem; - inc(DataStackCount3); - DataStackLock3.Release; - Break; + // lock and attempt to store the data + while Not Posted do + begin + DataStackLock1.Acquire; + if (DataStackCount1 < DataBufferSize1 - 2) then + begin + Move(ReadItem2[0], DataStack1[DataStackCount1].RawData + [0], SizeOf(ReadItem2)); + inc(DataStackCount1); + DataStackLock1.Release; + Posted := True; + end + else + begin + DataStackLock1.Release; + Sleep(1); // stack is full + end; + end; + If Posted then + Break; end - else // need to wait, hope for room later + else + begin + Sleep(1); // stack is full + end; + end; + end; + 1: + begin + while True do // break on success + begin + if (DataStackCount2 < DataBufferSize2 - DataBufferCushion) + then begin - DataStackLock3.Release; - while (DataStackCount3 > DataBufferSize3 - - DataBufferCushion) do + // lock and attempt to store the data + while Not Posted do begin - // nothing real -// TimeOut := DataStackCount3 mod 16001; -// TimeOut := DataStackCount3 +1; + DataStackLock2.Acquire; + if (DataStackCount2 < DataBufferSize2 - 2) then + begin + Move(ReadItem2[0], DataStack2[DataStackCount2].RawData + [0], SizeOf(ReadItem2)); + inc(DataStackCount2); + DataStackLock2.Release; + Posted := True; + end + else + begin + DataStackLock2.Release; + Sleep(1); // stack is full + end; end; + If Posted then + Break; + end + else + begin + Sleep(1); // stack is full end; end; - end - else // S and beyond + end; + 2: + begin + while True do // break on success + begin + if (DataStackCount3 < DataBufferSize3 - DataBufferCushion) + then + begin + // lock and attempt to store the data + while Not Posted do + begin + DataStackLock3.Acquire; + if (DataStackCount3 < DataBufferSize3 - 2) then + begin + Move(ReadItem2[0], DataStack3[DataStackCount3].RawData + [0], SizeOf(ReadItem2)); + inc(DataStackCount3); + DataStackLock3.Release; + Posted := True; + end + else + begin + DataStackLock3.Release; + Sleep(1); // stack is full + end; + end; + If Posted then + Break; + end + else + begin + Sleep(1); // stack is full + end; + end; + end; + 3: begin while True do // break on success begin - DataStackLock4.Acquire; - if (DataStackCount4 < DataBufferSize4 - 1) then - // there is room + if (DataStackCount4 < DataBufferSize4 - DataBufferCushion) + then begin - DataStack4[DataStackCount4] := DataStackItem; - inc(DataStackCount4); - DataStackLock4.Release; - Break; + // lock and attempt to store the data + while Not Posted do + begin + DataStackLock4.Acquire; + if (DataStackCount4 < DataBufferSize4 - 2) then + begin + Move(ReadItem2[0], DataStack4[DataStackCount4].RawData + [0], SizeOf(ReadItem2)); + inc(DataStackCount4); + DataStackLock4.Release; + Posted := True; + end + else + begin + DataStackLock4.Release; + Sleep(1); // stack is full + end; + end; + If Posted then + Break; end - else // need to wait, hope for room later + else + begin + Sleep(1); // stack is full + end; + end; + end; + 4: + begin + while True do // break on success + begin + if (DataStackCount5 < DataBufferSize5 - DataBufferCushion) + then begin - DataStackLock4.Release; - while (DataStackCount4 > DataBufferSize4 - - DataBufferCushion) do + // lock and attempt to store the data + while Not Posted do begin - // nothing real -// TimeOut := DataStackCount4 mod 16001; -// TimeOut := DataStackCount4 +1; + DataStackLock5.Acquire; + if (DataStackCount5 < DataBufferSize5 - 2) then + begin + Move(ReadItem2[0], DataStack5[DataStackCount5].RawData + [0], SizeOf(ReadItem2)); + inc(DataStackCount5); + DataStackLock5.Release; + Posted := True; + end + else + begin + DataStackLock5.Release; + Sleep(1); // stack is full + end; end; + If Posted then + Break; + end + else + begin + Sleep(1); // stack is full end; end; end; - end; - - // done processing, reset for next line - inc(LineCount); - DataIndex := 0; - end - else // accumulate bytes - begin - DataStackItem.RawData[DataIndex] := (ReadBuffer + BufferIndex)^; - inc(DataIndex); end; - inc(BufferIndex); - end; // of: while i < iBytesRead do - iBytesRead := Challenge.Read(ReadBuffer^, ReadBufferSize); + // done processing, reset for next line + inc(LineCount); + DataIndex := 0; + end + else // accumulate bytes + begin + ReadItem2[DataIndex] := (PReadBuffer2 + BufferIndex)^; + inc(DataIndex); + end; + inc(BufferIndex); + + end; // of: while i < iBytesRead do - end; // of while iBytesRead > 0 do --- implies end of file + end; // of while iBytesRead > 0 do --- implies end of file - ReadFile_Done := True; + ReadFile_Done2 := True; - finally - Challenge.Free; - // nothing here - end; end); - ReadingThread.Start; + ReadingThread2.Start; end; procedure LaunchTabulateThread1; var TabulateThread1: TThread; - DataStackItem1: TRawRecord; begin TabulateThread1 := TThread.CreateAnonymousThread( procedure var + DataStackItem: array [0 .. 107] of Byte; DataIndex: Integer; // index into raw data array SC: Boolean; // True after finding semi-colon - entry_name_array: array [0 .. 127] of Byte; // will hold place name + entry_name_array: array [0 .. 107] of Byte; // will hold place name entry_temp_array: array [0 .. 7] of Byte; // will hold place temperature entry_name_array_length: Integer; entry_temp_array_length: Integer; @@ -558,50 +1044,64 @@ procedure LaunchTabulateThread1; G: Cardinal; Hash: Cardinal; BTT: Integer; // temporary for bytes to temperature conversion - TimeOut: Integer; + HaveData: Boolean; begin - while True do + while True do // main loop begin - - // get item from stack - while True do // break on success + while True do // try for data from this stack begin - DataStackLock1.Acquire; - if (DataStackCount1 > 0) then // there is data + // does this stack have data + HaveData := False; + if (DataStackCount1 = 0) then begin - DataStackItem1 := DataStack1[DataStackCount1 - 1]; - dec(DataStackCount1); - DataStackLock1.Release; - Break; + // check to see if we are done with all stacks + if (DataStackCount1 + DataStackCount2 + DataStackCount3 + + DataStackCount4 + DataStackCount5 = 0) then + // might be done + begin + if (ReadFile_Done1 and ReadFile_Done2) then + // recheck to verify no more data + begin + // wait a moment to be sure + Sleep(1); + if (DataStackCount1 + DataStackCount2 + DataStackCount3 + + DataStackCount4 + DataStackCount5 = 0) then + begin + HaveData := False; + ParseData_Done := True; + Break; + end; + end; // of: if (ReadFile_Done1 and ReadFile_Done2) then + end; end - else + else // lock and check again begin - if ReadFile_Done then // no more data + DataStackLock1.Acquire; + if (DataStackCount1 > 0) then // there is data begin - ParseDataQ_Done1 := True; + Move(DataStack1[DataStackCount1 - 1].RawData[0], DataStackItem[0], + SizeOf(DataStackItem)); + dec(DataStackCount1); DataStackLock1.Release; + HaveData := True; Break; end; DataStackLock1.Release; - // nothing real - TimeOut := DataStackCount1 mod 16001; end; - end; - if ParseDataQ_Done1 then // escape - begin - Break; - end - else // tabulate + end; // of: while True do // try for data from this stack + + if HaveData then begin // split raw data into name and temperature SC := False; entry_name_array_length := 0; entry_temp_array_length := 0; - for DataIndex := 0 to SizeOf(DataStackItem1.RawData) - 1 do + DataIndex := 0; + while DataStackItem[DataIndex] <> 0 do begin - if (DataStackItem1.RawData[DataIndex] = 59) then + if (DataStackItem[DataIndex] = 59) then begin SC := True; end @@ -610,28 +1110,21 @@ procedure LaunchTabulateThread1; if SC then begin // skip decimal in number - if Not(DataStackItem1.RawData[DataIndex] = 46) then + if Not(DataStackItem[DataIndex] = 46) then begin - // skip trailing CR - if (DataStackItem1.RawData[DataIndex] = 13) then // done - begin - Break; - end - else - begin - entry_temp_array[entry_temp_array_length] := - DataStackItem1.RawData[DataIndex]; - inc(entry_temp_array_length); - end; + entry_temp_array[entry_temp_array_length] := + DataStackItem[DataIndex]; + inc(entry_temp_array_length); end; end else // don't skip period in name begin entry_name_array[entry_name_array_length] := - DataStackItem1.RawData[DataIndex]; + DataStackItem[DataIndex]; inc(entry_name_array_length); end; end; + inc(DataIndex); end; // convert characters in byte array to temperature @@ -665,26 +1158,21 @@ procedure LaunchTabulateThread1; entry_integer := BTT; - // convert name bytes to Unicode - // entry_name_array_length := Length(entry_name_array); - SetLength(DigestedString, entry_name_array_length); - for entry_array_index := 0 to entry_name_array_length - 1 do - begin - DigestedString[entry_array_index] := entry_name_array - [entry_array_index]; - end; - - entry_Unicode := TEncoding.UTF8.GetString(DigestedString); - // hash name bytes, adapted from: (* ****************************************************************** *) (* Tomes of Delphi: Algorithms and Data Structures *) (* ------------------------------------------------------------------ *) (* function TDPJWHash *) (* ****************************************************************** *) + // combine this with loop to setup for Unicode + SetLength(DigestedString, entry_name_array_length); + Hash := 0; for entry_array_index := 0 to entry_name_array_length - 1 do begin + DigestedString[entry_array_index] := entry_name_array + [entry_array_index]; + Hash := (Hash shl 4) + entry_name_array[entry_array_index]; G := Hash and $F000000; if (G <> 0) then @@ -692,8 +1180,13 @@ procedure LaunchTabulateThread1; end; entry_hash := Hash mod 18701; // Prime = 18701 - while True do // success is handled by breaking out of while loop + entry_Unicode := TEncoding.UTF8.GetString(DigestedString); + + // send to appropriate hash table + while True do + // success is handled by breaking out of while loop begin + HashRecordLock1.Acquire; if HashRecordMap1[entry_hash].DataCount = 0 then // this hash has not been used begin @@ -703,6 +1196,7 @@ procedure LaunchTabulateThread1; HashRecordMap1[entry_hash].DataMax := entry_integer; HashRecordMap1[entry_hash].DataMin := entry_integer; HashRecordMap1[entry_hash].DataName := entry_Unicode; + HashRecordLock1.Release; Break; end else // hash has been used, check for collision @@ -718,22 +1212,29 @@ procedure LaunchTabulateThread1; HashRecordMap1[entry_hash].DataMax := entry_integer; if (HashRecordMap1[entry_hash].DataMin > entry_integer) then HashRecordMap1[entry_hash].DataMin := entry_integer; + HashRecordLock1.Release; Break; end else // collision, try next spot in HashRecordMap array begin - entry_hash := entry_hash + 19; // try a jump - - // for Prime = 18701 + inc(entry_hash); if entry_hash >= 18701 then entry_hash := entry_hash - 18701; end; end; end; // of: while True do - end; // of: if DigestedData = nil then + end; // of: if HaveData then - end; // of: while True do + if ParseData_Done then + begin + ParseData_Done := True; + Break; + end; + + end; // of: while True do // main loop + + Sleep(1); end); TabulateThread1.Start; @@ -742,15 +1243,15 @@ procedure LaunchTabulateThread1; procedure LaunchTabulateThread2; var TabulateThread2: TThread; - DataStackItem2: TRawRecord; begin TabulateThread2 := TThread.CreateAnonymousThread( procedure var + DataStackItem: array [0 .. 107] of Byte; DataIndex: Integer; // index into raw data array SC: Boolean; // True after finding semi-colon - entry_name_array: array [0 .. 127] of Byte; // will hold place name + entry_name_array: array [0 .. 107] of Byte; // will hold place name entry_temp_array: array [0 .. 7] of Byte; // will hold place temperature entry_name_array_length: Integer; entry_temp_array_length: Integer; @@ -762,50 +1263,64 @@ procedure LaunchTabulateThread2; G: Cardinal; Hash: Cardinal; BTT: Integer; // temporary for bytes to temperature conversion - TimeOut: Integer; + HaveData: Boolean; begin - while True do + while True do // main loop begin - - // get item from stack - while True do // break on success + while True do // try for data from this stack begin - DataStackLock2.Acquire; - if (DataStackCount2 > 0) then // there is data + // does this stack have data + HaveData := False; + if (DataStackCount2 = 0) then begin - DataStackItem2 := DataStack2[DataStackCount2 - 1]; - dec(DataStackCount2); - DataStackLock2.Release; - Break; + // check to see if we are done with all stacks + if (DataStackCount1 + DataStackCount2 + DataStackCount3 + + DataStackCount4 + DataStackCount5 = 0) then + // might be done + begin + if (ReadFile_Done1 and ReadFile_Done2) then + // recheck to verify no more data + begin + // wait a moment to be sure + Sleep(1); + if (DataStackCount1 + DataStackCount2 + DataStackCount3 + + DataStackCount4 + DataStackCount5 = 0) then + begin + HaveData := False; + ParseData_Done := True; + Break; + end; + end; // of: if (ReadFile_Done1 and ReadFile_Done2) then + end; end - else + else // lock and check again begin - if ReadFile_Done then // no more data + DataStackLock2.Acquire; + if (DataStackCount2 > 0) then // there is data begin - ParseDataQ_Done2 := True; + Move(DataStack2[DataStackCount2 - 1].RawData[0], DataStackItem[0], + SizeOf(DataStackItem)); + dec(DataStackCount2); DataStackLock2.Release; + HaveData := True; Break; end; DataStackLock2.Release; - // nothing real - TimeOut := DataStackCount2 mod 16001; end; - end; - if ParseDataQ_Done2 then // escape - begin - Break; - end - else // tabulate + end; // of: while True do // try for data from this stack + + if HaveData then begin // split raw data into name and temperature SC := False; entry_name_array_length := 0; entry_temp_array_length := 0; - for DataIndex := 0 to SizeOf(DataStackItem2.RawData) - 1 do + DataIndex := 0; + while DataStackItem[DataIndex] <> 0 do begin - if (DataStackItem2.RawData[DataIndex] = 59) then + if (DataStackItem[DataIndex] = 59) then begin SC := True; end @@ -814,28 +1329,21 @@ procedure LaunchTabulateThread2; if SC then begin // skip decimal in number - if Not(DataStackItem2.RawData[DataIndex] = 46) then + if Not(DataStackItem[DataIndex] = 46) then begin - // skip trailing CR - if (DataStackItem2.RawData[DataIndex] = 13) then // done - begin - Break; - end - else - begin - entry_temp_array[entry_temp_array_length] := - DataStackItem2.RawData[DataIndex]; - inc(entry_temp_array_length); - end; + entry_temp_array[entry_temp_array_length] := + DataStackItem[DataIndex]; + inc(entry_temp_array_length); end; end else // don't skip period in name begin entry_name_array[entry_name_array_length] := - DataStackItem2.RawData[DataIndex]; + DataStackItem[DataIndex]; inc(entry_name_array_length); end; end; + inc(DataIndex); end; // convert characters in byte array to temperature @@ -869,26 +1377,21 @@ procedure LaunchTabulateThread2; entry_integer := BTT; - // convert name bytes to Unicode - // entry_name_array_length := Length(entry_name_array); - SetLength(DigestedString, entry_name_array_length); - for entry_array_index := 0 to entry_name_array_length - 1 do - begin - DigestedString[entry_array_index] := entry_name_array - [entry_array_index]; - end; - - entry_Unicode := TEncoding.UTF8.GetString(DigestedString); - // hash name bytes, adapted from: (* ****************************************************************** *) (* Tomes of Delphi: Algorithms and Data Structures *) (* ------------------------------------------------------------------ *) (* function TDPJWHash *) (* ****************************************************************** *) + // combine this with loop to setup for Unicode + SetLength(DigestedString, entry_name_array_length); + Hash := 0; for entry_array_index := 0 to entry_name_array_length - 1 do begin + DigestedString[entry_array_index] := entry_name_array + [entry_array_index]; + Hash := (Hash shl 4) + entry_name_array[entry_array_index]; G := Hash and $F000000; if (G <> 0) then @@ -896,8 +1399,13 @@ procedure LaunchTabulateThread2; end; entry_hash := Hash mod 18701; // Prime = 18701 - while True do // success is handled by breaking out of while loop + entry_Unicode := TEncoding.UTF8.GetString(DigestedString); + + // send to appropriate hash table + while True do + // success is handled by breaking out of while loop begin + HashRecordLock2.Acquire; if HashRecordMap2[entry_hash].DataCount = 0 then // this hash has not been used begin @@ -907,6 +1415,7 @@ procedure LaunchTabulateThread2; HashRecordMap2[entry_hash].DataMax := entry_integer; HashRecordMap2[entry_hash].DataMin := entry_integer; HashRecordMap2[entry_hash].DataName := entry_Unicode; + HashRecordLock2.Release; Break; end else // hash has been used, check for collision @@ -922,12 +1431,12 @@ procedure LaunchTabulateThread2; HashRecordMap2[entry_hash].DataMax := entry_integer; if (HashRecordMap2[entry_hash].DataMin > entry_integer) then HashRecordMap2[entry_hash].DataMin := entry_integer; + HashRecordLock2.Release; Break; end else // collision, try next spot in HashRecordMap array begin - entry_hash := entry_hash + 19; // try a jump - + inc(entry_hash); // for Prime = 18701 if entry_hash >= 18701 then entry_hash := entry_hash - 18701; @@ -935,9 +1444,17 @@ procedure LaunchTabulateThread2; end; end; // of: while True do - end; // of: if DigestedData = nil then + end; // of: if HaveData then + + if ParseData_Done then + begin + ParseData_Done := True; + Break; + end; + + end; // of: while True do // main loop - end; // of: while True do + Sleep(1); end); TabulateThread2.Start; @@ -946,15 +1463,15 @@ procedure LaunchTabulateThread2; procedure LaunchTabulateThread3; var TabulateThread3: TThread; - DataStackItem3: TRawRecord; begin TabulateThread3 := TThread.CreateAnonymousThread( procedure var + DataStackItem: array [0 .. 107] of Byte; DataIndex: Integer; // index into raw data array SC: Boolean; // True after finding semi-colon - entry_name_array: array [0 .. 127] of Byte; // will hold place name + entry_name_array: array [0 .. 107] of Byte; // will hold place name entry_temp_array: array [0 .. 7] of Byte; // will hold place temperature entry_name_array_length: Integer; entry_temp_array_length: Integer; @@ -966,50 +1483,64 @@ procedure LaunchTabulateThread3; G: Cardinal; Hash: Cardinal; BTT: Integer; // temporary for bytes to temperature conversion - TimeOut: Integer; + HaveData: Boolean; begin - while True do + while True do // main loop begin - - // get item from stack - while True do // break on success + while True do // try for data from this stack begin - DataStackLock3.Acquire; - if (DataStackCount3 > 0) then // there is data + // does this stack have data + HaveData := False; + if (DataStackCount3 = 0) then begin - DataStackItem3 := DataStack3[DataStackCount3 - 1]; - dec(DataStackCount3); - DataStackLock3.Release; - Break; + // check to see if we are done with all stacks + if (DataStackCount1 + DataStackCount2 + DataStackCount3 + + DataStackCount4 + DataStackCount5 = 0) then + // might be done + begin + if (ReadFile_Done1 and ReadFile_Done2) then + // recheck to verify no more data + begin + // wait a moment to be sure + Sleep(1); + if (DataStackCount1 + DataStackCount2 + DataStackCount3 + + DataStackCount4 + DataStackCount5 = 0) then + begin + HaveData := False; + ParseData_Done := True; + Break; + end; + end; // of: if (ReadFile_Done1 and ReadFile_Done2) then + end; end - else + else // lock and check again begin - if ReadFile_Done then // no more data + DataStackLock3.Acquire; + if (DataStackCount3 > 0) then // there is data begin - ParseDataQ_Done3 := True; + Move(DataStack3[DataStackCount3 - 1].RawData[0], DataStackItem[0], + SizeOf(DataStackItem)); + dec(DataStackCount3); DataStackLock3.Release; + HaveData := True; Break; end; DataStackLock3.Release; - // nothing real - TimeOut := DataStackCount3 mod 16001; end; - end; - if ParseDataQ_Done3 then // escape - begin - Break; - end - else // tabulate + end; // of: while True do // try for data from this stack + + if HaveData then begin // split raw data into name and temperature SC := False; entry_name_array_length := 0; entry_temp_array_length := 0; - for DataIndex := 0 to SizeOf(DataStackItem3.RawData) - 1 do + DataIndex := 0; + while DataStackItem[DataIndex] <> 0 do begin - if (DataStackItem3.RawData[DataIndex] = 59) then + if (DataStackItem[DataIndex] = 59) then begin SC := True; end @@ -1018,28 +1549,21 @@ procedure LaunchTabulateThread3; if SC then begin // skip decimal in number - if Not(DataStackItem3.RawData[DataIndex] = 46) then + if Not(DataStackItem[DataIndex] = 46) then begin - // skip trailing CR - if (DataStackItem3.RawData[DataIndex] = 13) then // done - begin - Break; - end - else - begin - entry_temp_array[entry_temp_array_length] := - DataStackItem3.RawData[DataIndex]; - inc(entry_temp_array_length); - end; + entry_temp_array[entry_temp_array_length] := + DataStackItem[DataIndex]; + inc(entry_temp_array_length); end; end else // don't skip period in name begin entry_name_array[entry_name_array_length] := - DataStackItem3.RawData[DataIndex]; + DataStackItem[DataIndex]; inc(entry_name_array_length); end; end; + inc(DataIndex); end; // convert characters in byte array to temperature @@ -1073,26 +1597,21 @@ procedure LaunchTabulateThread3; entry_integer := BTT; - // convert name bytes to Unicode - // entry_name_array_length := Length(entry_name_array); - SetLength(DigestedString, entry_name_array_length); - for entry_array_index := 0 to entry_name_array_length - 1 do - begin - DigestedString[entry_array_index] := entry_name_array - [entry_array_index]; - end; - - entry_Unicode := TEncoding.UTF8.GetString(DigestedString); - // hash name bytes, adapted from: (* ****************************************************************** *) (* Tomes of Delphi: Algorithms and Data Structures *) (* ------------------------------------------------------------------ *) (* function TDPJWHash *) (* ****************************************************************** *) + // combine this with loop to setup for Unicode + SetLength(DigestedString, entry_name_array_length); + Hash := 0; for entry_array_index := 0 to entry_name_array_length - 1 do begin + DigestedString[entry_array_index] := entry_name_array + [entry_array_index]; + Hash := (Hash shl 4) + entry_name_array[entry_array_index]; G := Hash and $F000000; if (G <> 0) then @@ -1100,8 +1619,13 @@ procedure LaunchTabulateThread3; end; entry_hash := Hash mod 18701; // Prime = 18701 - while True do // success is handled by breaking out of while loop + entry_Unicode := TEncoding.UTF8.GetString(DigestedString); + + // send to appropriate hash table + while True do + // success is handled by breaking out of while loop begin + HashRecordLock3.Acquire; if HashRecordMap3[entry_hash].DataCount = 0 then // this hash has not been used begin @@ -1111,6 +1635,7 @@ procedure LaunchTabulateThread3; HashRecordMap3[entry_hash].DataMax := entry_integer; HashRecordMap3[entry_hash].DataMin := entry_integer; HashRecordMap3[entry_hash].DataName := entry_Unicode; + HashRecordLock3.Release; Break; end else // hash has been used, check for collision @@ -1126,12 +1651,12 @@ procedure LaunchTabulateThread3; HashRecordMap3[entry_hash].DataMax := entry_integer; if (HashRecordMap3[entry_hash].DataMin > entry_integer) then HashRecordMap3[entry_hash].DataMin := entry_integer; + HashRecordLock3.Release; Break; end else // collision, try next spot in HashRecordMap array begin - entry_hash := entry_hash + 19; // try a jump - + inc(entry_hash); // for Prime = 18701 if entry_hash >= 18701 then entry_hash := entry_hash - 18701; @@ -1139,9 +1664,17 @@ procedure LaunchTabulateThread3; end; end; // of: while True do - end; // of: if DigestedData = nil then + end; // of: if HaveData then + + if ParseData_Done then + begin + ParseData_Done := True; + Break; + end; + + end; // of: while True do // main loop - end; // of: while True do + Sleep(1); end); TabulateThread3.Start; @@ -1152,15 +1685,15 @@ procedure LaunchTabulateThread3; procedure LaunchTabulateThread4; var TabulateThread4: TThread; - DataStackItem4: TRawRecord; begin TabulateThread4 := TThread.CreateAnonymousThread( procedure var + DataStackItem: array [0 .. 107] of Byte; DataIndex: Integer; // index into raw data array SC: Boolean; // True after finding semi-colon - entry_name_array: array [0 .. 127] of Byte; // will hold place name + entry_name_array: array [0 .. 107] of Byte; // will hold place name entry_temp_array: array [0 .. 7] of Byte; // will hold place temperature entry_name_array_length: Integer; entry_temp_array_length: Integer; @@ -1172,50 +1705,64 @@ procedure LaunchTabulateThread4; G: Cardinal; Hash: Cardinal; BTT: Integer; // temporary for bytes to temperature conversion - TimeOut: Integer; + HaveData: Boolean; begin - while True do + while True do // main loop begin - - // get item from stack - while True do // break on success + while True do // try for data from this stack begin - DataStackLock4.Acquire; - if (DataStackCount4 > 0) then // there is data + // does this stack have data + HaveData := False; + if (DataStackCount4 = 0) then begin - DataStackItem4 := DataStack4[DataStackCount4 - 1]; - dec(DataStackCount4); - DataStackLock4.Release; - Break; + // check to see if we are done with all stacks + if (DataStackCount1 + DataStackCount2 + DataStackCount3 + + DataStackCount4 + DataStackCount5 = 0) then + // might be done + begin + if (ReadFile_Done1 and ReadFile_Done2) then + // recheck to verify no more data + begin + // wait a moment to be sure + Sleep(1); + if (DataStackCount1 + DataStackCount2 + DataStackCount3 + + DataStackCount4 + DataStackCount5 = 0) then + begin + HaveData := False; + ParseData_Done := True; + Break; + end; + end; // of: if (ReadFile_Done1 and ReadFile_Done2) then + end; end - else + else // lock and check again begin - if ReadFile_Done then // no more data + DataStackLock4.Acquire; + if (DataStackCount4 > 0) then // there is data begin - ParseDataQ_Done4 := True; + Move(DataStack4[DataStackCount4 - 1].RawData[0], DataStackItem[0], + SizeOf(DataStackItem)); + dec(DataStackCount4); DataStackLock4.Release; + HaveData := True; Break; end; DataStackLock4.Release; - // nothing real - TimeOut := DataStackCount4 mod 16001; end; - end; - if ParseDataQ_Done4 then // escape - begin - Break; - end - else // tabulate + end; // of: while True do // try for data from this stack + + if HaveData then begin // split raw data into name and temperature SC := False; entry_name_array_length := 0; entry_temp_array_length := 0; - for DataIndex := 0 to SizeOf(DataStackItem4.RawData) - 1 do + DataIndex := 0; + while DataStackItem[DataIndex] <> 0 do begin - if (DataStackItem4.RawData[DataIndex] = 59) then + if (DataStackItem[DataIndex] = 59) then begin SC := True; end @@ -1224,28 +1771,21 @@ procedure LaunchTabulateThread4; if SC then begin // skip decimal in number - if Not(DataStackItem4.RawData[DataIndex] = 46) then + if Not(DataStackItem[DataIndex] = 46) then begin - // skip trailing CR - if (DataStackItem4.RawData[DataIndex] = 13) then // done - begin - Break; - end - else - begin - entry_temp_array[entry_temp_array_length] := - DataStackItem4.RawData[DataIndex]; - inc(entry_temp_array_length); - end; + entry_temp_array[entry_temp_array_length] := + DataStackItem[DataIndex]; + inc(entry_temp_array_length); end; end else // don't skip period in name begin entry_name_array[entry_name_array_length] := - DataStackItem4.RawData[DataIndex]; + DataStackItem[DataIndex]; inc(entry_name_array_length); end; end; + inc(DataIndex); end; // convert characters in byte array to temperature @@ -1279,26 +1819,21 @@ procedure LaunchTabulateThread4; entry_integer := BTT; - // convert name bytes to Unicode - // entry_name_array_length := Length(entry_name_array); - SetLength(DigestedString, entry_name_array_length); - for entry_array_index := 0 to entry_name_array_length - 1 do - begin - DigestedString[entry_array_index] := entry_name_array - [entry_array_index]; - end; - - entry_Unicode := TEncoding.UTF8.GetString(DigestedString); - // hash name bytes, adapted from: (* ****************************************************************** *) (* Tomes of Delphi: Algorithms and Data Structures *) (* ------------------------------------------------------------------ *) (* function TDPJWHash *) (* ****************************************************************** *) + // combine this with loop to setup for Unicode + SetLength(DigestedString, entry_name_array_length); + Hash := 0; for entry_array_index := 0 to entry_name_array_length - 1 do begin + DigestedString[entry_array_index] := entry_name_array + [entry_array_index]; + Hash := (Hash shl 4) + entry_name_array[entry_array_index]; G := Hash and $F000000; if (G <> 0) then @@ -1306,8 +1841,13 @@ procedure LaunchTabulateThread4; end; entry_hash := Hash mod 18701; // Prime = 18701 - while True do // success is handled by breaking out of while loop + entry_Unicode := TEncoding.UTF8.GetString(DigestedString); + + // send to appropriate hash table + while True do + // success is handled by breaking out of while loop begin + HashRecordLock4.Acquire; if HashRecordMap4[entry_hash].DataCount = 0 then // this hash has not been used begin @@ -1317,6 +1857,7 @@ procedure LaunchTabulateThread4; HashRecordMap4[entry_hash].DataMax := entry_integer; HashRecordMap4[entry_hash].DataMin := entry_integer; HashRecordMap4[entry_hash].DataName := entry_Unicode; + HashRecordLock4.Release; Break; end else // hash has been used, check for collision @@ -1332,12 +1873,12 @@ procedure LaunchTabulateThread4; HashRecordMap4[entry_hash].DataMax := entry_integer; if (HashRecordMap4[entry_hash].DataMin > entry_integer) then HashRecordMap4[entry_hash].DataMin := entry_integer; + HashRecordLock4.Release; Break; end else // collision, try next spot in HashRecordMap array begin - entry_hash := entry_hash + 19; // try a jump - + inc(entry_hash); // for Prime = 18701 if entry_hash >= 18701 then entry_hash := entry_hash - 18701; @@ -1345,12 +1886,239 @@ procedure LaunchTabulateThread4; end; end; // of: while True do - end; // of: if DigestedData = nil then + end; // of: if HaveData then + + if ParseData_Done then + begin + ParseData_Done := True; + Break; + end; + + end; // of: while True do // main loop - end; // of: while True do + Sleep(1); end); TabulateThread4.Start; end; +procedure LaunchTabulateThread5; +var + TabulateThread5: TThread; + +begin + TabulateThread5 := TThread.CreateAnonymousThread( + procedure + var + DataStackItem: array [0 .. 107] of Byte; + DataIndex: Integer; // index into raw data array + SC: Boolean; // True after finding semi-colon + entry_name_array: array [0 .. 107] of Byte; // will hold place name + entry_temp_array: array [0 .. 7] of Byte; // will hold place temperature + entry_name_array_length: Integer; + entry_temp_array_length: Integer; + entry_hash: Integer; + entry_integer: Integer; + entry_Unicode: String; + entry_array_index: Integer; // index for entry_temp_array + DigestedString: TArray; // prefer not to use TArray, but for now... + G: Cardinal; + Hash: Cardinal; + BTT: Integer; // temporary for bytes to temperature conversion + HaveData: Boolean; + + begin + while True do // main loop + begin + while True do // try for data from this stack + begin + // does this stack have data + HaveData := False; + if (DataStackCount5 = 0) then + begin + // check to see if we are done with all stacks + if (DataStackCount1 + DataStackCount2 + DataStackCount3 + + DataStackCount4 + DataStackCount5 = 0) then + // might be done + begin + if (ReadFile_Done1 and ReadFile_Done2) then + // recheck to verify no more data + begin + // wait a moment to be sure + Sleep(1); + if (DataStackCount1 + DataStackCount2 + DataStackCount3 + + DataStackCount4 + DataStackCount5 = 0) then + begin + HaveData := False; + ParseData_Done := True; + Break; + end; + end; // of: if (ReadFile_Done1 and ReadFile_Done2) then + end; + end + else // lock and check again + begin + DataStackLock5.Acquire; + if (DataStackCount5 > 0) then // there is data + begin + Move(DataStack5[DataStackCount5 - 1].RawData[0], DataStackItem[0], + SizeOf(DataStackItem)); + dec(DataStackCount5); + DataStackLock5.Release; + HaveData := True; + Break; + end; + DataStackLock5.Release; + end; + + end; // of: while True do // try for data from this stack + + if HaveData then + begin + // split raw data into name and temperature + SC := False; + entry_name_array_length := 0; + entry_temp_array_length := 0; + DataIndex := 0; + while DataStackItem[DataIndex] <> 0 do + begin + if (DataStackItem[DataIndex] = 59) then + begin + SC := True; + end + else + begin + if SC then + begin + // skip decimal in number + if Not(DataStackItem[DataIndex] = 46) then + begin + entry_temp_array[entry_temp_array_length] := + DataStackItem[DataIndex]; + inc(entry_temp_array_length); + end; + end + else // don't skip period in name + begin + entry_name_array[entry_name_array_length] := + DataStackItem[DataIndex]; + inc(entry_name_array_length); + end; + end; + inc(DataIndex); + end; + + // convert characters in byte array to temperature + if entry_temp_array[0] = 45 then + // dash (-), so negative number + begin + // BTT := -(entry_temp_array[count -1] -48) - 10 * (entry_temp_array[count -2] - 48); + // BTT := 48 - entry_temp_array[count -1] - 10 * (entry_temp_array[count -2] - 48); + // BTT := 48 + 480 - entry_temp_array[count -1] - 10 * (entry_temp_array[count -2]); + BTT := 528 - entry_temp_array[entry_temp_array_length - 1] - 10 * + (entry_temp_array[entry_temp_array_length - 2]); + if entry_temp_array_length = 4 then + // do one more digit + begin + BTT := BTT - 100 * (entry_temp_array[1] - 48); + end; + end + else + begin + // BTT := (entry_temp_array[count -1] -48) + 10 * (entry_temp_array[count -2] - 48); + // BTT := entry_temp_array[count -1] -48 + 10 * (entry_temp_array[count -2] - 48); + // BTT := entry_temp_array[count -1] -48 - 480 + 10 * (entry_temp_array[count -2]); + BTT := -528 + (entry_temp_array[entry_temp_array_length - 1]) + 10 * + (entry_temp_array[entry_temp_array_length - 2]); + if entry_temp_array_length = 3 then + // do one more digit + begin + BTT := BTT + 100 * (entry_temp_array[0] - 48); + end; + end; + + entry_integer := BTT; + + // hash name bytes, adapted from: + (* ****************************************************************** *) + (* Tomes of Delphi: Algorithms and Data Structures *) + (* ------------------------------------------------------------------ *) + (* function TDPJWHash *) + (* ****************************************************************** *) + // combine this with loop to setup for Unicode + SetLength(DigestedString, entry_name_array_length); + + Hash := 0; + for entry_array_index := 0 to entry_name_array_length - 1 do + begin + DigestedString[entry_array_index] := entry_name_array + [entry_array_index]; + + Hash := (Hash shl 4) + entry_name_array[entry_array_index]; + G := Hash and $F000000; + if (G <> 0) then + Hash := (Hash xor (G shr 24)) xor G; + end; + entry_hash := Hash mod 18701; // Prime = 18701 + + entry_Unicode := TEncoding.UTF8.GetString(DigestedString); + + // send to appropriate hash table + while True do + // success is handled by breaking out of while loop + begin + HashRecordLock5.Acquire; + if HashRecordMap5[entry_hash].DataCount = 0 then + // this hash has not been used + begin + // store the first entry for this station + HashRecordMap5[entry_hash].DataCount := 1; + HashRecordMap5[entry_hash].DataSum := entry_integer; + HashRecordMap5[entry_hash].DataMax := entry_integer; + HashRecordMap5[entry_hash].DataMin := entry_integer; + HashRecordMap5[entry_hash].DataName := entry_Unicode; + HashRecordLock5.Release; + Break; + end + else // hash has been used, check for collision + begin + if HashRecordMap5[entry_hash].DataName = entry_Unicode then + // OK, merge data + begin + HashRecordMap5[entry_hash].DataCount := + HashRecordMap5[entry_hash].DataCount + 1; + HashRecordMap5[entry_hash].DataSum := HashRecordMap5[entry_hash] + .DataSum + entry_integer; + if (HashRecordMap5[entry_hash].DataMax < entry_integer) then + HashRecordMap5[entry_hash].DataMax := entry_integer; + if (HashRecordMap5[entry_hash].DataMin > entry_integer) then + HashRecordMap5[entry_hash].DataMin := entry_integer; + HashRecordLock5.Release; + Break; + end + else // collision, try next spot in HashRecordMap array + begin + inc(entry_hash); + if entry_hash >= 18701 then + entry_hash := entry_hash - 18701; + end; + end; + end; // of: while True do + + end; // of: if HaveData then + + if ParseData_Done then + begin + ParseData_Done := True; + Break; + end; + + end; // of: while True do // main loop + + Sleep(1); + + end); + TabulateThread5.Start; +end; + end. diff --git a/entries/bfire/src/ProcessByHashUnit.pas b/entries/bfire/src/ProcessByHashUnit.pas deleted file mode 100644 index c8a1f3a..0000000 --- a/entries/bfire/src/ProcessByHashUnit.pas +++ /dev/null @@ -1,517 +0,0 @@ -unit ProcessByHashUnit; - -interface - -uses - System.SysUtils, - System.StrUtils, - Classes, - Math, - System.Generics.Collections; - -type - TStationRecord = record // each array item is compiled data for a place - DataSum: Integer; // temperature times ten - DataCount: Integer; // number of temperature measurements - DataMax: SmallInt; // temperature times ten - DataMin: SmallInt; // temperature times ten - end; - - TNameRecord = record // Station name and index to Station Data record - sName: WideString; - sIndex: Integer; - end; - - TStationForSortClass = class(TObject) - public - DataSum: Integer; - DataCount: Integer; - DataMax: Integer; - DataMin: Integer; - end; - - TStationForSort = class of TStationForSortClass; - -const - // see http://compoasso.free.fr/primelistweb/page/prime/liste_online_en.php - Prime: Integer = 75013; - -const - myBufferSize: Integer = 16384; // 32768 - 6 sec; 16384 -> 6 sec; 8192 -> 7 sec - -var - // for the challenge, set up for 41351 places (41343 entries, plus spare space) - StationData: array [0 .. 41350] of TStationRecord; - // not sorted, indexed with PlaceIndex array - StationName: array [0 .. 41350] of TNameRecord; - // not sorted, indexed with PlaceIndex array - - HashMap: array [0 .. 75012] of Integer; // for Prime := 75013 - - StationsForSort: TStringList; - - start: TDateTime; // for timing - - PlaceCount: Integer; - - // static arrays - // size entry_name_array to hold longest (by byte count) name - // Dolores Hidalgo Cuna de la Independencia Nacional = 98 bytes - // size entry_temp_array to hold -9999 - entry_name_array: array [0 .. 255] of Byte; // will hold place name - entry_temp_array: array [0 .. 15] of Byte; // will hold place temperature - entry_name_array_length: Integer; - entry_temp_array_length: Integer; - - // prepare empty arrays to initialize before each entry read - empty_entry_name_array: array [0 .. 255] of Byte; - empty_entry_temp_array: array [0 .. 15] of Byte; - -procedure FileToArrays(inFile: String; UseStdOut: Boolean); -procedure HashAndSave; -function BytesToTemp: Integer; -function StaticBytesToString: String; -function BytesToIndexHash: Integer; -procedure SortArrays; -procedure ArrayToFile(outFile: String; UseStdOut: Boolean); - -implementation - -function IntegerFixup(something: Integer): String; inline; -// fixup to adjust for storing tenths of degrees as integer -// e.g. convert integer 234 to string 23.4 and -167 to -16.7 -var - thing: String; -begin - thing := IntToStr(something); - thing := LeftStr(thing, Length(thing) - 1) + '.' + RightStr(thing, 1); - // make sure we have leading zero, where applicable - if thing[1] = '.' then - thing := '0' + thing; - IntegerFixup := thing.Replace('-.', '-0.'); -end; - -function MeanFixup(total: Integer; count: Integer): String; inline; -// fixup to adjust for storing tenths of degrees as integer -// calculate mean to one decimal place, rounded up/down depending on sign -var - temp: string; - ratio: Integer; - // remainder: Integer; - neg: Boolean; -begin - if total < 0 then - begin - neg := True; - total := -total; - end - else - begin - neg := False; - end; - ratio := total div count; - if (neg) then // no rounding needed - begin - temp := IntToStr(ratio); - if (ratio > 9) then - begin - // string modification equivalent to dividing by ten - temp := '-' + LeftStr(temp, Length(temp) - 1) + '.' + temp[Length(temp)]; - end - else - begin - if temp[1] = '0' then - begin - temp := '0.' + temp; - end - else - begin - temp := '-0.' + temp; - end; - end; - end - else // round up - begin - temp := IntToStr(ratio + 1); - if (ratio > 8) then // 8, rather than 9, since one was added - begin - // string modification equivalent to dividing by ten - temp := LeftStr(temp, Length(temp) - 1) + '.' + temp[Length(temp)]; - end - else - begin - temp := '0.' + temp; - end; - end; - MeanFixup := temp; // was temp; -end; - -function BytesToTemp: Integer; -// convert entry_chars_temp to signed integer, '0' is ascii 48 decimal -// note: we always have at least two bytes -// temperatures range from -99.9 to 99.8 -// that appears here as -999 to 998 -var - BTT: Integer; -begin - if entry_temp_array[0] = 45 then // negative number - begin - // BTT := -(entry_temp_array[count -1] -48) - 10 * (entry_temp_array[count -2] - 48); - // BTT := 48 - entry_temp_array[count -1] - 10 * (entry_temp_array[count -2] - 48); - // BTT := 48 + 480 - entry_temp_array[count -1] - 10 * (entry_temp_array[count -2]); - BTT := 528 - entry_temp_array[entry_temp_array_length - 1] - 10 * - (entry_temp_array[entry_temp_array_length - 2]); - if entry_temp_array_length = 4 then // do one more digit - begin - BTT := BTT - 100 * (entry_temp_array[1] - 48); - end; - end - else - begin - // BTT := (entry_temp_array[count -1] -48) + 10 * (entry_temp_array[count -2] - 48); - // BTT := entry_temp_array[count -1] -48 + 10 * (entry_temp_array[count -2] - 48); - // BTT := entry_temp_array[count -1] -48 - 480 + 10 * (entry_temp_array[count -2]); - BTT := -528 + (entry_temp_array[entry_temp_array_length - 1]) + 10 * - (entry_temp_array[entry_temp_array_length - 2]); - if entry_temp_array_length = 3 then // do one more digit - begin - BTT := BTT + 100 * (entry_temp_array[0] - 48); - end; - end; - BytesToTemp := BTT; -end; - -// adapted from: -(* ****************************************************************** *) -(* Tomes of Delphi: Algorithms and Data Structures *) -(* ------------------------------------------------------------------ *) -(* function TDPJWHash *) -(* ****************************************************************** *) -function BytesToIndexHash; -// convert entry_chars to hash -// use Prime: Integer = 75013; -var - G: Cardinal; - i: Integer; - Hash: Cardinal; -begin - Hash := 0; -// for i := 0 to entry_name_array_length - 1 do - for i := entry_name_array_length - 1 downto 0 do - begin - Hash := (Hash shl 4) + entry_name_array[i]; - G := Hash and $F0000000; -// G := Hash and $F000000; - if (G <> 0) then - Hash := (Hash xor (G shr 24)) xor G; - end; - BytesToIndexHash := Hash mod Prime; -end; - -// ============================================================================ -// ============================================================================ -// ============================================================================ - -procedure InitializeThings; inline; -var - i: Integer; -begin - // pre-fill arrays - for i := 0 to Prime - 1 do - HashMap[i] := -1; - for i := 0 to 255 do - empty_entry_name_array[i] := 0; - for i := 0 to 15 do - empty_entry_temp_array[i] := 0; - - StationsForSort := TStringList.Create; // and explicitly set options - // sorts strings using ANSI (Windows) or UTF-8 (Linux) order - StationsForSort.UseLocale := False; // True => ansi sort, NOT wanted - StationsForSort.Duplicates := dupIgnore; - // Ignore attempts to add duplicate strings to the list. - StationsForSort.CaseSensitive := True; // default is false - StationsForSort.OwnsObjects := True; - // so, Stations object destroys its data objects - StationsForSort.Sorted := False; // sort later -end; - -function StaticBytesToString; -var - ugly: TArray; // prefer not to use TArray, but for now... - i: Integer; -begin - SetLength(ugly, entry_name_array_length); - for i := 0 to entry_name_array_length - 1 do - begin - ugly[i] := entry_name_array[i]; - end; - StaticBytesToString := TEncoding.UTF8.GetString(ugly); -end; - - -procedure HashAndSave; -var - entry_integer: Integer; - entry_Unicode: String; - my_data_item: TStationForSortClass; - - entry_hash: Integer; - PlaceIndex: Integer; // location in index array - -begin - entry_integer := BytesToTemp; // only pass J - - entry_Unicode := StaticBytesToString; - // entry_name_array is global, only pass length - - entry_hash := BytesToIndexHash; - // entry_name_array is global, only pass length - - while True do // success is handled by breaking out of while loop - begin - PlaceIndex := HashMap[entry_hash]; - - if PlaceIndex = -1 then // this hash has not been used - begin - // store the first entry for this station - PlaceIndex := PlaceCount; - HashMap[entry_hash] := PlaceIndex; // set it - my_data_item := TStationForSort.Create(); - my_data_item.DataCount := 1; - my_data_item.DataSum := entry_integer; - my_data_item.DataMin := entry_integer; - my_data_item.DataMax := entry_integer; - StationsForSort.AddObject(entry_Unicode, TObject(my_data_item)); - - Inc(PlaceCount); - Break; - end - else // hash has been used, check for collision - begin - if StationsForSort[PlaceIndex] = entry_Unicode then // OK, merge data - begin - (StationsForSort.Objects[PlaceIndex] as TStationForSortClass).DataSum := - (StationsForSort.Objects[PlaceIndex] as TStationForSortClass).DataSum - + entry_integer; - - (StationsForSort.Objects[PlaceIndex] as TStationForSortClass).DataCount - := (StationsForSort.Objects[PlaceIndex] as TStationForSortClass) - .DataCount + 1; - - if (StationsForSort.Objects[PlaceIndex] as TStationForSortClass).DataMax - < entry_integer then - (StationsForSort.Objects[PlaceIndex] as TStationForSortClass).DataMax - := entry_integer; - - if (StationsForSort.Objects[PlaceIndex] as TStationForSortClass).DataMin - > entry_integer then - (StationsForSort.Objects[PlaceIndex] as TStationForSortClass).DataMin - := entry_integer; - - Break; - end - else // collision, try next spot in HashMap array - begin - entry_hash := entry_hash + 19; - if entry_hash >= Prime then // wrap around - entry_hash := entry_hash - Prime; - end; - end; - end; // of: while True do - -end; - -procedure FileToArrays(inFile: String; UseStdOut: Boolean); inline; -var - i: Integer; // used for temporary purposes - - LF: Boolean; // True after finding #10 - SC: Boolean; // True after finding semi-colon - - iBytesRead: Integer; - Buffer: PByte; - - Challenge: TFileStream; - -begin - - if Not(UseStdOut) then - begin - WriteLn('Making Hash...'); - end; - - // Load the data - PlaceCount := 0; - InitializeThings; - - if FileExists(inFile) then - begin - - try - Challenge := TFileStream.Create(inFile, fmOpenRead); - Challenge.Seek(0, soFromBeginning); - // Buffer := System.AllocMem(myBufferSize +1); // why +1 ?? - Buffer := System.AllocMem(myBufferSize); // seems OK - i := 0; - entry_temp_array_length := 0; - entry_name_array_length := 0; - LF := False; - SC := False; - move(empty_entry_name_array[0], entry_name_array[0], 256); - // will hold place name - move(empty_entry_temp_array[0], entry_temp_array[0], 16); - // will hold temperature - - iBytesRead := Challenge.Read(Buffer^, myBufferSize); - while iBytesRead > 0 do - begin - while Not LF do // read byte by byte - begin - if ((Buffer + i)^ = 10) then - begin - LF := True; // and done collecting bytes - end - else // accumulate bytes - begin - if ((Buffer + i)^ = 59) then - SC := True; - // skip line feed, carriage return and semi-colon - if Not(((Buffer + i)^ = 13) or ((Buffer + i)^ = 59)) then - begin - if SC then - begin - // skip decimal in number - if Not((Buffer + i)^ = 46) then - begin - entry_temp_array[entry_temp_array_length] := (Buffer + i)^; - Inc(entry_temp_array_length); - end; - end - else - begin - entry_name_array[entry_name_array_length] := (Buffer + i)^; // don't skip period in name - Inc(entry_name_array_length); - end; - end; - end; - Inc(i); // next byte - if i >= iBytesRead then // need next buffer - begin - iBytesRead := Challenge.Read(Buffer^, myBufferSize); - i := 0; // reset to beggining of buffer - end; - - if LF then // convert to something usable and process it - begin - HashAndSave; - end; - - end; // of: if LF then - - // reset to get next line - LF := False; - SC := False; - entry_temp_array_length := 0; - entry_name_array_length := 0; - move(empty_entry_name_array[0], entry_name_array[0], 256); - // will hold place name - move(empty_entry_temp_array[0], entry_temp_array[0], 16); - // will hold temperature - - end; // of while iBytesRead > 0 do --- implies end of file - Challenge.Free; - - finally - // nothing here - end; - - end - else - begin - raise Exception.Create(Format('File "%s" not found.', [inFile])); - end; - -end; - -procedure SortArrays; -begin - StationsForSort.Sort; -end; - -procedure ArrayToFile(outFile: String; UseStdOut: Boolean); -// Inline; // inline not help -var - outputFileStream: TFileStream; - PlaceIndex: Integer; // location in array - bufferStr: String; - EntrySum: Integer; - EntryCount: Integer; - EntryMax: Integer; - EntryMin: Integer; - -begin - try - if Not(UseStdOut) then - begin - outputFileStream := TFileStream.Create(outFile, fmCreate); - end; - - for PlaceIndex := 0 to PlaceCount - 1 do - begin - if PlaceIndex = 0 then - begin - bufferStr := '{'; - end - else - begin - bufferStr := ', '; - end; - - EntrySum := (StationsForSort.Objects[PlaceIndex] - as TStationForSortClass).DataSum; - EntryCount := (StationsForSort.Objects[PlaceIndex] - as TStationForSortClass).DataCount; - EntryMax := (StationsForSort.Objects[PlaceIndex] - as TStationForSortClass).DataMax; - EntryMin := (StationsForSort.Objects[PlaceIndex] - as TStationForSortClass).DataMin; - - bufferStr := bufferStr + StationsForSort[PlaceIndex] + '=' + - IntegerFixup(EntryMin) + '/' + MeanFixup(EntrySum, EntryCount) + '/' + - IntegerFixup(EntryMax); - - if UseStdOut then // send to STDOUT, where it gets mangled - begin - write(bufferStr); - end - else - begin - outputFileStream.WriteBuffer(TEncoding.UTF8.GetBytes(bufferStr), - TEncoding.UTF8.GetByteCount(bufferStr)); - end; - end; - - bufferStr := '}' + Chr(10); // linefeed appears at end of baseline file - if UseStdOut then // send to STDOUT, where it gets mangled - begin - write(bufferStr); - end - else - begin - outputFileStream.WriteBuffer(TEncoding.UTF8.GetBytes(bufferStr), - TEncoding.UTF8.GetByteCount(bufferStr)); - outputFileStream.Free; - end; - - finally - if Not(UseStdOut) then - begin - WriteLn; - WriteLn; - WriteLn('Done'); - end; - end; - -end; - -end. diff --git a/entries/bfire/src/README.md b/entries/bfire/src/README.md new file mode 100644 index 0000000..27e9a27 --- /dev/null +++ b/entries/bfire/src/README.md @@ -0,0 +1,62 @@ +# Brian Fire + +An Entry to the One Billion Row Challenge in Object Pascal using Delphi 12 by [EagleAglow](https://github.com/EagleAglow), Discord: briar.on.fire + +## Compiler + +**Delphi 12** Professional Edition + +### Dependencies + +Project uses Delphi System units: `Classes`, `SysUtils`, `StrUtils`, `Diagnostics`, +`Threading` and `SyncObjs`. + +### Execution +``` + Usage + bfire -h | Write this help message and exit + bfire -v | Write the version and exit + bfire -i | contains Weather Data + bfire -i -o | contains Weather Data + | contains result + If is not defined, result goes to CONSOLE (STDOUT) +``` + +#### Contest Mode + +To run the challenge, read from the 'challenge.csv' file: + +```console +C:> bfire -i challenge.csv +``` + +## Remarks + +I haven't used Delphi very much recently, really needed to work on this for a refresher. +I like TStringList self-sorting, but it is not as fast as other techniques. +Now that this entry is set up, I can play with improvements. Maybe even get a time under 15 minutes! :) + +Second version uses hash of station name to accumulate data and fill a TStringList for Unicode station name. +The list is initially unsorted and has linked objects for records holding accumulated data for each station. +Finally, the TStringList is sorted and used to output sorted data. + +Third version has a thread for the console (which waits for tabulation, then sorts and writes results), +two threads to read file, five threads to tabulate stations. Stations are grouped into five separate stacks, +so each tabulation thread has roughly the same work load. File is read byte-wise into "classic" byte array +for each file line ending in ascii 10. Each of these arrays is queued as a record in a last-in-first-out stack. +Tabulation threads split the data into station name and temperature, then hash station name and use hash +as index into one of five data arrays. After all data is read and tabulated, the five data arrays are added to an +initially unsorted TStringList that holds unsorted Unicode station name and has linked pointers to +tabulated data for each station. Finally, the TStringList is sorted, and the data is output. + +## History + +- Version 1.0: first working version, based on TStringList. +- Version 1.1: modified rounding to new baseline. +- Version 2.0: use hashing, sort later. +- Version 2.1: minor speed tweaks. +- Version 2.2: try hash functions modification. +- Version 3.0: Six threads: one to read, four to tabulate, one (console) to rule them all... +- Version 3.1: Safer locking strategy - didn't work. +- Version 3.2: Eigth threads: two to read, five to tabulate, one (console) to rule them all... + diff --git a/entries/bfire/src/bfire.dpr b/entries/bfire/src/bfire.dpr index daac5ec..44740ec 100644 --- a/entries/bfire/src/bfire.dpr +++ b/entries/bfire/src/bfire.dpr @@ -13,7 +13,6 @@ uses ConsoleUnit in 'ConsoleUnit.pas'; var - UseStdOut: Boolean; // True unless output file is defined start: TDateTime; // for timing begin @@ -49,14 +48,16 @@ begin if Not(UseStdOut) then // wait and report begin - while Not(ReadFile_Done and ParseDataQ_Done1 and ParseDataQ_Done2 and - ParseDataQ_Done3 and ParseDataQ_Done4) do + // while Not(ReadFile_Done1 and ReadFile_Done2 and ParseDataQ_Done1 and + // ParseDataQ_Done2 and ParseDataQ_Done3 and ParseDataQ_Done4 and + // ParseDataQ_Done5 and ParseDataQ_Done6) do + while Not(ReadFile_Done1 and ReadFile_Done2 and ParseData_Done) do begin Sleep(1000); WriteLn('Lines: ' + IntToStr(LineCount) + ' Stacks: ' + IntToStr(DataStackCount1) + ' / ' + IntToStr(DataStackCount2) + ' / ' + IntToStr(DataStackCount3) + ' / ' + - IntToStr(DataStackCount4)); + IntToStr(DataStackCount4) + ' / ' + IntToStr(DataStackCount5)); if DataStackCount1 > StackMax1 then StackMax1 := DataStackCount1; @@ -66,28 +67,30 @@ begin StackMax3 := DataStackCount3; if DataStackCount4 > StackMax4 then StackMax4 := DataStackCount4; - - if ReadFile_Done then - WriteLn('Done reading file'); - if ParseDataQ_Done1 then - WriteLn('Done with ParseDataQ1'); - if ParseDataQ_Done2 then - WriteLn('Done with ParseDataQ2'); - if ParseDataQ_Done3 then - WriteLn('Done with ParseDataQ3'); - if ParseDataQ_Done4 then - WriteLn('Done with ParseDataQ4'); + if DataStackCount5 > StackMax5 then + StackMax5 := DataStackCount5; + + if ReadFile_Done1 then + WriteLn('Done with reading thread 1'); + if ReadFile_Done2 then + WriteLn('Done with reading thread 2'); + if ParseData_Done then + WriteLn('Done with tabulating threads'); end; end else // just wait begin - while Not(ReadFile_Done and ParseDataQ_Done1 and ParseDataQ_Done2 and - ParseDataQ_Done3 and ParseDataQ_Done4) do + // while Not(ReadFile_Done1 and ReadFile_Done2 and ParseDataQ_Done2 and + // ParseDataQ_Done2 and ParseDataQ_Done3 and ParseDataQ_Done4 and + // ParseDataQ_Done5 and ParseDataQ_Done6) do + while Not(ReadFile_Done1 and ReadFile_Done2 and ParseData_Done) do begin Sleep(100); end; end; + Challenge.Free; + SortArrays; // sort ArrayToFile(outputFilename, UseStdOut); // output @@ -97,7 +100,8 @@ begin Now - start)])); WriteLn('Stack Max: ' + IntToStr(StackMax1) + '/' + IntToStr(StackMax2) - + '/' + IntToStr(StackMax3) + '/' + IntToStr(StackMax4)); + + '/' + IntToStr(StackMax3) + '/' + IntToStr(StackMax4) + '/' + + IntToStr(StackMax5)); WriteLn('Press ENTER to exit'); readln; diff --git a/entries/bfire/src/version.inc b/entries/bfire/src/version.inc index f402702..e045fad 100644 --- a/entries/bfire/src/version.inc +++ b/entries/bfire/src/version.inc @@ -1,2 +1,2 @@ const - cVersion = '3.1'; + cVersion = '3.2';