Skip to content
Permalink
Browse files

Issue #515: Handle encoding errors in apphelpers.ReadTextfileChunk ac…

…cording to how Delphi 10.3 raises exceptions. And probably fix the issue by increasing the new chunk size by 1M instead of only 4B.
  • Loading branch information...
ansgarbecker committed Apr 14, 2019
1 parent 141c348 commit 63212600ed51e0114453cd7b87028ce4bb79016e
Showing with 19 additions and 17 deletions.
  1. +15 −17 source/apphelpers.pas
  2. +4 −0 source/main.pas
@@ -1338,7 +1338,7 @@ function DetectEncoding(Stream: TStream): TEncoding;

function ReadTextfileChunk(Stream: TFileStream; Encoding: TEncoding; ChunkSize: Int64 = 0): String;
const
BufferPadding = 4;
BufferPadding = SIZE_MB;
var
DataLeft, StartPosition: Int64;
LBuffer: TBytes;
@@ -1356,24 +1356,22 @@ function ReadTextfileChunk(Stream: TFileStream; Encoding: TEncoding; ChunkSize:
ChunkSize := DataLeft;

i := 0;
while Length(LBuffer) = 0 do begin
SetLength(LBuffer, ChunkSize);
Stream.ReadBuffer(Pointer(LBuffer)^, ChunkSize);
LBuffer := Encoding.Convert(Encoding, TEncoding.Unicode, LBuffer);
if Length(LBuffer) = 0 then begin
// Now, TEncoding.Convert returns an empty TByte array in files with russion characters
// See http://www.heidisql.com/forum.php?t=13044
// Re-read the whole chunk + some more bytes
// See: Classes.TStreamReader.FillBuffer
// See: https://forums.embarcadero.com/message.jspa?messageID=368526
MainForm.LogSQL(f_('End of file block was cut within some multibyte character, at position %s. Increasing chunk size and retry reading...', [FormatNumber(Stream.Position)]), lcError);
Stream.Position := StartPosition;
Inc(ChunkSize, BufferPadding);
end else // Success, exit loop
Break;
while True do begin
Inc(i);
if i=10 then // Give up
try
SetLength(LBuffer, ChunkSize);
Stream.ReadBuffer(Pointer(LBuffer)^, ChunkSize);
LBuffer := Encoding.Convert(Encoding, TEncoding.Unicode, LBuffer);
// Success, exit loop
Break;
except
on E:EEncodingError do begin
if i=10 then // Give up
Raise;
Stream.Position := StartPosition;
Inc(ChunkSize, BufferPadding);
end;
end;
end;

Result := TEncoding.Unicode.GetString(LBuffer);
@@ -3816,6 +3816,10 @@ procedure TMainForm.RunQueryFile(FileName: String; Encoding: TEncoding);
f_('Notice: You can disable the "%s" option to ignore such errors', [actQueryStopOnErrors.Caption])
);
end;
on E:EEncodingError do begin
StopProgress;
ErrorDialog(E.Message);
end;
end;
end;

0 comments on commit 6321260

Please sign in to comment.
You can’t perform that action at this time.