Permalink
Browse files

more aggressive auto-resize of TTextWriter internal buffer

to use 1MB when 40MB is reached (instead of 64KB)
  • Loading branch information...
Arnaud Bouchez
Arnaud Bouchez committed Jan 31, 2019
1 parent e18422e commit e9af4cb0743fd58ba99bf1c6a384cd26aee0afb2
Showing with 28 additions and 20 deletions.
  1. +11 −9 SQLite3/mORMot.pas
  2. +15 −9 SynCommons.pas
  3. +1 −1 SynSQLite3.pas
  4. +1 −1 SynopseCommit.inc
/// read-only access to a particular field value, as VCL String
// - raise an ESQLTableException if called outside valid Step() sequence
// - similar to GetString() method, but for the current Step
function FieldAsString(const FieldName: String): String; overload;
function FieldAsString(const FieldName: RawUTF8): String; overload;
{$ifdef HASINLINE}inline;{$endif}
{$ifndef NOVARIANTS}
/// read-only access to a particular field value, as a variant
UTF8DecodeToString(buf,StrLen(buf),result);
end;

function TSQLTable.FieldAsString(const FieldName: String): String;
function TSQLTable.FieldAsString(const FieldName: RawUTF8): String;
var buf: PUTF8Char;
begin
buf := FieldBuffer(FieldName);
if self=nil then
exit;
with RecordProps do
serializer := CreateJSONWriter(JSON,Expand,withID,SimpleFieldsBits[Occasion],0);
serializer := CreateJSONWriter(JSON,Expand,withID,
SimpleFieldsBits[Occasion],{knownrows=}0);
serializer.SQLRecordOptions := SQLRecordOptions;
GetJSONValuesAndFree(serializer);
end;
begin
J := TRawByteStringStream.Create;
try
serializer := RecordProps.CreateJSONWriter(J,Expand,withID,Fields,0);
serializer := RecordProps.CreateJSONWriter(J,Expand,withID,Fields,{knownrows=}0);
serializer.SQLRecordOptions := SQLRecordOptions;
GetJSONValuesAndFree(serializer);
result := J.DataString;
rec := Table.CreateAndFillPrepare(Call.OutBody);
try
W := TableRecordProps.Props.CreateJSONWriter(
TRawByteStringStream.Create,true,FieldsCSV,0);
TRawByteStringStream.Create,true,FieldsCSV,{knownrows=}0);
try
include(W.fCustomOptions,twoForceJSONStandard); // force regular JSON
W.SQLRecordOptions := Options; // will do the magic
KnownRowsCount := fValue.Count else
KnownRowsCount := 0;
Stmt.SelectFieldBits(bits,withID);
W := fStoredClassRecordProps.CreateJSONWriter(Stream,Expand,withID,bits,KnownRowsCount);
W := fStoredClassRecordProps.CreateJSONWriter(Stream,Expand,withID,bits,
KnownRowsCount,{bufsize=}256 shl 10);
if W<>nil then
try
rec := pointer(fValue.List);
try
rec := pointer(fValue.List);
W := fStoredClassRecordProps.CreateJSONWriter(
Stream,Expand,true,ALL_FIELDS,fValue.Count,65536);
Stream,Expand,true,ALL_FIELDS,fValue.Count,{bufsize=}256 shl 10);
try
if Expand then
W.Add('[');
exit;
fLogTableStorage := THeapMemoryStream.Create;
fLogTableWriter := OneLog.RecordProps.CreateJSONWriter(
fLogTableStorage,false,true,ALL_ACCESS_RIGHTS,0);
fLogTableStorage,false,true,ALL_FIELDS,{knownrows=}0);
fLogTableWriter.FlushToStream;
P := pointer(aJSON);
if not CompareMem(fLogTableStorage.Memory,P,fLogTableStorage.Position) or
if not Assigned(fLogTableStorage) then begin
fLogTableStorage := THeapMemoryStream.Create;
fLogTableWriter := OneLog.RecordProps.CreateJSONWriter(
fLogTableStorage,false,true,ALL_ACCESS_RIGHTS,0);
fLogTableStorage,false,true,ALL_FIELDS,{knownrows=}0);
fLogTableRowCount := 1;
end else begin
fLogTableWriter.Add(',');
end;

function ToTextOS(osint32: integer): RawUTF8;
var osv: TOperatingSystemVersion;
var osv: TOperatingSystemVersion absolute osint32;
ost: ShortString;
begin
PInteger(@osv)^ := osint32;
ost := ToText(osv);
if (osv.os>=osLinux) and (osv.utsrelease[2]<>0) then
result := FormatUTF8('% %.%.%',[ost,osv.utsrelease[2],osv.utsrelease[1],osv.utsrelease[0]]) else

procedure TTextWriter.FlushToStream;
var i: PtrInt;
written: cardinal;
begin
if fEchos<>nil then begin
EchoFlush;
fStream.WriteBuffer(fTempBuf^,i);
inc(fTotalFileSize,i);
if not (twoFlushToStreamNoAutoResize in fCustomOptions) and
not (twoBufferIsExternal in fCustomOptions) and
(fTempBufSize<49152) and
(fTotalFileSize-fInitialStreamPosition>1 shl 18) then begin
FreeMem(fTempBuf); // with big content (256KB) comes bigger buffer (64KB)
fTempBufSize := 65536;
GetMem(fTempBuf,65536);
BEnd := fTempBuf+(65536-2);
not (twoBufferIsExternal in fCustomOptions) then begin
written := fTotalFileSize-fInitialStreamPosition;
if (fTempBufSize<49152) and (written>1 shl 18) then // 256KB -> 64KB buffer
written := 65536 else
if (fTempBufSize<1 shl 20) and (written>40 shl 20) then // 40MB -> 1MB buffer
written := 1 shl 20 else
written := 0;
if written>0 then begin
fTempBufSize := written;
FreeMem(fTempBuf); // with big content comes bigger buffer
GetMem(fTempBuf,fTempBufSize);
BEnd := fTempBuf+(fTempBufSize-2);
end;
end;
B := fTempBuf-1;
end;
@@ -5024,7 +5024,7 @@ function TSQLRequest.Execute(aDB: TSQLite3DB; const aSQL: RawUTF8; JSON: TStream
W: TJSONWriter;
begin
result := 0;
W := TJSONWriter.Create(JSON,Expand,false);
W := TJSONWriter.Create(JSON,Expand,false,nil,{bufsize=}65536);
try
// prepare the SQL request
if aSQL<>'' then // if not already prepared, reset and bound by caller
@@ -1 +1 @@
'1.18.5011'
'1.18.5012'

0 comments on commit e9af4cb

Please sign in to comment.