more aggressive auto-resize of TTextWriter internal buffer

to use 1MB when 40MB is reached (instead of 64KB)
synopse · Jan 31, 2019 · e9af4cb0743fd58ba99bf1c6a384cd26aee0afb2 · e9af4cb
1 parent e18422e
commit e9af4cb0743fd58ba99bf1c6a384cd26aee0afb2
Unified Split

Showing with 28 additions and 20 deletions.

+11 −9 SQLite3/mORMot.pas

+15 −9 SynCommons.pas

+1 −1 SynSQLite3.pas

+1 −1 SynopseCommit.inc
diff --git a/SQLite3/mORMot.pas b/SQLite3/mORMot.pas

    /// read-only access to a particular field value, as VCL String
    // - raise an ESQLTableException if called outside valid Step() sequence
    // - similar to GetString() method, but for the current Step
-  
-    function FieldAsString(const FieldName: String): String; overload;
+  
+    function FieldAsString(const FieldName: RawUTF8): String; overload;
      {$ifdef HASINLINE}inline;{$endif}
    {$ifndef NOVARIANTS}
    /// read-only access to a particular field value, as a variant

  UTF8DecodeToString(buf,StrLen(buf),result);
 end;
-  
-function TSQLTable.FieldAsString(const FieldName: String): String;
+  
+function TSQLTable.FieldAsString(const FieldName: RawUTF8): String;
 var buf: PUTF8Char;
 begin
  buf := FieldBuffer(FieldName);

  if self=nil then
    exit;
  with RecordProps do
-  
-    serializer := CreateJSONWriter(JSON,Expand,withID,SimpleFieldsBits[Occasion],0);
+  
+    serializer := CreateJSONWriter(JSON,Expand,withID,
+  
+      SimpleFieldsBits[Occasion],{knownrows=}0);
  serializer.SQLRecordOptions := SQLRecordOptions;
  GetJSONValuesAndFree(serializer);
 end;

 begin
  J := TRawByteStringStream.Create;
  try
-  
-    serializer := RecordProps.CreateJSONWriter(J,Expand,withID,Fields,0);
+  
+    serializer := RecordProps.CreateJSONWriter(J,Expand,withID,Fields,{knownrows=}0);
    serializer.SQLRecordOptions := SQLRecordOptions;
    GetJSONValuesAndFree(serializer);
    result := J.DataString;

    rec := Table.CreateAndFillPrepare(Call.OutBody);
    try
      W := TableRecordProps.Props.CreateJSONWriter(
-  
-        TRawByteStringStream.Create,true,FieldsCSV,0);
+  
+        TRawByteStringStream.Create,true,FieldsCSV,{knownrows=}0);
      try
        include(W.fCustomOptions,twoForceJSONStandard); // force regular JSON
        W.SQLRecordOptions := Options; // will do the magic

      KnownRowsCount := fValue.Count else
    KnownRowsCount := 0;
  Stmt.SelectFieldBits(bits,withID);
-  
-  W := fStoredClassRecordProps.CreateJSONWriter(Stream,Expand,withID,bits,KnownRowsCount);
+  
+  W := fStoredClassRecordProps.CreateJSONWriter(Stream,Expand,withID,bits,
+  
+    KnownRowsCount,{bufsize=}256 shl 10);
  if W<>nil then
  try
    rec := pointer(fValue.List);

  try
    rec := pointer(fValue.List);
    W := fStoredClassRecordProps.CreateJSONWriter(
-  
-      Stream,Expand,true,ALL_FIELDS,fValue.Count,65536);
+  
+      Stream,Expand,true,ALL_FIELDS,fValue.Count,{bufsize=}256 shl 10);
    try
      if Expand then
        W.Add('[');

    exit;
  fLogTableStorage := THeapMemoryStream.Create;
  fLogTableWriter := OneLog.RecordProps.CreateJSONWriter(
-  
-    fLogTableStorage,false,true,ALL_ACCESS_RIGHTS,0);
+  
+    fLogTableStorage,false,true,ALL_FIELDS,{knownrows=}0);
  fLogTableWriter.FlushToStream;
  P := pointer(aJSON);
  if not CompareMem(fLogTableStorage.Memory,P,fLogTableStorage.Position) or

  if not Assigned(fLogTableStorage) then begin
    fLogTableStorage := THeapMemoryStream.Create;
    fLogTableWriter := OneLog.RecordProps.CreateJSONWriter(
-  
-      fLogTableStorage,false,true,ALL_ACCESS_RIGHTS,0);
+  
+      fLogTableStorage,false,true,ALL_FIELDS,{knownrows=}0);
    fLogTableRowCount := 1;
  end else begin
    fLogTableWriter.Add(',');
diff --git a/SynCommons.pas b/SynCommons.pas

 end;
 function ToTextOS(osint32: integer): RawUTF8;
-  
-var osv: TOperatingSystemVersion;
+  
+var osv: TOperatingSystemVersion absolute osint32;
    ost: ShortString;
 begin
-  
-  PInteger(@osv)^ := osint32;
  ost := ToText(osv);
  if (osv.os>=osLinux) and (osv.utsrelease[2]<>0) then
    result := FormatUTF8('% %.%.%',[ost,osv.utsrelease[2],osv.utsrelease[1],osv.utsrelease[0]]) else

 procedure TTextWriter.FlushToStream;
 var i: PtrInt;
+  
+    written: cardinal;
 begin
  if fEchos<>nil then begin
    EchoFlush;

  fStream.WriteBuffer(fTempBuf^,i);
  inc(fTotalFileSize,i);
  if not (twoFlushToStreamNoAutoResize in fCustomOptions) and
-  
-     not (twoBufferIsExternal in fCustomOptions) and
-  
-     (fTempBufSize<49152) and
-  
-     (fTotalFileSize-fInitialStreamPosition>1 shl 18) then begin
-  
-    FreeMem(fTempBuf); // with big content (256KB) comes bigger buffer (64KB)
-  
-    fTempBufSize := 65536;
-  
-    GetMem(fTempBuf,65536);
-  
-    BEnd := fTempBuf+(65536-2);
+  
+     not (twoBufferIsExternal in fCustomOptions) then begin
+  
+    written := fTotalFileSize-fInitialStreamPosition;
+  
+    if (fTempBufSize<49152) and (written>1 shl 18) then // 256KB -> 64KB buffer
+  
+      written := 65536 else
+  
+    if (fTempBufSize<1 shl 20) and (written>40 shl 20) then // 40MB -> 1MB buffer
+  
+      written := 1 shl 20 else
+  
+      written := 0;
+  
+    if written>0 then begin
+  
+      fTempBufSize := written;
+  
+      FreeMem(fTempBuf); // with big content comes bigger buffer
+  
+      GetMem(fTempBuf,fTempBufSize);
+  
+      BEnd := fTempBuf+(fTempBufSize-2);
+  
+    end;
  end;
  B := fTempBuf-1;
 end;
diff --git a/SynSQLite3.pas b/SynSQLite3.pas
@@ -5024,7 +5024,7 @@ function TSQLRequest.Execute(aDB: TSQLite3DB; const aSQL: RawUTF8; JSON: TStream
    W: TJSONWriter;
 begin
  result := 0;
-  
-  W := TJSONWriter.Create(JSON,Expand,false);
+  
+  W := TJSONWriter.Create(JSON,Expand,false,nil,{bufsize=}65536);
  try
    // prepare the SQL request
    if aSQL<>'' then // if not already prepared, reset and bound by caller

diff --git a/SynopseCommit.inc b/SynopseCommit.inc
@@ -1 +1 @@
-  
-'1.18.5011'
+  
+'1.18.5012'