Skip to content

Commit

Permalink
allowed any column count limit for SynDB remote binary serialization
Browse files Browse the repository at this point in the history
  • Loading branch information
Arnaud Bouchez committed May 17, 2019
1 parent 93ae66c commit 0822a40
Show file tree
Hide file tree
Showing 4 changed files with 45 additions and 54 deletions.
4 changes: 3 additions & 1 deletion SQLite3/Samples/ThirdPartyDemos/EMartin/TSynRestDataset/SynRestMidasVCL.pas
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,9 @@ interface
mORMot,
mORMotHttpClient,
SynCommons,
SynDB, SynRestVCL,
SynTable,
SynDB,
SynRestVCL,
DB,
{$ifdef FPC}
BufDataset
Expand Down
24 changes: 10 additions & 14 deletions SQLite3/Samples/ThirdPartyDemos/EMartin/TSynRestDataset/SynRestVCL.pas
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,11 +172,10 @@ TSynRestSQLDataSet = class(TSynBinaryDataSet)

// JSON columns to binary from a TSQLTableJSON, is not ideal because this code is a almost repeated code.
procedure JSONColumnsToBinary(const aTable: TSQLTableJSON; W: TFileBufferWriter;
const Null: TSQLDBProxyStatementColumns;
const ColTypes: TSQLDBFieldTypeDynArray);
Null: pointer; const ColTypes: TSQLDBFieldTypeDynArray);
// convert to binary from a TSQLTableJSON, is not ideal because this code is a almost repeated code.
function JSONToBinary(const aTable: TSQLTableJSON; Dest: TStream; MaxRowCount: cardinal=0; DataRowPosition: PCardinalDynArray=nil;
const DefaultDataType: TSQLDBFieldType = SynTable.ftUTF8; const DefaultFieldSize: Integer = 255): cardinal;
const DefaultDataType: TSQLDBFieldType = SynTable.ftUTF8; const DefaultFieldSize: Integer = 255): cardinal;

implementation

Expand Down Expand Up @@ -285,15 +284,15 @@ implementation


procedure JSONColumnsToBinary(const aTable: TSQLTableJSON; W: TFileBufferWriter;
const Null: TSQLDBProxyStatementColumns; const ColTypes: TSQLDBFieldTypeDynArray);
Null: pointer; const ColTypes: TSQLDBFieldTypeDynArray);
var F: integer;
VDouble: double;
VCurrency: currency absolute VDouble;
VDateTime: TDateTime absolute VDouble;
colType: TSQLDBFieldType;
begin
for F := 0 to length(ColTypes)-1 do
if not (F in Null) then begin
if not GetBitPtr(Null,F) then begin
colType := ColTypes[F];
if colType<ftInt64 then begin // ftUnknown,ftNull
colType := SQLFIELDTYPETODBFIELDTYPE[aTable.FieldType(F)]; // per-row column type (SQLite3 only)
Expand Down Expand Up @@ -335,12 +334,11 @@ function JSONToBinary(const aTable: TSQLTableJSON; Dest: TStream; MaxRowCount: c
const DefaultDataType: TSQLDBFieldType = SynTable.ftUTF8; const DefaultFieldSize: Integer = 255): cardinal;
var F, FMax, FieldSize, NullRowSize: integer;
StartPos: cardinal;
Null: TSQLDBProxyStatementColumns;
Null: TByteDynArray;
W: TFileBufferWriter;
ColTypes: TSQLDBFieldTypeDynArray;
FieldType: TSQLDBFieldType;
begin
FillChar(Null,sizeof(Null),0);
result := 0;
W := TFileBufferWriter.Create(Dest);
try
Expand All @@ -364,10 +362,8 @@ function JSONToBinary(const aTable: TSQLTableJSON; Dest: TStream; MaxRowCount: c
W.WriteVarUInt32(FieldSize);
end;
// initialize null handling
NullRowSize := (FMax shr 3)+1;
if NullRowSize>sizeof(Null) then
raise ESQLDBException.CreateUTF8(
'JSONToBinary: too many columns', []);
SetLength(Null,(FMax shr 3)+1);
NullRowSize := 0;
// save all data rows
StartPos := W.TotalWritten;
if aTable.Step or (aTable.RowCount=1) then // Need step first or error is raised in Table.Field function.
Expand All @@ -380,19 +376,19 @@ function JSONToBinary(const aTable: TSQLTableJSON; Dest: TStream; MaxRowCount: c
end;
// first write null columns flags
if NullRowSize>0 then begin
FillChar(Null,NullRowSize,0);
FillChar(Null[0],NullRowSize,0);
NullRowSize := 0;
end;
for F := 0 to FMax do
begin
if VarIsNull(aTable.Field(F)) then begin
include(Null,F);
SetBitPtr(pointer(Null),F);
NullRowSize := (F shr 3)+1;
end;
end;
W.WriteVarUInt32(NullRowSize);
if NullRowSize>0 then
W.Write(@Null,NullRowSize);
W.Write(Null,NullRowSize);
// then write data values
JSONColumnsToBinary(aTable, W,Null,ColTypes);
inc(result);
Expand Down
69 changes: 31 additions & 38 deletions SynDB.pas
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1101,12 +1101,6 @@ TSQLDBDefinitionLimitClause = record
const FieldTypes: TSQLDBFieldTypeArray; RowCount: integer;
const FieldValues: TRawUTF8DynArrayDynArray) of object;

/// bit set to identify columns, e.g. null columns
TSQLDBProxyStatementColumns = set of 0..255;

/// pointer to a bit set to identify columns, e.g. null columns
PSQLDBProxyStatementColumns = ^TSQLDBProxyStatementColumns;

/// specify the class of TSQLDBConnectionProperties
// - sometimes used to create connection properties instances, from a set
// of available classes (see e.g. SynDBExplorer or sample 16)
Expand Down Expand Up @@ -2347,8 +2341,7 @@ TSQLDBStatement = class(TInterfacedObject, ISQLDBRows, ISQLDBStatement)
// - virtual method called by FetchAllToBinary()
// - follows the format expected by TSQLDBProxyStatement
procedure ColumnsToBinary(W: TFileBufferWriter;
const Null: TSQLDBProxyStatementColumns;
const ColTypes: TSQLDBFieldTypeDynArray); virtual;
Null: pointer; const ColTypes: TSQLDBFieldTypeDynArray); virtual;
published
/// the prepared SQL statement, as supplied to Prepare() method
property SQL: RawUTF8 read fSQL;
Expand Down Expand Up @@ -2794,9 +2787,9 @@ TSQLDBProxyStatementAbstract = class(TSQLDBStatementWithParamsAndColumns)
fDataRowCount: integer;
fDataRowReaderOrigin, fDataRowReader: PByte;
fDataRowNullSize: cardinal;
fDataCurrentRowNullLen: cardinal;
fDataCurrentRowNull: TSQLDBProxyStatementColumns;
fDataCurrentRowIndex: integer;
fDataCurrentRowNullLen: cardinal;
fDataCurrentRowNull: TByteDynArray;
fDataCurrentRowValues: array of pointer;
fDataCurrentRowValuesStart: pointer;
fDataCurrentRowValuesSize: Cardinal;
Expand Down Expand Up @@ -2838,8 +2831,7 @@ TSQLDBProxyStatementAbstract = class(TSQLDBStatementWithParamsAndColumns)
// - virtual method called by FetchAllToBinary()
// - follows the format expected by TSQLDBProxyStatement
procedure ColumnsToBinary(W: TFileBufferWriter;
const Null: TSQLDBProxyStatementColumns;
const ColTypes: TSQLDBFieldTypeDynArray); override;
Null: pointer; const ColTypes: TSQLDBFieldTypeDynArray); override;

/// read-only access to the number of data rows stored
property DataRowCount: integer read fDataRowCount;
Expand Down Expand Up @@ -4861,7 +4853,7 @@ function TSQLDBConnectionProperties.SharedTransaction(SessionID: cardinal;
else begin // (nested) commit/rollback
if InterlockedDecrement(fSharedTransactions[i].RefCount)=0 then begin
dec(n);
move(fSharedTransactions[i+1],fSharedTransactions[i],(n-i)*sizeof(fSharedTransactions[0]));
MoveFast(fSharedTransactions[i+1],fSharedTransactions[i],(n-i)*sizeof(fSharedTransactions[0]));
SetLength(fSharedTransactions,n);
case action of
transCommitWithException, transCommitWithoutException:
Expand Down Expand Up @@ -5179,7 +5171,7 @@ procedure TSQLDBConnectionProperties.GetFields(const aTableName: RawUTF8;
begin
FA.Init(TypeInfo(TSQLDBColumnDefineDynArray),Fields,@n);
FA.Compare := SortDynArrayAnsiStringI; // FA.Find() case insensitive
fillchar(F,sizeof(F),0);
FillCharFast(F,sizeof(F),0);
if fDBMS=dSQLite then begin // SQLite3 has a specific PRAGMA metadata query
try
with Execute('PRAGMA table_info(`'+aTableName+'`)',[]) do
Expand Down Expand Up @@ -5286,7 +5278,7 @@ procedure TSQLDBConnectionProperties.GetProcedureParameters(const aProcName: Raw
begin
FA.Init(TypeInfo(TSQLDBColumnDefineDynArray),Parameters,@n);
FA.Compare := SortDynArrayAnsiStringI; // FA.Find() case insensitive
fillchar(F,sizeof(F),0);
FillcharFast(F,sizeof(F),0);
SQL := SQLGetParameter(aProcName);
if SQL='' then
exit;
Expand Down Expand Up @@ -7121,15 +7113,15 @@ function TSQLDBStatement.FetchAllAsJSON(Expanded: boolean;
end;

procedure TSQLDBStatement.ColumnsToBinary(W: TFileBufferWriter;
const Null: TSQLDBProxyStatementColumns; const ColTypes: TSQLDBFieldTypeDynArray);
Null: pointer; const ColTypes: TSQLDBFieldTypeDynArray);
var F: integer;
VDouble: double;
VCurrency: currency absolute VDouble;
VDateTime: TDateTime absolute VDouble;
colType: TSQLDBFieldType;
begin
for F := 0 to length(ColTypes)-1 do
if not (F in Null) then begin
if not GetBitPtr(Null, F) then begin
colType := ColTypes[F];
if colType<ftInt64 then begin // ftUnknown,ftNull
colType := ColumnType(F); // per-row column type (SQLite3 only)
Expand Down Expand Up @@ -7168,11 +7160,10 @@ function TSQLDBStatement.FetchAllToBinary(Dest: TStream; MaxRowCount: cardinal;
DataRowPosition: PCardinalDynArray): cardinal;
var F, FMax, FieldSize, NullRowSize: integer;
StartPos: cardinal;
Null: TSQLDBProxyStatementColumns;
W: TFileBufferWriter;
ColTypes: TSQLDBFieldTypeDynArray;
Null: TByteDynArray;
begin
FillChar(Null,sizeof(Null),0);
result := 0;
W := TFileBufferWriter.Create(Dest);
try
Expand All @@ -7190,10 +7181,8 @@ function TSQLDBStatement.FetchAllToBinary(Dest: TStream; MaxRowCount: cardinal;
W.WriteVarUInt32(FieldSize);
end;
// initialize null handling
NullRowSize := (FMax shr 3)+1;
if NullRowSize>sizeof(Null) then
raise ESQLDBException.CreateUTF8(
'%.FetchAllToBinary: too many columns',[self]);
SetLength(Null,(FMax shr 3)+1);
NullRowSize := 0;
// save all data rows
StartPos := W.TotalWritten;
if (CurrentRow=1) or Step then // Step may already be done (e.g. TQuery.Open)
Expand All @@ -7206,19 +7195,19 @@ function TSQLDBStatement.FetchAllToBinary(Dest: TStream; MaxRowCount: cardinal;
end;
// first write null columns flags
if NullRowSize>0 then begin
FillChar(Null,NullRowSize,0);
FillCharFast(Null[0],NullRowSize,0);
NullRowSize := 0;
end;
for F := 0 to FMax do
if ColumnNull(F) then begin
include(Null,F);
SetBitPtr(pointer(Null),F);
NullRowSize := (F shr 3)+1;
end;
W.WriteVarUInt32(NullRowSize);
if NullRowSize>0 then
W.Write(@Null,NullRowSize);
W.Write(pointer(Null),NullRowSize);
// then write data values
ColumnsToBinary(W,Null,ColTypes);
ColumnsToBinary(W,pointer(Null),ColTypes);
inc(result);
if (MaxRowCount>0) and (result>=MaxRowCount) then
break;
Expand Down Expand Up @@ -8294,6 +8283,8 @@ procedure TSQLDBProxyStatementAbstract.IntHeaderProcess(Data: PByte; DataLen: in
fDataCurrentRowValuesStart := nil;
fDataCurrentRowValuesSize := 0;
fDataCurrentRowIndex := -1;
fDataCurrentRowNull := nil;
fDataCurrentRowNullLen := 0;
repeat
if DataLen<=5 then
break;
Expand All @@ -8314,12 +8305,12 @@ procedure TSQLDBProxyStatementAbstract.IntHeaderProcess(Data: PByte; DataLen: in
end;
if fColumnCount=0 then
exit; // no data returned
if (fColumnCount>sizeof(TSQLDBProxyStatementColumns)shl 3) or
(cardinal(fDataRowCount)>=cardinal(DataLen) div cardinal(fColumnCount)) then
break;
if cardinal(fDataRowCount)>=cardinal(DataLen) then
break; // obviously truncated
fDataRowReaderOrigin := Data;
fDataRowReader := Data;
fDataRowNullSize := ((fColumnCount-1) shr 3)+1;
SetLength(fDataCurrentRowNull,fDataRowNullSize);
exit;
until false;
fDataRowCount := 0;
Expand All @@ -8332,17 +8323,18 @@ procedure TSQLDBProxyStatementAbstract.IntFillDataCurrent(var Reader: PByte;
var F,Len: Integer;
begin // format match TSQLDBStatement.FetchAllToBinary()
if fDataCurrentRowNullLen>0 then
FillChar(fDataCurrentRowNull,fDataCurrentRowNullLen,0);
FillCharFast(fDataCurrentRowNull[0],fDataCurrentRowNullLen,0);
fDataCurrentRowNullLen := FromVarUInt32(Reader);
if fDataCurrentRowNullLen>fDataRowNullSize then
raise ESQLDBException.CreateUTF8('Invalid %.IntFillDataCurrent',[self]);
raise ESQLDBException.CreateUTF8('Invalid %.IntFillDataCurrent %>%',
[self,fDataCurrentRowNullLen,fDataRowNullSize]);
if fDataCurrentRowNullLen>0 then begin
Move(Reader^,fDataCurrentRowNull,fDataCurrentRowNullLen);
MoveFast(Reader^,fDataCurrentRowNull[0],fDataCurrentRowNullLen);
inc(Reader,fDataCurrentRowNullLen);
end;
fDataCurrentRowValuesStart := Reader;
for F := 0 to fColumnCount-1 do
if F in fDataCurrentRowNull then
if GetBitPtr(pointer(fDataCurrentRowNull),F) then
fDataCurrentRowValues[F] := nil else begin
fDataCurrentRowColTypes[F] := fColumns[F].ColumnType;
if fDataCurrentRowColTypes[F]<ftInt64 then begin
Expand Down Expand Up @@ -8415,7 +8407,7 @@ procedure TSQLDBProxyStatementAbstract.ColumnsToJSON(WR: TJSONWriter);
end;

procedure TSQLDBProxyStatementAbstract.ColumnsToBinary(W: TFileBufferWriter;
const Null: TSQLDBProxyStatementColumns; const ColTypes: TSQLDBFieldTypeDynArray);
Null: pointer; const ColTypes: TSQLDBFieldTypeDynArray);
begin
W.Write(fDataCurrentRowValuesStart,fDataCurrentRowValuesSize);
end;
Expand All @@ -8427,10 +8419,10 @@ function TSQLDBProxyStatementAbstract.ColumnData(Col: integer): pointer;
result := nil;
end;

function TSQLDBProxyStatementAbstract.ColumnType(Col: integer; FieldSize: PInteger=nil): TSQLDBFieldType;
function TSQLDBProxyStatementAbstract.ColumnType(Col: integer; FieldSize: PInteger): TSQLDBFieldType;
begin
if (fDataRowCount>0) and (cardinal(Col)<cardinal(fColumnCount)) then
if Col in fDataCurrentRowNull then
if GetBitPtr(pointer(fDataCurrentRowNull),Col) then
result := ftNull else
with fColumns[Col] do begin
if FieldSize<>nil then
Expand Down Expand Up @@ -8502,7 +8494,8 @@ function TSQLDBProxyStatementAbstract.ColumnInt(Col: integer): Int64;

function TSQLDBProxyStatementAbstract.ColumnNull(Col: integer): boolean;
begin
result := Col in fDataCurrentRowNull;
result := (cardinal(Col)>=cardinal(fColumnCount)) or
GetBitPtr(pointer(fDataCurrentRowNull),Col);
end;

function TSQLDBProxyStatementAbstract.ColumnBlob(Col: integer): RawByteString;
Expand Down
2 changes: 1 addition & 1 deletion SynopseCommit.inc
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
'1.18.5220'
'1.18.5221'

0 comments on commit 0822a40

Please sign in to comment.