Permalink
Browse files

Merge branch 'master' into master

  • Loading branch information...
RobWunderlich committed Jan 15, 2019
2 parents c3ef51e + 0dead9e commit e8fc047feeac43758b5193608d0d93d2b4aac8c9
Showing with 313 additions and 359 deletions.
  1. +3 −1 .gitignore
  2. BIN Doc/QvcDocumentation.qvw
  3. +111 −101 QVC_Source/Qvc_DbExtract.qvs
  4. +4 −2 QVC_Source/Qvc_Utility.qvs
  5. 0 Qvc_Runtime/{language → Language}/Qvc_Language_PL.qvs
  6. +187 −131 Qvc_Runtime/Qvc.qvs
  7. +8 −8 Qvc_Runtime/language/Qvc_Language_RU.qvs
  8. +0 −2 Tests/Test_Cleanup_log.txt
  9. +0 −5 Tests/Test_CreateWildMapExpression_log.txt
  10. +0 −3 Tests/Test_DataLineage_log.txt
  11. BIN Tests/Test_DefaultIfEmpty.qvw
  12. +0 −6 Tests/Test_DefaultIfEmpty_log.txt
  13. BIN Tests/Test_EmptyQvd.qvw
  14. BIN Tests/Test_ExpandInterval.qvw
  15. BIN Tests/Test_ExportModel.qvw
  16. BIN Tests/Test_FileExists.qvw
  17. +0 −8 Tests/Test_FileExists_log.txt
  18. +0 −12 Tests/Test_GetFieldValues_log.txt
  19. BIN Tests/Test_Icons.qvw
  20. BIN Tests/Test_InRange.qvw
  21. BIN Tests/Test_IncrementalReload.qvw
  22. BIN Tests/Test_JoinGenericTables.qvw
  23. +0 −4 Tests/Test_JoinGenericTables_log.txt
  24. BIN Tests/Test_LinkTable.qvw
  25. +0 −8 Tests/Test_LinkTable_log.txt
  26. BIN Tests/Test_ListDirectories.qvw
  27. +0 −5 Tests/Test_ListDirectories_log.txt
  28. BIN Tests/Test_ListFiles.qvw
  29. BIN Tests/Test_Log.qvw
  30. +0 −13 Tests/Test_Log_log.txt
  31. BIN Tests/Test_NVL.qvw
  32. BIN Tests/Test_PopulateVariables.qvw
  33. +0 −7 Tests/Test_PopulateVariables_log.txt
  34. BIN Tests/Test_QvcAvailableUpdate.qvw
  35. +0 −2 Tests/Test_QvcAvailableUpdate_log.txt
  36. BIN Tests/Test_QvdFieldNumber.qvw
  37. BIN Tests/Test_SegmentedStore.qvw
  38. BIN Tests/Test_TableExists.qvw
  39. +0 −5 Tests/Test_TableExists_log.txt
  40. BIN Tests/Test_TableStats.qvw
  41. +0 −19 Tests/Test_TableStats_log.txt
  42. BIN Tests/Test_UniqueId.qvw
  43. +0 −5 Tests/Test_UniqueId_log.txt
  44. BIN Tests/Test_UpdateQvd.qvw
  45. +0 −12 Tests/Test_UpdateQvd_log.txt
  46. BIN Tools/QvsEditor.qvw
@@ -12,4 +12,6 @@ Tests/*_log.txt
#Qvc_Runtime

# Office temp files
~$*
~$*
build/QvcValidate.qvw
build/ScriptRepository_V1.8.qvw
BIN -8.25 KB (97%) Doc/QvcDocumentation.qvw
Binary file not shown.
@@ -203,124 +203,134 @@ SET _qvctemp.den.qvdAge=; // Delete variable

IF $(_qvctemp.den.doExtract) THEN // If not turned off by previous test, then extract this table

//==============================================
// Find the connection. First look for a connectionString,
// then a connectionFile.
//==============================================
IF len('$(_qvctemp.den.connectionString)') > 0 THEN // We have a connectionString, use it.
SET _qvctemp.den.connect = $(_qvctemp.den.connectionString);

// No connectionString, look for a connectionFile.
ELSEIF len('$(_qvctemp.den.connectionFilename)') > 0 THEN // We have a connectionFile.
// Load the connection string.
_qvctemp.Conn_temp:
LOAD @1:n as _qvctemp.ConnectString
FROM [$(Qvc.Loader.v.ConnectionDir)\$(_qvctemp.den.connectionFilename)]
(fix, codepage is 1252);

LET _qvctemp.den.connect = peek('_qvctemp.ConnectString'); // Assign to connection string to variable
DROP TABLE _qvctemp.Conn_temp; // Drop temp table
ELSE
// Neither connectionString or connectionFile, it's an error.
CALL Qvc.LogError('No connection specified for $(_qvctemp.den.table). Extract skipped.');
SET _qvctemp.den.doExtract = 0; // Skip this table
END IF

//==============================================
// Find the connection. First look for a connectionString,
// then a connectionFile.
//==============================================
IF len('$(_qvctemp.den.connectionString)') > 0 THEN // We have a connectionString, use it.
SET _qvctemp.den.connect = $(_qvctemp.den.connectionString);

// No connectionString, look for a connectionFile.
ELSEIF len('$(_qvctemp.den.connectionFilename)') > 0 THEN // We have a connectionFile.
// Load the connection string.
_qvctemp.Conn_temp:
LOAD @1:n as _qvctemp.ConnectString
FROM [$(Qvc.Loader.v.ConnectionDir)\$(_qvctemp.den.connectionFilename)]
(fix, codepage is 1252);

LET _qvctemp.den.connect = peek('_qvctemp.ConnectString'); // Assign to connection string to variable
DROP TABLE _qvctemp.Conn_temp; // Drop temp table
ELSE
// Neither connectionString or connectionFile, it's an error.
CALL Qvc.LogError('No connection specified for $(_qvctemp.den.table). Extract skipped.');
SET _qvctemp.den.doExtract = 0; // Skip this table
ENDIF

IF $(_qvctemp.den.doExtract) THEN // If not turned off by previous test, then extract this table
$(_qvctemp.den.connect); // Execute the Connect statement
$(_qvctemp.den.connect); // Execute the Connect statement

LET _qvctemp.den.extractStart = num(now(1),'00000.0000000','.',','); // Record a start time for the Extract
LET _qvctemp.den.extractStart = num(now(1),'00000.0000000','.',','); // Record a start time for the Extract

QUALIFY ExtractTimestamp; // File ExtractTimestamp will record date of extract in final table.
QUALIFY ExtractTimestamp; // File ExtractTimestamp will record date of extract in final table.

// Calc fullreload flag
LET _qvctemp.den.fullReload = if('$(_qvctemp.den.extractMode)'='F', -1, 0); // Set full reload boolean flag
// Call IncrementalSetup().
// Calling parameters are Tablename, UpdateColumn, QVD_PrimaryKey.
CALL Qvc.IncrementalSetup ('$(_qvctemp.den.qvdName)', '$(_qvctemp.den.updateField)', '$(_qvctemp.den.keyField)', $(_qvctemp.den.fullReload));
// Calc fullreload flag
LET _qvctemp.den.fullReload = if('$(_qvctemp.den.extractMode)'='F', -1, 0); // Set full reload boolean flag
// Call IncrementalSetup().
// Calling parameters are Tablename, UpdateColumn, QVD_PrimaryKey.
CALL Qvc.IncrementalSetup ('$(_qvctemp.den.qvdName)', '$(_qvctemp.den.updateField)', '$(_qvctemp.den.keyField)', $(_qvctemp.den.fullReload));

LET _qvctemp.den.vSqlStart = num(now(1),'00000.0000000','.',','); // Record SQL start time
LET _qvctemp.den.vSqlStart = num(now(1),'00000.0000000','.',','); // Record SQL start time

//==========================================================
// Execute the SQL Select.
//==========================================================
[$(Qvc.Loader.v.Tablename)]: // vTablename is set by the IncrementalSetup routine.
REM; // Fix syntax error from table label with variable
//==========================================================
// Execute the SQL Select.
//==========================================================
// This IF is written this way to satisfy the script parser requirement that a table label be paired with a
// LOAD statement. Otherwise, we could just generate the table label at the top of the section.
//
// If LOAD Statement provided, use it
IF len('$(_qvctemp.den.loadStatement)')>0 THEN
$(_qvctemp.den.loadStatement);
ELSE // No LOAD Statement, build a SQL SELECT
IF $(Qvc.Loader.v.CreateExtractTimestampField) THEN
LOAD *, now(1) as ExtractTimestamp;
IF len('$(_qvctemp.den.loadStatement)')>0 THEN
[$(Qvc.Loader.v.Tablename)]:
$(_qvctemp.den.loadStatement);
ELSE // No LOAD Statement, build a SQL SELECT
IF $(Qvc.Loader.v.CreateExtractTimestampField) THEN
[$(Qvc.Loader.v.Tablename)]:
LOAD *, now(1) as ExtractTimestamp;
ENDIF

IF $(Qvc.Loader.v.CreateExtractTimestampField) THEN // Dont' generate tablename label
SQL SELECT $(_qvctemp.den.columns) FROM $(_qvctemp.den.table)
WHERE $(Qvc.Loader.v.IncrementalExpression) // IncrementalExpression is set up the Qvc.IncrementalSetup routine.
;
ELSE //Generate tablename label
[$(Qvc.Loader.v.Tablename)]:
SQL SELECT $(_qvctemp.den.columns) FROM $(_qvctemp.den.table)
WHERE $(Qvc.Loader.v.IncrementalExpression) // IncrementalExpression is set up the Qvc.IncrementalSetup routine.
;
ENDIF


ENDIF

UNQUALIFY ExtractTimestamp; // Unqualify the currency field.
DISCONNECT; // Disconnect from database

//==========================================================
// Validate returned data
//==========================================================
// If a Delta extract but keyfield was not included, then log a warning.
IF '$(_qvctemp.den.extractMode)' = 'D' AND FieldNumber('$(_qvctemp.den.keyField)', '$(Qvc.Loader.v.Tablename)')=0 THEN
CALL Qvc.LogWarning('Warning: Keyfield "$(_qvctemp.den.keyField)" not found in table "$(Qvc.Loader.v.Tablename)".');
ENDIF
SQL SELECT $(_qvctemp.den.columns) FROM $(_qvctemp.den.table)
WHERE $(Qvc.Loader.v.IncrementalExpression) // IncrementalExpression is set up the Qvc.IncrementalSetup routine.
;
ENDIF

UNQUALIFY ExtractTimestamp; // Unqualify the currency field.
DISCONNECT; // Disconnect from database
//==========================================================
// Compute and log SQL statistics.
//==========================================================
// Using RangeMax because very short durations can calc as -0 because of different precision.
LET _qvctemp.den.vSqlDuration = Interval(RangeMax(0, Now(1)-$(_qvctemp.den.vSqlStart)), 'hh:mm:ss');
CALL Qvc.Log('SQL SELECT Duration=$(_qvctemp.den.vSqlDuration)');

//==========================================================
// Validate returned data
//==========================================================
// If a Delta extract but keyfield was not included, then log a warning.
IF '$(_qvctemp.den.extractMode)' = 'D' AND FieldNumber('$(_qvctemp.den.keyField)', '$(Qvc.Loader.v.Tablename)')=0 THEN
CALL Qvc.LogWarning('Warning: Keyfield "$(_qvctemp.den.keyField)" not found in table "$(Qvc.Loader.v.Tablename)".');
ENDIF

//==========================================================
// Compute and log SQL statistics.
//==========================================================
// Using RangeMax because very short durations can calc as -0 because of different precision.
LET _qvctemp.den.vSqlDuration = Interval(RangeMax(0, Now(1)-$(_qvctemp.den.vSqlStart)), 'hh:mm:ss');
CALL Qvc.Log('SQL SELECT Duration=$(_qvctemp.den.vSqlDuration)');
LET _qvctemp.den.vNoOfRows = num(NoOfRows('$(Qvc.Loader.v.Tablename)'),'#,##0');
CALL Qvc.Log('$(_qvctemp.den.vNoOfRows) rows loaded from Table $(_qvctemp.den.table)');

LET _qvctemp.den.vNoOfRows = num(NoOfRows('$(Qvc.Loader.v.Tablename)'),'#,##0');
CALL Qvc.Log('$(_qvctemp.den.vNoOfRows) rows loaded from Table $(_qvctemp.den.table)');

//==========================================================
// Prepare and store the QVD.
//==========================================================
// Apply any Table Comment
IF len('$(_qvctemp.den.tableComment)') > 0 THEN
COMMENT TABLES [$(Qvc.Loader.v.Tablename)] WITH '$(_qvctemp.den.tableComment)';
ENDIF

//==========================================================
// Prepare and store the QVD.
//==========================================================
// Apply any Table Comment
IF len('$(_qvctemp.den.tableComment)') > 0 THEN
COMMENT TABLES [$(Qvc.Loader.v.Tablename)] WITH '$(_qvctemp.den.tableComment)';
ENDIF
// Specify if we are using non-unique (generic) key. This has to be specified before IncrementalStore is called.
SET Qvc.Loader.v.KeyFieldIsUnique = $(_qvctemp.den.uniqueKey);

// If we are doing Delete processing, get the keys from the DB Table
IF '$(_qvctemp.den.deletes)'='Y' THEN
[_qvctemp.den.DeleteKeys]:
LOAD [$(_qvctemp.den.keyField)] as [_qvctemp.den.DeleteKeysField]
;
SQL SELECT $(_qvctemp.den.keyField) FROM $(_qvctemp.den.table);
// Call IncrementalStore to update the QVD, with deletes.
CALL Qvc.IncrementalStore('_qvctemp.den.DeleteKeys');
ELSE
// Call IncrementalStore to update the QVD, no deletes.
CALL Qvc.IncrementalStore
ENDIF

// Specify if we are using non-unique (generic) key. This has to be specified before IncrementalStore is called.
SET Qvc.Loader.v.KeyFieldIsUnique = $(_qvctemp.den.uniqueKey);

// If we are doing Delete processing, get the keys from the DB Table
IF '$(_qvctemp.den.deletes)'='Y' THEN
[_qvctemp.den.DeleteKeys]:
LOAD [$(_qvctemp.den.keyField)] as [_qvctemp.den.DeleteKeysField]
//==========================================================
// Record statistics about this load.
//==========================================================
ExtractStats:
LOAD
'$(_qvctemp.den.recid)' as RecId,
timestamp($(_qvctemp.den.extractStart)) as [Extract Start],
'$(_qvctemp.den.vSqlDuration)' as [SQL Duration],
'$(_qvctemp.den.vNoOfRows)' as [SQL Rows],
num(QvdNoOfRecords('$(_qvctemp.den.qvdPath)'),'#,##0') as [QVD Rows],
QvdNoOfFields('$(_qvctemp.den.qvdPath)') as [QVD Fields]
AUTOGENERATE 1
;
SQL SELECT $(_qvctemp.den.keyField) FROM $(_qvctemp.den.table);
// Call IncrementalStore to update the QVD, with deletes.
CALL Qvc.IncrementalStore('_qvctemp.den.DeleteKeys');
ELSE
// Call IncrementalStore to update the QVD, no deletes.
CALL Qvc.IncrementalStore
ENDIF


//==========================================================
// Record statistics about this load.
//==========================================================
ExtractStats:
LOAD
'$(_qvctemp.den.recid)' as RecId,
timestamp($(_qvctemp.den.extractStart)) as [Extract Start],
'$(_qvctemp.den.vSqlDuration)' as [SQL Duration],
'$(_qvctemp.den.vNoOfRows)' as [SQL Rows],
num(QvdNoOfRecords('$(_qvctemp.den.qvdPath)'),'#,##0') as [QVD Rows],
QvdNoOfFields('$(_qvctemp.den.qvdPath)') as [QVD Fields]
AUTOGENERATE 1
;

ENDIF // END for IF $(_qvctemp.den.doExtract) THEN
ENDIF // END for IF $(_qvctemp.den.doExtract) THEN

//==========================================================
@@ -203,12 +203,14 @@ The optional third parameter is a variable name used to return the result. If om
*/
UNQUALIFY "_qvctemp.*"; // UNQUALIFY all qvctemp fields

IF len('$(_qvctemp.retvar)')=0 THEN; SET _qvctemp.retvar = 'Qvc.v.Return'; ENDIF
IF len('$(_qvctemp.retvar)')=0 THEN
SET _qvctemp.retvar = 'Qvc.v.Return';
ENDIF
// Using "0" as not found is consistent with the FieldNumber() function.
SET $(_qvctemp.retvar) = 0; // Default return value is "not found"

// If the QVD does not exist, exit with 0. This is consistent with table-not-found condition in FieldNumber().
IF NOT $(Qvc.FileExists('$(_qvctemp.qvdname)')) THEN ;
IF NOT $(Qvc.FileExists('$(_qvctemp.qvdname)')) THEN
SET $(_qvctemp.retvar) = 0;
EXIT Sub
ENDIF
Oops, something went wrong.

0 comments on commit e8fc047

Please sign in to comment.