From 17d1bd64a30a41dcd393f7cc1d9c05ce5aab33a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20R=C3=B6nnb=C3=A4ck?= Date: Fri, 28 Apr 2017 14:46:32 +0200 Subject: [PATCH] Added BIML generation code for anchor/attribute loading. --- Example/biml/target.biml | 375 ++++++++++++++++++++++++++++++++ Example/sources/source.sql | 114 ++++------ Example/targets/target.sql | 8 +- Example/workflows/workflow.sql | 8 +- Sisulate.bat | 12 + biml.directive | 8 + sisulets/biml/CreatePackages.js | 346 +++++++++++++++++++++++++++++ sisulets/biml/Helpers.js | 60 +++++ sisulets/biml/Naming.js | 6 + sisulets/biml/Variables.js | 32 +++ 10 files changed, 884 insertions(+), 85 deletions(-) create mode 100644 Example/biml/target.biml create mode 100644 biml.directive create mode 100644 sisulets/biml/CreatePackages.js create mode 100644 sisulets/biml/Helpers.js create mode 100644 sisulets/biml/Naming.js create mode 100644 sisulets/biml/Variables.js diff --git a/Example/biml/target.biml b/Example/biml/target.biml new file mode 100644 index 0000000..8c7cf04 --- /dev/null +++ b/Example/biml/target.biml @@ -0,0 +1,375 @@ + + + + + + + + + + + -- preparations can be put here + + + + + DISABLE TRIGGER ALL ON [dbo].[ST_NAM_Street_Name]; + ALTER TABLE [dbo].[ST_NAM_Street_Name] NOCHECK CONSTRAINT ALL; + + + + + + + DECLARE @known INT = 0; + MERGE [Traffic].[dbo].[ST_Street] [ST] + USING ( + SELECT + l.ST_ID, + t.StreetName, + t.metadata_CO_ID + FROM ( + select + StreetName, + min(metadata_CO_ID) as metadata_CO_ID + from ( + select distinct + IntersectingStreet as StreetName, + metadata_CO_ID + from + etl.NYPD_Vehicle_Collision_Typed + union + select distinct + CrossStreet as StreetName, + metadata_CO_ID + from + etl.NYPD_Vehicle_Collision_Typed + ) s + group by + StreetName + ) t + LEFT JOIN + [Traffic].[dbo].[lST_Street] l + ON + t.StreetName = l.ST_NAM_Street_Name + ) src + ON + src.ST_ID = [ST].ST_ID + WHEN NOT MATCHED THEN + INSERT ( Metadata_ST ) + VALUES ( src.metadata_CO_ID ) + WHEN MATCHED THEN + UPDATE SET @known = @known + 1 + OUTPUT + isnull(src.ST_ID, inserted.ST_ID) as ST_ID, + src.StreetName, + src.metadata_CO_ID, + left($action, 1) as Operation; + + + + + + [Operation]=="U" + + + [Operation]=="I" + + + + + + + + + + + + + + + + + + + + + + + ENABLE TRIGGER ALL ON [dbo].[ST_NAM_Street_Name]; + ALTER TABLE [dbo].[ST_NAM_Street_Name] WITH NOCHECK CHECK CONSTRAINT ALL; + + + + + -- post processing can be put here + + + + + + + + + + + DECLARE @known INT = 0; + MERGE [Traffic].[dbo].[IS_Intersection] [IS] + USING ( + select + src.IntersectingStreet, + src.CrossStreet, + src.metadata_CO_ID, + stst.IS_ID_of + from ( + select + IntersectingStreet, + CrossStreet, + min(metadata_CO_ID) as metadata_CO_ID + from + etl.NYPD_Vehicle_Collision_Typed + group by + IntersectingStreet, + CrossStreet + ) src + left join + [Traffic].dbo.lST_Street st_i + on + st_i.ST_NAM_Street_Name = src.IntersectingStreet + left join + [Traffic].dbo.lST_Street st_c + on + st_c.ST_NAM_Street_Name = src.CrossStreet + left join + [Traffic].dbo.ST_intersecting_IS_of_ST_crossing stst + on + stst.ST_ID_intersecting = st_i.ST_ID + and + stst.ST_ID_crossing = st_c.ST_ID + ) src + ON + src.IS_ID_of = [IS].IS_ID + WHEN NOT MATCHED THEN + INSERT ( Metadata_IS ) + VALUES ( src.metadata_CO_ID ) + WHEN MATCHED THEN + UPDATE SET @known = @known + 1 + OUTPUT + isnull(src.IS_ID_of, inserted.IS_ID) as IS_ID, + src.metadata_CO_ID, + left($action, 1) as Operation; + + + + + + + + + + + + + + + + + + + DISABLE TRIGGER ALL ON [dbo].[IS_COL_Intersection_CollisionCount]; + ALTER TABLE [dbo].[IS_COL_Intersection_CollisionCount] NOCHECK CONSTRAINT ALL; + DISABLE TRIGGER ALL ON [dbo].[IS_VEH_Intersection_VehicleCount]; + ALTER TABLE [dbo].[IS_VEH_Intersection_VehicleCount] NOCHECK CONSTRAINT ALL; + DISABLE TRIGGER ALL ON [dbo].[IS_INJ_Intersection_InjuredCount]; + ALTER TABLE [dbo].[IS_INJ_Intersection_InjuredCount] NOCHECK CONSTRAINT ALL; + DISABLE TRIGGER ALL ON [dbo].[IS_KIL_Intersection_KilledCount]; + ALTER TABLE [dbo].[IS_KIL_Intersection_KilledCount] NOCHECK CONSTRAINT ALL; + + + + + + + DECLARE @known INT = 0; + MERGE [Traffic].[dbo].[IS_Intersection] [IS] + USING ( + select + md.ChangedAt, + stst.IS_ID_of, + count(*) as CollisionCount, + sum(src.CollisionVehicleCount) as CollisionVehicleCount, + sum(src.CollisionInjuredCount) as CollisionInjuredCount, + sum(src.CollisionKilledCount) as CollisionKilledCount + from + etl.NYPD_Vehicle_Collision_Typed src + join + etl.NYPD_Vehicle_CollisionMetadata_Typed md + on + md.metadata_CO_ID = src.metadata_CO_ID + join + [Traffic].dbo.lST_Street st_i + on + st_i.ST_NAM_Street_Name = src.IntersectingStreet + join + [Traffic].dbo.lST_Street st_c + on + st_c.ST_NAM_Street_Name = src.CrossStreet + join + [Traffic].dbo.ST_intersecting_IS_of_ST_crossing stst + on + stst.ST_ID_intersecting = st_i.ST_ID + and + stst.ST_ID_crossing = st_c.ST_ID + group by + md.ChangedAt, + stst.IS_ID_of + ) src + ON + src.IS_ID_of = [IS].IS_ID + WHEN MATCHED THEN + UPDATE SET @known = @known + 1 + OUTPUT + isnull(src.IS_ID_of, inserted.IS_ID) as IS_ID, + src.CollisionCount, + src.ChangedAt, + src.CollisionVehicleCount, + src.CollisionInjuredCount, + src.CollisionKilledCount, + 0 as Metadata_IS, + left($action, 1) as Operation; + + + + + + [Operation]=="U" + + + [Operation]=="I" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ENABLE TRIGGER ALL ON [dbo].[IS_COL_Intersection_CollisionCount]; + ALTER TABLE [dbo].[IS_COL_Intersection_CollisionCount] WITH NOCHECK CHECK CONSTRAINT ALL; + ENABLE TRIGGER ALL ON [dbo].[IS_VEH_Intersection_VehicleCount]; + ALTER TABLE [dbo].[IS_VEH_Intersection_VehicleCount] WITH NOCHECK CHECK CONSTRAINT ALL; + ENABLE TRIGGER ALL ON [dbo].[IS_INJ_Intersection_InjuredCount]; + ALTER TABLE [dbo].[IS_INJ_Intersection_InjuredCount] WITH NOCHECK CHECK CONSTRAINT ALL; + ENABLE TRIGGER ALL ON [dbo].[IS_KIL_Intersection_KilledCount]; + ALTER TABLE [dbo].[IS_KIL_Intersection_KilledCount] WITH NOCHECK CHECK CONSTRAINT ALL; + + + + + + diff --git a/Example/sources/source.sql b/Example/sources/source.sql index f53e421..77a1544 100644 --- a/Example/sources/source.sql +++ b/Example/sources/source.sql @@ -19,6 +19,7 @@ GO IF Object_Id('etl.ToLocalTime', 'FS') IS NOT NULL DROP FUNCTION [etl].[ToLocalTime]; GO +-- BEGIN! LEGACY -- IF EXISTS ( SELECT * @@ -28,92 +29,51 @@ IF EXISTS ( name = 'etlUtilities' ) DROP ASSEMBLY etlUtilities; -IF NOT EXISTS ( - SELECT - * - FROM - sys.assemblies - WHERE - name = 'etlUtilities' -) -BEGIN TRY -- using Microsoft.SQLServer.Types version 13 (2016) - CREATE ASSEMBLY etlUtilities - AUTHORIZATION dbo - FROM 'C:\sisula\code\Utilities2016.dll' - WITH PERMISSION_SET = SAFE; - PRINT 'The .NET CLR for SQL Server 2016 was installed.' -END TRY -BEGIN CATCH - PRINT 'The .NET CLR for SQL Server 2016 was NOT installed.' -END CATCH -IF NOT EXISTS ( - SELECT - * - FROM - sys.assemblies - WHERE - name = 'etlUtilities' -) -BEGIN TRY -- using Microsoft.SQLServer.Types version 12 (2014) - CREATE ASSEMBLY etlUtilities - AUTHORIZATION dbo - FROM 'C:\sisula\code\Utilities2014.dll' - WITH PERMISSION_SET = SAFE; - PRINT 'The .NET CLR for SQL Server 2014 was installed.' -END TRY -BEGIN CATCH - PRINT 'The .NET CLR for SQL Server 2014 was NOT installed.' -END CATCH -IF NOT EXISTS ( +-- END! LEGACY -- +declare @version char(4) = + case + when @@VERSION like '% 2016 %' then '2016' + when @@VERSION like '% 2014 %' then '2014' + when @@VERSION like '% 2012 %' then '2012' + when @@VERSION like '% 2008 %' then '2008' + else '????' + end +IF EXISTS ( SELECT * FROM sys.assemblies WHERE - name = 'etlUtilities' + name = 'Utilities' ) -BEGIN TRY -- using Microsoft.SQLServer.Types version 11 (2012) - CREATE ASSEMBLY etlUtilities - AUTHORIZATION dbo - FROM 'C:\sisula\code\Utilities2012.dll' +BEGIN TRY + ALTER ASSEMBLY Utilities + FROM 'C:\sisula\code\Utilities' + @version + '.dll' WITH PERMISSION_SET = SAFE; - PRINT 'The .NET CLR for SQL Server 2012 was installed.' -END TRY -BEGIN CATCH - PRINT 'The .NET CLR for SQL Server 2012 was NOT installed.' -END CATCH -IF NOT EXISTS ( - SELECT - * - FROM - sys.assemblies - WHERE - name = 'etlUtilities' -) -BEGIN TRY -- using Microsoft.SQLServer.Types version 10 (2008) - CREATE ASSEMBLY etlUtilities + PRINT 'The .NET CLR for SQL Server ' + @version + ' was updated.' +END TRY BEGIN CATCH END CATCH +ELSE -- assembly does not exist +BEGIN TRY + CREATE ASSEMBLY Utilities AUTHORIZATION dbo - FROM 'C:\sisula\code\Utilities2008.dll' + FROM 'C:\sisula\code\Utilities' + @version + '.dll' WITH PERMISSION_SET = SAFE; - PRINT 'The .NET CLR for SQL Server 2008 was installed.' -END TRY -BEGIN CATCH - PRINT 'The .NET CLR for SQL Server 2008 was NOT installed.' -END CATCH + PRINT 'The .NET CLR for SQL Server ' + @version + ' was installed.' +END TRY BEGIN CATCH END CATCH GO CREATE FUNCTION [etl].Splitter(@row AS nvarchar(max), @pattern AS nvarchar(4000)) RETURNS TABLE ( [match] nvarchar(max), [index] int -) AS EXTERNAL NAME etlUtilities.Splitter.InitMethod; +) AS EXTERNAL NAME Utilities.Splitter.InitMethod; GO CREATE FUNCTION [etl].IsType(@dataValue AS nvarchar(max), @dataType AS nvarchar(4000)) RETURNS bit -AS EXTERNAL NAME etlUtilities.IsType.InitMethod; +AS EXTERNAL NAME Utilities.IsType.InitMethod; GO CREATE FUNCTION [etl].ToLocalTime(@sqlDatetime AS datetime) RETURNS datetime -AS EXTERNAL NAME etlUtilities.ToLocalTime.InitMethod; +AS EXTERNAL NAME Utilities.ToLocalTime.InitMethod; GO CREATE PROCEDURE [etl].ColumnSplitter( @table AS nvarchar(4000), @@ -121,7 +81,7 @@ CREATE PROCEDURE [etl].ColumnSplitter( @pattern AS nvarchar(4000), @includeColumns AS nvarchar(4000) = null ) -AS EXTERNAL NAME etlUtilities.ColumnSplitter.InitMethod; +AS EXTERNAL NAME Utilities.ColumnSplitter.InitMethod; GO IF NOT EXISTS ( SELECT value @@ -157,7 +117,7 @@ GO -- _timestamp -- The time the row was created. -- --- Generated: Mon May 2 11:10:30 UTC+0200 2016 by e-lronnback +-- Generated: Fri Apr 28 14:44:02 UTC+0200 2017 by e-lronnback -- From: TSE-9B50TY1 in the CORPNET domain -------------------------------------------------------------------------- CREATE PROCEDURE [etl].[NYPD_Vehicle_CreateRawTable] ( @@ -226,7 +186,7 @@ GO -- the target of the BULK INSERT operation, since it cannot insert -- into a table with multiple columns without a format file. -- --- Generated: Mon May 2 11:10:30 UTC+0200 2016 by e-lronnback +-- Generated: Fri Apr 28 14:44:02 UTC+0200 2017 by e-lronnback -- From: TSE-9B50TY1 in the CORPNET domain -------------------------------------------------------------------------- CREATE PROCEDURE [etl].[NYPD_Vehicle_CreateInsertView] ( @@ -296,7 +256,7 @@ GO -- This job may called multiple times in a workflow when more than -- one file matching a given filename pattern is found. -- --- Generated: Mon May 2 11:10:30 UTC+0200 2016 by e-lronnback +-- Generated: Fri Apr 28 14:44:02 UTC+0200 2017 by e-lronnback -- From: TSE-9B50TY1 in the CORPNET domain -------------------------------------------------------------------------- CREATE PROCEDURE [etl].[NYPD_Vehicle_BulkInsert] ( @@ -425,7 +385,7 @@ GO -- Create: NYPD_Vehicle_Collision_Split -- Create: NYPD_Vehicle_CollisionMetadata_Split -- --- Generated: Mon May 2 11:10:30 UTC+0200 2016 by e-lronnback +-- Generated: Fri Apr 28 14:44:02 UTC+0200 2017 by e-lronnback -- From: TSE-9B50TY1 in the CORPNET domain -------------------------------------------------------------------------- CREATE PROCEDURE [etl].[NYPD_Vehicle_CreateSplitViews] ( @@ -545,7 +505,7 @@ BEGIN TRY FROM ( SELECT [match], - ROW_NUMBER() OVER (ORDER BY [index] ASC) AS idx + ROW_NUMBER() OVER (ORDER BY (SELECT 1)) AS idx FROM [etl].Splitter(ISNULL(forcedMaterializationTrick.[row], ''''), N''(.*?);[0-9]{4}([0-9]{9})[^;]*;(.*?);(.*?);(.*?);(.*?);(.*?);(.*?);(.*?);(.*?);'') ) s @@ -646,7 +606,7 @@ BEGIN TRY FROM ( SELECT [match], - ROW_NUMBER() OVER (ORDER BY [index] ASC) AS idx + ROW_NUMBER() OVER (ORDER BY (SELECT 1)) AS idx FROM [etl].Splitter(ISNULL(forcedMaterializationTrick.[row], ''''), N''(?=.*?(\w+)\s+[0-9]{4})?(?=.*?\w+\s+([0-9]{4}))?(?=.*?NOTES[^:]*:(.*))?'') ) s @@ -707,7 +667,7 @@ GO -- Create: NYPD_Vehicle_Collision_Error -- Create: NYPD_Vehicle_CollisionMetadata_Error -- --- Generated: Mon May 2 11:10:30 UTC+0200 2016 by e-lronnback +-- Generated: Fri Apr 28 14:44:02 UTC+0200 2017 by e-lronnback -- From: TSE-9B50TY1 in the CORPNET domain -------------------------------------------------------------------------- CREATE PROCEDURE [etl].[NYPD_Vehicle_CreateErrorViews] ( @@ -817,7 +777,7 @@ GO -- Create: NYPD_Vehicle_Collision_Typed -- Create: NYPD_Vehicle_CollisionMetadata_Typed -- --- Generated: Mon May 2 11:10:30 UTC+0200 2016 by e-lronnback +-- Generated: Fri Apr 28 14:44:02 UTC+0200 2017 by e-lronnback -- From: TSE-9B50TY1 in the CORPNET domain -------------------------------------------------------------------------- CREATE PROCEDURE [etl].[NYPD_Vehicle_CreateTypedTables] ( @@ -926,7 +886,7 @@ GO -- Load: NYPD_Vehicle_Collision_Split into NYPD_Vehicle_Collision_Typed -- Load: NYPD_Vehicle_CollisionMetadata_Split into NYPD_Vehicle_CollisionMetadata_Typed -- --- Generated: Mon May 2 11:10:30 UTC+0200 2016 by e-lronnback +-- Generated: Fri Apr 28 14:44:02 UTC+0200 2017 by e-lronnback -- From: TSE-9B50TY1 in the CORPNET domain -------------------------------------------------------------------------- CREATE PROCEDURE [etl].[NYPD_Vehicle_SplitRawIntoTyped] ( @@ -1106,7 +1066,7 @@ GO -- Key: CrossStreet (as primary key) -- Key: CollisionOrder (as primary key) -- --- Generated: Mon May 2 11:10:30 UTC+0200 2016 by e-lronnback +-- Generated: Fri Apr 28 14:44:02 UTC+0200 2017 by e-lronnback -- From: TSE-9B50TY1 in the CORPNET domain -------------------------------------------------------------------------- CREATE PROCEDURE [etl].[NYPD_Vehicle_AddKeysToTyped] ( diff --git a/Example/targets/target.sql b/Example/targets/target.sql index bc3d02a..39e8cf1 100644 --- a/Example/targets/target.sql +++ b/Example/targets/target.sql @@ -11,7 +11,7 @@ GO -- Map: StreetName to ST_NAM_Street_Name (as natural key) -- Map: metadata_CO_ID to Metadata_ST (as metadata) -- --- Generated: Mon May 2 11:10:30 UTC+0200 2016 by e-lronnback +-- Generated: Fri Apr 28 14:44:02 UTC+0200 2017 by e-lronnback -- From: TSE-9B50TY1 in the CORPNET domain -------------------------------------------------------------------------- CREATE PROCEDURE [etl].[lST_Street__NYPD_Vehicle_Collision_Typed] ( @@ -134,7 +134,7 @@ GO -- Map: IS_ID_of to IS_ID (as surrogate key) -- Map: metadata_CO_ID to Metadata_IS (as metadata) -- --- Generated: Mon May 2 11:10:30 UTC+0200 2016 by e-lronnback +-- Generated: Fri Apr 28 14:44:02 UTC+0200 2017 by e-lronnback -- From: TSE-9B50TY1 in the CORPNET domain -------------------------------------------------------------------------- CREATE PROCEDURE [etl].[lIS_Intersection__NYPD_Vehicle_Collision_Typed__1] ( @@ -262,7 +262,7 @@ GO -- Map: IS_ID_of to IS_ID_of -- Map: metadata_CO_ID to Metadata_ST_intersecting_IS_of_ST_crossing (as metadata) -- --- Generated: Mon May 2 11:10:30 UTC+0200 2016 by e-lronnback +-- Generated: Fri Apr 28 14:44:02 UTC+0200 2017 by e-lronnback -- From: TSE-9B50TY1 in the CORPNET domain -------------------------------------------------------------------------- CREATE PROCEDURE [etl].[lST_intersecting_IS_of_ST_crossing__NYPD_Vehicle_Collision_Typed] ( @@ -432,7 +432,7 @@ GO -- Map: CollisionKilledCount to IS_KIL_Intersection_KilledCount -- Map: ChangedAt to IS_KIL_ChangedAt -- --- Generated: Mon May 2 11:10:30 UTC+0200 2016 by e-lronnback +-- Generated: Fri Apr 28 14:44:02 UTC+0200 2017 by e-lronnback -- From: TSE-9B50TY1 in the CORPNET domain -------------------------------------------------------------------------- CREATE PROCEDURE [etl].[lIS_Intersection__NYPD_Vehicle_Collision_Typed__2] ( diff --git a/Example/workflows/workflow.sql b/Example/workflows/workflow.sql index 823a45d..00855e5 100644 --- a/Example/workflows/workflow.sql +++ b/Example/workflows/workflow.sql @@ -25,7 +25,7 @@ OPEN schedules; IF EXISTS (select job_id from [dbo].[sysjobs] where name = 'NYPD_Vehicle_Staging') BEGIN FETCH FIRST FROM schedules INTO @scheduleId, @scheduleName; - WHILE(@@FETCH_STATUS = 0) + WHILE(@@FETCH_STATUS = 0) BEGIN --PRINT 'Detaching schedule "' + @scheduleName + '" from NYPD_Vehicle_Staging'; EXEC msdb.dbo.sp_detach_schedule @job_name = 'NYPD_Vehicle_Staging', @schedule_id = @scheduleId; @@ -233,7 +233,7 @@ EXEC sp_update_jobstep @on_success_action = 4, -- go to step with id @on_success_step_id = 11; FETCH FIRST FROM schedules INTO @scheduleId, @scheduleName; -WHILE(@@FETCH_STATUS = 0) +WHILE(@@FETCH_STATUS = 0) BEGIN --PRINT 'Attaching schedule "' + @scheduleName + '" to NYPD_Vehicle_Staging'; EXEC msdb.dbo.sp_attach_schedule @job_name = 'NYPD_Vehicle_Staging', @schedule_id = @scheduleId; @@ -264,7 +264,7 @@ OPEN schedules; IF EXISTS (select job_id from [dbo].[sysjobs] where name = 'NYPD_Vehicle_Loading') BEGIN FETCH FIRST FROM schedules INTO @scheduleId, @scheduleName; - WHILE(@@FETCH_STATUS = 0) + WHILE(@@FETCH_STATUS = 0) BEGIN --PRINT 'Detaching schedule "' + @scheduleName + '" from NYPD_Vehicle_Loading'; EXEC msdb.dbo.sp_detach_schedule @job_name = 'NYPD_Vehicle_Loading', @schedule_id = @scheduleId; @@ -372,7 +372,7 @@ EXEC sp_update_jobstep @on_success_action = 4, -- go to step with id @on_success_step_id = 6; FETCH FIRST FROM schedules INTO @scheduleId, @scheduleName; -WHILE(@@FETCH_STATUS = 0) +WHILE(@@FETCH_STATUS = 0) BEGIN --PRINT 'Attaching schedule "' + @scheduleName + '" to NYPD_Vehicle_Loading'; EXEC msdb.dbo.sp_attach_schedule @job_name = 'NYPD_Vehicle_Loading', @schedule_id = @scheduleId; diff --git a/Sisulate.bat b/Sisulate.bat index e9f4aea..0cdfce3 100644 --- a/Sisulate.bat +++ b/Sisulate.bat @@ -85,6 +85,18 @@ for %%f in (%FolderPath%\targets\*.xml) do ( set SQLFiles[!i!]=!OutputFile! ) +REM ------------------------------------------------------------------- +REM Create BIML files +REM ------------------------------------------------------------------- +for %%f in (%FolderPath%\targets\*.xml) do ( + set OutputFile=%FolderPath%\biml\%%~nf.biml + echo * Transforming target to BIML file: + echo %%~f ... + echo !OutputFile! + Sisulator.js -x "%%~f" -m Target -d biml.directive -o "!OutputFile!" + IF ERRORLEVEL 1 GOTO ERROR +) + REM ------------------------------------------------------------------- REM Create SQL Server Agent job code REM ------------------------------------------------------------------- diff --git a/biml.directive b/biml.directive new file mode 100644 index 0000000..351e1ab --- /dev/null +++ b/biml.directive @@ -0,0 +1,8 @@ +# These are the sisulets that create the loading logic. +sisulets/Polyfills.js +sisulets/Variables.js +sisulets/Metadata.js +sisulets/biml/Variables.js +sisulets/biml/Helpers.js +sisulets/biml/Naming.js +sisulets/biml/CreatePackages.js diff --git a/sisulets/biml/CreatePackages.js b/sisulets/biml/CreatePackages.js new file mode 100644 index 0000000..dfcbfd2 --- /dev/null +++ b/sisulets/biml/CreatePackages.js @@ -0,0 +1,346 @@ +// Create loading logic +var load, map, sql, i; +/*~ + + + + + + +~*/ +while(load = target.nextLoad()) { +/*~ + + +~*/ + if(sql = load.sql ? load.sql.before : null) { +/*~ + + $sql._sql + +~*/ + } + var naturalKeys = [], + surrogateKeys = [], + metadata = [], + others = []; + + while(map = load.nextMap()) { + switch (map.as) { + case 'natural key': + naturalKeys.push(map); + break; + case 'surrogate key': + surrogateKeys.push(map); + break; + case 'metadata': + metadata.push(map); + break; + default: + others.push(map); + } + } + + var attributeMappings = []; + var historizedAttributesExist = false; + if(load.toAnchor()) { + while(map = load.nextMap()) { + if(map.toAttribute()) { + var otherMap; + var attributeMnemonic = map.target.match(/^(..\_...)\_.*/)[1]; + for(i = 0; otherMap = others[i]; i++) { + if(otherMap.target == attributeMnemonic + '_ChangedAt') { + map.isHistorized = true; + historizedAttributesExist = true; + } + } + attributeMappings.push(map); + } + } + } + + if(attributeMappings.length > 0) { +/*~ + + +~*/ + for(i = 0; map = attributeMappings[i]; i++) { +/*~ + DISABLE TRIGGER ALL ON [${VARIABLES.TargetSchema}$].[${map.target}$]; + ALTER TABLE [${VARIABLES.TargetSchema}$].[${map.target}$] NOCHECK CONSTRAINT ALL; +~*/ + } +/*~ + + +~*/ + } +/*~ + + +~*/ + var commaStr = ',', + andStr = 'AND'; + + var loadingSQL = load._load ? load._load : "SELECT * FROM " + load.source; + if(naturalKeys.length == 0 && surrogateKeys.length == 0) { +/*~ + + $loadingSQL + +~*/ + } + else if(naturalKeys.length > 0 && load.toAnchor()) { +/*~ + + + DECLARE @known INT = 0; + MERGE [${VARIABLES.TargetDatabase}$].[${VARIABLES.TargetSchema}$].[${load.targetTable}$] [${load.anchorMnemonic}$] + USING ( + SELECT + l.${load.anchorMnemonic}$_ID, +~*/ + while(map = load.nextMap()) { + commaStr = load.hasMoreMaps() ? ',' : ''; +/*~ + t.${map.source + commaStr}$ +~*/ + } +/*~ + FROM ( + $loadingSQL + ) t + LEFT JOIN + [${VARIABLES.TargetDatabase}$].[${VARIABLES.TargetSchema}$].[${load.target}$] l + ON +~*/ + for(i = 0; map = naturalKeys[i]; i++) { + andStr = naturalKeys[i+1] ? 'AND' : ''; +/*~ + t.$map.source = l.$map.target $andStr +~*/ + } +/*~ + ) src + ON + src.${load.anchorMnemonic}$_ID = [${load.anchorMnemonic}$].${load.anchorMnemonic}$_ID +~*/ + if(metadata[0]) { +/*~ + WHEN NOT MATCHED THEN + INSERT ( Metadata_${load.anchorMnemonic}$ ) + VALUES ( src.${metadata[0].source}$ ) +~*/ + } +/*~ + WHEN MATCHED THEN + UPDATE SET @known = @known + 1 + OUTPUT + isnull(src.${load.anchorMnemonic}$_ID, inserted.${load.anchorMnemonic}$_ID) as ${load.anchorMnemonic}$_ID, +~*/ + var uniqueSourceColumns = []; + while(map = load.nextMap()) { + if(uniqueSourceColumns.indexOf(map.source) < 0) { + uniqueSourceColumns.push(map.source); +/*~ + src.$map.source, +~*/ + } + } + if(!metadata[0]) { +/*~ + 0 as Metadata_${load.anchorMnemonic}$, +~*/ + } +/*~ + left($$action, 1) as Operation; + + +~*/ + } + else if(surrogateKeys.length > 0 && load.toAnchor()) { +/*~ + + + DECLARE @known INT = 0; + MERGE [${VARIABLES.TargetDatabase}$].[${VARIABLES.TargetSchema}$].[${load.targetTable}$] [${load.anchorMnemonic}$] + USING ( + $loadingSQL + ) src + ON + src.${surrogateKeys[0].source}$ = [${load.anchorMnemonic}$].${surrogateKeys[0].target}$ +~*/ + if(metadata[0]) { +/*~ + WHEN NOT MATCHED THEN + INSERT ( Metadata_${load.anchorMnemonic}$ ) + VALUES ( src.${metadata[0].source}$ ) +~*/ + } +/*~ + WHEN MATCHED THEN + UPDATE SET @known = @known + 1 + OUTPUT + isnull(src.${surrogateKeys[0].source}$, inserted.${load.anchorMnemonic}$_ID) as ${load.anchorMnemonic}$_ID, +~*/ + var uniqueSourceColumns = []; + while(map = load.nextMap()) { + if(uniqueSourceColumns.indexOf(map.source) < 0 && surrogateKeys.indexOf(map) < 0) { + uniqueSourceColumns.push(map.source); +/*~ + src.$map.source, +~*/ + } + } + if(!metadata[0]) { +/*~ + 0 as Metadata_${load.anchorMnemonic}$, +~*/ + } +/*~ + left($$action, 1) as Operation; + + +~*/ + } + + if(attributeMappings.length > 0) { +/*~ + + + + [Operation]=="U" + + + [Operation]=="I" + + + +~*/ + if(historizedAttributesExist) { +/*~ + + +~*/ + for(i = 0; map = attributeMappings[i]; i++) { + if(map.isHistorized) { +/*~ + +~*/ + } + } +/*~ + + + +~*/ + for(i = 0; map = attributeMappings[i]; i++) { + var attributeMnemonic = map.target.match(/^(..\_...)\_.*/)[1]; + if(map.isHistorized) { +/*~ + + + + + + +~*/ + var attributeMap; + while(attributeMap = load.nextMap()) { + if(attributeMap.target.indexOf(attributeMnemonic) >= 0) { +/*~ + +~*/ + } + } + if(!metadata[0]) { +/*~ + +~*/ + } +/*~ + + +~*/ + } + } + } // end of if historized attributes exist +/*~ + + +~*/ + for(i = 0; map = attributeMappings[i]; i++) { +/*~ + +~*/ + } +/*~ + + + +~*/ + for(i = 0; map = attributeMappings[i]; i++) { + var attributeMnemonic = map.target.match(/^(..\_...)\_.*/)[1]; +/*~ + + + + + + +~*/ + var attributeMap; + while(attributeMap = load.nextMap()) { + if(attributeMap.target.indexOf(attributeMnemonic) >= 0) { +/*~ + +~*/ + } + } + if(!metadata[0]) { +/*~ + +~*/ + } +/*~ + + +~*/ + } + } // end of if attributes exist +/*~ + + +~*/ + if(attributeMappings.length > 0) { +/*~ + + +~*/ + for(i = 0; map = attributeMappings[i]; i++) { +/*~ + ENABLE TRIGGER ALL ON [${VARIABLES.TargetSchema}$].[${map.target}$]; + ALTER TABLE [${VARIABLES.TargetSchema}$].[${map.target}$] WITH NOCHECK CHECK CONSTRAINT ALL; +~*/ + } +/*~ + + +~*/ + } + if(sql = load.sql ? load.sql.after : null) { +/*~ + + $sql._sql + +~*/ + } +/*~ + + +~*/ +} +/*~ + + +~*/ diff --git a/sisulets/biml/Helpers.js b/sisulets/biml/Helpers.js new file mode 100644 index 0000000..09369bd --- /dev/null +++ b/sisulets/biml/Helpers.js @@ -0,0 +1,60 @@ +// set up some iterators for the different components +target._iterator = {}; +target._iterator.load = 0; +target._iterator.map = 0; + +// set up helpers for loads +target.nextLoad = function() { + if(!this.loads) return null; + if(target._iterator.load == this.loads.length) { + target._iterator.load = 0; + return null; + } + return this.load[this.loads[target._iterator.load++]]; +}; +target.hasMoreLoads = function() { + if(!this.loads) return false; + return target._iterator.load < this.loads.length; +}; +target.isFirstLoad = function() { + return target._iterator.load == 1; +}; +target.hasLoad = function(load) { + return this.loads.indexOf(load.name) >= 0; +}; + +var load, map; +while(load = target.nextLoad()) { + load.targetTable = load.target.match(/l(.*)/)[1]; + load.anchorMnemonic = load.target.match(/l(..)\_.*/)[1]; + load.toAnchor = function() { + return load.target.match(/^l..\_[^\_]*$/); + } + load.nextMap = function() { + if(!this.maps) return null; + if(target._iterator.map == this.maps.length) { + target._iterator.map = 0; + return null; + } + return this.map[this.maps[target._iterator.map++]]; + }; + load.hasMoreMaps = function() { + if(!this.maps) return false; + return target._iterator.map < this.maps.length; + }; + load.isFirstMap = function() { + return target._iterator.map == 1; + }; + load.hasMap = function(map) { + return this.maps.indexOf(map.name) >= 0; + }; + load.condition = load.condition && load.condition.singleton && load.condition.singleton._condition ? load.condition.singleton._condition : null; +} + +while(load = target.nextLoad()) { + while(map = load.nextMap()) { + map.toAttribute = function() { + return !map.target.match(/^.*\_(ChangedAt|ID)$/) && !map.target.match(/^Metadata\_.*$/); + } + } +} diff --git a/sisulets/biml/Naming.js b/sisulets/biml/Naming.js new file mode 100644 index 0000000..528c5d7 --- /dev/null +++ b/sisulets/biml/Naming.js @@ -0,0 +1,6 @@ +// create some qualified names +var load, pass; +while(load = target.nextLoad()) { + pass = load.pass ? '__' + load.pass : ''; + load.qualified = load.target + '__' + load.source + pass; +} diff --git a/sisulets/biml/Variables.js b/sisulets/biml/Variables.js new file mode 100644 index 0000000..feb7289 --- /dev/null +++ b/sisulets/biml/Variables.js @@ -0,0 +1,32 @@ +// local copies with additions and overrides (target level) +target.VARIABLES = copyVariables(VARIABLES); +if(target.variables && target.variables.length > 0) { + var name, value; + for(var v = 0; v < target.variables.length; v++) { + name = target.variables[v]; + value = target.variable[name].value; + target.VARIABLES[name] = value; + } +} + +// local copies with additions and overrides (load level) +var load, j = 0; +while(load = target.load[target.loads[j++]]) { + load.VARIABLES = copyVariables(target.VARIABLES); + if(load.variables && load.variables.length > 0) { + var name, value; + for(var v = 0; v < load.variables.length; v++) { + name = load.variables[v]; + value = load.variable[name].value; + load.VARIABLES[name] = value; + } + } +} + +// do the actual replacement +replaceVariables(VARIABLES, target); + +// global +var MAXLEN = 2147483647; +var S_SCHEMA = VARIABLES.SourceSchema ? VARIABLES.SourceSchema : 'dbo'; +var T_SCHEMA = VARIABLES.TargetSchema ? VARIABLES.TargetSchema : 'dbo';