Skip to content
Permalink
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
362 lines (362 sloc) 16.5 KB
{
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"factoryName": {
"type": "string",
"metadata": "Data Factory Name",
"defaultValue": "angryadf"
},
"blobstagingaccount_connectionString": {
"type": "secureString",
"metadata": "Secure string for 'connectionString' of 'blobstagingaccount'"
},
"sqlAdventureWorksSource_connectionString": {
"type": "secureString",
"metadata": "Secure string for 'connectionString' of 'sqlAdventureWorksSource'"
},
"sqldwAdventureWorksDestination_connectionString": {
"type": "secureString",
"metadata": "Secure string for 'connectionString' of 'sqldwAdventureWorksDestination'"
}
},
"variables": {
"factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
},
"resources": [
{
"name": "[concat(parameters('factoryName'), '/delta_load_w_updates')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"description": "Copy new or updated rows only from database using external control table",
"activities": [
{
"name": "LookupLastWaterMark",
"description": "Retrieve the last high-watermark value stored in external control table",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "SqlDWSource"
},
"dataset": {
"referenceName": "sqldwAdventureWorksDestTable",
"type": "DatasetReference",
"parameters": {
"TableName": {
"value": "@pipeline().parameters.Control_Table_Table_Name",
"type": "Expression"
}
}
}
}
},
{
"name": "LookupCurrentWaterMark",
"description": "Retrieve the current maximum value in watermark column of source data store",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "SqlSource",
"sqlReaderQuery": {
"value": "@concat('select MAX(',pipeline().parameters.Data_Source_WaterMarkColumn, ') as NewWatermarkValue from ',pipeline().parameters.Data_Source_Table_Name)",
"type": "Expression"
}
},
"dataset": {
"referenceName": "sqlAdventureWorksSourceTable",
"type": "DatasetReference",
"parameters": {
"TableName": {
"value": "@pipeline().parameters.Data_Source_Table_Name",
"type": "Expression"
}
}
}
}
},
{
"name": "DeltaCopyfromDB",
"description": "Copy activity to use query to filter the delta data by > last high-watermark and <= current high-watermark from source database, and then only copy the changes to the destination store.",
"type": "Copy",
"dependsOn": [
{
"activity": "LookupCurrentWaterMark",
"dependencyConditions": [
"Succeeded"
]
},
{
"activity": "LookupLastWaterMark",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "SqlSource",
"sqlReaderQuery": {
"value": "select * from \n@{pipeline().parameters.Data_Source_Table_Name} where @{pipeline().parameters.Data_Source_WaterMarkColumn} > '@{activity('LookupLastWaterMark').output.firstRow.WatermarkValue}' and @{pipeline().parameters.Data_Source_WaterMarkColumn} <= '@{activity('LookupCurrentWaterMark').output.firstRow.NewWatermarkValue}'",
"type": "Expression"
}
},
"sink": {
"type": "SqlDWSink",
"allowPolyBase": true,
"writeBatchSize": 10000,
"preCopyScript": {
"value": "@{concat('TRUNCATE TABLE ', pipeline().parameters.Data_Destination_Staging_Table_Name)}",
"type": "Expression"
},
"polyBaseSettings": {
"rejectValue": 0,
"rejectType": "value",
"useTypeDefault": false
}
},
"enableStaging": true,
"stagingSettings": {
"linkedServiceName": {
"referenceName": "blobstagingaccount",
"type": "LinkedServiceReference"
},
"path": "adfstagingcopydata",
"enableCompression": true
},
"dataIntegrationUnits": 0
},
"inputs": [
{
"referenceName": "sqlAdventureWorksSourceTable",
"type": "DatasetReference",
"parameters": {
"TableName": {
"value": "@pipeline().parameters.Data_Source_Table_Name",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "sqldwAdventureWorksDestTable",
"type": "DatasetReference",
"parameters": {
"TableName": {
"value": "@pipeline().parameters.Data_Destination_Staging_Table_Name",
"type": "Expression"
}
}
}
]
},
{
"name": "DeltaLoadFromStaging",
"description": "Stored procedure activity to find the existing records in the destination table and delete them prior to inserting the records from staging. ",
"type": "SqlServerStoredProcedure",
"dependsOn": [
{
"activity": "DeltaCopyfromDB",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30
},
"userProperties": [],
"typeProperties": {
"storedProcedureName": "[concat('[', 'ANGRY].[delta_load_w_updates]')]",
"storedProcedureParameters": {
"LastModifiedDatetime": {
"value": {
"value": "@activity('LookupCurrentWaterMark').output.firstRow.NewWatermarkValue",
"type": "Expression"
},
"type": "DateTime"
},
"StagingTableName": {
"value": {
"value": "@pipeline().parameters.Data_Destination_Staging_Table_Name",
"type": "Expression"
},
"type": "String"
},
"TableName": {
"value": {
"value": "@pipeline().parameters.Data_Destination_Table_Name",
"type": "Expression"
},
"type": "String"
}
}
},
"linkedServiceName": {
"referenceName": "sqldwAdventureWorksDestination",
"type": "LinkedServiceReference"
}
}
],
"parameters": {
"Data_Source_Table_Name": {
"type": "String",
"defaultValue": "ANGRY.FactResellerSales"
},
"Data_Source_WaterMarkColumn": {
"type": "String",
"defaultValue": "ModifiedDate"
},
"Data_Destination_Table_Name": {
"type": "String",
"defaultValue": "ANGRY.FactResellerSales"
},
"Control_Table_Table_Name": {
"type": "String",
"defaultValue": "ANGRY.watermarktable"
},
"Control_Table_Column_Name": {
"type": "String",
"defaultValue": "WatermarkValue"
},
"Data_Destination_Staging_Table_Name": {
"type": "String",
"defaultValue": "ANGRY.FactResellerSales_Staging"
}
},
"folder": {
"name": "delta_w_updates"
},
"annotations": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/sqldwAdventureWorksDestTable')]",
"[concat(variables('factoryId'), '/datasets/sqlAdventureWorksSourceTable')]",
"[concat(variables('factoryId'), '/linkedServices/sqldwAdventureWorksDestination')]",
"[concat(variables('factoryId'), '/linkedServices/blobstagingaccount')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/sqlAdventureWorksSourceTable')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "sqlAdventureWorksSource",
"type": "LinkedServiceReference"
},
"parameters": {
"TableName": {
"type": "String"
}
},
"folder": {
"name": "delta_w_updates"
},
"annotations": [],
"type": "AzureSqlTable",
"typeProperties": {
"tableName": {
"value": "@dataset().TableName",
"type": "Expression"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/sqlAdventureWorksSource')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/sqldwAdventureWorksDestTable')]",
"type": "Microsoft.DataFactory/factories/datasets",
"apiVersion": "2018-06-01",
"properties": {
"linkedServiceName": {
"referenceName": "sqldwAdventureWorksDestination",
"type": "LinkedServiceReference"
},
"parameters": {
"TableName": {
"type": "String"
}
},
"folder": {
"name": "delta_w_updates"
},
"annotations": [],
"type": "AzureSqlDWTable",
"typeProperties": {
"tableName": {
"value": "@dataset().TableName",
"type": "Expression"
}
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/sqldwAdventureWorksDestination')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/blobstagingaccount')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureBlobStorage",
"typeProperties": {
"connectionString": "[parameters('blobstagingaccount_connectionString')]"
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/sqlAdventureWorksSource')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureSqlDatabase",
"typeProperties": {
"connectionString": "[parameters('sqlAdventureWorksSource_connectionString')]"
}
},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/sqldwAdventureWorksDestination')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"annotations": [],
"type": "AzureSqlDW",
"typeProperties": {
"connectionString": "[parameters('sqldwAdventureWorksDestination_connectionString')]"
}
},
"dependsOn": []
}
]
}
You can’t perform that action at this time.