AzureDataFactory/Dynamic Filter Activity Deployment Template.json at master · AussieRobSQL/AzureDataFactory · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
{
    "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
    "contentVersion": "1.0.0.0",
    "parameters": {
        "factoryName": {
            "defaultValue": "Azure Data Factory",
            "type": "String",
            "metadata": "Data Factory Name"
        },
        "dynamicFilters_fileName": {
            "defaultValue": "dynamicFilter.json",
            "type": "String"
        },
        "dynamicFilters_folderPath": {
            "type": "String"
        },
        "storageLinkedServiceName": {
            "type": "String",
            "metadata": "Storage Account for the dynamicFilters Dataset"
        }
    },
    "variables": {
        "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]"
    },
    "resources": [
        {
            "type": "Microsoft.DataFactory/factories/pipelines",
            "name": "[concat(parameters('factoryName'), '/DynamicFilterTest')]",
            "apiVersion": "2018-06-01",
            "properties": {
                "activities": [
                    {
                        "name": "Get Dynamic Filter Sample Data",
                        "description": "This will read the Source Dataset in the Settings tab.",
                        "type": "Lookup",
                        "dependsOn": [],
                        "policy": {
                            "timeout": "7.00:00:00",
                            "retry": 0,
                            "retryIntervalInSeconds": 30,
                            "secureOutput": false
                        },
                        "userProperties": [],
                        "typeProperties": {
                            "source": {
                                "type": "BlobSource",
                                "recursive": true
                            },
                            "dataset": {
                                "referenceName": "dynamicFilters_input",
                                "type": "DatasetReference",
                                "parameters": {}
                            },
                            "firstRowOnly": false
                        }
                    },
                    {
                        "name": "Dynamic Filter - Apply Filters",
                        "description": "Apply the Dynamic Filters to the Output of the previous activity",
                        "type": "Filter",
                        "dependsOn": [
                            {
                                "activity": "Get Dynamic Filter Sample Data",
                                "dependencyConditions": [
                                    "Succeeded"
                                ]
                            }
                        ],
                        "userProperties": [],
                        "typeProperties": {
                            "items": {
                                "value": "@activity('Get Dynamic Filter Sample Data').output.value",
                                "type": "Expression"
                            },
                            "condition": {
                                "value": "@if(equals('equals',pipeline().parameters.filterCondition),if(equals(item().date,pipeline().parameters.filterDate),true,false),if(equals('greater',pipeline().parameters.filterCondition),if(greater(item().date,pipeline().parameters.filterDate),true,false),if(equals('greaterOrEquals',pipeline().parameters.filterCondition),if(greaterOrEquals(item().date,pipeline().parameters.filterDate),true,false),if(equals('less',pipeline().parameters.filterCondition),if(less(item().date,pipeline().parameters.filterDate),true,false),if(equals('lessOrEquals',pipeline().parameters.filterCondition),if(lessOrEquals(item().date,pipeline().parameters.filterDate),true,false),false)))))",
                                "type": "Expression"
                            }
                        }
                    }
                ],
                "parameters": {
                    "filterDate": {
                        "type": "String",
                        "defaultValue": "formatDateTime(trigger().startTime,'yyyy-MM-dd')"
                    },
                    "filterCondition": {
                        "type": "String",
                        "defaultValue": "equals"
                    }
                }
            },
            "dependsOn": [
                "[concat(variables('factoryId'), '/datasets/dynamicFilters_input')]"
            ]
        },
        {
            "type": "Microsoft.DataFactory/factories/datasets",
            "name": "[concat(parameters('factoryName'), '/dynamicFilters_input')]",
            "apiVersion": "2018-06-01",
            "properties": {
                "linkedServiceName": {
                    "referenceName": "[parameters('storageLinkedServiceName')]",
                    "type": "LinkedServiceReference"
                },
                "type": "AzureBlob",
                "typeProperties": {
                    "format": {
                        "type": "JsonFormat",
                        "filePattern": "arrayOfObjects"
                    },
                    "fileName": "[parameters('dynamicFilters_fileName')]",
                    "folderPath": "[parameters('dynamicFilters_folderPath')]"
                }
            }
        }
    ]
}