From 20a05af65e181c7d1b61632f39ec0800c43cd915 Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Thu, 16 Jan 2020 00:00:59 +0100 Subject: [PATCH 01/24] Synapse integration --- .../manage/synapse-analytics/spark/deploy.ps1 | 88 +++++++++++++++++++ .../synapse-analytics/spark/parameters.json | 53 +++++++++++ .../synapse-analytics/spark/template.json | 88 +++++++++++++++++++ .../storage/azuredeploy.json | 77 ++++++++++++++++ .../storage/azuredeploy.parameters.json | 9 ++ .../synapse-analytics/storage/metadata.json | 10 +++ .../synapse-analytics/storage/readme.md | 12 +++ 7 files changed, 337 insertions(+) create mode 100644 samples/manage/synapse-analytics/spark/deploy.ps1 create mode 100644 samples/manage/synapse-analytics/spark/parameters.json create mode 100644 samples/manage/synapse-analytics/spark/template.json create mode 100644 samples/manage/synapse-analytics/storage/azuredeploy.json create mode 100644 samples/manage/synapse-analytics/storage/azuredeploy.parameters.json create mode 100644 samples/manage/synapse-analytics/storage/metadata.json create mode 100644 samples/manage/synapse-analytics/storage/readme.md diff --git a/samples/manage/synapse-analytics/spark/deploy.ps1 b/samples/manage/synapse-analytics/spark/deploy.ps1 new file mode 100644 index 0000000000..4425c69ea5 --- /dev/null +++ b/samples/manage/synapse-analytics/spark/deploy.ps1 @@ -0,0 +1,88 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "sparkPoolName": { + "type": "string" + }, + "workspaceName": { + "type": "string" + }, + "location": { + "type": "string" + }, + "sparkPoolTags": { + "type": "object", + "defaultValue": {} + }, + "sparkAutoScaleEnabled": { + "type": "bool" + }, + "sparkMinNodeCount": { + "type": "int", + "defaultValue": 0 + }, + "sparkMaxNodeCount": { + "type": "int", + "defaultValue": 0 + }, + "sparkNodeCount": { + "type": "int", + "defaultValue": 0 + }, + "sparkNodeSizeFamily": { + "type": "string" + }, + "sparkNodeSize": { + "type": "string" + }, + "sparkAutoPauseEnabled": { + "type": "bool" + }, + "sparkAutoPauseDelayInMinutes": { + "type": "int", + "defaultValue": 0 + }, + "sparkVersion": { + "type": "string" + }, + "packagesRequirementsFileName": { + "type": "string", + "defaultValue": "" + }, + "packagesRequirementsContent": { + "type": "string", + "defaultValue": "" + } + }, + "variables": {}, + "resources": [ + { + "type": "Microsoft.Synapse/workspaces/bigDataPools", + "apiVersion": "2019-06-01-preview", + "name": "[concat(parameters('workspaceName'), '/', parameters('sparkPoolName'))]", + "location": "[parameters('location')]", + "tags": "[parameters('sparkPoolTags')]", + "properties": { + "nodeCount": "[parameters('sparkNodeCount')]", + "nodeSizeFamily": "[parameters('sparkNodeSizeFamily')]", + "nodeSize": "[parameters('sparkNodeSize')]", + "autoScale": { + "enabled": "[parameters('sparkAutoScaleEnabled')]", + "minNodeCount": "[parameters('sparkMinNodeCount')]", + "maxNodeCount": "[parameters('sparkMaxNodeCount')]" + }, + "autoPause": { + "enabled": "[parameters('sparkAutoPauseEnabled')]", + "delayInMinutes": "[parameters('sparkAutoPauseDelayInMinutes')]" + }, + "sparkVersion": "[parameters('sparkVersion')]", + "libraryRequirements": { + "filename": "[parameters('packagesRequirementsFileName')]", + "content": "[parameters('packagesRequirementsContent')]" + } + } + } + ], + "outputs": {} +} \ No newline at end of file diff --git a/samples/manage/synapse-analytics/spark/parameters.json b/samples/manage/synapse-analytics/spark/parameters.json new file mode 100644 index 0000000000..b055e5b071 --- /dev/null +++ b/samples/manage/synapse-analytics/spark/parameters.json @@ -0,0 +1,53 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "sparkPoolName": { + "value": "jovanpopspark" + }, + "workspaceName": { + "value": "jovanpopws1" + }, + "location": { + "value": "westeurope" + }, + "sparkPoolTags": { + "value": { + "Owner": "jovanpop" + } + }, + "sparkAutoScaleEnabled": { + "value": true + }, + "sparkMinNodeCount": { + "value": 3 + }, + "sparkMaxNodeCount": { + "value": 3 + }, + "sparkNodeCount": { + "value": 0 + }, + "sparkNodeSizeFamily": { + "value": "MemoryOptimized" + }, + "sparkNodeSize": { + "value": "Small" + }, + "sparkAutoPauseEnabled": { + "value": true + }, + "sparkAutoPauseDelayInMinutes": { + "value": 15 + }, + "sparkVersion": { + "value": "2.4" + }, + "packagesRequirementsFileName": { + "value": "" + }, + "packagesRequirementsContent": { + "value": "" + } + } +} \ No newline at end of file diff --git a/samples/manage/synapse-analytics/spark/template.json b/samples/manage/synapse-analytics/spark/template.json new file mode 100644 index 0000000000..4425c69ea5 --- /dev/null +++ b/samples/manage/synapse-analytics/spark/template.json @@ -0,0 +1,88 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "sparkPoolName": { + "type": "string" + }, + "workspaceName": { + "type": "string" + }, + "location": { + "type": "string" + }, + "sparkPoolTags": { + "type": "object", + "defaultValue": {} + }, + "sparkAutoScaleEnabled": { + "type": "bool" + }, + "sparkMinNodeCount": { + "type": "int", + "defaultValue": 0 + }, + "sparkMaxNodeCount": { + "type": "int", + "defaultValue": 0 + }, + "sparkNodeCount": { + "type": "int", + "defaultValue": 0 + }, + "sparkNodeSizeFamily": { + "type": "string" + }, + "sparkNodeSize": { + "type": "string" + }, + "sparkAutoPauseEnabled": { + "type": "bool" + }, + "sparkAutoPauseDelayInMinutes": { + "type": "int", + "defaultValue": 0 + }, + "sparkVersion": { + "type": "string" + }, + "packagesRequirementsFileName": { + "type": "string", + "defaultValue": "" + }, + "packagesRequirementsContent": { + "type": "string", + "defaultValue": "" + } + }, + "variables": {}, + "resources": [ + { + "type": "Microsoft.Synapse/workspaces/bigDataPools", + "apiVersion": "2019-06-01-preview", + "name": "[concat(parameters('workspaceName'), '/', parameters('sparkPoolName'))]", + "location": "[parameters('location')]", + "tags": "[parameters('sparkPoolTags')]", + "properties": { + "nodeCount": "[parameters('sparkNodeCount')]", + "nodeSizeFamily": "[parameters('sparkNodeSizeFamily')]", + "nodeSize": "[parameters('sparkNodeSize')]", + "autoScale": { + "enabled": "[parameters('sparkAutoScaleEnabled')]", + "minNodeCount": "[parameters('sparkMinNodeCount')]", + "maxNodeCount": "[parameters('sparkMaxNodeCount')]" + }, + "autoPause": { + "enabled": "[parameters('sparkAutoPauseEnabled')]", + "delayInMinutes": "[parameters('sparkAutoPauseDelayInMinutes')]" + }, + "sparkVersion": "[parameters('sparkVersion')]", + "libraryRequirements": { + "filename": "[parameters('packagesRequirementsFileName')]", + "content": "[parameters('packagesRequirementsContent')]" + } + } + } + ], + "outputs": {} +} \ No newline at end of file diff --git a/samples/manage/synapse-analytics/storage/azuredeploy.json b/samples/manage/synapse-analytics/storage/azuredeploy.json new file mode 100644 index 0000000000..38ed618695 --- /dev/null +++ b/samples/manage/synapse-analytics/storage/azuredeploy.json @@ -0,0 +1,77 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "storageAccount": { + "defaultValue": "jovanpopsynapse", + "type": "String" + } + }, + "variables": {}, + "resources": [ + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2019-04-01", + "name": "[parameters('storageAccount')]", + "location": "westeurope", + "tags": { + "owner": "SynapsePreview" + }, + "sku": { + "name": "Standard_RAGRS", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "isHnsEnabled": true, + "networkAcls": { + "bypass": "AzureServices", + "virtualNetworkRules": [], + "ipRules": [], + "defaultAction": "Allow" + }, + "supportsHttpsTrafficOnly": true, + "encryption": { + "services": { + "file": { + "enabled": true + }, + "blob": { + "enabled": true + } + }, + "keySource": "Microsoft.Storage" + }, + "accessTier": "Hot" + } + }, + { + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2019-04-01", + "name": "[concat(parameters('storageAccount'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccount'))]" + ], + "properties": { + "cors": { + "corsRules": [] + }, + "deleteRetentionPolicy": { + "enabled": false + } + } + }, + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2019-04-01", + "name": "[concat(parameters('storageAccount'), '/default/workspace')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', parameters('storageAccount'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccount'))]" + ], + "properties": { + "publicAccess": "None" + } + } + ] +} \ No newline at end of file diff --git a/samples/manage/synapse-analytics/storage/azuredeploy.parameters.json b/samples/manage/synapse-analytics/storage/azuredeploy.parameters.json new file mode 100644 index 0000000000..b34f2f426b --- /dev/null +++ b/samples/manage/synapse-analytics/storage/azuredeploy.parameters.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "clusterStorageAccountName": { + "value": "GEN-UNIQUE" + } + } +} \ No newline at end of file diff --git a/samples/manage/synapse-analytics/storage/metadata.json b/samples/manage/synapse-analytics/storage/metadata.json new file mode 100644 index 0000000000..c90fdbbbde --- /dev/null +++ b/samples/manage/synapse-analytics/storage/metadata.json @@ -0,0 +1,10 @@ +{ + "$schema": "https://aka.ms/azure-quickstart-templates-metadata-schema#", + "type": "QuickStart", + "itemDisplayName": "Deploy Azure Storage account for Synapse Workspace", + "description": "This template allows you to deploy Azure Storage account for Synapse Workspace.", + "summary": "Deploy Azure Storage account for Synapse Workspace.", + "githubUsername": "jocapc", + "dateUpdated": "2019-10-10" +} + diff --git a/samples/manage/synapse-analytics/storage/readme.md b/samples/manage/synapse-analytics/storage/readme.md new file mode 100644 index 0000000000..ca4b9ccb29 --- /dev/null +++ b/samples/manage/synapse-analytics/storage/readme.md @@ -0,0 +1,12 @@ +# Deploy Azure Storage account for Synapse workspace + +This template deploys Azure Storage account for Synapse workspace + + + + + + + + +`Tags: Azure, Synapse, Storage` \ No newline at end of file From 63f65474ddae05d62b8de40032049ef23584ac42 Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Thu, 16 Jan 2020 00:09:43 +0100 Subject: [PATCH 02/24] Update readme.md --- samples/manage/synapse-analytics/storage/readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/manage/synapse-analytics/storage/readme.md b/samples/manage/synapse-analytics/storage/readme.md index ca4b9ccb29..a5e70b273a 100644 --- a/samples/manage/synapse-analytics/storage/readme.md +++ b/samples/manage/synapse-analytics/storage/readme.md @@ -2,10 +2,10 @@ This template deploys Azure Storage account for Synapse workspace - + - + From b44d405a503c88653c98273cb04b205b33f3b7ff Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Thu, 16 Jan 2020 11:50:29 +0100 Subject: [PATCH 03/24] Added location --- samples/manage/synapse-analytics/storage/azuredeploy.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/manage/synapse-analytics/storage/azuredeploy.json b/samples/manage/synapse-analytics/storage/azuredeploy.json index 38ed618695..796262ef6e 100644 --- a/samples/manage/synapse-analytics/storage/azuredeploy.json +++ b/samples/manage/synapse-analytics/storage/azuredeploy.json @@ -4,7 +4,7 @@ "parameters": { "storageAccount": { "defaultValue": "jovanpopsynapse", - "type": "String" + "type": "string" } }, "variables": {}, @@ -13,7 +13,7 @@ "type": "Microsoft.Storage/storageAccounts", "apiVersion": "2019-04-01", "name": "[parameters('storageAccount')]", - "location": "westeurope", + "location": "[resourceGroup().location]", "tags": { "owner": "SynapsePreview" }, From 4bd462d64e90fa68f4ebc0131a355734f4bd7287 Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Thu, 16 Jan 2020 12:23:52 +0100 Subject: [PATCH 04/24] Added location parameter --- samples/manage/synapse-analytics/storage/azuredeploy.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/samples/manage/synapse-analytics/storage/azuredeploy.json b/samples/manage/synapse-analytics/storage/azuredeploy.json index 796262ef6e..b36a3967f9 100644 --- a/samples/manage/synapse-analytics/storage/azuredeploy.json +++ b/samples/manage/synapse-analytics/storage/azuredeploy.json @@ -5,6 +5,10 @@ "storageAccount": { "defaultValue": "jovanpopsynapse", "type": "string" + }, + "location": { + "defaultValue": "[resourceGroup().location]", + "type": "string" } }, "variables": {}, @@ -13,7 +17,7 @@ "type": "Microsoft.Storage/storageAccounts", "apiVersion": "2019-04-01", "name": "[parameters('storageAccount')]", - "location": "[resourceGroup().location]", + "location": "[parameters('location')]", "tags": { "owner": "SynapsePreview" }, From e23061522f06ba1c26b63893483cef35c18f31b8 Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Thu, 16 Jan 2020 15:37:22 +0100 Subject: [PATCH 05/24] Added workspace deployment --- .../storage/azuredeploy.json | 1 - .../workspace/azuredeploy.json | 129 ++++++++++++++++++ .../workspace/azuredeploy.parameters.json | 45 ++++++ .../synapse-analytics/workspace/metadata.json | 10 ++ .../synapse-analytics/workspace/readme.md | 12 ++ 5 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 samples/manage/synapse-analytics/workspace/azuredeploy.json create mode 100644 samples/manage/synapse-analytics/workspace/azuredeploy.parameters.json create mode 100644 samples/manage/synapse-analytics/workspace/metadata.json create mode 100644 samples/manage/synapse-analytics/workspace/readme.md diff --git a/samples/manage/synapse-analytics/storage/azuredeploy.json b/samples/manage/synapse-analytics/storage/azuredeploy.json index b36a3967f9..03c1d94a96 100644 --- a/samples/manage/synapse-analytics/storage/azuredeploy.json +++ b/samples/manage/synapse-analytics/storage/azuredeploy.json @@ -3,7 +3,6 @@ "contentVersion": "1.0.0.0", "parameters": { "storageAccount": { - "defaultValue": "jovanpopsynapse", "type": "string" }, "location": { diff --git a/samples/manage/synapse-analytics/workspace/azuredeploy.json b/samples/manage/synapse-analytics/workspace/azuredeploy.json new file mode 100644 index 0000000000..b4b136c022 --- /dev/null +++ b/samples/manage/synapse-analytics/workspace/azuredeploy.json @@ -0,0 +1,129 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.1", + "parameters": { + "name": { + "type": "String" + }, + "location": { + "defaultValue": "[resourceGroup().location]", + "type": "String" + }, + "defaultDataLakeStorageAccountName": { + "type": "String" + }, + "defaultDataLakeStorageFilesystemName": { + "type": "String" + }, + "sqlAdministratorLogin": { + "type": "String" + }, + "sqlAdministratorPassword": { + "type": "SecureString" + }, + "setWorkspaceIdentityRbacOnFilesystem": { + "type": "Bool" + }, + "allowAllConnections": { + "defaultValue": true, + "type": "Bool" + }, + "poolSubnetId": { + "defaultValue": "", + "type": "String" + }, + "tagValues": { + "defaultValue": {}, + "type": "Object" + }, + "storageSubscriptionID": { + "defaultValue": "[subscription().subscriptionId]", + "type": "String" + }, + "storageResourceGroupName": { + "defaultValue": "[resourceGroup().name]", + "type": "String" + }, + "storageLocation": { + "defaultValue": "[resourceGroup().location]", + "type": "String" + } + }, + "variables": { + "storageBlobDataContributorRoleID": "ba92f5b4-2d11-453d-a403-e96b0029c9fe", + "defaultDataLakeStorageAccountUrl": "[concat('https://', parameters('defaultDataLakeStorageAccountName'), '.dfs.core.windows.net')]" + }, + "resources": [ + { + "type": "Microsoft.Synapse/workspaces", + "apiVersion": "2019-06-01-preview", + "name": "[parameters('name')]", + "location": "[parameters('location')]", + "tags": "[parameters('tagValues')]", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "defaultDataLakeStorage": { + "accountUrl": "[variables('defaultDataLakeStorageAccountUrl')]", + "filesystem": "[parameters('defaultDataLakeStorageFilesystemName')]" + }, + "virtualNetworkProfile": { + "computeSubnetId": "[parameters('poolSubnetId')]" + }, + "sqlAdministratorLogin": "[parameters('sqlAdministratorLogin')]", + "sqlAdministratorLoginPassword": "[parameters('sqlAdministratorLoginPassword')]" + }, + "resources": [ + { + "type": "firewallrules", + "apiVersion": "2019-06-01-preview", + "name": "allowAll", + "location": "[parameters('location')]", + "dependsOn": [ + "[concat('Microsoft.Synapse/workspaces/', parameters('name'))]" + ], + "properties": { + "startIpAddress": "0.0.0.0", + "endIpAddress": "255.255.255.255" + }, + "condition": "[parameters('allowAllConnections')]" + } + ] + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2019-05-01", + "name": "storageRoleDeploymentResource", + "dependsOn": [ + "[concat('Microsoft.Synapse/workspaces/', parameters('name'))]" + ], + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": {}, + "variables": {}, + "resources": [ + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers/providers/roleAssignments", + "apiVersion": "2018-09-01-preview", + "name": "[concat(parameters('defaultDataLakeStorageAccountName'), '/default/', parameters('defaultDataLakeStorageFilesystemName'), '/Microsoft.Authorization/', guid(concat(resourceGroup().id, '/', variables('storageBlobDataContributorRoleID'), '/', parameters('name'))))]", + "location": "[parameters('storageLocation')]", + "properties": { + "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', variables('storageBlobDataContributorRoleID'))]", + "principalId": "[reference(concat('Microsoft.Synapse/workspaces/', parameters('name')), '2019-06-01-preview', 'Full').identity.principalId]", + "principalType": "ServicePrincipal" + } + } + ] + } + }, + "subscriptionId": "[parameters('storageSubscriptionID')]", + "resourceGroup": "[parameters('storageResourceGroupName')]", + "condition": "[parameters('setWorkspaceIdentityRbacOnFilesystem')]" + } + ], + "outputs": {} +} \ No newline at end of file diff --git a/samples/manage/synapse-analytics/workspace/azuredeploy.parameters.json b/samples/manage/synapse-analytics/workspace/azuredeploy.parameters.json new file mode 100644 index 0000000000..b6ae8b2c4b --- /dev/null +++ b/samples/manage/synapse-analytics/workspace/azuredeploy.parameters.json @@ -0,0 +1,45 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "name": { + "value": "" + }, + "location": { + "value": "" + }, + "defaultDataLakeStorageAccountName": { + "value": "" + }, + "defaultDataLakeStorageFilesystemName": { + "value": "workspace" + }, + "sqlAdministratorLogin": { + "value": "" + }, + "sqlAdministratorLoginPassword": { + "value": null + }, + "setWorkspaceIdentityRbacOnFilesystem": { + "value": true + }, + "allowAllConnections": { + "value": true + }, + "poolSubnetId": { + "value": "" + }, + "tagValues": { + "value": {} + }, + "storageSubscriptionID": { + "value": "" + }, + "storageResourceGroupName": { + "value": "" + }, + "storageLocation": { + "value": "" + } + } +} \ No newline at end of file diff --git a/samples/manage/synapse-analytics/workspace/metadata.json b/samples/manage/synapse-analytics/workspace/metadata.json new file mode 100644 index 0000000000..28521f7e38 --- /dev/null +++ b/samples/manage/synapse-analytics/workspace/metadata.json @@ -0,0 +1,10 @@ +{ + "$schema": "https://aka.ms/azure-quickstart-templates-metadata-schema#", + "type": "QuickStart", + "itemDisplayName": "Deploy Azure Synapse Workspace", + "description": "This template allows you to deploy Azure Synapse Workspace.", + "summary": "Deploy Azure Synapse Workspace.", + "githubUsername": "jocapc", + "dateUpdated": "2019-10-10" +} + diff --git a/samples/manage/synapse-analytics/workspace/readme.md b/samples/manage/synapse-analytics/workspace/readme.md new file mode 100644 index 0000000000..7031e6f0ae --- /dev/null +++ b/samples/manage/synapse-analytics/workspace/readme.md @@ -0,0 +1,12 @@ +# Deploy Azure Synapse workspace + +This template deploys Azure Synapse workspace + + + + + + + + +`Tags: Azure, Synapse, Storage` \ No newline at end of file From 3a38913aa97901c87a76574f072f89173c056dff Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Thu, 16 Jan 2020 15:56:52 +0100 Subject: [PATCH 06/24] Minor fixes --- samples/manage/synapse-analytics/workspace/azuredeploy.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/samples/manage/synapse-analytics/workspace/azuredeploy.json b/samples/manage/synapse-analytics/workspace/azuredeploy.json index b4b136c022..a4ca9e0c69 100644 --- a/samples/manage/synapse-analytics/workspace/azuredeploy.json +++ b/samples/manage/synapse-analytics/workspace/azuredeploy.json @@ -13,6 +13,7 @@ "type": "String" }, "defaultDataLakeStorageFilesystemName": { + "defaultValue": "workspace", "type": "String" }, "sqlAdministratorLogin": { @@ -22,6 +23,7 @@ "type": "SecureString" }, "setWorkspaceIdentityRbacOnFilesystem": { + "defaultValue": true, "type": "Bool" }, "allowAllConnections": { @@ -72,7 +74,7 @@ "computeSubnetId": "[parameters('poolSubnetId')]" }, "sqlAdministratorLogin": "[parameters('sqlAdministratorLogin')]", - "sqlAdministratorLoginPassword": "[parameters('sqlAdministratorLoginPassword')]" + "sqlAdministratorLoginPassword": "[parameters('sqlAdministratorPassword')]" }, "resources": [ { From 1b14cb9cbc706d4650c296e759961d3fa701bd09 Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Thu, 16 Jan 2020 17:25:18 +0100 Subject: [PATCH 07/24] Updates --- .../synapse-analytics/storage/readme.md | 3 +++ .../workspace/azuredeploy.json | 26 +++++++++---------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/samples/manage/synapse-analytics/storage/readme.md b/samples/manage/synapse-analytics/storage/readme.md index a5e70b273a..a75c54e6ec 100644 --- a/samples/manage/synapse-analytics/storage/readme.md +++ b/samples/manage/synapse-analytics/storage/readme.md @@ -9,4 +9,7 @@ This template deploys Azure Storage account for Synapse workspace +When you create storage, use [this link](https://portal.azure.com/?feature.customportal=false&feature.canmodifystamps=true&SqlAzureExtension=synapseµsoft_azure_synapse_assettypeoptions=%7B%22SynapseWorkspace%22%3A%7B%22options%22%3A%22%22%7D%2C%22SparkPool%22%3A%7B%22options%22%3A%22hideassettype%22%7D%7Dµsoft_azure_marketplace_ItemHideKey=SynapseExtensionPreview#blade/Microsoft_Azure_Marketplace/MarketplaceOffersBlade/selectedMenuItemId/home/searchQuery/Azure%2BSynapse%2BAnalytics%2B(preview) +) to create Azure Synapse Analytics workspace (not former DataWarehouse) and assign the Azure Storage account created with this template. + `Tags: Azure, Synapse, Storage` \ No newline at end of file diff --git a/samples/manage/synapse-analytics/workspace/azuredeploy.json b/samples/manage/synapse-analytics/workspace/azuredeploy.json index a4ca9e0c69..e557af2849 100644 --- a/samples/manage/synapse-analytics/workspace/azuredeploy.json +++ b/samples/manage/synapse-analytics/workspace/azuredeploy.json @@ -22,6 +22,14 @@ "sqlAdministratorPassword": { "type": "SecureString" }, + "tagValues": { + "defaultValue": {}, + "type": "Object" + } + }, + "variables": { + "storageBlobDataContributorRoleID": "ba92f5b4-2d11-453d-a403-e96b0029c9fe", + "defaultDataLakeStorageAccountUrl": "[concat('https://', parameters('defaultDataLakeStorageAccountName'), '.dfs.core.windows.net')]", "setWorkspaceIdentityRbacOnFilesystem": { "defaultValue": true, "type": "Bool" @@ -34,10 +42,6 @@ "defaultValue": "", "type": "String" }, - "tagValues": { - "defaultValue": {}, - "type": "Object" - }, "storageSubscriptionID": { "defaultValue": "[subscription().subscriptionId]", "type": "String" @@ -51,10 +55,6 @@ "type": "String" } }, - "variables": { - "storageBlobDataContributorRoleID": "ba92f5b4-2d11-453d-a403-e96b0029c9fe", - "defaultDataLakeStorageAccountUrl": "[concat('https://', parameters('defaultDataLakeStorageAccountName'), '.dfs.core.windows.net')]" - }, "resources": [ { "type": "Microsoft.Synapse/workspaces", @@ -71,7 +71,7 @@ "filesystem": "[parameters('defaultDataLakeStorageFilesystemName')]" }, "virtualNetworkProfile": { - "computeSubnetId": "[parameters('poolSubnetId')]" + "computeSubnetId": "[variables('poolSubnetId')]" }, "sqlAdministratorLogin": "[parameters('sqlAdministratorLogin')]", "sqlAdministratorLoginPassword": "[parameters('sqlAdministratorPassword')]" @@ -89,7 +89,7 @@ "startIpAddress": "0.0.0.0", "endIpAddress": "255.255.255.255" }, - "condition": "[parameters('allowAllConnections')]" + "condition": "[variables('allowAllConnections')]" } ] }, @@ -122,9 +122,9 @@ ] } }, - "subscriptionId": "[parameters('storageSubscriptionID')]", - "resourceGroup": "[parameters('storageResourceGroupName')]", - "condition": "[parameters('setWorkspaceIdentityRbacOnFilesystem')]" + "subscriptionId": "[variables('storageSubscriptionID')]", + "resourceGroup": "[variables('storageResourceGroupName')]", + "condition": "[variables('setWorkspaceIdentityRbacOnFilesystem')]" } ], "outputs": {} From f67e565e9c0bbc81c7f2f1cc6d6ecb4c69c813e0 Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Thu, 16 Jan 2020 17:54:00 +0100 Subject: [PATCH 08/24] moved some variables to parameters --- .../workspace/azuredeploy.json | 32 +++++++------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/samples/manage/synapse-analytics/workspace/azuredeploy.json b/samples/manage/synapse-analytics/workspace/azuredeploy.json index e557af2849..cf29963647 100644 --- a/samples/manage/synapse-analytics/workspace/azuredeploy.json +++ b/samples/manage/synapse-analytics/workspace/azuredeploy.json @@ -25,22 +25,6 @@ "tagValues": { "defaultValue": {}, "type": "Object" - } - }, - "variables": { - "storageBlobDataContributorRoleID": "ba92f5b4-2d11-453d-a403-e96b0029c9fe", - "defaultDataLakeStorageAccountUrl": "[concat('https://', parameters('defaultDataLakeStorageAccountName'), '.dfs.core.windows.net')]", - "setWorkspaceIdentityRbacOnFilesystem": { - "defaultValue": true, - "type": "Bool" - }, - "allowAllConnections": { - "defaultValue": true, - "type": "Bool" - }, - "poolSubnetId": { - "defaultValue": "", - "type": "String" }, "storageSubscriptionID": { "defaultValue": "[subscription().subscriptionId]", @@ -55,6 +39,14 @@ "type": "String" } }, + "variables": { + "storageBlobDataContributorRoleID": "ba92f5b4-2d11-453d-a403-e96b0029c9fe", + "defaultDataLakeStorageAccountUrl": "[concat('https://', parameters('defaultDataLakeStorageAccountName'), '.dfs.core.windows.net')]", + "poolSubnetId": { + "defaultValue": "", + "type": "String" + } + }, "resources": [ { "type": "Microsoft.Synapse/workspaces", @@ -89,7 +81,7 @@ "startIpAddress": "0.0.0.0", "endIpAddress": "255.255.255.255" }, - "condition": "[variables('allowAllConnections')]" + "condition": true } ] }, @@ -122,9 +114,9 @@ ] } }, - "subscriptionId": "[variables('storageSubscriptionID')]", - "resourceGroup": "[variables('storageResourceGroupName')]", - "condition": "[variables('setWorkspaceIdentityRbacOnFilesystem')]" + "subscriptionId": "[parameters('storageSubscriptionID')]", + "resourceGroup": "[parameters('storageResourceGroupName')]", + "condition": true } ], "outputs": {} From e6a99e4809a983aa271111a5aeb208e1b1dba8ce Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Thu, 16 Jan 2020 19:37:21 +0100 Subject: [PATCH 09/24] Update azuredeploy.json --- .../workspace/azuredeploy.json | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/samples/manage/synapse-analytics/workspace/azuredeploy.json b/samples/manage/synapse-analytics/workspace/azuredeploy.json index cf29963647..fc4ca50021 100644 --- a/samples/manage/synapse-analytics/workspace/azuredeploy.json +++ b/samples/manage/synapse-analytics/workspace/azuredeploy.json @@ -10,11 +10,17 @@ "type": "String" }, "defaultDataLakeStorageAccountName": { - "type": "String" + "type": "String", + "metadata": { + "description": "Data Lake Storage account that you will use for Synapse Workspace." + } }, "defaultDataLakeStorageFilesystemName": { "defaultValue": "workspace", - "type": "String" + "type": "String", + "metadata": { + "description": "container in Data Lake Storage account that you will use for Synapse Workspace." + } }, "sqlAdministratorLogin": { "type": "String" @@ -23,20 +29,29 @@ "type": "SecureString" }, "tagValues": { - "defaultValue": {}, + "defaultValue": {"createdWith":"Azure Resource Manager template"}, "type": "Object" }, "storageSubscriptionID": { "defaultValue": "[subscription().subscriptionId]", - "type": "String" + "type": "String", + "metadata": { + "description": "Do not change this value if Data Lake Storage is placed in the same subscription as Synapse Workspace(recommended)." + } }, "storageResourceGroupName": { "defaultValue": "[resourceGroup().name]", - "type": "String" + "type": "String", + "metadata": { + "description": "Do not change this value if Data Lake Storage is placed in the same resource group as Synapse Workspace." + } }, "storageLocation": { "defaultValue": "[resourceGroup().location]", - "type": "String" + "type": "String", + "metadata": { + "description": "Do not change this value if Data Lake Storage is placed in the same region as Synapse Workspace(recommended)." + } } }, "variables": { From 35b9066a2c1c066f92de03724bc5510df537edf9 Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Fri, 17 Jan 2020 15:39:07 +0100 Subject: [PATCH 10/24] Removing condition: true --- samples/manage/synapse-analytics/workspace/azuredeploy.json | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/samples/manage/synapse-analytics/workspace/azuredeploy.json b/samples/manage/synapse-analytics/workspace/azuredeploy.json index fc4ca50021..c69dded568 100644 --- a/samples/manage/synapse-analytics/workspace/azuredeploy.json +++ b/samples/manage/synapse-analytics/workspace/azuredeploy.json @@ -95,8 +95,7 @@ "properties": { "startIpAddress": "0.0.0.0", "endIpAddress": "255.255.255.255" - }, - "condition": true + } } ] }, @@ -130,8 +129,7 @@ } }, "subscriptionId": "[parameters('storageSubscriptionID')]", - "resourceGroup": "[parameters('storageResourceGroupName')]", - "condition": true + "resourceGroup": "[parameters('storageResourceGroupName')]" } ], "outputs": {} From f3c7cbc4f48fff22cf2bbe83a0e0681c32759af1 Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Fri, 17 Jan 2020 16:31:59 +0100 Subject: [PATCH 11/24] Reverted potentialy valid verison --- .../workspace/azuredeploy.json | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/samples/manage/synapse-analytics/workspace/azuredeploy.json b/samples/manage/synapse-analytics/workspace/azuredeploy.json index c69dded568..d49f6d57b7 100644 --- a/samples/manage/synapse-analytics/workspace/azuredeploy.json +++ b/samples/manage/synapse-analytics/workspace/azuredeploy.json @@ -16,11 +16,7 @@ } }, "defaultDataLakeStorageFilesystemName": { - "defaultValue": "workspace", - "type": "String", - "metadata": { - "description": "container in Data Lake Storage account that you will use for Synapse Workspace." - } + "type": "String" }, "sqlAdministratorLogin": { "type": "String" @@ -28,6 +24,17 @@ "sqlAdministratorPassword": { "type": "SecureString" }, + "setWorkspaceIdentityRbacOnFilesystem": { + "type": "Bool" + }, + "allowAllConnections": { + "defaultValue": true, + "type": "Bool" + }, + "poolSubnetId": { + "defaultValue": "", + "type": "String" + }, "tagValues": { "defaultValue": {"createdWith":"Azure Resource Manager template"}, "type": "Object" @@ -81,7 +88,7 @@ "computeSubnetId": "[variables('poolSubnetId')]" }, "sqlAdministratorLogin": "[parameters('sqlAdministratorLogin')]", - "sqlAdministratorLoginPassword": "[parameters('sqlAdministratorPassword')]" + "sqlAdministratorLoginPassword": "[parameters('sqlAdministratorLoginPassword')]" }, "resources": [ { From 1667b91610c414077eccaeaac850421c24249475 Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Fri, 17 Jan 2020 16:44:38 +0100 Subject: [PATCH 12/24] Adding downloaded template --- .../workspace/azuredeploy.json | 39 ++++++------------- .../workspace/azuredeploy.parameters.json | 17 ++++---- 2 files changed, 19 insertions(+), 37 deletions(-) diff --git a/samples/manage/synapse-analytics/workspace/azuredeploy.json b/samples/manage/synapse-analytics/workspace/azuredeploy.json index d49f6d57b7..670537dc45 100644 --- a/samples/manage/synapse-analytics/workspace/azuredeploy.json +++ b/samples/manage/synapse-analytics/workspace/azuredeploy.json @@ -6,14 +6,10 @@ "type": "String" }, "location": { - "defaultValue": "[resourceGroup().location]", "type": "String" }, "defaultDataLakeStorageAccountName": { - "type": "String", - "metadata": { - "description": "Data Lake Storage account that you will use for Synapse Workspace." - } + "type": "String" }, "defaultDataLakeStorageFilesystemName": { "type": "String" @@ -21,7 +17,7 @@ "sqlAdministratorLogin": { "type": "String" }, - "sqlAdministratorPassword": { + "sqlAdministratorLoginPassword": { "type": "SecureString" }, "setWorkspaceIdentityRbacOnFilesystem": { @@ -36,38 +32,25 @@ "type": "String" }, "tagValues": { - "defaultValue": {"createdWith":"Azure Resource Manager template"}, + "defaultValue": {}, "type": "Object" }, "storageSubscriptionID": { "defaultValue": "[subscription().subscriptionId]", - "type": "String", - "metadata": { - "description": "Do not change this value if Data Lake Storage is placed in the same subscription as Synapse Workspace(recommended)." - } + "type": "String" }, "storageResourceGroupName": { "defaultValue": "[resourceGroup().name]", - "type": "String", - "metadata": { - "description": "Do not change this value if Data Lake Storage is placed in the same resource group as Synapse Workspace." - } + "type": "String" }, "storageLocation": { "defaultValue": "[resourceGroup().location]", - "type": "String", - "metadata": { - "description": "Do not change this value if Data Lake Storage is placed in the same region as Synapse Workspace(recommended)." - } + "type": "String" } }, "variables": { "storageBlobDataContributorRoleID": "ba92f5b4-2d11-453d-a403-e96b0029c9fe", - "defaultDataLakeStorageAccountUrl": "[concat('https://', parameters('defaultDataLakeStorageAccountName'), '.dfs.core.windows.net')]", - "poolSubnetId": { - "defaultValue": "", - "type": "String" - } + "defaultDataLakeStorageAccountUrl": "[concat('https://', parameters('defaultDataLakeStorageAccountName'), '.dfs.core.windows.net')]" }, "resources": [ { @@ -85,7 +68,7 @@ "filesystem": "[parameters('defaultDataLakeStorageFilesystemName')]" }, "virtualNetworkProfile": { - "computeSubnetId": "[variables('poolSubnetId')]" + "computeSubnetId": "[parameters('poolSubnetId')]" }, "sqlAdministratorLogin": "[parameters('sqlAdministratorLogin')]", "sqlAdministratorLoginPassword": "[parameters('sqlAdministratorLoginPassword')]" @@ -102,7 +85,8 @@ "properties": { "startIpAddress": "0.0.0.0", "endIpAddress": "255.255.255.255" - } + }, + "condition": "[parameters('allowAllConnections')]" } ] }, @@ -136,7 +120,8 @@ } }, "subscriptionId": "[parameters('storageSubscriptionID')]", - "resourceGroup": "[parameters('storageResourceGroupName')]" + "resourceGroup": "[parameters('storageResourceGroupName')]", + "condition": "[parameters('setWorkspaceIdentityRbacOnFilesystem')]" } ], "outputs": {} diff --git a/samples/manage/synapse-analytics/workspace/azuredeploy.parameters.json b/samples/manage/synapse-analytics/workspace/azuredeploy.parameters.json index b6ae8b2c4b..b40cd50dd3 100644 --- a/samples/manage/synapse-analytics/workspace/azuredeploy.parameters.json +++ b/samples/manage/synapse-analytics/workspace/azuredeploy.parameters.json @@ -3,22 +3,19 @@ "contentVersion": "1.0.0.0", "parameters": { "name": { - "value": "" + "value": "jphome1" }, "location": { - "value": "" + "value": "westeurope" }, "defaultDataLakeStorageAccountName": { - "value": "" + "value": "jphome1" }, "defaultDataLakeStorageFilesystemName": { "value": "workspace" }, "sqlAdministratorLogin": { - "value": "" - }, - "sqlAdministratorLoginPassword": { - "value": null + "value": "cloudSA" }, "setWorkspaceIdentityRbacOnFilesystem": { "value": true @@ -33,13 +30,13 @@ "value": {} }, "storageSubscriptionID": { - "value": "" + "value": "9932f414-50cd-474d-8a1a-bf0649225d73" }, "storageResourceGroupName": { - "value": "" + "value": "jphome1" }, "storageLocation": { - "value": "" + "value": "westeurope" } } } \ No newline at end of file From 4924c875e0b3c5cc5bea90c843dcbe255336b3b0 Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Fri, 17 Jan 2020 17:09:57 +0100 Subject: [PATCH 13/24] Added comments on valid sample --- .../workspace/azuredeploy.json | 30 ++++++++++++++----- .../workspace/azuredeploy.parameters.json | 14 ++++----- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/samples/manage/synapse-analytics/workspace/azuredeploy.json b/samples/manage/synapse-analytics/workspace/azuredeploy.json index 670537dc45..06b66aa912 100644 --- a/samples/manage/synapse-analytics/workspace/azuredeploy.json +++ b/samples/manage/synapse-analytics/workspace/azuredeploy.json @@ -9,15 +9,22 @@ "type": "String" }, "defaultDataLakeStorageAccountName": { - "type": "String" + "type": "String", + "metadata": { + "description": "Data Lake Storage account that you will use for Synapse Workspace." + } }, "defaultDataLakeStorageFilesystemName": { - "type": "String" + "defaultValue": "workspace", + "type": "String", + "metadata": { + "description": "Container in Data Lake Storage account that you will use for Synapse Workspace." + } }, "sqlAdministratorLogin": { "type": "String" }, - "sqlAdministratorLoginPassword": { + "sqlAdministratorPassword": { "type": "SecureString" }, "setWorkspaceIdentityRbacOnFilesystem": { @@ -37,15 +44,24 @@ }, "storageSubscriptionID": { "defaultValue": "[subscription().subscriptionId]", - "type": "String" + "type": "String", + "metadata": { + "description": "Do not change this value if Data Lake Storage is placed in the same subscription as Synapse Workspace(recommended)." + } }, "storageResourceGroupName": { "defaultValue": "[resourceGroup().name]", - "type": "String" + "type": "String", + "metadata": { + "description": "Do not change this value if Data Lake Storage is placed in the same resource group as Synapse Workspace." + } }, "storageLocation": { "defaultValue": "[resourceGroup().location]", - "type": "String" + "type": "String", + "metadata": { + "description": "Do not change this value if Data Lake Storage is placed in the same region as Synapse Workspace(recommended)." + } } }, "variables": { @@ -71,7 +87,7 @@ "computeSubnetId": "[parameters('poolSubnetId')]" }, "sqlAdministratorLogin": "[parameters('sqlAdministratorLogin')]", - "sqlAdministratorLoginPassword": "[parameters('sqlAdministratorLoginPassword')]" + "sqlAdministratorLoginPassword": "[parameters('sqlAdministratorPassword')]" }, "resources": [ { diff --git a/samples/manage/synapse-analytics/workspace/azuredeploy.parameters.json b/samples/manage/synapse-analytics/workspace/azuredeploy.parameters.json index b40cd50dd3..f8629371f4 100644 --- a/samples/manage/synapse-analytics/workspace/azuredeploy.parameters.json +++ b/samples/manage/synapse-analytics/workspace/azuredeploy.parameters.json @@ -3,19 +3,19 @@ "contentVersion": "1.0.0.0", "parameters": { "name": { - "value": "jphome1" + "value": "" }, "location": { - "value": "westeurope" + "value": "" }, "defaultDataLakeStorageAccountName": { - "value": "jphome1" + "value": "" }, "defaultDataLakeStorageFilesystemName": { - "value": "workspace" + "value": "" }, "sqlAdministratorLogin": { - "value": "cloudSA" + "value": "" }, "setWorkspaceIdentityRbacOnFilesystem": { "value": true @@ -33,10 +33,10 @@ "value": "9932f414-50cd-474d-8a1a-bf0649225d73" }, "storageResourceGroupName": { - "value": "jphome1" + "value": "" }, "storageLocation": { - "value": "westeurope" + "value": "" } } } \ No newline at end of file From 73bd99b0e2025f7bacee9a398cdda7237ac1c5eb Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Fri, 17 Jan 2020 17:16:40 +0100 Subject: [PATCH 14/24] Added defualt values on working version --- .../manage/synapse-analytics/workspace/azuredeploy.json | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/samples/manage/synapse-analytics/workspace/azuredeploy.json b/samples/manage/synapse-analytics/workspace/azuredeploy.json index 06b66aa912..7ba84017f9 100644 --- a/samples/manage/synapse-analytics/workspace/azuredeploy.json +++ b/samples/manage/synapse-analytics/workspace/azuredeploy.json @@ -6,7 +6,11 @@ "type": "String" }, "location": { - "type": "String" + "defaultValue": "[resourceGroup().location]", + "type": "String", + "metadata": { + "description": "Do not change this value." + } }, "defaultDataLakeStorageAccountName": { "type": "String", @@ -28,6 +32,7 @@ "type": "SecureString" }, "setWorkspaceIdentityRbacOnFilesystem": { + "defaultValue": true, "type": "Bool" }, "allowAllConnections": { @@ -39,7 +44,7 @@ "type": "String" }, "tagValues": { - "defaultValue": {}, + "defaultValue": {"Created with":"Azure Resource Manager deploment template"}, "type": "Object" }, "storageSubscriptionID": { From baecada84a7004538ca6de4295e2151fa147a887 Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Fri, 17 Jan 2020 17:31:46 +0100 Subject: [PATCH 15/24] Removed constnts from valid version --- .../workspace/azuredeploy.json | 20 +++---------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/samples/manage/synapse-analytics/workspace/azuredeploy.json b/samples/manage/synapse-analytics/workspace/azuredeploy.json index 7ba84017f9..5944c638e2 100644 --- a/samples/manage/synapse-analytics/workspace/azuredeploy.json +++ b/samples/manage/synapse-analytics/workspace/azuredeploy.json @@ -31,18 +31,6 @@ "sqlAdministratorPassword": { "type": "SecureString" }, - "setWorkspaceIdentityRbacOnFilesystem": { - "defaultValue": true, - "type": "Bool" - }, - "allowAllConnections": { - "defaultValue": true, - "type": "Bool" - }, - "poolSubnetId": { - "defaultValue": "", - "type": "String" - }, "tagValues": { "defaultValue": {"Created with":"Azure Resource Manager deploment template"}, "type": "Object" @@ -89,7 +77,7 @@ "filesystem": "[parameters('defaultDataLakeStorageFilesystemName')]" }, "virtualNetworkProfile": { - "computeSubnetId": "[parameters('poolSubnetId')]" + "computeSubnetId": "" }, "sqlAdministratorLogin": "[parameters('sqlAdministratorLogin')]", "sqlAdministratorLoginPassword": "[parameters('sqlAdministratorPassword')]" @@ -106,8 +94,7 @@ "properties": { "startIpAddress": "0.0.0.0", "endIpAddress": "255.255.255.255" - }, - "condition": "[parameters('allowAllConnections')]" + } } ] }, @@ -141,8 +128,7 @@ } }, "subscriptionId": "[parameters('storageSubscriptionID')]", - "resourceGroup": "[parameters('storageResourceGroupName')]", - "condition": "[parameters('setWorkspaceIdentityRbacOnFilesystem')]" + "resourceGroup": "[parameters('storageResourceGroupName')]" } ], "outputs": {} From 410284a6a0b97a3a2f3b45d29a14a0995cd5c53e Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Fri, 17 Jan 2020 22:55:28 +0100 Subject: [PATCH 16/24] Removing duplicate location on valid deployment --- .../manage/synapse-analytics/workspace/azuredeploy.json | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/samples/manage/synapse-analytics/workspace/azuredeploy.json b/samples/manage/synapse-analytics/workspace/azuredeploy.json index 5944c638e2..d7dfc089f0 100644 --- a/samples/manage/synapse-analytics/workspace/azuredeploy.json +++ b/samples/manage/synapse-analytics/workspace/azuredeploy.json @@ -5,13 +5,6 @@ "name": { "type": "String" }, - "location": { - "defaultValue": "[resourceGroup().location]", - "type": "String", - "metadata": { - "description": "Do not change this value." - } - }, "defaultDataLakeStorageAccountName": { "type": "String", "metadata": { @@ -66,7 +59,7 @@ "type": "Microsoft.Synapse/workspaces", "apiVersion": "2019-06-01-preview", "name": "[parameters('name')]", - "location": "[parameters('location')]", + "location": "[resourceGroup().location]", "tags": "[parameters('tagValues')]", "identity": { "type": "SystemAssigned" From e52b876488b03a0590d147a61283495d0b82e2d5 Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Fri, 17 Jan 2020 23:09:05 +0100 Subject: [PATCH 17/24] Additional change --- .../manage/synapse-analytics/storage/azuredeploy.json | 9 +++------ .../manage/synapse-analytics/workspace/azuredeploy.json | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/samples/manage/synapse-analytics/storage/azuredeploy.json b/samples/manage/synapse-analytics/storage/azuredeploy.json index 03c1d94a96..e1a49f2adf 100644 --- a/samples/manage/synapse-analytics/storage/azuredeploy.json +++ b/samples/manage/synapse-analytics/storage/azuredeploy.json @@ -4,10 +4,6 @@ "parameters": { "storageAccount": { "type": "string" - }, - "location": { - "defaultValue": "[resourceGroup().location]", - "type": "string" } }, "variables": {}, @@ -16,9 +12,10 @@ "type": "Microsoft.Storage/storageAccounts", "apiVersion": "2019-04-01", "name": "[parameters('storageAccount')]", - "location": "[parameters('location')]", + "location": "[resourceGroup().location]", "tags": { - "owner": "SynapsePreview" + "Type": "Synapse Data Lake Storage", + "Created with":"Synapse Azure Resource Manager deploment template" }, "sku": { "name": "Standard_RAGRS", diff --git a/samples/manage/synapse-analytics/workspace/azuredeploy.json b/samples/manage/synapse-analytics/workspace/azuredeploy.json index d7dfc089f0..3f47da8c2a 100644 --- a/samples/manage/synapse-analytics/workspace/azuredeploy.json +++ b/samples/manage/synapse-analytics/workspace/azuredeploy.json @@ -25,7 +25,7 @@ "type": "SecureString" }, "tagValues": { - "defaultValue": {"Created with":"Azure Resource Manager deploment template"}, + "defaultValue": {"Created with":"Synapse Azure Resource Manager deploment template"}, "type": "Object" }, "storageSubscriptionID": { From 8ba1e19a7d293af2b0e844ef50873ee892b06611 Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Fri, 17 Jan 2020 23:18:25 +0100 Subject: [PATCH 18/24] Fixed blocking bug with location parameter --- samples/manage/synapse-analytics/workspace/azuredeploy.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/manage/synapse-analytics/workspace/azuredeploy.json b/samples/manage/synapse-analytics/workspace/azuredeploy.json index 3f47da8c2a..24dc72021e 100644 --- a/samples/manage/synapse-analytics/workspace/azuredeploy.json +++ b/samples/manage/synapse-analytics/workspace/azuredeploy.json @@ -80,7 +80,7 @@ "type": "firewallrules", "apiVersion": "2019-06-01-preview", "name": "allowAll", - "location": "[parameters('location')]", + "location": "[resourceGroup().location]", "dependsOn": [ "[concat('Microsoft.Synapse/workspaces/', parameters('name'))]" ], From ceb6ea78be0400b749e1c69513f55a2030749434 Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Fri, 17 Jan 2020 23:46:27 +0100 Subject: [PATCH 19/24] Blank workspace template --- .../manage/synapse-analytics/azuredeploy.json | 59 +++++++++++++++++++ samples/manage/synapse-analytics/readme.md | 12 ++++ 2 files changed, 71 insertions(+) create mode 100644 samples/manage/synapse-analytics/azuredeploy.json create mode 100644 samples/manage/synapse-analytics/readme.md diff --git a/samples/manage/synapse-analytics/azuredeploy.json b/samples/manage/synapse-analytics/azuredeploy.json new file mode 100644 index 0000000000..557bb89450 --- /dev/null +++ b/samples/manage/synapse-analytics/azuredeploy.json @@ -0,0 +1,59 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.1", + "parameters": { + "name": { + "type": "String" + }, + "sqlAdministratorLogin": { + "type": "String" + }, + "sqlAdministratorPassword": { + "type": "SecureString" + }, + "tagValues": { + "defaultValue": {"Created with":"Synapse Azure Resource Manager deploment template"}, + "type": "Object" + } + }, + "resources": [ + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2018-05-01", + "name": "storage", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "[uri(deployment().properties.templateLink.uri, '/storage/azuredeployment.json')]", + "contentVersion": "1.0.0.0" + }, + "parameters":{ + "storageAccount":{"value": "[parameters('name')]"} + } + } + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2018-05-01", + "name": "workspace", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "[uri(deployment().properties.templateLink.uri, '/workspace/azuredeploy.json')]", + "contentVersion": "1.0.0.0" + }, + "parameters":{ + "name":{"value": "[parameters('name')]"}, + "sqlAdministratorLogin":{"value": "[parameters('sqlAdministratorLogin')]"}, + "sqlAdministratorPassword":{"value": "[parameters('sqlAdministratorPassword')]"}, + "defaultDataLakeStorageAccountName":{"value": "[parameters('name')]"}, + "tagValues":{"value": "[parameters('tagValues')]"} + } + }, + "dependsOn": [ + "storage" + ] + } + ], + "outputs": {} +} \ No newline at end of file diff --git a/samples/manage/synapse-analytics/readme.md b/samples/manage/synapse-analytics/readme.md new file mode 100644 index 0000000000..ca22ec4010 --- /dev/null +++ b/samples/manage/synapse-analytics/readme.md @@ -0,0 +1,12 @@ +# Deploy Azure Synapse workspace + +This template deploys Azure Synapse workspace with underlying Data Lake Storage. + + + + + + + + +`Tags: Azure, Synapse, Analytics` \ No newline at end of file From b71b922e0a711d8ad470baa01a23689d26151c33 Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Fri, 17 Jan 2020 23:57:17 +0100 Subject: [PATCH 20/24] Fixed relative Url in main template --- samples/manage/synapse-analytics/azuredeploy.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/manage/synapse-analytics/azuredeploy.json b/samples/manage/synapse-analytics/azuredeploy.json index 557bb89450..4fd7db4e4c 100644 --- a/samples/manage/synapse-analytics/azuredeploy.json +++ b/samples/manage/synapse-analytics/azuredeploy.json @@ -24,7 +24,7 @@ "properties": { "mode": "Incremental", "templateLink": { - "uri": "[uri(deployment().properties.templateLink.uri, '/storage/azuredeployment.json')]", + "uri": "[uri(deployment().properties.templateLink.uri, '/JocaPC/sql-server-samples/synapse/samples/manage/synapse-analytics/storage/azuredeployment.json')]", "contentVersion": "1.0.0.0" }, "parameters":{ @@ -39,7 +39,7 @@ "properties": { "mode": "Incremental", "templateLink": { - "uri": "[uri(deployment().properties.templateLink.uri, '/workspace/azuredeploy.json')]", + "uri": "[uri(deployment().properties.templateLink.uri, '/JocaPC/sql-server-samples/synapse/samples/manage/synapse-analytics/workspace/azuredeploy.json')]", "contentVersion": "1.0.0.0" }, "parameters":{ From 8d165eff28656b44bd339fd0fac7f055cd9a50fe Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Sat, 18 Jan 2020 00:03:58 +0100 Subject: [PATCH 21/24] Sixed name of storage linke Url template --- samples/manage/synapse-analytics/azuredeploy.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/manage/synapse-analytics/azuredeploy.json b/samples/manage/synapse-analytics/azuredeploy.json index 4fd7db4e4c..3d57f293e5 100644 --- a/samples/manage/synapse-analytics/azuredeploy.json +++ b/samples/manage/synapse-analytics/azuredeploy.json @@ -24,7 +24,7 @@ "properties": { "mode": "Incremental", "templateLink": { - "uri": "[uri(deployment().properties.templateLink.uri, '/JocaPC/sql-server-samples/synapse/samples/manage/synapse-analytics/storage/azuredeployment.json')]", + "uri": "[uri(deployment().properties.templateLink.uri, '/JocaPC/sql-server-samples/synapse/samples/manage/synapse-analytics/storage/azuredeploy.json')]", "contentVersion": "1.0.0.0" }, "parameters":{ From fa4cd714c7586c8466fb8e65d90710b05b56975c Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Sat, 18 Jan 2020 00:09:00 +0100 Subject: [PATCH 22/24] Corrected content version --- samples/manage/synapse-analytics/workspace/azuredeploy.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/manage/synapse-analytics/workspace/azuredeploy.json b/samples/manage/synapse-analytics/workspace/azuredeploy.json index 24dc72021e..5b8a80782e 100644 --- a/samples/manage/synapse-analytics/workspace/azuredeploy.json +++ b/samples/manage/synapse-analytics/workspace/azuredeploy.json @@ -1,6 +1,6 @@ { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.1", + "contentVersion": "1.0.0.0", "parameters": { "name": { "type": "String" From 1ccc10163b262ab0a5efab73db4ae2c741244c46 Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Sat, 18 Jan 2020 00:12:11 +0100 Subject: [PATCH 23/24] Anoter content update --- samples/manage/synapse-analytics/azuredeploy.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/manage/synapse-analytics/azuredeploy.json b/samples/manage/synapse-analytics/azuredeploy.json index 3d57f293e5..d7a398a020 100644 --- a/samples/manage/synapse-analytics/azuredeploy.json +++ b/samples/manage/synapse-analytics/azuredeploy.json @@ -1,6 +1,6 @@ { "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.1", + "contentVersion": "1.0.0.0", "parameters": { "name": { "type": "String" From de7c885b701d048a628bdbf7100fdd38f8634d50 Mon Sep 17 00:00:00 2001 From: Jovan Popovic Date: Sun, 19 Jan 2020 00:38:41 +0100 Subject: [PATCH 24/24] Prepared templates for MSFT deployment --- .../manage/synapse-analytics/azuredeploy.json | 4 +- samples/manage/synapse-analytics/readme.md | 4 +- .../manage/synapse-analytics/spark/deploy.ps1 | 88 ------------------- .../synapse-analytics/spark/parameters.json | 53 ----------- .../synapse-analytics/spark/template.json | 88 ------------------- .../synapse-analytics/storage/readme.md | 8 +- .../synapse-analytics/workspace/readme.md | 8 +- 7 files changed, 12 insertions(+), 241 deletions(-) delete mode 100644 samples/manage/synapse-analytics/spark/deploy.ps1 delete mode 100644 samples/manage/synapse-analytics/spark/parameters.json delete mode 100644 samples/manage/synapse-analytics/spark/template.json diff --git a/samples/manage/synapse-analytics/azuredeploy.json b/samples/manage/synapse-analytics/azuredeploy.json index d7a398a020..869ce60b07 100644 --- a/samples/manage/synapse-analytics/azuredeploy.json +++ b/samples/manage/synapse-analytics/azuredeploy.json @@ -24,7 +24,7 @@ "properties": { "mode": "Incremental", "templateLink": { - "uri": "[uri(deployment().properties.templateLink.uri, '/JocaPC/sql-server-samples/synapse/samples/manage/synapse-analytics/storage/azuredeploy.json')]", + "uri": "[uri(deployment().properties.templateLink.uri, '/microsoft/sql-server-samples/master/samples/manage/synapse-analytics/storage/azuredeploy.json')]", "contentVersion": "1.0.0.0" }, "parameters":{ @@ -39,7 +39,7 @@ "properties": { "mode": "Incremental", "templateLink": { - "uri": "[uri(deployment().properties.templateLink.uri, '/JocaPC/sql-server-samples/synapse/samples/manage/synapse-analytics/workspace/azuredeploy.json')]", + "uri": "[uri(deployment().properties.templateLink.uri, '/microsoft/sql-server-samples/master/samples/manage/synapse-analytics/workspace/azuredeploy.json')]", "contentVersion": "1.0.0.0" }, "parameters":{ diff --git a/samples/manage/synapse-analytics/readme.md b/samples/manage/synapse-analytics/readme.md index ca22ec4010..1be32bf732 100644 --- a/samples/manage/synapse-analytics/readme.md +++ b/samples/manage/synapse-analytics/readme.md @@ -2,10 +2,10 @@ This template deploys Azure Synapse workspace with underlying Data Lake Storage. - + - + diff --git a/samples/manage/synapse-analytics/spark/deploy.ps1 b/samples/manage/synapse-analytics/spark/deploy.ps1 deleted file mode 100644 index 4425c69ea5..0000000000 --- a/samples/manage/synapse-analytics/spark/deploy.ps1 +++ /dev/null @@ -1,88 +0,0 @@ -{ - "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "sparkPoolName": { - "type": "string" - }, - "workspaceName": { - "type": "string" - }, - "location": { - "type": "string" - }, - "sparkPoolTags": { - "type": "object", - "defaultValue": {} - }, - "sparkAutoScaleEnabled": { - "type": "bool" - }, - "sparkMinNodeCount": { - "type": "int", - "defaultValue": 0 - }, - "sparkMaxNodeCount": { - "type": "int", - "defaultValue": 0 - }, - "sparkNodeCount": { - "type": "int", - "defaultValue": 0 - }, - "sparkNodeSizeFamily": { - "type": "string" - }, - "sparkNodeSize": { - "type": "string" - }, - "sparkAutoPauseEnabled": { - "type": "bool" - }, - "sparkAutoPauseDelayInMinutes": { - "type": "int", - "defaultValue": 0 - }, - "sparkVersion": { - "type": "string" - }, - "packagesRequirementsFileName": { - "type": "string", - "defaultValue": "" - }, - "packagesRequirementsContent": { - "type": "string", - "defaultValue": "" - } - }, - "variables": {}, - "resources": [ - { - "type": "Microsoft.Synapse/workspaces/bigDataPools", - "apiVersion": "2019-06-01-preview", - "name": "[concat(parameters('workspaceName'), '/', parameters('sparkPoolName'))]", - "location": "[parameters('location')]", - "tags": "[parameters('sparkPoolTags')]", - "properties": { - "nodeCount": "[parameters('sparkNodeCount')]", - "nodeSizeFamily": "[parameters('sparkNodeSizeFamily')]", - "nodeSize": "[parameters('sparkNodeSize')]", - "autoScale": { - "enabled": "[parameters('sparkAutoScaleEnabled')]", - "minNodeCount": "[parameters('sparkMinNodeCount')]", - "maxNodeCount": "[parameters('sparkMaxNodeCount')]" - }, - "autoPause": { - "enabled": "[parameters('sparkAutoPauseEnabled')]", - "delayInMinutes": "[parameters('sparkAutoPauseDelayInMinutes')]" - }, - "sparkVersion": "[parameters('sparkVersion')]", - "libraryRequirements": { - "filename": "[parameters('packagesRequirementsFileName')]", - "content": "[parameters('packagesRequirementsContent')]" - } - } - } - ], - "outputs": {} -} \ No newline at end of file diff --git a/samples/manage/synapse-analytics/spark/parameters.json b/samples/manage/synapse-analytics/spark/parameters.json deleted file mode 100644 index b055e5b071..0000000000 --- a/samples/manage/synapse-analytics/spark/parameters.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "sparkPoolName": { - "value": "jovanpopspark" - }, - "workspaceName": { - "value": "jovanpopws1" - }, - "location": { - "value": "westeurope" - }, - "sparkPoolTags": { - "value": { - "Owner": "jovanpop" - } - }, - "sparkAutoScaleEnabled": { - "value": true - }, - "sparkMinNodeCount": { - "value": 3 - }, - "sparkMaxNodeCount": { - "value": 3 - }, - "sparkNodeCount": { - "value": 0 - }, - "sparkNodeSizeFamily": { - "value": "MemoryOptimized" - }, - "sparkNodeSize": { - "value": "Small" - }, - "sparkAutoPauseEnabled": { - "value": true - }, - "sparkAutoPauseDelayInMinutes": { - "value": 15 - }, - "sparkVersion": { - "value": "2.4" - }, - "packagesRequirementsFileName": { - "value": "" - }, - "packagesRequirementsContent": { - "value": "" - } - } -} \ No newline at end of file diff --git a/samples/manage/synapse-analytics/spark/template.json b/samples/manage/synapse-analytics/spark/template.json deleted file mode 100644 index 4425c69ea5..0000000000 --- a/samples/manage/synapse-analytics/spark/template.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "sparkPoolName": { - "type": "string" - }, - "workspaceName": { - "type": "string" - }, - "location": { - "type": "string" - }, - "sparkPoolTags": { - "type": "object", - "defaultValue": {} - }, - "sparkAutoScaleEnabled": { - "type": "bool" - }, - "sparkMinNodeCount": { - "type": "int", - "defaultValue": 0 - }, - "sparkMaxNodeCount": { - "type": "int", - "defaultValue": 0 - }, - "sparkNodeCount": { - "type": "int", - "defaultValue": 0 - }, - "sparkNodeSizeFamily": { - "type": "string" - }, - "sparkNodeSize": { - "type": "string" - }, - "sparkAutoPauseEnabled": { - "type": "bool" - }, - "sparkAutoPauseDelayInMinutes": { - "type": "int", - "defaultValue": 0 - }, - "sparkVersion": { - "type": "string" - }, - "packagesRequirementsFileName": { - "type": "string", - "defaultValue": "" - }, - "packagesRequirementsContent": { - "type": "string", - "defaultValue": "" - } - }, - "variables": {}, - "resources": [ - { - "type": "Microsoft.Synapse/workspaces/bigDataPools", - "apiVersion": "2019-06-01-preview", - "name": "[concat(parameters('workspaceName'), '/', parameters('sparkPoolName'))]", - "location": "[parameters('location')]", - "tags": "[parameters('sparkPoolTags')]", - "properties": { - "nodeCount": "[parameters('sparkNodeCount')]", - "nodeSizeFamily": "[parameters('sparkNodeSizeFamily')]", - "nodeSize": "[parameters('sparkNodeSize')]", - "autoScale": { - "enabled": "[parameters('sparkAutoScaleEnabled')]", - "minNodeCount": "[parameters('sparkMinNodeCount')]", - "maxNodeCount": "[parameters('sparkMaxNodeCount')]" - }, - "autoPause": { - "enabled": "[parameters('sparkAutoPauseEnabled')]", - "delayInMinutes": "[parameters('sparkAutoPauseDelayInMinutes')]" - }, - "sparkVersion": "[parameters('sparkVersion')]", - "libraryRequirements": { - "filename": "[parameters('packagesRequirementsFileName')]", - "content": "[parameters('packagesRequirementsContent')]" - } - } - } - ], - "outputs": {} -} \ No newline at end of file diff --git a/samples/manage/synapse-analytics/storage/readme.md b/samples/manage/synapse-analytics/storage/readme.md index a75c54e6ec..fe5cb1226d 100644 --- a/samples/manage/synapse-analytics/storage/readme.md +++ b/samples/manage/synapse-analytics/storage/readme.md @@ -1,15 +1,15 @@ # Deploy Azure Storage account for Synapse workspace -This template deploys Azure Storage account for Synapse workspace +This template deploys configured Azure Storage account for Synapse workspace. - + - + When you create storage, use [this link](https://portal.azure.com/?feature.customportal=false&feature.canmodifystamps=true&SqlAzureExtension=synapseµsoft_azure_synapse_assettypeoptions=%7B%22SynapseWorkspace%22%3A%7B%22options%22%3A%22%22%7D%2C%22SparkPool%22%3A%7B%22options%22%3A%22hideassettype%22%7D%7Dµsoft_azure_marketplace_ItemHideKey=SynapseExtensionPreview#blade/Microsoft_Azure_Marketplace/MarketplaceOffersBlade/selectedMenuItemId/home/searchQuery/Azure%2BSynapse%2BAnalytics%2B(preview) -) to create Azure Synapse Analytics workspace (not former DataWarehouse) and assign the Azure Storage account created with this template. +) to create Azure Synapse Analytics workspace **(not former DataWarehouse)** and assign the Azure Storage account created with this template. `Tags: Azure, Synapse, Storage` \ No newline at end of file diff --git a/samples/manage/synapse-analytics/workspace/readme.md b/samples/manage/synapse-analytics/workspace/readme.md index 7031e6f0ae..3e5780da25 100644 --- a/samples/manage/synapse-analytics/workspace/readme.md +++ b/samples/manage/synapse-analytics/workspace/readme.md @@ -1,12 +1,12 @@ # Deploy Azure Synapse workspace -This template deploys Azure Synapse workspace +This template deploys Azure Synapse workspace. - + - + -`Tags: Azure, Synapse, Storage` \ No newline at end of file +`Tags: Azure, Synapse, Analytics` \ No newline at end of file