From b8a5ce7f6eab27b875d3bb8cd93e9ff4db7bf95d Mon Sep 17 00:00:00 2001 From: Richin Jain Date: Fri, 3 Jun 2022 13:24:17 -0700 Subject: [PATCH 1/6] Adding EventHub creation to the deployment script --- .../azure_resource_provision.json | 177 +++++++++++++----- 1 file changed, 125 insertions(+), 52 deletions(-) diff --git a/docs/how-to-guides/azure_resource_provision.json b/docs/how-to-guides/azure_resource_provision.json index 78bd5baee..185e30299 100644 --- a/docs/how-to-guides/azure_resource_provision.json +++ b/docs/how-to-guides/azure_resource_provision.json @@ -2,47 +2,66 @@ "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { - - "resourcePrefix": { - "type": "string", + "minLength": 3, + "maxLength": 15, + "type": "String", "metadata": { "description": "Resource prefix for all the resource provisioned. This should be an alphanumeric string." - }, - "minLength": 3, - "maxLength": 15 + } }, "principalId": { - "type": "string", + "type": "String", "metadata": { "description": "Specifies the principal ID assigned to the role. You can find it by logging into 'https://shell.azure.com/bash' and run 'az ad signed-in-user show --query objectId -o tsv'" } }, "allowAllConnections": { - "type": "string", - "allowedValues": [ "true", "false" ], "defaultValue": "true", + "allowedValues": [ + "true", + "false" + ], + "type": "String", "metadata": { "description": "Specifies whether to allow client IPs to connect to Synapse" } }, "provisionPurview": { - "type": "string", - "allowedValues": [ "true", "false" ], "defaultValue": "true", + "allowedValues": [ + "true", + "false" + ], + "type": "String", "metadata": { "description": "Whether or not put purview in the provision script" } + }, + "provisionEventHub": { + "defaultValue": "true", + "allowedValues": [ + "true", + "false" + ], + "type": "String", + "metadata": { + "description": "Whether or not to deploy eventhub provision script" + } } }, - "functions": [], "variables": { "location": "[resourceGroup().location]", "tenantId": "[subscription().tenantId]", "redisCacheName": "[concat(parameters('resourcePrefix'),'redis' )]", "keyVaultName": "[concat(parameters('resourcePrefix'),'kv')]", + "eventhubNameSpaceName": "[concat(parameters('resourcePrefix'),'ehnamespace')]", + "eventhubName": "[concat(parameters('resourcePrefix'),'eh')]", + "eventhubSku": "Standard", + "eventhubSkuCapacity": 1, "keyVault": "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]", "redisCache": "[resourceId('Microsoft.Cache/redis', variables('redisCacheName'))]", + "eventhubNameSpace": "[resourceId('Microsoft.EventHub/namespaces/', variables('eventhubNameSpaceName'))]", "sparkPoolName": "spark31", "workspaceName": "[toLower(concat(parameters('resourcePrefix'),'syws'))]", "dlsName": "[toLower(concat(parameters('resourcePrefix'), 'dls'))]", @@ -54,6 +73,7 @@ "roleAssignmentNameForBlobContributor": "[guid(parameters('principalId'), variables('roleDefinitionIdForBlobContributor'), resourceGroup().id)]", "roleAssignmentNameForKeyVaultSecretsUser": "[guid(parameters('principalId'), variables('roleDefinitionIdForKeyVaultSecretsUser'), resourceGroup().id)]" }, + "functions": [], "resources": [ { "type": "Microsoft.KeyVault/vaults", @@ -73,24 +93,22 @@ "resources": [ { "type": "Microsoft.KeyVault/vaults/secrets", - "name": "[concat(variables('keyVaultName'), '/FEATHR-PREFIX')]", "apiVersion": "2021-10-01", + "name": "[concat(variables('keyVaultName'), '/FEATHR-PREFIX')]", "location": "[resourceGroup().location]", - "properties": { - "value": "[parameters('resourcePrefix')]" - }, "dependsOn": [ "[variables('keyVault')]" - ] + ], + "properties": { + "value": "[parameters('resourcePrefix')]" + } } - ] }, - { "type": "Microsoft.Cache/redis", - "name": "[variables('redisCacheName')]", "apiVersion": "2021-06-01", + "name": "[variables('redisCacheName')]", "location": "[resourceGroup().location]", "tags": { "displayName": "Feathr Online Store" @@ -106,23 +124,29 @@ "resources": [ { "type": "Microsoft.KeyVault/vaults/secrets", - "name": "[concat(variables('keyVaultName'), '/FEATHR-ONLINE-STORE-CONN')]", "apiVersion": "2021-10-01", + "name": "[concat(variables('keyVaultName'), '/FEATHR-ONLINE-STORE-CONN')]", "location": "[resourceGroup().location]", + "dependsOn": [ + "[variables('keyVault')]", + "[variables('redisCache')]" + ], "properties": { "value": "[concat(variables('redisCacheName'),'.redis.cache.windows.net:6380,password=', listKeys(concat('Microsoft.Cache/redis/', variables('redisCacheName')), '2021-06-01').primaryKey, ',ssl=True')]" - }, - "dependsOn": [ "[variables('keyVault')]", "[variables('redisCache')]" ] + } }, { "type": "Microsoft.KeyVault/vaults/secrets", - "name": "[concat(variables('keyVaultName'), '/REDIS-PASSWORD')]", "apiVersion": "2021-10-01", + "name": "[concat(variables('keyVaultName'), '/REDIS-PASSWORD')]", "location": "[resourceGroup().location]", + "dependsOn": [ + "[variables('keyVault')]", + "[variables('redisCache')]" + ], "properties": { "value": "[listKeys(concat('Microsoft.Cache/redis/', variables('redisCacheName')), '2021-06-01').primaryKey]" - }, - "dependsOn": [ "[variables('keyVault')]", "[variables('redisCache')]" ] + } } ] }, @@ -143,21 +167,22 @@ }, "resources": [ { - "name": "[concat('default/', variables('dlsFsName'))]", "type": "blobServices/containers", "apiVersion": "2021-08-01", - "dependsOn": [ "[variables('dlsName')]" ], + "name": "[concat('default/', variables('dlsFsName'))]", + "dependsOn": [ + "[variables('dlsName')]" + ], "properties": { "publicAccess": "None" } } ] }, - { + "condition": "[equals(parameters('provisionPurview'),'true')]", "type": "Microsoft.Purview/accounts", "apiVersion": "2021-07-01", - "condition": "[equals(parameters('provisionPurview'),'true')]", "name": "[variables('purviewName')]", "location": "[variables('location')]", "sku": { @@ -172,16 +197,61 @@ "publicNetworkAccess": "Enabled" } }, - + { + "condition": "[equals(parameters('provisionEventHub'),'true')]", + "type": "Microsoft.EventHub/namespaces", + "apiVersion": "2018-01-01-preview", + "name": "[variables('eventhubNameSpaceName')]", + "location": "[variables('location')]", + "sku": { + "name": "[variables('eventhubSku')]", + "tier": "[variables('eventhubSku')]", + "capacity": "[variables('eventhubSkuCapacity')]" + }, + "properties": {}, + "resources": [ + { + "condition": "[equals(parameters('provisionEventHub'),'true')]", + "type": "eventhubs", + "apiVersion": "2017-04-01", + "name": "[variables('eventhubName')]", + "dependsOn": [ + "[variables('eventhubNameSpace')]" + ], + "properties": {} + }, + { + "condition": "[equals(parameters('provisionEventHub'),'true')]", + "type": "Microsoft.KeyVault/vaults/secrets", + "apiVersion": "2021-10-01", + "name": "[concat(variables('keyVaultName'), '/EVENTHUB-POLICY-KEY')]", + "condition": "[equals(parameters('provisionEventHub'),'true')]", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[variables('keyVault')]", + "[variables('eventhubNameSpace')]" + ], + "properties": { + "value": "[if(equals(parameters('provisionEventHub'),'true'),listKeys(resourceId('Microsoft.EventHub/namespaces/AuthorizationRules',variables('eventhubNameSpaceName'),'RootManageSharedAccessKey'),'2015-08-01').primaryConnectionString)]", + "dependsOn": [ + "[variables('eventhubNameSpace')]" + ] + } + } + ] + }, { "type": "Microsoft.Synapse/workspaces", "apiVersion": "2021-06-01", "name": "[variables('workspaceName')]", "location": "[variables('location')]", + "dependsOn": [ + "[variables('dlsName')]", + "[variables('dlsFsName')]" + ], "identity": { "type": "SystemAssigned" }, - "dependsOn": [ "[variables('dlsName')]", "[variables('dlsFsName')]" ], "properties": { "defaultDataLakeStorage": { "accountUrl": "[reference(variables('dlsName')).primaryEndpoints.dfs]", @@ -191,23 +261,27 @@ }, "resources": [ { - "condition": "[equals(parameters('allowAllConnections'),'true')]", "type": "firewallrules", "apiVersion": "2021-06-01", "name": "allowAll", "location": "[variables('location')]", - "dependsOn": [ "[variables('workspaceName')]" ], + "dependsOn": [ + "[variables('workspaceName')]" + ], "properties": { "startIpAddress": "0.0.0.0", "endIpAddress": "255.255.255.255" - } + }, + "condition": "[equals(parameters('allowAllConnections'),'true')]" }, { "type": "firewallrules", "apiVersion": "2021-06-01", "name": "AllowAllWindowsAzureIps", "location": "[variables('location')]", - "dependsOn": [ "[variables('workspaceName')]" ], + "dependsOn": [ + "[variables('workspaceName')]" + ], "properties": { "startIpAddress": "0.0.0.0", "endIpAddress": "0.0.0.0" @@ -218,7 +292,9 @@ "apiVersion": "2021-06-01", "name": "default", "location": "[variables('location')]", - "dependsOn": [ "[variables('workspaceName')]" ], + "dependsOn": [ + "[variables('workspaceName')]" + ], "properties": { "grantSqlControlToManagedIdentity": { "desiredState": "Enabled" @@ -232,7 +308,9 @@ "apiVersion": "2021-06-01", "name": "[concat(variables('workspaceName'), '/', variables('sparkPoolName'))]", "location": "[variables('location')]", - "dependsOn": [ "[variables('workspaceName')]" ], + "dependsOn": [ + "[variables('workspaceName')]" + ], "properties": { "autoPause": { "delayInMinutes": 30, @@ -249,37 +327,32 @@ "nodeSize": "Medium" } }, - - - { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2020-10-01-preview", "name": "[variables('roleAssignmentNameForBlobContributor')]", + "dependsOn": [ + "[variables('dlsAccount')]" + ], "properties": { "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', variables('roleDefinitionIdForBlobContributor'))]", "principalId": "[parameters('principalId')]", "scope": "[resourceGroup().id]" - }, - "dependsOn": [ - "[variables('dlsAccount')]" - ] + } }, { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2020-10-01-preview", "name": "[variables('roleAssignmentNameForKeyVaultSecretsUser')]", + "dependsOn": [ + "[variables('keyVault')]" + ], "properties": { "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', variables('roleDefinitionIdForKeyVaultSecretsUser'))]", "principalId": "[parameters('principalId')]", "scope": "[resourceGroup().id]" - }, - "dependsOn": [ - "[variables('keyVault')]" - ] + } } - - ], "outputs": {} -} +} \ No newline at end of file From 8d162f0830fcb8d4b0698624a087ae64416a3041 Mon Sep 17 00:00:00 2001 From: Richin Jain Date: Fri, 3 Jun 2022 14:41:17 -0700 Subject: [PATCH 2/6] Upgrading API versions and fixing the if clause of KV property creation for EH --- docs/how-to-guides/azure_resource_provision.json | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/how-to-guides/azure_resource_provision.json b/docs/how-to-guides/azure_resource_provision.json index 185e30299..33117f60d 100644 --- a/docs/how-to-guides/azure_resource_provision.json +++ b/docs/how-to-guides/azure_resource_provision.json @@ -55,7 +55,7 @@ "tenantId": "[subscription().tenantId]", "redisCacheName": "[concat(parameters('resourcePrefix'),'redis' )]", "keyVaultName": "[concat(parameters('resourcePrefix'),'kv')]", - "eventhubNameSpaceName": "[concat(parameters('resourcePrefix'),'ehnamespace')]", + "eventhubNameSpaceName": "[concat(parameters('resourcePrefix'),'ehns')]", "eventhubName": "[concat(parameters('resourcePrefix'),'eh')]", "eventhubSku": "Standard", "eventhubSkuCapacity": 1, @@ -200,7 +200,7 @@ { "condition": "[equals(parameters('provisionEventHub'),'true')]", "type": "Microsoft.EventHub/namespaces", - "apiVersion": "2018-01-01-preview", + "apiVersion": "2021-11-01", "name": "[variables('eventhubNameSpaceName')]", "location": "[variables('location')]", "sku": { @@ -213,7 +213,7 @@ { "condition": "[equals(parameters('provisionEventHub'),'true')]", "type": "eventhubs", - "apiVersion": "2017-04-01", + "apiVersion": "2021-11-01", "name": "[variables('eventhubName')]", "dependsOn": [ "[variables('eventhubNameSpace')]" @@ -232,10 +232,7 @@ "[variables('eventhubNameSpace')]" ], "properties": { - "value": "[if(equals(parameters('provisionEventHub'),'true'),listKeys(resourceId('Microsoft.EventHub/namespaces/AuthorizationRules',variables('eventhubNameSpaceName'),'RootManageSharedAccessKey'),'2015-08-01').primaryConnectionString)]", - "dependsOn": [ - "[variables('eventhubNameSpace')]" - ] + "value": "[if(equals(parameters('provisionEventHub'),'true'),listKeys(resourceId('Microsoft.EventHub/namespaces/AuthorizationRules',variables('eventhubNameSpaceName'),'RootManageSharedAccessKey'),'2021-11-01').primaryConnectionString,'null' )]" } } ] From cb517f4d78c520e4ff4d93a9148db55f17f09ae6 Mon Sep 17 00:00:00 2001 From: Richin Jain Date: Thu, 7 Jul 2022 07:22:32 -0700 Subject: [PATCH 3/6] Removing the need to ask for Principal Id from user and creating it in the template. Also removing bacpac file url parameter, and copying it to user's storage account as part of the template. This simplifies the flow for user --- .../azure_resource_provision.json | 88 ++++++++++++------- 1 file changed, 57 insertions(+), 31 deletions(-) diff --git a/docs/how-to-guides/azure_resource_provision.json b/docs/how-to-guides/azure_resource_provision.json index 58788bbea..4b266dc6d 100644 --- a/docs/how-to-guides/azure_resource_provision.json +++ b/docs/how-to-guides/azure_resource_provision.json @@ -10,12 +10,6 @@ "description": "Resource prefix for all the resource provisioned. This should be an alphanumeric string." } }, - "principalId": { - "type": "String", - "metadata": { - "description": "Specifies the principal ID assigned to the role. You can find it by logging into 'https://shell.azure.com/bash' and run 'az ad signed-in-user show --query id -o tsv'" - } - }, "allowAllConnections": { "defaultValue": "true", "allowedValues": [ @@ -61,19 +55,6 @@ "description": "Specifies the password for admin" } }, - "storageAccountKey": { - "type": "string", - "metadata": { - "description": "Specifies the key of the storage account where the BACPAC file is stored." - } - }, - "bacpacFileUrl": { - "type": "string", - "defaultValue": "https://azurefeathrstorage.blob.core.windows.net/public/feathr-registry-schema.bacpac", - "metadata": { - "description": "This is the pre-created BACPAC file that contains required schemas by the registry server." - } - }, "preBuiltFeathrUIDockerImage": { "defaultValue": "feathrfeaturestore/sql-registry", "type": "String", @@ -112,16 +93,21 @@ "dlsFsName": "[toLower(concat(parameters('resourcePrefix'),'fs'))]", "dlsAccount": "[resourceId('Microsoft.Storage/storageAccounts', variables('dlsName'))]", "purviewName": "[concat(parameters('resourcePrefix'),'purview' )]", + "identityName": "[concat(parameters('resourcePrefix'),'identity' )]", "roleDefinitionIdForBlobContributor": "ba92f5b4-2d11-453d-a403-e96b0029c9fe", "roleDefinitionIdForKeyVaultSecretsUser": "4633458b-17de-408a-b874-0445c86b69e6", - "roleAssignmentNameForBlobContributor": "[guid(parameters('principalId'), variables('roleDefinitionIdForBlobContributor'), resourceGroup().id)]", - "roleAssignmentNameForKeyVaultSecretsUser": "[guid(parameters('principalId'), variables('roleDefinitionIdForKeyVaultSecretsUser'), resourceGroup().id)]", + "roleAssignmentNameForBlobContributor": "[guid(variables('roleDefinitionIdForBlobContributor'), resourceGroup().id)]", + "roleAssignmentNameForKeyVaultSecretsUser": "[guid(variables('roleDefinitionIdForKeyVaultSecretsUser'), resourceGroup().id)]", "webAppName": "[concat(parameters('resourcePrefix'),'webapp' )]", "webAppPlanName": "[concat(parameters('resourcePrefix'),'appplan' )]", "webAppPlanSku": "P1v2", "webAppAPIVersion": "2021-03-01", "sqlServerName": "[concat(parameters('resourcePrefix'),'dbserver' )]", - "sqlDatabaseName": "[concat(parameters('resourcePrefix'),'db' )]" + "sqlDatabaseName": "[concat(parameters('resourcePrefix'),'db' )]", + "sourceBacpacBlobUrl": "https://azurefeathrstorage.blob.core.windows.net/public/feathr-registry-schema.bacpac", + "bacpacBlobName": "feathr-registry-schema.bacpac", + "destinationBacpacBlobUrl": "[concat('https://',variables('dlsName'),'.blob.core.windows.net/',variables('dlsFsName'),'/',variables('bacpacBlobName'))]", + "deploymentScriptName": "CopyBacpacFile" }, "functions": [], "resources": [ @@ -432,17 +418,26 @@ "nodeSize": "Medium" } }, + { + "type": "Microsoft.ManagedIdentity/userAssignedIdentities", + "apiVersion": "2018-11-30", + "name": "[variables('identityName')]", + "location": "[variables('location')]" + }, { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2020-10-01-preview", "name": "[variables('roleAssignmentNameForBlobContributor')]", "dependsOn": [ - "[variables('dlsAccount')]" + "[variables('dlsAccount')]", + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('identityName'))]" ], "properties": { "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', variables('roleDefinitionIdForBlobContributor'))]", - "principalId": "[parameters('principalId')]", - "scope": "[resourceGroup().id]" + "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('identityName')), '2018-11-30').principalId]", + "scope": "[resourceGroup().id]", + "principalType": "ServicePrincipal" + } }, { @@ -450,12 +445,16 @@ "apiVersion": "2020-10-01-preview", "name": "[variables('roleAssignmentNameForKeyVaultSecretsUser')]", "dependsOn": [ - "[variables('keyVault')]" + "[variables('keyVault')]", + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('identityName'))]" + ], "properties": { "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', variables('roleDefinitionIdForKeyVaultSecretsUser'))]", - "principalId": "[parameters('principalId')]", - "scope": "[resourceGroup().id]" + "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('identityName')), '2018-11-30').principalId]", + "scope": "[resourceGroup().id]", + "principalType": "ServicePrincipal" + } }, { @@ -484,6 +483,31 @@ } ] }, + { + "type": "Microsoft.Resources/deploymentScripts", + "apiVersion": "2020-10-01", + "name": "[variables('deploymentScriptName')]", + "location": "[variables('location')]", + "kind": "AzureCLI", + "identity": { + "type": "UserAssigned", + "userAssignedIdentities": { + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('identityName'))]": {} + } + }, + "dependsOn": [ + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('identityName'))]", + "[resourceId('Microsoft.Storage/storageAccounts/blobServices/containers', variables('dlsName'), 'default', variables('dlsFsName'))]" + ], + "properties": { + "AzCliVersion": "2.2.0", + "timeout": "PT30M", + "arguments": "[concat(listKeys(variables('dlsAccount'),'2019-04-01').keys[0].value, ' ',variables('dlsName'), ' ', variables('bacpacBlobName'), ' ', variables('dlsFsName'),' ' ,variables('sourceBacpacBlobUrl'))]", + "scriptContent": "az storage blob copy start --account-key $1 --account-name $2 --destination-blob $3 --destination-container $4 --source-uri $5", + "cleanupPreference": "OnSuccess", + "retentionInterval": "P1D" + } +}, { "type": "Microsoft.Sql/servers/databases", "apiVersion": "2021-11-01-preview", @@ -498,12 +522,14 @@ "apiVersion": "2021-11-01-preview", "name": "Import", "dependsOn": [ - "[resourceId('Microsoft.Sql/servers/databases', variables('sqlServerName'), variables('sqlDatabaseName'))]" + "[resourceId('Microsoft.Sql/servers/databases', variables('sqlServerName'), variables('sqlDatabaseName'))]", + "[resourceId('Microsoft.Resources/deploymentScripts', variables('deploymentScriptName'))]" + ], "properties": { "storageKeyType": "StorageAccessKey", - "storageKey": "[parameters('storageAccountKey')]", - "storageUri": "[parameters('bacpacFileUrl')]", + "storageKey": "[listKeys(variables('dlsAccount'),'2019-04-01').keys[0].value]", + "storageUri": "[variables('destinationBacpacBlobUrl')]", "administratorLogin": "[parameters('sqlAdminUsername')]", "administratorLoginPassword": "[parameters('sqlAdminPassword')]", "operationMode": "Import" From ddb8245842efb23df4fac45b6e8c765ef49336a8 Mon Sep 17 00:00:00 2001 From: Richin Jain Date: Thu, 7 Jul 2022 13:53:15 -0700 Subject: [PATCH 4/6] Making template more organized and updated documentaton --- docs/README.md | 29 +++++++++++++--- docs/how-to-guides/azure-deployment.md | 11 ++----- .../azure_resource_provision.json | 33 ++++++++----------- 3 files changed, 40 insertions(+), 33 deletions(-) diff --git a/docs/README.md b/docs/README.md index d663d0a87..fc9b6dcbb 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,15 +28,34 @@ Feathr automatically computes your feature values and joins them to your trainin - **Native cloud integration** with simplified and scalable architecture, which is illustrated in the next section. - **Feature sharing and reuse made easy:** Feathr has built-in feature registry so that features can be easily shared across different teams and boost team productivity. -## Running Feathr on Azure with 3 Simple Steps +## Running Feathr on Azure with few Simple Steps -Feathr has native cloud integration. To use Feathr on Azure, you only need three steps: +1. To enable authentication on the Feathr UI (which gets created as part of the deployment script) we need to create an Azure Active Directory (AAD) application. Currently it is not possible to create one through ARM template but you can easily create one by running the following CLI commands in the [Cloud Shell](https://shell.azure.com/bash) -1. Get the `Principal ID` of your account by running `az ad signed-in-user show --query id -o tsv` in the link below (Select "Bash" if asked), and write down that value (something like `b65ef2e0-42b8-44a7-9b55-abbccddeefff`). Think this ID as something representing you when accessing Azure, and it will be used to grant permissions in the next step in the UI. +```bash +# This is the prefix you want to name your resources with, make a note of it, you will need it during deployment. +prefix="YOUR_RESOURCE_PREFIX" + +# Please don't change this name, a corresponding webapp with same name gets created in subsequent steps. +sitename="${prefix}webapp" + +# This will create the Azure AD application, note that we need to create an AAD app of platform type Single Page Application(SPA). By default passing the redirect-uris with create command creates an app of type web. +az ad app create --display-name $sitename --sign-in-audience AzureADMyOrg --web-home-page-url "https://$sitename.azurewebsites.net" --enable-id-token-issuance true + +#Fetch the ClientId, TenantId and ObjectId for the created app +aad_clientId=$(az ad app list --display-name $sitename --query [].appId -o tsv) +aad_tenantId=$(az account tenant list --query [].tenantId -o tsv) +aad_objectId=$(az ad app list --display-name $sitename --query [].id -o tsv) + +# Updating the SPA app created above, currently there is no CLI support to add redirectUris to a SPA, so we have to patch manually via az rest +az rest --method PATCH --uri "https://graph.microsoft.com/v1.0/applications/$aad_objectId" --headers "Content-Type=application/json" --body "{spa:{redirectUris:['https://$sitename.azurewebsites.net/.auth/login/aad/callback']}}" -[Launch Cloud Shell](https://shell.azure.com/bash) +# Make a note of the ClientId and TenantId, you will need it during deployment. +echo "AAD_CLIENT_ID: $aad_clientId" +echo "AZURE_TENANT_ID: $aad_tenantId" +``` -2. Click the button below to deploy a minimal set of Feathr resources for demo purpose. You will need to fill in the `Principal ID` and `Resource Prefix`. You will need "Owner" permission of the selected subscription. +2. Click the button below to deploy a minimal set of Feathr resources. This is not for production use as we choose a minimal set of resources, but treat it as a template that you can modify for further use. Note that you should have "Owner" access in your subscription to perform some of the actions. [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Flinkedin%2Ffeathr%2Fmain%2Fdocs%2Fhow-to-guides%2Fazure_resource_provision.json) diff --git a/docs/how-to-guides/azure-deployment.md b/docs/how-to-guides/azure-deployment.md index 6c87d9b23..0aed607cb 100644 --- a/docs/how-to-guides/azure-deployment.md +++ b/docs/how-to-guides/azure-deployment.md @@ -11,14 +11,9 @@ Due to the complexity of the possible cloud environment, it is almost impossible ## Method 1: Provision Azure Resources with Current User's Identity: -Feathr has native cloud integration and getting started with Feathr is very straightforward. You only need three steps: +Feathr has native cloud integration and getting started with Feathr is very straightforward. Here are the instructions: -1. Get the principal ID of your account by running `az ad signed-in-user show --query id -o tsv` in the link below (Select "Bash" if you are asked to choose one), and write down that value (will be something like `b65ef2e0-42b8-44a7-9b55-abbccddeefff`) - -[Launch Cloud Shell](https://shell.azure.com/bash) - - -2. To enable authentication on the Feathr UI (which gets created as part of the deployment script) we need to create an Azure Active Directory (AAD) application. Currently it is not possible to create one through ARM template but you can easily create one by running the following CLI commands in the [Cloud Shell](https://shell.azure.com/bash) +1. To enable authentication on the Feathr UI (which gets created as part of the deployment script) we need to create an Azure Active Directory (AAD) application. Currently it is not possible to create one through ARM template but you can easily create one by running the following CLI commands in the [Cloud Shell](https://shell.azure.com/bash) ```bash # This is the prefix you want to name your resources with, make a note of it, you will need it during deployment. @@ -43,7 +38,7 @@ echo "AAD_CLIENT_ID: $aad_clientId" echo "AZURE_TENANT_ID: $aad_tenantId" ``` -3. Click the button below to deploy a minimal set of Feathr resources. This is not for production use as we choose a minimal set of resources, but treat it as a template that you can modify for further use. Note that you should have "Owner" access in your subscription to perform some of the actions. +2. Click the button below to deploy a minimal set of Feathr resources. This is not for production use as we choose a minimal set of resources, but treat it as a template that you can modify for further use. Note that you should have "Owner" access in your subscription to perform some of the actions. [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Flinkedin%2Ffeathr%2Fmain%2Fdocs%2Fhow-to-guides%2Fazure_resource_provision.json) diff --git a/docs/how-to-guides/azure_resource_provision.json b/docs/how-to-guides/azure_resource_provision.json index 4b266dc6d..c7a544e90 100644 --- a/docs/how-to-guides/azure_resource_provision.json +++ b/docs/how-to-guides/azure_resource_provision.json @@ -10,17 +10,6 @@ "description": "Resource prefix for all the resource provisioned. This should be an alphanumeric string." } }, - "allowAllConnections": { - "defaultValue": "true", - "allowedValues": [ - "true", - "false" - ], - "type": "String", - "metadata": { - "description": "Specifies whether to allow client IPs to connect to Synapse" - } - }, "provisionPurview": { "defaultValue": "true", "allowedValues": [ @@ -29,7 +18,7 @@ ], "type": "String", "metadata": { - "description": "Whether or not put purview in the provision script" + "description": "Whether or not to provision purview as part of deployment script" } }, "provisionEventHub": { @@ -40,7 +29,7 @@ ], "type": "String", "metadata": { - "description": "Whether or not to deploy eventhub provision script" + "description": "Whether or not to provision EventHub as part of deployment script" } }, "sqlAdminUsername": { @@ -55,11 +44,15 @@ "description": "Specifies the password for admin" } }, - "preBuiltFeathrUIDockerImage": { - "defaultValue": "feathrfeaturestore/sql-registry", + "registryBackend": { + "defaultValue": "Azure-SQL", + "allowedValues": [ + "Azure-SQL", + "Azure-Purview" + ], "type": "String", "metadata": { - "description": "Pre-Built Docker Image of Feathr UI (from dockerhub) with support for specific registry backends. Current supported registry backends are Azure Purview and Azure SQL" + "description": "Backend for registry metadata storage. Current supported registry backends are Azure SQL and Azure Purview" } }, "azureADClientId": { @@ -107,7 +100,8 @@ "sourceBacpacBlobUrl": "https://azurefeathrstorage.blob.core.windows.net/public/feathr-registry-schema.bacpac", "bacpacBlobName": "feathr-registry-schema.bacpac", "destinationBacpacBlobUrl": "[concat('https://',variables('dlsName'),'.blob.core.windows.net/',variables('dlsFsName'),'/',variables('bacpacBlobName'))]", - "deploymentScriptName": "CopyBacpacFile" + "deploymentScriptName": "CopyBacpacFile", + "preBuiltdockerImage": "feathrfeaturestore/sql-registry" }, "functions": [], "resources": [ @@ -303,8 +297,7 @@ "properties": { "startIpAddress": "0.0.0.0", "endIpAddress": "255.255.255.255" - }, - "condition": "[equals(parameters('allowAllConnections'),'true')]" + } }, { "type": "firewallrules", @@ -366,7 +359,7 @@ "httpsOnly": "true", "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('webAppPlanName'))]", "siteConfig": { - "linuxFxVersion": "[concat('DOCKER|', parameters('preBuiltFeathrUIDockerImage'))]", + "linuxFxVersion": "[concat('DOCKER|', variables('preBuiltdockerImage'))]", "alwaysOn": true, "ftpsState": "Disabled", "appSettings": [ From 2bd02bca41c0ca64a1b340e3850aabee5e74ed2c Mon Sep 17 00:00:00 2001 From: Richin Jain Date: Mon, 11 Jul 2022 07:54:40 -0700 Subject: [PATCH 5/6] Addressing Blair's feedback to change registry image and app variable names --- docs/how-to-guides/azure_resource_provision.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/how-to-guides/azure_resource_provision.json b/docs/how-to-guides/azure_resource_provision.json index c7a544e90..27ca0eab9 100644 --- a/docs/how-to-guides/azure_resource_provision.json +++ b/docs/how-to-guides/azure_resource_provision.json @@ -101,7 +101,7 @@ "bacpacBlobName": "feathr-registry-schema.bacpac", "destinationBacpacBlobUrl": "[concat('https://',variables('dlsName'),'.blob.core.windows.net/',variables('dlsFsName'),'/',variables('bacpacBlobName'))]", "deploymentScriptName": "CopyBacpacFile", - "preBuiltdockerImage": "feathrfeaturestore/sql-registry" + "preBuiltdockerImage": "feathrfeaturestore/feathr-registry" }, "functions": [], "resources": [ @@ -368,11 +368,11 @@ "value": "https://index.docker.io/v1" }, { - "name": "AZURE_CLIENT_ID", + "name": "REACT_APP_AZURE_CLIENT_ID", "value": "[parameters('azureADClientId')]" }, { - "name": "AZURE_TENANT_ID", + "name": "REACT_APP_AZURE_TENANT_ID", "value": "[parameters('azureADTenantId')]" }, { @@ -380,6 +380,7 @@ "value": "api/v1" }, { + "condition": "[equals(parameters('registryBackend'),'Azure-SQL')]", "name": "CONNECTION_STR", "value": "[concat('Server=tcp:',reference(variables('sqlServerName')).fullyQualifiedDomainName,',1433;Initial Catalog=',variables('sqlDatabaseName'),';Persist Security Info=False;User ID=',reference(variables('sqlserverName')).administratorLogin,';Password=',parameters('sqlAdminPassword'),';MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;')]" } From 901bc520f51572a589c2bf1463516e0b15efc054 Mon Sep 17 00:00:00 2001 From: Richin Jain Date: Mon, 11 Jul 2022 14:41:10 -0700 Subject: [PATCH 6/6] Fix for redirect URI, the callback one stopped working --- docs/how-to-guides/azure-deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to-guides/azure-deployment.md b/docs/how-to-guides/azure-deployment.md index a90df3c03..3defe37b1 100644 --- a/docs/how-to-guides/azure-deployment.md +++ b/docs/how-to-guides/azure-deployment.md @@ -31,7 +31,7 @@ aad_tenantId=$(az account tenant list --query [].tenantId -o tsv) aad_objectId=$(az ad app list --display-name $sitename --query [].id -o tsv) # Updating the SPA app created above, currently there is no CLI support to add redirectUris to a SPA, so we have to patch manually via az rest -az rest --method PATCH --uri "https://graph.microsoft.com/v1.0/applications/$aad_objectId" --headers "Content-Type=application/json" --body "{spa:{redirectUris:['https://$sitename.azurewebsites.net/.auth/login/aad/callback']}}" +az rest --method PATCH --uri "https://graph.microsoft.com/v1.0/applications/$aad_objectId" --headers "Content-Type=application/json" --body "{spa:{redirectUris:['https://$sitename.azurewebsites.net']}}" # Make a note of the ClientId and TenantId, you will need it during deployment. echo "AAD_CLIENT_ID: $aad_clientId"