Skip to content

Latest commit

 

History

History
211 lines (175 loc) · 15.8 KB

README.md

File metadata and controls

211 lines (175 loc) · 15.8 KB
Raw

Azure Databricks Workspace Terraform module

Terraform module for creation Azure Databricks Workspace

Usage

This module provides an ability to deploy Azure Databricks Workspace. Here is an example how to provision Azure Databricks Workspace in managed network.

Currently, it is only possible to provision Databricks Workspace in managed network with help of this module.

# Prerequisite resources for Databricks Workspace Deployment
data "azurerm_virtual_network" "example" {
  name                = "example-vnet"
  resource_group_name = "example-rg"
  location            = "eastus"
}

data "azurerm_network_security_group" "default_nsg" {
  name                = "example-eastus-sg"
  resource_group_name = "example-rg"
}

data "azurerm_key_vault" "example" {
  name                = "example-key-vault"
  resource_group_name = "example-rg"
}

data "azurerm_key_vault_key" "example" {
  name         = "cmk-example"
  key_vault_id = data.azurerm_key_vault.example.id
}

data "azurerm_log_analytics_workspace" "example" {
  name                = "example"
  resource_group_name = "example-rg"
}

module "databricks_public" {
  source  = "data-platform-hq/subnet/azurerm"
  version = "~> 1.0"

  name                = "databricks-public"
  resource_group_name = "example-rg"
  network             = data.azurerm_virtual_network.example.name
  cidr                = cidrsubnet(data.azurerm_virtual_network.example.address_space[0], 6, 0) 
  nsg_id              = data.azurerm_network_security_group.default_nsg.id

  delegations = [{
    name = "Microsoft.Databricks/workspaces"
    actions = [
      "Microsoft.Network/virtualNetworks/subnets/join/action",
      "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action",
      "Microsoft.Network/virtualNetworks/subnets/unprepareNetworkPolicies/action"
    ]
  }]
}

module "databricks_private" {
  source  = "data-platform-hq/subnet/azurerm"
  version = "~> 1.0"

  name                = "databricks-private"
  resource_group_name = "example-rg"
  network             = data.azurerm_virtual_network.example.name
  cidr                = cidrsubnet(data.azurerm_virtual_network.example.address_space[0], 6, 1) 
  nsg_id              = data.azurerm_network_security_group.default_nsg.id

  delegations = [{
    name = "Microsoft.Databricks/workspaces"
    actions = [
      "Microsoft.Network/virtualNetworks/subnets/join/action",
      "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action",
      "Microsoft.Network/virtualNetworks/subnets/unprepareNetworkPolicies/action"
    ]
  }]
}

# Databricks Workspace module usage with prerequisite resources mentioned above
module "databricks_workspace" {
  source  = "data-platform-hq/databricks-ws/azurerm"
  version = "~> 1.0"

  workspace_name              = "example-workspace"
  location                    = "eastus"
  sku                         = "premium"
  resource_group              = "example-rg"
  managed_resource_group_name = "example-managed-rg"

  # Custom resources names
  access_connector_name = "example-databricks-connector"
  diagnostics_name      = "example-databricks-diagnostics"

  # Vnet injection block
  network_id                        = data.azurerm_virtual_network.example.id
  public_subnet_name                = module.databricks_public.name
  private_subnet_name               = module.databricks_private.name
  public_subnet_nsg_association_id  = module.databricks_public.nsg_association_id
  private_subnet_nsg_association_id = module.databricks_private.nsg_association_id
  nsg_rules_required                = "AllRules"

  # CMK Encryption
  key_vault_id     = data.azurerm_key_vault.example.id
  
  # Databricks Services encryption
  managed_services_cmk_enabled          = true
  managed_services_cmk_key_vault_key_id = data.azurerm_key_vault_key.example.id

  # Data Plane Cluster Disks CMK Encryption
  managed_disk_cmk_enabled          = true
  managed_disk_cmk_policy_enabled   = true
  managed_disk_cmk_key_vault_key_id = data.azurerm_key_vault_key.example.id

  # DBFS
  managed_dbfs_cmk_enabled          = true
  managed_dbfs_cmk_key_vault_key_id = data.azurerm_key_vault_key.example.id

  # Other
  access_connector_enabled = true
  log_analytics_workspace  = { (data.azurerm_log_analytics_workspace.example.name) = data.azurerm_log_analytics_workspace.example.id }
}

Requirements

Name Version
terraform >= 1.0.0
azurerm >= 4.0.1

Providers

Name Version
azurerm >= 4.0.1

Modules

No modules.

Resources

Name Type
azurerm_databricks_access_connector.this resource
azurerm_databricks_workspace.this resource
azurerm_databricks_workspace_root_dbfs_customer_managed_key.this resource
azurerm_key_vault_access_policy.databricks_storage_account_msi resource
azurerm_key_vault_access_policy.databricks_ws_disk resource
azurerm_key_vault_access_policy.databricks_ws_service resource
azurerm_monitor_diagnostic_setting.this resource
azurerm_client_config.current data source
azurerm_monitor_diagnostic_categories.this data source
azurerm_storage_account.this data source
azurerm_user_assigned_identity.this data source

Inputs

Name Description Type Default Required
access_connector_enabled Provides an ability to provision Databricks Access Connector which is required for Unity Catalog feature bool true no
access_connector_name Databricks Access Connector optional name string null no
analytics_destination_type Log analytics destination type string "Dedicated" no
diagnostics_name Diagnostic Settings optional name string null no
global_databricks_object_id Global 'AzureDatabricks' SP object id string "9b38785a-6e08-4087-a0c4-20634343f21f" no
key_vault_id Key Vault ID string null no
key_vault_key_permissions List of key vault key permissions for Databricks Global Service Principal list(string) 
[
  "Get",
  "List",
  "Encrypt",
  "Decrypt",
  "WrapKey",
  "UnwrapKey",
  "GetRotationPolicy"
]
 no
key_vault_secret_permissions List of key vault secret permissions for Databricks Global Service Principal list(string) 
[
  "Get",
  "List"
]
 no
location Azure location string n/a yes
log_analytics_workspace Log Analytics Workspace Name to ID map map(string) {} no
managed_dbfs_cmk_enabled Boolean flag that determines whether Workspace DBFS is encrypted with CMK key bool false no
managed_dbfs_cmk_key_vault_key_id Key Vault key ID used for Databricks DBFS encryption string null no
managed_disk_cmk_enabled Boolean flag that determines whether Data Plane Disks are encrypted with CMK key bool false no
managed_disk_cmk_key_vault_key_id Key Vault key ID used for Data Plane Disks encryption string null no
managed_disk_cmk_policy_enabled Create Key Vault Policy for Databricks Workspace Managed Disk identity.
Upon initial creation of Workspace with Disk CMK encryption, Disk Encryption Set with managed identity is created, it is used for cluster's disks encryption.

However, if Workspace already provisioned and have to updated to use Managed Disk encryption, then 'Disk Encryption Set' is known after creation.
Which means, that you have to first apply with 'managed_disk_cmk_enabled = true' only and set 'managed_disk_cmk_policy_enabled' to false, because identity is unknown.
On next apply, set 'managed_disk_cmk_policy_enabled' to true, because identity of Managed Disk is created and known.		 bool true no
managed_resource_group_name The name of the managed resource group string null no
managed_services_cmk_enabled Encrypts Databricks Workspaces Services like Notebooks and Queries, once CMK type of encryption is enabled it won't be possible to switch back to default Microsoft Managed Encryption. bool false no
managed_services_cmk_key_vault_key_id Key Vault key ID used for Databricks Managed Services encryption string null no
managed_storage_account_identity_enabled Prerequisite for DBFS encryption. Enabled managed Storage Account identity to create Key Vault Policy to access encryption keys bool true no
network_id The ID of a Virtual Network where this Databricks Cluster should be created string n/a yes
no_public_ip Are public IP Addresses not allowed?: [true|false] bool true no
nsg_rules_required Does the data plane to control plane communication happen over private link endpoint only or publicly?: [AllRules, NoAzureDatabricksRules, NoAzureServiceRules] string "AllRules" no
private_subnet_name The name of the Private Subnet within the Virtual Network. Required if virtual_network_id is set string n/a yes
private_subnet_nsg_association_id The resource ID of the azurerm_subnet_network_security_group_association resource which is referred to by the private_subnet_name field. Required if virtual_network_id is set string n/a yes
public_network_access_enabled Allow public access for accessing workspace: [true|false] bool true no
public_subnet_name The name of the Public Subnet within the Virtual Network. Required if virtual_network_id is set string n/a yes
public_subnet_nsg_association_id The resource ID of the azurerm_subnet_network_security_group_association resource which is referred to by the public_subnet_name field. Required if virtual_network_id is set string n/a yes
resource_group The name of the resource group in which to create the storage account string n/a yes
sku The sku to use for the Databricks Workspace: [standard|premium|trial] string "premium" no
storage_firewall_enabled Boolean flag that determines whether public access is disallowed bool false no
tags A mapping of tags to assign to the resource map(string) {} no
workspace_name Databricks Workspace name string n/a yes

Outputs

Name Description
access_connector_id Databricks Access Connector's Id
access_connector_identity Databricks Access Connector's Identities list
databricks_client_id_identity The Client ID of the User Assigned Identity.
databricks_disk_encryption_set_id The ID of Managed Disk Encryption Set created by the Databricks Workspace.
databricks_managed_storage_account_id Azure Databricks Workspace Managed Storage Account ID.
databricks_principal_id_identity The Service Principal ID of the User Assigned Identity.
id Azure Databricks Resource ID
sku Azure Databricks Workspace SKU type
workspace_id Azure Databricks Workspace ID
workspace_url Azure Databricks Workspace URL

License

Apache 2 Licensed. For more information please see LICENSE