Add Ingestion Resources

.github/workflows/_terraformDestroyTemplate.yml

@@ -85,6 +85,15 @@ jobs:
         id: checkout_repository
         uses: actions/checkout@v4
 
+      # Azure login
+      - name: Azure login
+        id: azure_login
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.CLIENT_ID }}
+          tenant-id: ${{ inputs.tenant_id }}
+          subscription-id: ${{ inputs.subscription_id }}
+
       # Terraform Init
       - name: Terraform Init
         working-directory: ${{ inputs.working_directory }}

.github/workflows/_terraformEnvironmentTemplate.yml

@@ -229,6 +229,15 @@ jobs:
         id: checkout_repository
         uses: actions/checkout@v4
 
+      # Azure login
+      - name: Azure login
+        id: azure_login
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.CLIENT_ID }}
+          tenant-id: ${{ inputs.tenant_id }}
+          subscription-id: ${{ inputs.subscription_id }}
+
       # Terraform Init
       - name: Terraform Init
         working-directory: ${{ inputs.working_directory }}

code/infra/aoai.tf

@@ -7,7 +7,7 @@ module "azure_open_ai" {
 
   location                                                = var.location_openai
   location_private_endpoint                               = var.location
-  resource_group_name                                     = azurerm_resource_group.resource_group.name
+  resource_group_name                                     = azurerm_resource_group.resource_group_consumption.name
   tags                                                    = var.tags
   cognitive_account_name                                  = "${local.prefix}-aoai001"
   cognitive_account_kind                                  = "OpenAI"

code/infra/applicationinsights.tf

@@ -5,7 +5,7 @@ module "application_insights" {
   }
 
   location                                        = var.location
-  resource_group_name                             = azurerm_resource_group.resource_group.name
+  resource_group_name                             = azurerm_resource_group.resource_group_consumption.name
   tags                                            = var.tags
   application_insights_name                       = "${local.prefix}-appi001"
   application_insights_application_type           = "web"

code/infra/appserviceplan.tf

@@ -5,7 +5,7 @@ module "app_service_plan" {
   }
 
   location                                  = var.location
-  resource_group_name                       = azurerm_resource_group.resource_group.name
+  resource_group_name                       = azurerm_resource_group.resource_group_consumption.name
   tags                                      = var.tags
   service_plan_name                         = "${local.prefix}-asp001"
   service_plan_maximum_elastic_worker_count = null

code/infra/botservice.tf

@@ -6,7 +6,7 @@ module "bot_service" {
   }
 
   location             = var.location
-  resource_group_name  = azurerm_resource_group.resource_group.name
+  resource_group_name  = azurerm_resource_group.resource_group_consumption.name
   tags                 = var.tags
   bot_service_name     = "${local.prefix}-bot001"
   bot_service_location = "global"
@@ -38,7 +38,7 @@ resource "azurerm_bot_connection" "bot_connection_aadv2_oauth" {
   name                = local.bot_connection_aadv2_oauth_name
   bot_name            = module.bot_service.bot_service_name
   location            = "global"
-  resource_group_name = azurerm_resource_group.resource_group.name
+  resource_group_name = azurerm_resource_group.resource_group_consumption.name
 
   client_id     = var.bot_oauth_client_id
   client_secret = var.bot_oauth_client_secret

code/infra/cosmosdb.tf

@@ -6,7 +6,7 @@ module "cosmosdb_account" {
   }
 
   location                                            = var.location
-  resource_group_name                                 = azurerm_resource_group.resource_group.name
+  resource_group_name                                 = azurerm_resource_group.resource_group_consumption.name
   tags                                                = var.tags
   cosmosdb_account_name                               = "${local.prefix}-cosmos001"
   cosmosdb_account_access_key_metadata_writes_enabled = true
@@ -58,7 +58,7 @@ module "cosmosdb_account" {
 resource "azurerm_cosmosdb_sql_database" "cosmosdb_sql_database" {
   name                = "BotDb"
   account_name        = module.cosmosdb_account.cosmosdb_account_name
-  resource_group_name = azurerm_resource_group.resource_group.name
+  resource_group_name = azurerm_resource_group.resource_group_consumption.name
 
   autoscale_settings {
     max_throughput = 1000

code/infra/datafactory.tf

@@ -0,0 +1,36 @@
+module "data_factory" {
+  source = "github.com/PerfectThymeTech/terraform-azurerm-modules//modules/datafactory?ref=main"
+  providers = {
+    azurerm = azurerm
+    azapi   = azapi
+    time    = time
+  }
+
+  location                                          = var.location
+  resource_group_name                               = azurerm_resource_group.resource_group_ingestion.name
+  tags                                              = var.tags
+  data_factory_name                                 = "${local.prefix}-adf001"
+  data_factory_purview_id                           = null
+  data_factory_azure_devops_repo                    = {}
+  data_factory_github_repo                          = {}
+  data_factory_global_parameters                    = {}
+  data_factory_published_content                    = {}
+  data_factory_published_content_template_variables = {}
+  data_factory_triggers_start                       = []
+  data_factory_pipelines_run                        = []
+  data_factory_managed_private_endpoints = {
+    "storage-blob" = {
+      subresource_name   = "blob"
+      target_resource_id = module.storage_account.storage_account_id
+    }
+    "keyvault-vault" = {
+      subresource_name   = "vault"
+      target_resource_id = module.key_vault_ingestion.key_vault_id
+    }
+  }
+  diagnostics_configurations       = local.diagnostics_configurations
+  subnet_id                        = azapi_resource.subnet_private_endpoints.id
+  connectivity_delay_in_seconds    = var.connectivity_delay_in_seconds
+  private_dns_zone_id_data_factory = var.private_dns_zone_id_data_factory
+  customer_managed_key             = local.customer_managed_key
+}

code/infra/keyvault.tf

@@ -1,12 +1,12 @@
-module "key_vault" {
+module "key_vault_consumption" {
   source = "github.com/PerfectThymeTech/terraform-azurerm-modules//modules/keyvault?ref=main"
   providers = {
     azurerm = azurerm
     time    = time
   }
 
   location                             = var.location
-  resource_group_name                  = azurerm_resource_group.resource_group.name
+  resource_group_name                  = azurerm_resource_group.resource_group_consumption.name
   tags                                 = var.tags
   key_vault_name                       = "${local.prefix}-kv001"
   key_vault_sku_name                   = "standard"
@@ -16,3 +16,22 @@ module "key_vault" {
   connectivity_delay_in_seconds        = var.connectivity_delay_in_seconds
   private_dns_zone_id_vault            = var.private_dns_zone_id_vault
 }
+
+module "key_vault_ingestion" {
+  source = "github.com/PerfectThymeTech/terraform-azurerm-modules//modules/keyvault?ref=main"
+  providers = {
+    azurerm = azurerm
+    time    = time
+  }
+
+  location                             = var.location
+  resource_group_name                  = azurerm_resource_group.resource_group_ingestion.name
+  tags                                 = var.tags
+  key_vault_name                       = "${local.prefix}-ngst-kv001"
+  key_vault_sku_name                   = "standard"
+  key_vault_soft_delete_retention_days = 7
+  diagnostics_configurations           = local.diagnostics_configurations
+  subnet_id                            = azapi_resource.subnet_private_endpoints.id
+  connectivity_delay_in_seconds        = var.connectivity_delay_in_seconds
+  private_dns_zone_id_vault            = var.private_dns_zone_id_vault
+}

code/infra/locals.tf

@@ -63,6 +63,10 @@ locals {
     name                = split("/", var.log_analytics_workspace_id)[8]
   }
 
+  # Storage locals
+  storage_account_container_raw_name     = "raw"
+  storage_account_container_curated_name = "curated"
+
   # Logging locals
   diagnostics_configurations = [
     {

code/infra/main.tf

@@ -1,5 +1,11 @@
-resource "azurerm_resource_group" "resource_group" {
-  name     = "${local.prefix}-bot-rg"
+resource "azurerm_resource_group" "resource_group_consumption" {
+  name     = "${local.prefix}-bot-cnsm-rg"
+  location = var.location
+  tags     = var.tags
+}
+
+resource "azurerm_resource_group" "resource_group_ingestion" {
+  name     = "${local.prefix}-bot-ngst-rg"
   location = var.location
   tags     = var.tags
 }

code/infra/roleassignments_datafactory.tf

@@ -0,0 +1,15 @@
+resource "azurerm_role_assignment" "data_factory_roleassignment_storage_blob_data_owner" {
+  description          = "Required for reding and writing data from data factory."
+  scope                = module.storage_account.storage_account_id
+  role_definition_name = "Storage Blob Data Owner"
+  principal_id         = module.data_factory.data_factory_principal_id
+  principal_type       = "ServicePrincipal"
+}
+
+resource "azurerm_role_assignment" "data_factory_roleassignment_key_vault_secrets_user" {
+  description          = "Required for accessing secrets in the key vault from the data factory."
+  scope                = module.key_vault_ingestion.key_vault_id
+  role_definition_name = "Key Vault Secrets User"
+  principal_id         = module.data_factory.data_factory_principal_id
+  principal_type       = "ServicePrincipal"
+}

code/infra/roleassignments_uai.tf

@@ -7,8 +7,8 @@ resource "azurerm_role_assignment" "uai_roleassignment_open_ai_contributor" {
 }
 
 resource "azurerm_role_assignment" "uai_roleassignment_key_vault_secrets_user" {
-  description          = "Required for accessing secrets in teh key vault from teh web app app settings."
-  scope                = module.key_vault.key_vault_id
+  description          = "Required for accessing secrets in the key vault from teh web app app settings."
+  scope                = module.key_vault_consumption.key_vault_id
   role_definition_name = "Key Vault Secrets User"
   principal_id         = module.user_assigned_identity.user_assigned_identity_principal_id
   principal_type       = "ServicePrincipal"

code/infra/storage.tf

@@ -0,0 +1,47 @@
+module "storage_account" {
+  source = "github.com/PerfectThymeTech/terraform-azurerm-modules//modules/storage?ref=main"
+  providers = {
+    azurerm = azurerm
+    time    = time
+  }
+
+  location            = var.location
+  resource_group_name = azurerm_resource_group.resource_group_ingestion.name
+  tags                = var.tags
+
+  storage_account_name                            = replace("${local.prefix}-stg001", "-", "")
+  storage_access_tier                             = "Hot"
+  storage_account_type                            = "StorageV2"
+  storage_account_tier                            = "Standard"
+  storage_account_replication_type                = "ZRS"
+  storage_account_allowed_copy_scope              = "AAD"
+  storage_blob_change_feed_enabled                = false
+  storage_blob_container_delete_retention_in_days = 7
+  storage_blob_delete_retention_in_days           = 7
+  storage_blob_cors_rules                         = {}
+  storage_blob_last_access_time_enabled           = false
+  storage_blob_versioning_enabled                 = false
+  storage_is_hns_enabled                          = false
+  storage_network_bypass                          = ["None"]
+  storage_network_private_link_access = [
+    "/subscriptions/${data.azurerm_client_config.current.subscription_id}/providers/Microsoft.Security/datascanners/storageDataScanner",
+    "/subscriptions/${data.azurerm_client_config.current.subscription_id}/resourceGroups/*/providers/Microsoft.CognitiveServices/accounts/*",
+  ]
+  storage_public_network_access_enabled = true
+  storage_nfsv3_enabled                 = false
+  storage_sftp_enabled                  = false
+  storage_shared_access_key_enabled     = false
+  storage_container_names               = [local.storage_account_container_raw_name, local.storage_account_container_curated_name, ]
+  storage_static_website                = []
+  diagnostics_configurations            = local.diagnostics_configurations
+  subnet_id                             = azapi_resource.subnet_private_endpoints.id
+  connectivity_delay_in_seconds         = var.connectivity_delay_in_seconds
+  private_endpoint_subresource_names    = ["blob"]
+  private_dns_zone_id_blob              = var.private_dns_zone_id_blob
+  private_dns_zone_id_file              = ""
+  private_dns_zone_id_table             = ""
+  private_dns_zone_id_queue             = ""
+  private_dns_zone_id_web               = ""
+  private_dns_zone_id_dfs               = ""
+  customer_managed_key                  = local.customer_managed_key
+}

code/infra/userassignedidentity.tf

@@ -5,7 +5,7 @@ module "user_assigned_identity" {
   }
 
   location                                              = var.location
-  resource_group_name                                   = azurerm_resource_group.resource_group.name
+  resource_group_name                                   = azurerm_resource_group.resource_group_consumption.name
   tags                                                  = var.tags
   user_assigned_identity_name                           = "${local.prefix}-uai001"
   user_assigned_identity_federated_identity_credentials = {}

code/infra/variables.tf

@@ -138,6 +138,7 @@ variable "subnet_cidr_private_endpoints" {
   }
 }
 
+# DNS variables
 variable "private_dns_zone_id_vault" {
   description = "Specifies the resource ID of the private DNS zone for Azure Key Vault. Not required if DNS A-records get created via Azure Policy."
   type        = string
@@ -203,3 +204,25 @@ variable "private_dns_zone_id_cosmos_sql" {
     error_message = "Please specify a valid resource ID for the private DNS Zone."
   }
 }
+
+variable "private_dns_zone_id_blob" {
+  description = "Specifies the resource ID of the private DNS zone for blob storage. Not required if DNS A-records get created via Azure Policy."
+  type        = string
+  sensitive   = false
+  default     = ""
+  validation {
+    condition     = var.private_dns_zone_id_blob == "" || (length(split("/", var.private_dns_zone_id_blob)) == 9 && endswith(var.private_dns_zone_id_blob, "privatelink.blob.core.windows.net"))
+    error_message = "Please specify a valid resource ID for the private DNS Zone."
+  }
+}
+
+variable "private_dns_zone_id_data_factory" {
+  description = "Specifies the resource ID of the private DNS zone for Azure Data Factory. Not required if DNS A-records get created via Azure Policy."
+  type        = string
+  sensitive   = false
+  default     = ""
+  validation {
+    condition     = var.private_dns_zone_id_data_factory == "" || (length(split("/", var.private_dns_zone_id_data_factory)) == 9 && endswith(var.private_dns_zone_id_data_factory, "privatelink.datafactory.azure.net"))
+    error_message = "Please specify a valid resource ID for the private DNS Zone."
+  }
+}

code/infra/webapplinux.tf

@@ -1,7 +1,7 @@
 resource "azurerm_linux_web_app" "linux_web_app" {
   name                = "${local.prefix}-app001"
   location            = var.location
-  resource_group_name = azurerm_resource_group.resource_group.name
+  resource_group_name = azurerm_resource_group.resource_group_consumption.name
   tags                = var.tags
   identity {
     type = "UserAssigned"
@@ -89,7 +89,7 @@ resource "azurerm_monitor_diagnostic_setting" "diagnostic_setting_linux_web_app"
 resource "azurerm_private_endpoint" "linux_web_app_private_endpoint" {
   name                = "${azurerm_linux_web_app.linux_web_app.name}-pe"
   location            = var.location
-  resource_group_name = azurerm_resource_group.resource_group.name
+  resource_group_name = azurerm_resource_group.resource_group_consumption.name
   tags                = var.tags
 
   custom_network_interface_name = "${azurerm_linux_web_app.linux_web_app.name}-nic"

config/PerfectThymeTech/vars.tfvars

@@ -33,3 +33,5 @@ private_dns_zone_id_bot_framework_directline = "/subscriptions/e82c5267-9dc4-4f4
 private_dns_zone_id_bot_framework_token      = "/subscriptions/e82c5267-9dc4-4f45-ac13-abdd5e130d27/resourceGroups/ptt-dev-privatedns-rg/providers/Microsoft.Network/privateDnsZones/privatelink.token.botframework.com"
 private_dns_zone_id_open_ai                  = "/subscriptions/e82c5267-9dc4-4f45-ac13-abdd5e130d27/resourceGroups/ptt-dev-privatedns-rg/providers/Microsoft.Network/privateDnsZones/privatelink.openai.azure.com"
 private_dns_zone_id_cosmos_sql               = "/subscriptions/e82c5267-9dc4-4f45-ac13-abdd5e130d27/resourceGroups/ptt-dev-privatedns-rg/providers/Microsoft.Network/privateDnsZones/privatelink.documents.azure.com"
+private_dns_zone_id_blob                     = "/subscriptions/e82c5267-9dc4-4f45-ac13-abdd5e130d27/resourceGroups/ptt-dev-privatedns-rg/providers/Microsoft.Network/privateDnsZones/privatelink.blob.core.windows.net"
+private_dns_zone_id_data_factory             = "/subscriptions/e82c5267-9dc4-4f45-ac13-abdd5e130d27/resourceGroups/ptt-dev-privatedns-rg/providers/Microsoft.Network/privateDnsZones/privatelink.datafactory.azure.net"

docs/prereqs/locals.tf

@@ -27,5 +27,7 @@ locals {
     bot_framework_token      = "privatelink.token.botframework.com",
     open_ai                  = "privatelink.openai.azure.com",
     cosmos_sql               = "privatelink.documents.azure.com",
+    blob                     = "privatelink.blob.core.windows.net",
+    data_factory             = "privatelink.datafactory.azure.net",
   }
 }