From 86d8d9eb5c60fad5ad2345540445f253683bfa08 Mon Sep 17 00:00:00 2001 From: KyleWerts Date: Wed, 27 Mar 2024 00:31:55 +0000 Subject: [PATCH 01/10] Added Bing Search Key to keyvault --- infra/core/ai/bingSearch/bingSearch.tf | 6 ++++++ infra/core/ai/bingSearch/variables.tf | 6 +++++- infra/core/host/enrichmentapp/enrichmentapp.tf | 1 + infra/core/host/functions/functions.tf | 1 + infra/core/host/webapp/webapp.tf | 11 ++++++----- infra/main.tf | 1 + scripts/json-to-env.function.debug.sh | 2 +- scripts/json-to-env.sh | 2 +- scripts/json-to-env.webapp.debug.sh | 2 +- 9 files changed, 23 insertions(+), 9 deletions(-) diff --git a/infra/core/ai/bingSearch/bingSearch.tf b/infra/core/ai/bingSearch/bingSearch.tf index fe3bcf1ce..db515a1e1 100644 --- a/infra/core/ai/bingSearch/bingSearch.tf +++ b/infra/core/ai/bingSearch/bingSearch.tf @@ -23,4 +23,10 @@ resource "azurerm_resource_group_template_deployment" "bing_search" { # this ensures the keys are up-to-date name = "bingsearch-${filemd5(local.arm_file_path)}" deployment_mode = "Incremental" +} + +resource "azurerm_key_vault_secret" "search_service_key" { + name = "BINGSEARCH-KEY" + value = azurerm_resource_group_template_deployment.bing_search.primary_key + key_vault_id = var.keyVaultId } \ No newline at end of file diff --git a/infra/core/ai/bingSearch/variables.tf b/infra/core/ai/bingSearch/variables.tf index 11b3dd28e..38a666769 100644 --- a/infra/core/ai/bingSearch/variables.tf +++ b/infra/core/ai/bingSearch/variables.tf @@ -16,7 +16,11 @@ variable "sku" { type = string } - variable "arm_template_schema_mgmt_api" { type = string +} + +variable "keyVaultId" { + type = string + sensitive = true } \ No newline at end of file diff --git a/infra/core/host/enrichmentapp/enrichmentapp.tf b/infra/core/host/enrichmentapp/enrichmentapp.tf index b0d8a8c25..815336672 100644 --- a/infra/core/host/enrichmentapp/enrichmentapp.tf +++ b/infra/core/host/enrichmentapp/enrichmentapp.tf @@ -103,6 +103,7 @@ resource "azurerm_linux_web_app" "app_service" { "AZURE_SEARCH_SERVICE_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-SEARCH-SERVICE-KEY)" "COSMOSDB_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/COSMOSDB-KEY)" "ENRICHMENT_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/ENRICHMENT-KEY)" + "BINGSEARCH_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/BINGSEARCH-KEY)" "AZURE_BLOB_STORAGE_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-BLOB-STORAGE-KEY)" "BLOB_CONNECTION_STRING" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/BLOB-CONNECTION-STRING)" "AZURE_STORAGE_CONNECTION_STRING" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/BLOB-CONNECTION-STRING)" diff --git a/infra/core/host/functions/functions.tf b/infra/core/host/functions/functions.tf index e467ae92b..df24bd973 100644 --- a/infra/core/host/functions/functions.tf +++ b/infra/core/host/functions/functions.tf @@ -153,6 +153,7 @@ resource "azurerm_linux_function_app" "function_app" { EMBEDDINGS_QUEUE = var.EMBEDDINGS_QUEUE AZURE_SEARCH_SERVICE_KEY = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-SEARCH-SERVICE-KEY)" COSMOSDB_KEY = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/COSMOSDB-KEY)" + BINGSEARCH_KEY = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/BINGSEARCH-KEY)" AZURE_SEARCH_SERVICE_ENDPOINT = var.azureSearchServiceEndpoint AZURE_SEARCH_INDEX = var.azureSearchIndex AZURE_AI_TRANSLATION_DOMAIN = var.azure_ai_translation_domain diff --git a/infra/core/host/webapp/webapp.tf b/infra/core/host/webapp/webapp.tf index e90f34afc..47eb01e31 100644 --- a/infra/core/host/webapp/webapp.tf +++ b/infra/core/host/webapp/webapp.tf @@ -102,11 +102,12 @@ resource "azurerm_linux_web_app" "app_service" { "SCM_DO_BUILD_DURING_DEPLOYMENT" = lower(tostring(var.scmDoBuildDuringDeployment)) "ENABLE_ORYX_BUILD" = lower(tostring(var.enableOryxBuild)) "APPLICATIONINSIGHTS_CONNECTION_STRING" = var.applicationInsightsConnectionString - "AZURE_SEARCH_SERVICE_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-SEARCH-SERVICE-KEY)" - "COSMOSDB_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/COSMOSDB-KEY)" - "AZURE_BLOB_STORAGE_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-BLOB-STORAGE-KEY)" - "ENRICHMENT_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/ENRICHMENT-KEY)" - "AZURE_OPENAI_SERVICE_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-OPENAI-SERVICE-KEY)" + "AZURE_SEARCH_SERVICE_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-SEARCH-SERVICE-KEY)" + "COSMOSDB_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/COSMOSDB-KEY)" + "BINGSEARCH_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/BINGSEARCH-KEY)" + "AZURE_BLOB_STORAGE_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-BLOB-STORAGE-KEY)" + "ENRICHMENT_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/ENRICHMENT-KEY)" + "AZURE_OPENAI_SERVICE_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-OPENAI-SERVICE-KEY)" } ) diff --git a/infra/main.tf b/infra/main.tf index ba74674a0..4c4b988d6 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -483,6 +483,7 @@ module "bingSearch" { tags = local.tags sku = "S1" //supported SKUs can be found at https://www.microsoft.com/en-us/bing/apis/pricing arm_template_schema_mgmt_api = var.arm_template_schema_mgmt_api + keyVaultId = module.kvModule.keyVaultId } // DEPLOYMENT OF AZURE CUSTOMER ATTRIBUTION TAG diff --git a/scripts/json-to-env.function.debug.sh b/scripts/json-to-env.function.debug.sh index 5a01b8d42..46b6b678c 100755 --- a/scripts/json-to-env.function.debug.sh +++ b/scripts/json-to-env.function.debug.sh @@ -20,7 +20,7 @@ secrets="{" keyVaultName=$(cat inf_output.json | jq -r .DEPLOYMENT_KEYVAULT_NAME.value) # Names of your secrets -secretNames=("AZURE-SEARCH-SERVICE-KEY" "AZURE-BLOB-STORAGE-KEY" "BLOB-CONNECTION-STRING" "COSMOSDB-KEY" "AZURE-FORM-RECOGNIZER-KEY" "ENRICHMENT-KEY") +secretNames=("AZURE-SEARCH-SERVICE-KEY" "AZURE-BLOB-STORAGE-KEY" "BLOB-CONNECTION-STRING" "COSMOSDB-KEY" "BINGSEARCH-KEY" "AZURE-FORM-RECOGNIZER-KEY" "ENRICHMENT-KEY") azWebJobSecretName="BLOB-CONNECTION-STRING" azWebJobVarName="AzureWebJobsStorage" diff --git a/scripts/json-to-env.sh b/scripts/json-to-env.sh index 2ac411b88..1a79a176d 100755 --- a/scripts/json-to-env.sh +++ b/scripts/json-to-env.sh @@ -147,7 +147,7 @@ fi # Name of your Key Vault keyVaultName=$(cat inf_output.json | jq -r .DEPLOYMENT_KEYVAULT_NAME.value) # Names of your secrets -secretNames=("AZURE-SEARCH-SERVICE-KEY" "AZURE-BLOB-STORAGE-KEY" "BLOB-CONNECTION-STRING" "COSMOSDB-KEY" "AZURE-OPENAI-SERVICE-KEY") +secretNames=("AZURE-SEARCH-SERVICE-KEY" "AZURE-BLOB-STORAGE-KEY" "BLOB-CONNECTION-STRING" "COSMOSDB-KEY" "BINGSEARCH-KEY" "AZURE-OPENAI-SERVICE-KEY") # Retrieve and export each secret for secretName in "${secretNames[@]}"; do diff --git a/scripts/json-to-env.webapp.debug.sh b/scripts/json-to-env.webapp.debug.sh index 1a4642f90..b3897639f 100755 --- a/scripts/json-to-env.webapp.debug.sh +++ b/scripts/json-to-env.webapp.debug.sh @@ -196,7 +196,7 @@ fi keyVaultName=$(cat inf_output.json | jq -r .DEPLOYMENT_KEYVAULT_NAME.value) # Names of your secrets -secretNames=("AZURE-SEARCH-SERVICE-KEY" "AZURE-BLOB-STORAGE-KEY" "BLOB-CONNECTION-STRING" "COSMOSDB-KEY" "AZURE-OPENAI-SERVICE-KEY" "AZURE-CLIENT-SECRET" "ENRICHMENT-KEY") +secretNames=("AZURE-SEARCH-SERVICE-KEY" "AZURE-BLOB-STORAGE-KEY" "BLOB-CONNECTION-STRING" "COSMOSDB-KEY" "BINGSEARCH-KEY" "AZURE-OPENAI-SERVICE-KEY" "AZURE-CLIENT-SECRET" "ENRICHMENT-KEY") # Retrieve and export each secret for secretName in "${secretNames[@]}"; do From 8cadbf09fa0d645df66007681338a570ef16d127 Mon Sep 17 00:00:00 2001 From: KyleWerts Date: Wed, 27 Mar 2024 21:56:16 +0000 Subject: [PATCH 02/10] Update naming convention and fix key vault reference --- infra/core/ai/bingSearch/bingSearch.tf | 2 +- infra/core/host/enrichmentapp/enrichmentapp.tf | 2 +- infra/core/host/functions/functions.tf | 2 +- infra/core/host/webapp/webapp.tf | 2 +- infra/main.tf | 1 - 5 files changed, 4 insertions(+), 5 deletions(-) diff --git a/infra/core/ai/bingSearch/bingSearch.tf b/infra/core/ai/bingSearch/bingSearch.tf index db515a1e1..ecbf0ab5b 100644 --- a/infra/core/ai/bingSearch/bingSearch.tf +++ b/infra/core/ai/bingSearch/bingSearch.tf @@ -27,6 +27,6 @@ resource "azurerm_resource_group_template_deployment" "bing_search" { resource "azurerm_key_vault_secret" "search_service_key" { name = "BINGSEARCH-KEY" - value = azurerm_resource_group_template_deployment.bing_search.primary_key + value = jsondecode(azurerm_resource_group_template_deployment.bing_search.output_content).key1.value key_vault_id = var.keyVaultId } \ No newline at end of file diff --git a/infra/core/host/enrichmentapp/enrichmentapp.tf b/infra/core/host/enrichmentapp/enrichmentapp.tf index 815336672..89523c496 100644 --- a/infra/core/host/enrichmentapp/enrichmentapp.tf +++ b/infra/core/host/enrichmentapp/enrichmentapp.tf @@ -103,7 +103,7 @@ resource "azurerm_linux_web_app" "app_service" { "AZURE_SEARCH_SERVICE_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-SEARCH-SERVICE-KEY)" "COSMOSDB_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/COSMOSDB-KEY)" "ENRICHMENT_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/ENRICHMENT-KEY)" - "BINGSEARCH_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/BINGSEARCH-KEY)" + "BING_SEARCH_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/BINGSEARCH-KEY)" "AZURE_BLOB_STORAGE_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-BLOB-STORAGE-KEY)" "BLOB_CONNECTION_STRING" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/BLOB-CONNECTION-STRING)" "AZURE_STORAGE_CONNECTION_STRING" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/BLOB-CONNECTION-STRING)" diff --git a/infra/core/host/functions/functions.tf b/infra/core/host/functions/functions.tf index df24bd973..afe1edbea 100644 --- a/infra/core/host/functions/functions.tf +++ b/infra/core/host/functions/functions.tf @@ -153,7 +153,7 @@ resource "azurerm_linux_function_app" "function_app" { EMBEDDINGS_QUEUE = var.EMBEDDINGS_QUEUE AZURE_SEARCH_SERVICE_KEY = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-SEARCH-SERVICE-KEY)" COSMOSDB_KEY = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/COSMOSDB-KEY)" - BINGSEARCH_KEY = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/BINGSEARCH-KEY)" + BING_SEARCH_KEY = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/BINGSEARCH-KEY)" AZURE_SEARCH_SERVICE_ENDPOINT = var.azureSearchServiceEndpoint AZURE_SEARCH_INDEX = var.azureSearchIndex AZURE_AI_TRANSLATION_DOMAIN = var.azure_ai_translation_domain diff --git a/infra/core/host/webapp/webapp.tf b/infra/core/host/webapp/webapp.tf index 47eb01e31..e70ebcaee 100644 --- a/infra/core/host/webapp/webapp.tf +++ b/infra/core/host/webapp/webapp.tf @@ -104,7 +104,7 @@ resource "azurerm_linux_web_app" "app_service" { "APPLICATIONINSIGHTS_CONNECTION_STRING" = var.applicationInsightsConnectionString "AZURE_SEARCH_SERVICE_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-SEARCH-SERVICE-KEY)" "COSMOSDB_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/COSMOSDB-KEY)" - "BINGSEARCH_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/BINGSEARCH-KEY)" + "BING_SEARCH_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/BINGSEARCH-KEY)" "AZURE_BLOB_STORAGE_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-BLOB-STORAGE-KEY)" "ENRICHMENT_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/ENRICHMENT-KEY)" "AZURE_OPENAI_SERVICE_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-OPENAI-SERVICE-KEY)" diff --git a/infra/main.tf b/infra/main.tf index 4c4b988d6..2f2ce84b8 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -171,7 +171,6 @@ module "backend" { AZURE_AI_TRANSLATION_DOMAIN = var.azure_ai_translation_domain USE_SEMANTIC_RERANKER = var.use_semantic_reranker BING_SEARCH_ENDPOINT = var.enableWebChat ? module.bingSearch[0].endpoint : "" - BING_SEARCH_KEY = var.enableWebChat ? module.bingSearch[0].key : "" ENABLE_WEB_CHAT = var.enableWebChat ENABLE_BING_SAFE_SEARCH = var.enableBingSafeSearch ENABLE_UNGROUNDED_CHAT = var.enableUngroundedChat From c7d41bd4ba6fef6db82f6abe4762c3c5311b2bd8 Mon Sep 17 00:00:00 2001 From: KyleWerts Date: Thu, 28 Mar 2024 15:49:13 +0000 Subject: [PATCH 03/10] PR request changes and removed code that is not needed --- infra/core/ai/bingSearch/bingSearch.tf | 2 +- infra/core/host/enrichmentapp/enrichmentapp.tf | 1 - infra/core/host/functions/functions.tf | 1 - scripts/json-to-env.function.debug.sh | 2 +- scripts/json-to-env.sh | 2 +- 5 files changed, 3 insertions(+), 5 deletions(-) diff --git a/infra/core/ai/bingSearch/bingSearch.tf b/infra/core/ai/bingSearch/bingSearch.tf index ecbf0ab5b..06ebcd2a4 100644 --- a/infra/core/ai/bingSearch/bingSearch.tf +++ b/infra/core/ai/bingSearch/bingSearch.tf @@ -25,7 +25,7 @@ resource "azurerm_resource_group_template_deployment" "bing_search" { deployment_mode = "Incremental" } -resource "azurerm_key_vault_secret" "search_service_key" { +resource "azurerm_key_vault_secret" "bing_search_key" { name = "BINGSEARCH-KEY" value = jsondecode(azurerm_resource_group_template_deployment.bing_search.output_content).key1.value key_vault_id = var.keyVaultId diff --git a/infra/core/host/enrichmentapp/enrichmentapp.tf b/infra/core/host/enrichmentapp/enrichmentapp.tf index 89523c496..b0d8a8c25 100644 --- a/infra/core/host/enrichmentapp/enrichmentapp.tf +++ b/infra/core/host/enrichmentapp/enrichmentapp.tf @@ -103,7 +103,6 @@ resource "azurerm_linux_web_app" "app_service" { "AZURE_SEARCH_SERVICE_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-SEARCH-SERVICE-KEY)" "COSMOSDB_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/COSMOSDB-KEY)" "ENRICHMENT_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/ENRICHMENT-KEY)" - "BING_SEARCH_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/BINGSEARCH-KEY)" "AZURE_BLOB_STORAGE_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-BLOB-STORAGE-KEY)" "BLOB_CONNECTION_STRING" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/BLOB-CONNECTION-STRING)" "AZURE_STORAGE_CONNECTION_STRING" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/BLOB-CONNECTION-STRING)" diff --git a/infra/core/host/functions/functions.tf b/infra/core/host/functions/functions.tf index afe1edbea..e467ae92b 100644 --- a/infra/core/host/functions/functions.tf +++ b/infra/core/host/functions/functions.tf @@ -153,7 +153,6 @@ resource "azurerm_linux_function_app" "function_app" { EMBEDDINGS_QUEUE = var.EMBEDDINGS_QUEUE AZURE_SEARCH_SERVICE_KEY = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-SEARCH-SERVICE-KEY)" COSMOSDB_KEY = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/COSMOSDB-KEY)" - BING_SEARCH_KEY = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/BINGSEARCH-KEY)" AZURE_SEARCH_SERVICE_ENDPOINT = var.azureSearchServiceEndpoint AZURE_SEARCH_INDEX = var.azureSearchIndex AZURE_AI_TRANSLATION_DOMAIN = var.azure_ai_translation_domain diff --git a/scripts/json-to-env.function.debug.sh b/scripts/json-to-env.function.debug.sh index 46b6b678c..5a01b8d42 100755 --- a/scripts/json-to-env.function.debug.sh +++ b/scripts/json-to-env.function.debug.sh @@ -20,7 +20,7 @@ secrets="{" keyVaultName=$(cat inf_output.json | jq -r .DEPLOYMENT_KEYVAULT_NAME.value) # Names of your secrets -secretNames=("AZURE-SEARCH-SERVICE-KEY" "AZURE-BLOB-STORAGE-KEY" "BLOB-CONNECTION-STRING" "COSMOSDB-KEY" "BINGSEARCH-KEY" "AZURE-FORM-RECOGNIZER-KEY" "ENRICHMENT-KEY") +secretNames=("AZURE-SEARCH-SERVICE-KEY" "AZURE-BLOB-STORAGE-KEY" "BLOB-CONNECTION-STRING" "COSMOSDB-KEY" "AZURE-FORM-RECOGNIZER-KEY" "ENRICHMENT-KEY") azWebJobSecretName="BLOB-CONNECTION-STRING" azWebJobVarName="AzureWebJobsStorage" diff --git a/scripts/json-to-env.sh b/scripts/json-to-env.sh index 1a79a176d..2ac411b88 100755 --- a/scripts/json-to-env.sh +++ b/scripts/json-to-env.sh @@ -147,7 +147,7 @@ fi # Name of your Key Vault keyVaultName=$(cat inf_output.json | jq -r .DEPLOYMENT_KEYVAULT_NAME.value) # Names of your secrets -secretNames=("AZURE-SEARCH-SERVICE-KEY" "AZURE-BLOB-STORAGE-KEY" "BLOB-CONNECTION-STRING" "COSMOSDB-KEY" "BINGSEARCH-KEY" "AZURE-OPENAI-SERVICE-KEY") +secretNames=("AZURE-SEARCH-SERVICE-KEY" "AZURE-BLOB-STORAGE-KEY" "BLOB-CONNECTION-STRING" "COSMOSDB-KEY" "AZURE-OPENAI-SERVICE-KEY") # Retrieve and export each secret for secretName in "${secretNames[@]}"; do From e48b8a9650f7f4c165aa35a266a3162e052f957e Mon Sep 17 00:00:00 2001 From: KyleWerts Date: Thu, 28 Mar 2024 18:35:50 +0000 Subject: [PATCH 04/10] Passed the local env variable to set the key to "" if set to false --- infra/core/ai/bingSearch/bingSearch.tf | 3 ++- infra/core/ai/bingSearch/outputs.tf | 6 +++--- infra/core/ai/bingSearch/variables.tf | 4 ++++ infra/core/host/webapp/webapp.tf | 2 +- infra/main.tf | 2 +- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/infra/core/ai/bingSearch/bingSearch.tf b/infra/core/ai/bingSearch/bingSearch.tf index 06ebcd2a4..9a609d74f 100644 --- a/infra/core/ai/bingSearch/bingSearch.tf +++ b/infra/core/ai/bingSearch/bingSearch.tf @@ -11,6 +11,7 @@ data "template_file" "workflow" { } resource "azurerm_resource_group_template_deployment" "bing_search" { + count = var.enableWebChat ? 1 : 0 resource_group_name = var.resourceGroupName parameters_content = jsonencode({ "name" = { value = "${var.name}" }, @@ -27,6 +28,6 @@ resource "azurerm_resource_group_template_deployment" "bing_search" { resource "azurerm_key_vault_secret" "bing_search_key" { name = "BINGSEARCH-KEY" - value = jsondecode(azurerm_resource_group_template_deployment.bing_search.output_content).key1.value + value = var.enableWebChat ? jsondecode(azurerm_resource_group_template_deployment.bing_search[0].output_content).key1.value : "" key_vault_id = var.keyVaultId } \ No newline at end of file diff --git a/infra/core/ai/bingSearch/outputs.tf b/infra/core/ai/bingSearch/outputs.tf index 723e019f2..9f0f1d217 100644 --- a/infra/core/ai/bingSearch/outputs.tf +++ b/infra/core/ai/bingSearch/outputs.tf @@ -1,11 +1,11 @@ output "id" { - value = jsondecode(azurerm_resource_group_template_deployment.bing_search.output_content).id.value + value = var.enableWebChat ? jsondecode(azurerm_resource_group_template_deployment.bing_search[0].output_content).id.value : null } output "endpoint" { - value = jsondecode(azurerm_resource_group_template_deployment.bing_search.output_content).endpoint.value + value = var.enableWebChat ? jsondecode(azurerm_resource_group_template_deployment.bing_search[0].output_content).endpoint.value : null } output "key" { - value = jsondecode(azurerm_resource_group_template_deployment.bing_search.output_content).key1.value + value = var.enableWebChat ? jsondecode(azurerm_resource_group_template_deployment.bing_search[0].output_content).key1.value : null } \ No newline at end of file diff --git a/infra/core/ai/bingSearch/variables.tf b/infra/core/ai/bingSearch/variables.tf index 38a666769..71b61f3fa 100644 --- a/infra/core/ai/bingSearch/variables.tf +++ b/infra/core/ai/bingSearch/variables.tf @@ -23,4 +23,8 @@ variable "arm_template_schema_mgmt_api" { variable "keyVaultId" { type = string sensitive = true +} + +variable "enableWebChat" { + type = bool } \ No newline at end of file diff --git a/infra/core/host/webapp/webapp.tf b/infra/core/host/webapp/webapp.tf index e70ebcaee..c9c838c38 100644 --- a/infra/core/host/webapp/webapp.tf +++ b/infra/core/host/webapp/webapp.tf @@ -104,7 +104,7 @@ resource "azurerm_linux_web_app" "app_service" { "APPLICATIONINSIGHTS_CONNECTION_STRING" = var.applicationInsightsConnectionString "AZURE_SEARCH_SERVICE_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-SEARCH-SERVICE-KEY)" "COSMOSDB_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/COSMOSDB-KEY)" - "BING_SEARCH_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/BINGSEARCH-KEY)" + "BING_SEARCH_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/BINGSEARCH-KEY)" "AZURE_BLOB_STORAGE_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-BLOB-STORAGE-KEY)" "ENRICHMENT_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/ENRICHMENT-KEY)" "AZURE_OPENAI_SERVICE_KEY" = "@Microsoft.KeyVault(SecretUri=${var.keyVaultUri}secrets/AZURE-OPENAI-SERVICE-KEY)" diff --git a/infra/main.tf b/infra/main.tf index 0ce54b403..3004ee577 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -474,7 +474,7 @@ module "kvModule" { } module "bingSearch" { - count = var.enableWebChat ? 1 : 0 + enableWebChat = var.enableWebChat source = "./core/ai/bingSearch" name = "infoasst-bing-${random_string.random.result}" resourceGroupName = azurerm_resource_group.rg.name From b99f0eb8b8e5dece11fb0751464f6ae8baea13f2 Mon Sep 17 00:00:00 2001 From: KyleWerts Date: Fri, 29 Mar 2024 00:50:15 +0000 Subject: [PATCH 05/10] Fixed conditional logic --- infra/core/ai/bingSearch/outputs.tf | 6 +++--- infra/core/ai/bingSearch/variables.tf | 3 ++- infra/main.tf | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/infra/core/ai/bingSearch/outputs.tf b/infra/core/ai/bingSearch/outputs.tf index 9f0f1d217..43b310d71 100644 --- a/infra/core/ai/bingSearch/outputs.tf +++ b/infra/core/ai/bingSearch/outputs.tf @@ -1,11 +1,11 @@ output "id" { - value = var.enableWebChat ? jsondecode(azurerm_resource_group_template_deployment.bing_search[0].output_content).id.value : null + value = var.enableWebChat ? jsondecode(azurerm_resource_group_template_deployment.bing_search[0].output_content).id.value : "" } output "endpoint" { - value = var.enableWebChat ? jsondecode(azurerm_resource_group_template_deployment.bing_search[0].output_content).endpoint.value : null + value = var.enableWebChat ? jsondecode(azurerm_resource_group_template_deployment.bing_search[0].output_content).endpoint.value : "" } output "key" { - value = var.enableWebChat ? jsondecode(azurerm_resource_group_template_deployment.bing_search[0].output_content).key1.value : null + value = var.enableWebChat ? jsondecode(azurerm_resource_group_template_deployment.bing_search[0].output_content).key1.value : "" } \ No newline at end of file diff --git a/infra/core/ai/bingSearch/variables.tf b/infra/core/ai/bingSearch/variables.tf index 71b61f3fa..c0e0c2041 100644 --- a/infra/core/ai/bingSearch/variables.tf +++ b/infra/core/ai/bingSearch/variables.tf @@ -26,5 +26,6 @@ variable "keyVaultId" { } variable "enableWebChat" { - type = bool + description = "A flag to enable or disable web chat." + type = bool } \ No newline at end of file diff --git a/infra/main.tf b/infra/main.tf index 3004ee577..611be55a3 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -474,7 +474,7 @@ module "kvModule" { } module "bingSearch" { - enableWebChat = var.enableWebChat + count = var.enableWebChat ? 1 : 0 source = "./core/ai/bingSearch" name = "infoasst-bing-${random_string.random.result}" resourceGroupName = azurerm_resource_group.rg.name @@ -482,6 +482,7 @@ module "bingSearch" { sku = "S1" //supported SKUs can be found at https://www.microsoft.com/en-us/bing/apis/pricing arm_template_schema_mgmt_api = var.arm_template_schema_mgmt_api keyVaultId = module.kvModule.keyVaultId + enableWebChat = var.enableWebChat } // DEPLOYMENT OF AZURE CUSTOMER ATTRIBUTION TAG From 75b71456fc0ba572b52a5553046ad2a9f8a61efe Mon Sep 17 00:00:00 2001 From: George Earl Date: Wed, 27 Mar 2024 18:34:36 +0000 Subject: [PATCH 06/10] Update chunking parameters in FileLayoutParsingOther/__init__.py --- functions/FileLayoutParsingOther/__init__.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/functions/FileLayoutParsingOther/__init__.py b/functions/FileLayoutParsingOther/__init__.py index 211530d4a..93d5e432b 100644 --- a/functions/FileLayoutParsingOther/__init__.py +++ b/functions/FileLayoutParsingOther/__init__.py @@ -31,10 +31,6 @@ text_enrichment_queue = os.environ["TEXT_ENRICHMENT_QUEUE"] CHUNK_TARGET_SIZE = int(os.environ["CHUNK_TARGET_SIZE"]) -NEW_AFTER_N_CHARS = 1500 -COMBINE_UNDER_N_CHARS = 500 -MAX_CHARACTERS = 1500 - utilities = Utilities(azure_blob_storage_account, azure_blob_storage_endpoint, azure_blob_drop_storage_container, azure_blob_content_storage_container, azure_blob_storage_key) function_name = "FileLayoutParsingOther" @@ -160,9 +156,10 @@ def main(msg: func.QueueMessage) -> None: # Chunk the file from unstructured.chunking.title import chunk_by_title - # chunks = chunk_by_title(elements, multipage_sections=True, new_after_n_chars=NEW_AFTER_N_CHARS, combine_under_n_chars=COMBINE_UNDER_N_CHARS) - # chunks = chunk_by_title(elements, multipage_sections=True, new_after_n_chars=NEW_AFTER_N_CHARS, combine_under_n_chars=COMBINE_UNDER_N_CHARS, max_characters=MAX_CHARACTERS) - chunks = chunk_by_title(elements, multipage_sections=True, new_after_n_chars=NEW_AFTER_N_CHARS, combine_text_under_n_chars=COMBINE_UNDER_N_CHARS) + NEW_AFTER_N_CHARS = 2000 + COMBINE_UNDER_N_CHARS = 1000 + MAX_CHARACTERS = 2750 + chunks = chunk_by_title(elements, multipage_sections=True, new_after_n_chars=NEW_AFTER_N_CHARS, combine_text_under_n_chars=COMBINE_UNDER_N_CHARS, max_characters=MAX_CHARACTERS) statusLog.upsert_document(blob_name, f'{function_name} - chunking complete. {len(chunks)} chunks created', StatusClassification.DEBUG) subtitle_name = '' From 136b20e41f8e908322d05d77a90f28d68ad95784 Mon Sep 17 00:00:00 2001 From: dayland Date: Thu, 28 Mar 2024 23:07:58 +0000 Subject: [PATCH 07/10] whitespace change to trigger new build --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e037d375c..031877276 100644 --- a/Makefile +++ b/Makefile @@ -65,4 +65,4 @@ functional-tests: extract-env ## Run functional tests to check the processing pi @./scripts/functional-tests.sh run-migration: ## Migrate from bicep to terraform - python ./scripts/merge-databases.py + python ./scripts/merge-databases.py \ No newline at end of file From 4fa985ce78701192461fd309ddc644fac344092c Mon Sep 17 00:00:00 2001 From: dayland <48474707+dayland@users.noreply.github.com> Date: Fri, 29 Mar 2024 08:33:28 +0000 Subject: [PATCH 08/10] Remove count parameter from bingSearch module --- infra/main.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/infra/main.tf b/infra/main.tf index 611be55a3..6f37ca7f1 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -474,7 +474,6 @@ module "kvModule" { } module "bingSearch" { - count = var.enableWebChat ? 1 : 0 source = "./core/ai/bingSearch" name = "infoasst-bing-${random_string.random.result}" resourceGroupName = azurerm_resource_group.rg.name From 0168c41ba7ed8d5eb2b0630c021def684e77019a Mon Sep 17 00:00:00 2001 From: KyleWerts Date: Fri, 29 Mar 2024 15:50:11 +0000 Subject: [PATCH 09/10] Added Count in Bing Module --- infra/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/infra/main.tf b/infra/main.tf index 6f37ca7f1..611be55a3 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -474,6 +474,7 @@ module "kvModule" { } module "bingSearch" { + count = var.enableWebChat ? 1 : 0 source = "./core/ai/bingSearch" name = "infoasst-bing-${random_string.random.result}" resourceGroupName = azurerm_resource_group.rg.name From 5ddde4a2756a2950fcb81b82b6c15e06de0f43cd Mon Sep 17 00:00:00 2001 From: KyleWerts Date: Fri, 29 Mar 2024 16:26:28 +0000 Subject: [PATCH 10/10] Fixed indexer error --- infra/main.tf | 3 +-- infra/outputs.tf | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/infra/main.tf b/infra/main.tf index 611be55a3..cd7f567c2 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -170,7 +170,7 @@ module "backend" { APPLICATION_TITLE = var.applicationtitle == "" ? "Information Assistant, built with Azure OpenAI" : var.applicationtitle AZURE_AI_TRANSLATION_DOMAIN = var.azure_ai_translation_domain USE_SEMANTIC_RERANKER = var.use_semantic_reranker - BING_SEARCH_ENDPOINT = var.enableWebChat ? module.bingSearch[0].endpoint : "" + BING_SEARCH_ENDPOINT = var.enableWebChat ? module.bingSearch.endpoint : "" ENABLE_WEB_CHAT = var.enableWebChat ENABLE_BING_SAFE_SEARCH = var.enableBingSafeSearch ENABLE_UNGROUNDED_CHAT = var.enableUngroundedChat @@ -474,7 +474,6 @@ module "kvModule" { } module "bingSearch" { - count = var.enableWebChat ? 1 : 0 source = "./core/ai/bingSearch" name = "infoasst-bing-${random_string.random.result}" resourceGroupName = azurerm_resource_group.rg.name diff --git a/infra/outputs.tf b/infra/outputs.tf index 0dcc50228..dedcb86a2 100644 --- a/infra/outputs.tf +++ b/infra/outputs.tf @@ -164,11 +164,11 @@ output "AZURE_ENVIRONMENT" { } output "BING_SEARCH_ENDPOINT" { - value = var.enableWebChat ? module.bingSearch[0].endpoint : "" + value = var.enableWebChat ? module.bingSearch.endpoint : "" } output "BING_SEARCH_KEY" { - value = var.enableWebChat ? module.bingSearch[0].key : "" + value = var.enableWebChat ? module.bingSearch.key : "" } output "ENABLE_BING_SAFE_SEARCH" {