Merge pull request #675 from microsoft/vNext-Dev

v1.1 Release from vNext-Dev to main
microsoft · Apr 29, 2024 · 878475d · 878475d
2 parents f4b3f3a + 8806594
commit 878475d
Show file tree

Hide file tree

Showing 235 changed files with 13,662 additions and 4,640 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/vscode/devcontainers/python:0-3.10-bullseye
+FROM mcr.microsoft.com/vscode/devcontainers/python:3.10-bullseye
 
 # Avoid warnings by switching to noninteractive
 ENV DEBIAN_FRONTEND=noninteractive
@@ -48,11 +48,26 @@ RUN /tmp/docker-client.sh $USERNAME
 RUN git clone https://github.com/magicmonty/bash-git-prompt.git ~/.bash-git-prompt --depth=1 \
     && echo "if [ -f \"$HOME/.bash-git-prompt/gitprompt.sh\" ]; then GIT_PROMPT_ONLY_IN_REPO=1 && source $HOME/.bash-git-prompt/gitprompt.sh; fi" >> "/home/$USERNAME/.bashrc"
 
+# terraform + tflint
+ARG TERRAFORM_VERSION=1.3.7
+ARG TFLINT_VERSION=0.44.1
+RUN mkdir -p /tmp/docker-downloads \
+    && curl -sSL -o /tmp/docker-downloads/terraform.zip https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip \
+    && unzip /tmp/docker-downloads/terraform.zip \
+    && mv terraform /usr/local/bin \
+    && rm /tmp/docker-downloads/terraform.zip \
+    && echo "alias tf=terraform" >> "/home/$USERNAME/.bashrc"
+
+RUN curl -sSL -o /tmp/docker-downloads/tflint.zip https://github.com/wata727/tflint/releases/download/v${TFLINT_VERSION}/tflint_linux_amd64.zip \
+    && unzip /tmp/docker-downloads/tflint.zip \
+    && mv tflint /usr/local/bin \
+    && rm /tmp/docker-downloads/tflint.zip
+
 # azure-cli
 COPY ./scripts/azure-cli.sh /tmp/
 RUN /tmp/azure-cli.sh
 
-# Install dotnet 6 & Azure Functions Core Tools
+# Install Azure Functions Core Tools
 RUN echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-debian-$(lsb_release -cs)-prod $(lsb_release -cs) main" > /etc/apt/sources.list.d/microsoft.list \
     && echo "deb [arch=amd64] https://packages.microsoft.com/debian/$(lsb_release -rs | cut -d'.' -f 1)/prod $(lsb_release -cs) main" > /etc/apt/sources.list.d/dotnetdev.list \
     && wget https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb \
@@ -78,4 +93,10 @@ RUN if [ -z "$TZ" ]; then TZ="Europe/London"; fi && sudo ln -snf /usr/share/zone
 
 # Install gettext-base so that we have envsubst
 RUN sudo apt-get update \
-    && sudo apt-get -y install gettext-base
+    && sudo apt-get -y install gettext-base
+
+# Install python packages for migration
+RUN pip install azure-cosmos
+RUN pip install pyfiglet
+RUN pip install azure-identity
+RUN pip install azure-keyvault-secrets
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -23,58 +23,57 @@
 		"type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock"
 	],
 
-	// Set *default* container specific settings.json values on container create.
-	"settings": { 
-		"python.pythonPath": "/opt/conda/envs/development/bin/python",
-		"python.languageServer": "Pylance",
-		"python.linting.enabled": true,
-		"python.linting.pylintEnabled": true,
-		"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
-		"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
-		"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
-		"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
-		"python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
-		"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
-		"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
-		"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
-		"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint",
-        "files.watcherExclude": {
-            "**/.git/objects/**": true,
-            "**/.git/subtree-cache/**": true,
-            "**/node_modules/*/**": true,
-            "**/.python_packages/*/**": true
-          },
-		"files.associations": {
-			"*.workbook": "[jsonc]"
+	// Set *default* container specific settings values on container create.
+	"customizations": {
+		"vscode": {
+			"settings": { 
+				"python.pythonPath": "/opt/conda/envs/development/bin/python",
+				"python.languageServer": "Pylance",
+				"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
+				"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+				"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
+				"files.watcherExclude": {
+          			"**/.git/objects/**": true,
+         			"**/.git/subtree-cache/**": true,
+					"**/node_modules/*/**": true,
+           			"**/.python_packages/*/**": true
+				},
+				"files.associations": {
+					"*.workbook": "[jsonc]"
+				}
+			},
+			// Add extensions you want installed when the container is created into this array
+			"extensions": [
+				"ms-python.python",
+				"ms-python.vscode-pylance",
+				"IronGeek.vscode-env",
+				"ms-azuretools.vscode-docker",
+				"ms-toolsai.jupyter",
+				"humao.rest-client",
+				"ms-dotnettools.csharp",
+				"ms-vsliveshare.vsliveshare-pack",
+				"ms-vscode.powershell",
+				"DavidAnson.vscode-markdownlint",
+				"redhat.vscode-yaml",
+				"ms-azure-devops.azure-pipelines",
+				"k--kato.docomment",
+				"hediet.vscode-drawio",
+				"msazurermtools.azurerm-vscode-tools",
+				"ms-azuretools.vscode-azurestorage",
+				"GitHub.copilot",
+				"GitHub.copilot-chat",
+				"BelkacemBerras.spellcheck",
+				"ms-azuretools.vscode-azureresourcegroups",
+				"ms-azuretools.vscode-azurefunctions",
+				"ms-python.pylint",
+				"ms-python.mypy",
+				"HashiCorp.terraform",
+				"mhutchie.git-graph",
+				"esbenp.prettier-vscode",
+				"mutantdino.resourcemonitor"
+			]
 		}
 	},
 
-	// Add the IDs of extensions you want installed when the container is created.
-	"extensions": [
-		"ms-python.python",
-		"ms-python.vscode-pylance",
-		"irongeek.vscode-env",
-		"ms-azuretools.vscode-docker",
-		"ms-toolsai.jupyter",
-		"humao.rest-client",
-		"ms-dotnettools.csharp",
-		"ms-vsliveshare.vsliveshare-pack",
-		"ms-vscode.powershell",
-		"DavidAnson.vscode-markdownlint",
-		"redhat.vscode-yaml",
-		"ms-azure-devops.azure-pipelines",
-		"k--kato.docomment",
-		"hediet.vscode-drawio",
-		"msazurermtools.azurerm-vscode-tools",
-    	"ms-azuretools.vscode-azurestorage",
-		"ms-azuretools.vscode-bicep",
-		"GitHub.copilot",
-		"BelkacemBerras.spellcheck",
-		"ms-azuretools.vscode-azureresourcegroups",
-		"ms-azuretools.vscode-azurefunctions",
-		"ms-python.python",
-		"ms-python.pylint"
-	],
-
 	"remoteUser": "vscode"
-}
+}
diff --git a/.gitignore b/.gitignore
@@ -369,6 +369,7 @@ main.parameters.json
 infrastructure.env
 infrastructure.debug.env
 infra_output.json
+inf_output.json
 random.txt
 .state
 
@@ -385,4 +386,13 @@ app/backend/shared_code
 packages-microsoft*
 
 # docker container build artifacts
-app/enrichment/shared_code
+app/enrichment/shared_code
+
+#terraform
+.terraform
+infra/.terraform*
+terraform.tfstate
+terraform.tfstate.d
+.tfplan.txt
+infra/infoasst*
+infra/sp_config/config.json
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -4,24 +4,18 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
-
         {
             "name": "Python: WebApp backend",
             "type": "python",
             "request": "launch",
-            "module": "flask",
-            "cwd": "${workspaceFolder}/app/backend",
-            "env": {
-                "FLASK_APP": "app.py",
-                "FLASK_ENV": "development",
-                "FLASK_DEBUG": "0"
-            },
+            "module": "uvicorn",
             "args": [
-                "run",
-                "--no-debugger",
-                "--no-reload",
-                "-p 5000"
+                "app:app",
+                "--reload",
+                "--port",
+                "5000"
             ],
+            "cwd": "${workspaceFolder}/app/backend",
             "console": "integratedTerminal",
             "justMyCode": true,
             "envFile": "${workspaceFolder}/scripts/environments/infrastructure.debug.env",

diff --git a/Makefile b/Makefile
@@ -25,7 +25,7 @@ infrastructure: check-subscription ## Deploy infrastructure
 	@./scripts/inf-create.sh
 
 extract-env: extract-env-debug-webapp extract-env-debug-functions ## Extract infrastructure.env file from BICEP output
-	 @./scripts/json-to-env.sh < infra_output.json > ./scripts/environments/infrastructure.env
+	 @./scripts/json-to-env.sh < inf_output.json > ./scripts/environments/infrastructure.env
 
 deploy-webapp: extract-env ## Deploys the web app code to Azure App Service
 	@./scripts/deploy-webapp.sh
@@ -40,10 +40,10 @@ deploy-search-indexes: extract-env ## Deploy search indexes
 	@./scripts/deploy-search-indexes.sh
 
 extract-env-debug-webapp: ## Extract infrastructure.debug.env file from BICEP output
-	@./scripts/json-to-env.webapp.debug.sh < infra_output.json > ./scripts/environments/infrastructure.debug.env
+	@./scripts/json-to-env.webapp.debug.sh < inf_output.json > ./scripts/environments/infrastructure.debug.env
 
 extract-env-debug-functions: ## Extract local.settings.json to debug functions from BICEP output
-	@./scripts/json-to-env.function.debug.sh < infra_output.json > ./functions/local.settings.json
+	@./scripts/json-to-env.function.debug.sh < inf_output.json > ./functions/local.settings.json
 
 # Utils (used by other Makefile rules)
 check-subscription:
@@ -53,8 +53,16 @@ check-subscription:
 take-dir-ownership:
 	@sudo chown -R vscode .
 
+terraform-remote-backend:
+	@./scripts/terraform-remote-backend.sh
+
+infrastructure-remote-backend: terraform-remote-backend infrastructure
+
 destroy-inf: check-subscription
 	@./scripts/inf-destroy.sh
 
 functional-tests: extract-env ## Run functional tests to check the processing pipeline is working
 	@./scripts/functional-tests.sh	
+
+run-migration: ## Migrate from bicep to terraform
+	python ./scripts/merge-databases.py
diff --git a/README.md b/README.md
@@ -3,8 +3,9 @@
 > [!IMPORTANT]  
 > As of November 15, 2023, Azure Cognitive Search has been renamed to Azure AI Search. Azure Cognitive Services have also been renamed to Azure AI Services.
 
-## Table of Contents
+## Table of Contents 
 
+- [Response Generation Approaches](#response-generation-approaches) 
 - [Features](#features)
 - [Azure account requirements](#azure-account-requirements)
 - [Azure Deployment](./docs/deployment/deployment.md)
@@ -19,6 +20,7 @@
 - [Using the app](/docs/deployment/using_ia_first_time.md)
 - [Responsible AI](#responsible-ai)
   - [Transparency Note](#transparency-note)
+  - [Content Safety](#content-safety)
 - [Data Collection Notice](#data-collection-notice)
 - [Resources](#resources)
   - [Known Issues](./docs/knownissues.md)
@@ -39,17 +41,40 @@ The accelerator adapts prompts based on the model type for enhanced performance.
 
 Please [see this video](https://aka.ms/InfoAssist/video) for use cases that may be achievable with this accelerator.
 
+# Response Generation Approaches
+
+## Work(Grounded)
+It utilizes a retrieval-augmented generation (RAG) pattern to generate responses grounded in specific data sourced from your own dataset. By combining retrieval of relevant information with generative capabilities, It can produce responses that are not only contextually relevant but also grounded in verified data. The RAG pipeline accesses your dataset to retrieve relevant information before generating responses, ensuring accuracy and reliability. Additionally, each response includes a citation to the document chunk from which the answer is derived, providing transparency and allowing users to verify the source. This approach is particularly advantageous in domains where precision and factuality are paramount. Users can trust that the responses generated are based on reliable data sources, enhancing the credibility and usefulness of the application. Specific information on our Grounded (RAG) can be found in [RAG](docs/features/cognitive_search.md#azure-ai-search-integration)
+
+## Ungrounded
+It leverages the capabilities of a large language model (LLM) to generate responses in an ungrounded manner, without relying on external data sources or retrieval-augmented generation techniques. The LLM has been trained on a vast corpus of text data, enabling it to generate coherent and contextually relevant responses solely based on the input provided. This approach allows for open-ended and creative generation, making it suitable for tasks such as ideation, brainstorming, and exploring hypothetical scenarios. It's important to note that the generated responses are not grounded in specific factual data and should be evaluated critically, especially in domains where accuracy and verifiability are paramount.
+
+## Work and Web 
+It offers 3 response options: one generated through our retrieval-augmented generation (RAG) pipeline, and the other grounded in content directly from the web. When users opt for the RAG response, they receive a grounded answer sourced from your data, complete with citations to document chunks for transparency and verification. Conversely, selecting the web response provides access to a broader range of sources, potentially offering more diverse perspectives. Each web response is grounded in content from the web accompanied by citations of web links, allowing users to explore the original sources for further context and validation. Upon request, It can also generate a final response that compares and contrasts both responses. This comparative analysis allows users to make informed decisions based on the reliability, relevance, and context of the information provided.
+Specific information about our Grounded and Web can be found in [Web](/docs/features/features.md#bing-search-and-compare)
+
+## Assistants 
+It generates response by using LLM as a reasoning engine. The key strength lies in agent's ability to autonomously reason about tasks, decompose them into steps, and determine the appropriate tools and data sources to leverage, all without the need for predefined task definitions or rigid workflows. This approach allows for a dynamic and adaptive response generation process without predefining set of tasks. It harnesses the capabilities of LLM to understand natural language queries and generate responses tailored to specific tasks. These Agents are being released in preview mode as we continue to evaluate and mitigate the potential risks associated with autonomous reasoning, such as misuse of external tools, lack of transparency, biased outputs, privacy concerns, and remote code execution vulnerabilities. With future releases, we plan to work to enhance the safety and robustness of these autonomous reasoning capabilities. Specific information on our preview agents can be found in [Assistants](/docs/features/features.md#autonomous-reasoning-with-assistants-agents).
+
+
 ## Features
 
 The IA Accelerator contains several features, many of which have their own documentation.
 
 - Examples of custom Retrieval Augmented Generation (RAG), Prompt Engineering, and Document Pre-Processing
 - Azure AI Search Integration to include text search of both text documents and images
 - Customization and Personalization to enable enhanced AI interaction
+- Preview into autonomous agents
 
 For a detailed review see our [Features](./docs/features/features.md) page.
 
-![Process Flow](/docs/process_flow.png)
+### Process Flow for Work(Grounded), Ungrounded, and Work and Web
+
+![Process Flow for Chat](/docs/process_flow_chat.png)
+
+### Process Flow for Assistants
+
+![Process Flow for Assistants](/docs/process_flow_agent.png)
 
 ## Azure account requirements
 
@@ -61,7 +86,7 @@ For a detailed review see our [Features](./docs/features/features.md) page.
 
     Model Name | Supported Versions
     ---|---
-    gpt-35-turbo | 0301, 0613
+    gpt-35-turbo | current version
     **gpt-35-turbo-16k** | current version
     **gpt-4** | current version
     gpt-4-32k | current version
@@ -97,6 +122,20 @@ The Information Assistant (IA) Accelerator and Microsoft are committed to the ad
 
 Find out more with Microsoft's [Responsible AI resources](https://www.microsoft.com/en-us/ai/responsible-ai)
 
+### Content Safety
+
+Content safety is provided through Azure Open AI service. The Azure OpenAI Service includes a content filtering system that runs alongside the core AI models. This system uses an ensemble of classification models to detect four categories of potentially harmful content (violence, hate, sexual, and self-harm) at four severity levels (safe, low, medium, high).These 4 categories may not be sufficient for all use cases, especially for minors. Please read our [Transaparncy Note](/docs/transparency.md)
+
+By default, the content filters are set to filter out prompts and completions that are detected as medium or high severity for those four harm categories. Content labeled as low or safe severity is not filtered.
+
+There are optional binary classifiers/filters that can detect jailbreak risk (trying to bypass filters) as well as existing text or code pulled from public repositories. These are turned off by default, but some scenarios may require enabling the public content detection models to retain coverage under the customer copyright commitment.
+
+The filtering configuration can be customized at the resource level, allowing customers to adjust the severity thresholds for filtering each harm category separately for prompts and completions. 
+
+This provides controls for Azure customers to tailor the content filtering behavior to their needs while aiming to prevent potentially harmful generated content and any copyright violations from public content.
+
+Instructions on how to confiure content filters via Azure OpenAI Studio can be found here <https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/content-filters#configuring-content-filters-via-azure-openai-studio-preview>
+
 ## Data Collection Notice
 
 The software may collect information about you and your use of the software and send it to Microsoft. Microsoft may use this information to provide services and improve our products and services. You may turn off the telemetry as described in the repository. There are also some features in the software that may enable you and Microsoft to collect data from users of your applications. If you use these features, you must comply with applicable law, including providing appropriate notices to users of your applications together with a copy of Microsoft’s privacy statement. Our privacy statement is located at <https://go.microsoft.com/fwlink/?LinkID=824704>. You can learn more about data collection and use in the help documentation and our privacy statement. Your use of the software operates as your consent to these practices.
@@ -154,3 +193,4 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope
 ### Reporting Security Issues
 
 For security concerns, please see [Security Guidelines](./SECURITY.md)
+